senset/vnpy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #10 from zedyang/master

Browse Source 
通联历史数据模块。
This commit is contained in:
vn.py 2015-08-14 16:27:50 +08:00
commit d4fb389a27
16 changed files with 5422 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
vn.datayes/__init__.py Normal file
View File

1561
vn.datayes/api.py Normal file
View File

File diff suppressed because it is too large Load Diff

2801
vn.datayes/config/db_EQU_D1.csv Normal file
View File

File diff suppressed because it is too large Load Diff

51
vn.datayes/download.sh Executable file
View File

@ -0,0 +1,51 @@
#!/bin/bash
echo [API]: Prepare to construct DATAYES_FUTURE_D1, {20150101, 20150801}...
python - << EOF
from storage import *
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.download_future_D1('20150101','20150801')
EOF
echo [MONGOD]: DATAYES_FUTURE_D1 constructed.
echo [API]: Prepare to construct DATAYES_OPTION_D1, {20150101, 20150801}...
python - << EOF
from storage import *
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.download_option_D1('20150101','20150801')
EOF
echo [MONGOD]: DATAYES_OPTION_D1 constructed.
echo [API]: Prepare to construct DATAYES_INDEX_D1, {20150101, 20150801}...
python - << EOF
from storage import *
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.download_index_D1('20150101','20150801')
EOF
echo [MONGOD]: DATAYES_INDEX_D1 constructed.
echo [API]: Prepare to construct DATAYES_FUND_D1, {20150101, 20150801}...
python - << EOF
from storage import *
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.download_fund_D1('20150101','20150801')
EOF
echo [MONGOD]: DATAYES_FUND_D1 constructed.
echo [API]: Prepare to construct DATAYES_EQUITY_D1, {20130101, 20150801}...
python - << EOF
from storage import *
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.download_equity_D1('20130101','20150801')
EOF
echo [MONGOD]: DATAYES_EQUITY_D1 constructed.

33
vn.datayes/errors.py Normal file
View File

@ -0,0 +1,33 @@
class VNPAST_ConfigError(Exception):
"""
Config file error, raised when config.json or python object
is broken or invalid.
"""
pass
class VNPAST_RequestError(Exception):
"""
HTTP Request Error, raised when response is not properly gotten:
* GET response.status code != 200.
* POST response.status code != 201.
* Connection timed out.
* ...
"""
pass
class VNPAST_DatabaseError(Exception):
"""
"""
pass
class VNPAST_DataConstructorError(Exception):
"""
"""
pass

58
vn.datayes/fun/fetch.R Normal file
View File

@ -0,0 +1,58 @@
ensure_pkgs = function(){
if (!'data.table' %in% installed.packages()){
install.packages('data.table')
}
if (!'rmongodb' %in% installed.packages()){
install.packages('rmongodb')
}
require(data.table)
require(rmongodb)
return(1)
}
get_connection = function(){
client = mongo.create()
return(client)
}
get_dbs = function(){
client = mongo.create()
if(mongo.is.connected(client) == TRUE) {
dbs = mongo.get.databases(client)
}
return(dbs)
}
get_colls = function(db){
client = mongo.create()
if(mongo.is.connected(client) == TRUE) {
colls = mongo.get.database.collections(client, db)
}
return(colls)
}
view_doc = function(coll, one=1){
client = mongo.create()
if(mongo.is.connected(client) == TRUE) {
if(one==1){
doc = mongo.find.one(client, coll)
}
else{
doc = mongo.find.all(client, coll)
}
}
return(doc)
}
fetch = function(coll, start, end){
client = mongo.create()
if(mongo.is.connected(client) == TRUE) {
docs = mongo.find.all(client, coll,
query = list(
'date'=list('lte'=end),
'date'=list('gte'=start)
))
}
return(docs)
}

1
vn.datayes/names/equTicker.json Normal file
View File

File diff suppressed because one or more lines are too long

1
vn.datayes/names/fudTicker.json Normal file
View File

@ -0,0 +1 @@
["150001", "150008", "150009", "150012", "150013", "150016", "150017", "150018", "150019", "150020", "150021", "150022", "150023", "150027", "150028", "150029", "150030", "150031", "150032", "150033", "150034", "150035", "150036", "150037", "150039", "150040", "150042", "150047", "150048", "150049", "150050", "150051", "150052", "150053", "150054", "150055", "150056", "150057", "150058", "150059", "150060", "150064", "150065", "150066", "150067", "150073", "150075", "150076", "150077", "150080", "150083", "150084", "150085", "150086", "150088", "150089", "150090", "150091", "150092", "150093", "150094", "150095", "150096", "150097", "150098", "150099", "150100", "150101", "150102", "150104", "150105", "150106", "150107", "150108", "150109", "150112", "150113", "150114", "150117", "150118", "150120", "150121", "150122", "150123", "150124", "150128", "150129", "150130", "150131", "150133", "150134", "150135", "150136", "150137", "150138", "150139", "150140", "150141", "150142", "150143", "150144", "150145", "150146", "150147", "150148", "150149", "150150", "150151", "150152", "150153", "150154", "150156", "150157", "150158", "150160", "150161", "150164", "150165", "150167", "150168", "150169", "150170", "150171", "150172", "150173", "150174", "150175", "150176", "150177", "150178", "150179", "150180", "150181", "150182", "150184", "150185", "150186", "150187", "150188", "150189", "150190", "150191", "150192", "150193", "150194", "150195", "150196", "150197", "150198", "150199", "150200", "150201", "150203", "150204", "150205", "150206", "150207", "150208", "150209", "150210", "150211", "150212", "150213", "150214", "150215", "150216", "150217", "150218", "150219", "150220", "150221", "150222", "150223", "150224", "150227", "150228", "150229", "150230", "150241", "150242", "159001", "159003", "159005", "159901", "159902", "159903", "159905", "159906", "159907", "159908", "159909", "159910", "159911", "159912", "159913", "159915", "159916", "159917", "159918", "159919", "159920", "159921", "159922", "159923", "159924", "159925", "159926", "159927", "159928", "159929", "159930", "159931", "159932", "159933", "159934", "159935", "159936", "159937", "159938", "159939", "159940", "160105", "160106", "160119", "160123", "160125", "160128", "160130", "160131", "160133", "160211", "160212", "160215", "160216", "160220", "160311", "160314", "160415", "160416", "160505", "160512", "160513", "160515", "160607", "160610", "160611", "160613", "160615", "160616", "160617", "160618", "160621", "160706", "160716", "160717", "160719", "160720", "160805", "160807", "160810", "160812", "160813", "160910", "160915", "160916", "160918", "160919", "161005", "161010", "161015", "161017", "161019", "161115", "161116", "161117", "161119", "161210", "161213", "161216", "161217", "161219", "161222", "161224", "161505", "161607", "161610", "161614", "161706", "161713", "161714", "161716", "161810", "161813", "161815", "161820", "161821", "161831", "161903", "161907", "161908", "161911", "162006", "162105", "162207", "162307", "162308", "162411", "162605", "162607", "162703", "162711", "162712", "162715", "163001", "163110", "163208", "163210", "163302", "163402", "163407", "163409", "163412", "163415", "163503", "163801", "163819", "163821", "163824", "163827", "164105", "164208", "164606", "164701", "164702", "164705", "164808", "164810", "164814", "164815", "164902", "165309", "165311", "165313", "165508", "165509", "165510", "165512", "165513", "165516", "165517", "165705", "165806", "166001", "166006", "166007", "166008", "166009", "166011", "166401", "166902", "166904", "167901", "169101", "184721", "184722", "184728", "500038", "500056", "500058", "502000", "502001", "502002", "502048", "502049", "502050", "505888", "510010", "510020", "510030", "510050", "510060", "510070", "510090", "510110", "510120", "510130", "510150", "510160", "510170", "510180", "510190", "510210", "510220", "510230", "510260", "510270", "510280", "510290", "510300", "510310", "510330", "510410", "510420", "510430", "510440", "510450", "510500", "510510", "510520", "510610", "510620", "510630", "510650", "510660", "510680", "510700", "510880", "510900", "511010", "511210", "511220", "511800", "511810", "511860", "511880", "511990", "512010", "512070", "512110", "512120", "512210", "512220", "512230", "512300", "512310", "512600", "512610", "512640", "512990", "513030", "513100", "513500", "513600", "513660", "518800", "518880"]

1
vn.datayes/names/futTicker.json Normal file
View File

@ -0,0 +1 @@
["a1505", "a1507", "a1509", "a1511", "a1601", "a1603", "a1605", "a1607", "a1609", "Ag(T+D)", "ag1505", "ag1506", "ag1507", "ag1508", "ag1509", "ag1510", "ag1511", "ag1512", "ag1601", "ag1602", "ag1603", "ag1604", "al1505", "al1506", "al1507", "al1508", "al1509", "al1510", "al1511", "al1512", "al1601", "al1602", "al1603", "al1604", "Au(T+D)", "au1505", "au1506", "au1507", "au1508", "au1510", "au1512", "au1602", "au1604", "b1505", "b1507", "b1509", "b1511", "b1601", "b1603", "bb1505", "bb1506", "bb1507", "bb1508", "bb1509", "bb1510", "bb1511", "bb1512", "bb1601", "bb1602", "bb1603", "bb1604", "bu1505", "bu1506", "bu1507", "bu1508", "bu1509", "bu1510", "bu1512", "bu1603", "bu1606", "bu1609", "bu1612", "bu1703", "c1505", "c1507", "c1509", "c1511", "c1601", "c1603", "CBOZF", "CBOZL", "CBOZN", "CBOZT", "CF505", "CF507", "CF509", "CF511", "CF601", "CF603", "COMGC", "COMHG", "COMSI", "cs1505", "cs1507", "cs1509", "cs1511", "cs1601", "cs1603", "cu1505", "cu1506", "cu1507", "cu1508", "cu1509", "cu1510", "cu1511", "cu1512", "cu1601", "cu1602", "cu1603", "cu1604", "fb1505", "fb1506", "fb1507", "fb1508", "fb1509", "fb1510", "fb1511", "fb1512", "fb1601", "fb1602", "fb1603", "fb1604", "FG505", "FG506", "FG507", "FG508", "FG509", "FG510", "FG511", "FG512", "FG601", "FG602", "FG603", "FG604", "fu1506", "fu1507", "fu1508", "fu1509", "fu1510", "fu1511", "fu1512", "fu1601", "fu1603", "fu1604", "fu1605", "hc1505", "hc1506", "hc1507", "hc1508", "hc1509", "hc1510", "hc1511", "hc1512", "hc1601", "hc1602", "hc1603", "hc1604", "i1505", "i1506", "i1507", "i1508", "i1509", "i1510", "i1511", "i1512", "i1601", "i1602", "i1603", "i1604", "IC1505", "IC1506", "IC1509", "IC1512", "IF1505", "IF1506", "IF1509", "IF1512", "IH1505", "IH1506", "IH1509", "IH1512", "j1505", "j1506", "j1507", "j1508", "j1509", "j1510", "j1511", "j1512", "j1601", "j1602", "j1603", "j1604", "jd1505", "jd1506", "jd1509", "jd1510", "jd1511", "jd1512", "jd1601", "jd1602", "jd1603", "jd1604", "jm1505", "jm1506", "jm1507", "jm1508", "jm1509", "jm1510", "jm1511", "jm1512", "jm1601", "jm1602", "jm1603", "jm1604", "JR505", "JR507", "JR509", "JR511", "JR601", "JR603", "l1505", "l1506", "l1507", "l1508", "l1509", "l1510", "l1511", "l1512", "l1601", "l1602", "l1603", "l1604", "LLG", "LR505", "LR507", "LR509", "LR511", "LR601", "LR603", "m1505", "m1507", "m1508", "m1509", "m1511", "m1512", "m1601", "m1603", "MA506", "MA507", "MA508", "MA509", "MA510", "MA511", "MA512", "MA601", "MA602", "MA603", "MA604", "ME505", "ni1507", "ni1508", "ni1509", "ni1510", "ni1511", "ni1512", "ni1601", "ni1602", "ni1603", "ni1604", "NYMBZ", "NYMNG", "OI505", "OI507", "OI509", "OI511", "OI601", "OI603", "p1505", "p1506", "p1507", "p1508", "p1509", "p1510", "p1511", "p1512", "p1601", "p1602", "p1603", "p1604", "pb1505", "pb1506", "pb1507", "pb1508", "pb1509", "pb1510", "pb1511", "pb1512", "pb1601", "pb1602", "pb1603", "pb1604", "PM505", "PM507", "PM509", "PM511", "PM601", "PM603", "pp1505", "pp1506", "pp1507", "pp1508", "pp1509", "pp1510", "pp1511", "pp1512", "pp1601", "pp1602", "pp1603", "pp1604", "rb1505", "rb1506", "rb1507", "rb1508", "rb1509", "rb1510", "rb1511", "rb1512", "rb1601", "rb1602", "rb1603", "rb1604", "RI505", "RI507", "RI509", "RI511", "RI601", "RI603", "RM505", "RM507", "RM508", "RM509", "RM511", "RM601", "RM603", "RS507", "RS508", "RS509", "RS511", "ru1505", "ru1506", "ru1507", "ru1508", "ru1509", "ru1510", "ru1511", "ru1601", "ru1603", "ru1604", "SF505", "SF506", "SF507", "SF508", "SF509", "SF510", "SF511", "SF512", "SF601", "SF602", "SF603", "SF604", "SM505", "SM506", "SM507", "SM508", "SM509", "SM510", "SM511", "SM512", "SM601", "SM602", "SM603", "SM604", "sn1507", "sn1508", "sn1509", "sn1510", "sn1511", "sn1512", "sn1601", "sn1602", "sn1603", "sn1604", "SR505", "SR507", "SR509", "SR511", "SR601", "SR603", "SR605", "SR607", "SR609", "T1509", "T1512", "T1603", "TA505", "TA506", "TA507", "TA508", "TA509", "TA510", "TA511", "TA512", "TA601", "TA602", "TA603", "TA604", "TC506", "TC507", "TC508", "TC509", "TC510", "TC511", "TC512", "TC601", "TC602", "TC603", "TC604", "TF1506", "TF1509", "TF1512", "TOCRU", "v1505", "v1506", "v1507", "v1508", "v1509", "v1510", "v1511", "v1512", "v1601", "v1602", "v1603", "v1604", "WH505", "WH507", "WH509", "WH511", "WH601", "WH603", "wr1505", "wr1506", "wr1507", "wr1508", "wr1509", "wr1510", "wr1511", "wr1512", "wr1601", "wr1602", "wr1603", "wr1604", "y1505", "y1507", "y1508", "y1509", "y1511", "y1512", "y1601", "y1603", "zn1505", "zn1506", "zn1507", "zn1508", "zn1509", "zn1510", "zn1511", "zn1512", "zn1601", "zn1602", "zn1603", "zn1604"]

1
vn.datayes/names/idxTicker.json Normal file
View File

File diff suppressed because one or more lines are too long

1
vn.datayes/names/optTicker.json Normal file
View File

@ -0,0 +1 @@
["510050C1512M02950", "510050P1512M03500", "510050C1512M02900", "510050P1512M02950", "510050P1512M02900", "510050C1505M03200", "510050P1505M03200", "510050C1506M03200", "510050P1506M03200", "510050C1509M03200", "510050P1509M03200", "510050C1506M03500", "510050P1505M03500", "510050C1509M03500", "510050P1506M03500", "510050P1512M03300", "510050P1512M03200", "510050C1505M03500", "510050P1512M03400", "510050P1506M02500", "510050C1509M02500", "510050P1509M02500", "510050C1512M03500", "510050P1509M03500", "510050C1506M02500", "510050C1506M03400", "510050P1505M03400", "510050C1509M03300", "510050P1506M03300", "510050C1506M03300", "510050P1505M03300", "510050C1505M03400", "510050P1509M03300", "510050C1505M03300", "510050P1509M02450", "510050C1509M02450", "510050P1506M02450", "510050C1506M02450", "510050P1509M02400", "510050P1506M03000", "510050C1509M03000", "510050P1505M03000", "510050C1506M03000", "510050C1505M03000", "510050P1509M02950", "510050C1509M02950", "510050P1509M02900", "510050P1509M02600", "510050P1506M02600", "510050C1509M02600", "510050C1506M02650", "510050P1506M02650", "510050C1509M02650", "510050P1509M03000", "510050C1505M03100", "510050C1506M03100", "510050P1505M03100", "510050C1509M03100", "510050P1506M03100", "510050P1509M03100", "510050C1506M02750", "510050P1506M02750", "510050C1509M02750", "510050P1509M02750", "510050P1509M02550", "510050C1506M02550", "510050C1509M02550", "510050P1506M02550", "510050C1506M02600", "510050P1506M02850", "510050C1509M02850", "510050P1509M02850", "510050P1509M02700", "510050C1505M02850", "510050P1505M02800", "510050P1505M02850", "510050C1506M02850", "510050P1509M02650", "510050C1509M02700", "510050P1506M02700", "510050C1506M02700", "510050P1509M02200", "510050P1509M02250", "510050C1509M02350", "510050C1509M02400", "510050C1509M02250", "510050C1509M02300", "510050P1506M02400", "510050C1509M02200", "510050P1509M02300", "510050P1509M02350", "510050P1506M02900", "510050C1506M02950", "510050C1506M02900", "510050P1505M02950", "510050P1505M02900", "510050C1505M02950", "510050C1505M02900", "510050C1509M02900", "510050P1506M02950", "510050C1505M02650", "510050C1505M02700", "510050P1506M02800", "510050C1509M02800", "510050C1506M02800", "510050C1505M02550", "510050C1505M02600", "510050P1509M02800", "510050C1505M02500", "510050P1509M03400", "510050P1506M02300", "510050C1505M02800", "510050C1505M02750", "510050P1505M02700", "510050P1505M02750", "510050P1505M02550", "510050P1505M02500", "510050P1505M02650", "510050P1505M02600", "510050C1506M02300", "510050C1506M02250", "510050P1506M02350", "510050C1512M03000", "510050P1506M03400", "510050C1509M03400", "510050C1512M03300", "510050C1512M03400", "510050C1512M03100", "510050C1512M03200", "510050C1506M02200", "510050P1512M03000", "510050P1512M03100", "510050C1506M02400", "510050C1506M02350", "510050P1506M02250", "510050P1506M02200"]

1
vn.datayes/names/secID.json Normal file
View File

File diff suppressed because one or more lines are too long

84
vn.datayes/prepare.sh Executable file
View File

@ -0,0 +1,84 @@
#!/bin/bash
dir_n=names
if [ ! -d $dir_n ]; then
mkdir $dir_n
fi
dir_c=config
if [ ! -d $dir_c ]; then
mkdir $dir_c
fi
echo [vn-past]: Configuration starts.
python - << EOF
from storage import *
import pandas as pd
import os
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc._collNames['equTicker'] = mc._allEquTickers()
print '[MONGOD]: Equity tickers collected.'
mc._collNames['secID'] = mc._allSecIds()
print '[MONGOD]: Security IDs collected.'
mc._collNames['futTicker'] = mc._allFutTickers()
print '[MONGOD]: Future tickers collected.'
mc._collNames['optTicker'] = mc._allOptTickers()
print '[MONGOD]: Option symbols collected.'
mc._collNames['fudTicker'] = mc._allFudTickers()
print '[MONGOD]: Mutual Fund symbols collected.'
mc._collNames['idxTicker'] = mc._allIdxTickers()
print '[MONGOD]: Index symbols collected.'
mc._ensure_index()
EOF
echo [vn-past]: Configuration finished.
echo [vn-past]: Selected databases:
cd ./names
ls -l
echo [vn-past]: Prepare to construct[c]/update[u] databases...
read -r -p "[vn-past]: Waiting for orders[c/u]: " response
if [[ $response =~ ^([uU][pP][dD][aA][tT][eE]|[uU])$ ]]
then
echo [API]: Prepare to update data...
read -r -p "[API]: Confirm? [y/N] " response
if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]]
then
cd -
chmod +x update.sh
./update.sh
else
echo [vn-past]: Do not update.
:
fi
else
echo [API]: Prepare to download Bars...
read -r -p "[API]: Confirm? [y/N] " response
if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]]
then
cd -
chmod +x download.sh
./download.sh
else
echo [vn-past]: Do not download.
:
fi
fi
echo [vn-past]: Finished.

657
vn.datayes/storage.py Normal file
View File

@ -0,0 +1,657 @@
import os
import json
import pymongo
import pandas as pd
from datetime import datetime, timedelta
from api import Config, PyApi
from api import BaseDataContainer, History, Bar
from errors import (VNPAST_ConfigError, VNPAST_RequestError,
VNPAST_DataConstructorError, VNPAST_DatabaseError)
class DBConfig(Config):
"""
Json-like config object; inherits from Config()
Contains all kinds of settings relating to database settings.
privates
--------
Inherited from api.Config, plus:
* client: pymongo.MongoClient object, the connection
that is to be used for this session.
* body: dictionary; the main content of config.
- client: pymongo.MongoClient(), refers to self.client.
- dbs: dictionary, is a mapping from database alias
to another dictionary, which inclues configurations
and themselves(i.e. pymongo.database entity)
Concretely, dbs has the structure like:
{
alias1 : {
'self': client[dbName1],
'index': dbIndex1,
'collNames': collectionNameType1
},
alias2 : {
'self': client[dbName2],
'index': dbIndex2,
'collNames': collectionNameType2
}, ...
}
where alias#: string;
dbs.alias#.self: pymongo.database;
dbs.alias#.index: string;
dbs.alias#.collNames: string;
- dbNames: list; a list of database alias.
"""
head = 'DB config'
client = pymongo.MongoClient()
body = {
'client': client,
'dbs': {
'EQU_M1': {
'self': client['DATAYES_EQUITY_M1'],
'index': 'dateTime',
'collNames': 'secID'
},
'EQU_D1': {
'self': client['DATAYES_EQUITY_D1'],
'index': 'date',
'collNames': 'equTicker'
},
'FUT_D1': {
'self': client['DATAYES_FUTURE_D1'],
'index': 'date',
'collNames': 'futTicker'
},
'OPT_D1': {
'self': client['DATAYES_OPTION_D1'],
'index': 'date',
'collNames': 'optTicker'
},
'FUD_D1': {
'self': client['DATAYES_FUND_D1'],
'index': 'date',
'collNames': 'fudTicker'
},
'IDX_D1': {
'self': client['DATAYES_INDEX_D1'],
'index': 'date',
'collNames': 'idxTicker'
}
},
'dbNames': ['EQU_M1', 'EQU_D1', 'FUT_D1',
'OPT_D1', 'FUD_D1', 'IDX_D1']
}
def __init__(self, head=None, token=None, body=None):
"""
Inherited constructor.
parameters
----------
* head: string; the name of config file. Default is None.
* token: string; user's token.
* body: dictionary; the main content of config
"""
super(DBConfig, self).__init__(head, token, body)
def view(self):
""" Reloaded Prettify printing method. """
config_view = {
'dbConfig_head' : self.head,
'dbConfig_body' : str(self.body),
}
print json.dumps(config_view,
indent=4,
sort_keys=True)
#----------------------------------------------------------------------
# MongoDB Controller class
class MongodController(object):
"""
The MongoDB controller interface.
MongodController is initialized with a DBConfig configuration
object and a PyApi object, which has already been contructed with
its own Config json. The default version of constructor actually
does nothing special about the database. Yet if user executes shell
script prepare.sh to prepare the connection, MongodController will
firstly gather symbols that are going to become collection names
in corresponding databases. This process is done one database by another,
user can skip useless databases by editing the scripts.
Then, it ensures the index of each collection due to the 'index' value
in DBConfig.body.dbs. Concretely, for D1 bars, the index will be 'date',
and for intraday bars, it will be 'dateTime'; both take the form of
datetime.datetime timestamp.
download() and update() methods of controller dynamically construct
and maintain the databases, requesting data via PyApi. Once the database
is constructed, MongodController can access required data via its fetch()
method.
privates
--------
* _config: DBConfig object; a container of all useful settings for the
databases.
* _api: PyApi object; is responsible for making requests.
* _client: pymongo.MongoClient object; the connection to MongoDB.
* _dbs: dictionary; a mapping from database names to another dictionary,
which includes configurations of the database and the pymongo.database
entity. Inherited from _config.body.['dbs']. Note that keys
self._dbs are mere strings, only self._dbs[key]['self'] refers to the
pymongo.Database object.
* _dbNames: list; a list of names of databases.
* _collNames: dictionary; mapping from self._db[key]['collNames'] attribute
to the names of collections(i.e. tickers) within.
- example: _collNames['equTicker'] = ['000001', '000002', ...]
* _connected: boolean; whether the MongoClient was connected to or not.
* _mapTickersToSecIDs: dictionary; mapping from stock tickers to
its security ID.
example
-------
>> myApi = PyApi(Config())
>> mydbs = DBConfig()
>> controller = MongodController(mydbs, myApi)
>> controller._get_coll_names()
>> controller._ensure_index()
>> controller.download_equity_D1(20130101, 20150801)
>> controller.update_equity_D1()
"""
_config = DBConfig()
_api = None
_client = None
_dbs = None
_dbNames = []
_collNames = dict()
_connected = False
_mapTickersToSecIDs = dict()
def __init__(self, config, api):
"""
Constructor.
parameters
----------
* config: DBConfig object; specifies database configs.
* api: PyApi object.
"""
self._api = api # Set Datayes PyApi.
if config.body:
try:
self._config = config.body
self._client = config.body['client']
self._dbs = config.body['dbs']
self._dbNames = config.body['dbNames']
self._connected = True
except KeyError:
msg = '[MONGOD]: Unable to configure database; ' + \
'config file is incomplete.'
raise VNPAST_ConfigError(msg)
except Exception,e:
msg = '[MONGOD]: Unable to configure database; ' + str(e)
raise VNPAST_ConfigError(msg)
if self._connected:
#self._get_coll_names()
#self._ensure_index()
pass
def view(self):
"""
NOT IMPLEMENTED
"""
return
#----------------------------------------------------------------------
# Get collection names methods.
"""
Decorator;
Targeting at path dName, if exists, read data from this file;
if not, execute handle() which returns a json-like data and
stores the data at dName path.
parameters
----------
* dName: string; the specific path of file that __md looks at.
"""
def __md(dName):
def _md(get):
def handle(*args, **kwargs):
try:
if os.path.isfile(dName):
# if directory exists, read from it.
jsonFile = open(dName,'r')
data = json.loads(jsonFile.read())
jsonFile.close()
else:
# if not, get data via *get method,
# then write to the file.
data = get(*args, **kwargs)
jsonFile = open(dName, 'w+')
jsonFile.write(json.dumps(data))
jsonFile.close()
#print data
return data
except Exception,e:
raise e
return handle
return _md
@__md('names/equTicker.json')
def _allEquTickers(self):
"""get all equity tickers, decorated by @__md()."""
data = self._api.get_equity_D1()
allEquTickers = list(data.body['ticker'])
return allEquTickers
@__md('names/secID.json')
def _allSecIds(self):
"""get all security IDs, decorated by @__md()."""
data = self._api.get_equity_D1()
allTickers = list(data.body['ticker'])
exchangeCDs = list(data.body['exchangeCD'])
allSecIds = [allTickers[k]+'.'+exchangeCDs[k] for k in range(
len(allTickers))]
return allSecIds
@__md('names/futTicker.json')
def _allFutTickers(self):
"""get all future tickers, decorated by @__md()."""
data = self._api.get_future_D1()
allFutTickers = list(data.body['ticker'])
return allFutTickers
@__md('names/optTicker.json')
def _allOptTickers(self):
"""get all option tickers, decorated by @__md()."""
data = self._api.get_option_D1()
allOptTickers = list(data.body['ticker'])
return allOptTickers
@__md('names/fudTicker.json')
def _allFudTickers(self):
"""get all fund tickers, decorated by @__md()."""
data = self._api.get_fund_D1()
allFudTickers = list(data.body['ticker'])
return allFudTickers
@__md('names/idxTicker.json')
def _allIdxTickers(self):
"""get all index tickers, decorated by @__md()."""
data = self._api.get_index_D1()
allIdxTickers = list(data.body['ticker'])
return allIdxTickers
@__md('names/bndTicker.json')
def _allBndTickers(self):
"""get all bond tickers, decorated by @__md()."""
data = self._api.get_bond_D1()
allBndTickers = list(data.body['ticker'])
return allBndTickers
def _get_coll_names(self):
"""
get all instruments'names and store them in self._collNames.
"""
try:
if not os.path.exists('names'):
os.makedirs('names')
self._collNames['equTicker'] = self._allEquTickers()
self._collNames['fudTicker'] = self._allFudTickers()
self._collNames['secID'] = self._allSecIds()
self._collNames['futTicker'] = self._allFutTickers()
self._collNames['optTicker'] = self._allOptTickers()
self._collNames['idxTicker'] = self._allIdxTickers()
print '[MONGOD]: Collection names gotten.'
return 1
except AssertionError:
warning = '[MONGOD]: Warning, collection names ' + \
'is an empty list.'
print warning
except Exception, e:
msg = '[MONGOD]: Unable to set collection names; ' + \
str(e)
raise VNPAST_DatabaseError(msg)
#----------------------------------------------------------------------
# Ensure collection index method.
def _ensure_index(self):
"""
Ensure indices for all databases and collections.
first access self._dbs config to get index column names;
then get collection names from self._collNames and loop
over all collections.
"""
if self._collNames and self._dbs:
try:
for dbName in self._dbs:
# Iterate over database configurations.
db = self._dbs[dbName]
dbSelf = db['self']
index = db['index']
collNames = self._collNames[db['collNames']]
# db['self'] is the pymongo.Database object.
for name in collNames:
coll = dbSelf[name]
coll.ensure_index([(index,
pymongo.DESCENDING)], unique=True)
print '[MONGOD]: MongoDB index set.'
return 1
except KeyError:
msg = '[MONGOD]: Unable to set collection indices; ' + \
'infomation in Config.body["dbs"] is incomplete.'
raise VNPAST_DatabaseError(msg)
except Exception, e:
msg = '[MONGOD]: Unable to set collection indices; ' + str(e)
raise VNPAST_DatabaseError(msg)
#----------------------------------------------------------------------
# Download method.
def download_equity_D1(self, start, end, sessionNum=30):
"""
"""
try:
db = self._dbs['EQU_D1']['self']
self._api.get_equity_D1_mongod(db, start, end, sessionNum)
except Exception, e:
msg = '[MONGOD]: Unable to download data; ' + str(e)
raise VNPAST_DatabaseError(msg)
def download_equity_M1(self, tasks, startYr=2012, endYr=2015):
"""
"""
try:
# map equity tickers to security IDs.
if self._mapTickersToSecIDs:
maps = self._mapTickersToSecIDs
else:
assert os.isfile('./names/secID.json')
jsonFile = open(dName,'r')
allSecIds = json.loads(jsonFile.read())
jsonFile.close()
allTickers = [s.split('.')[0] for s in allSecIds]
maps = dict(zip(allTickers, allSecIds))
self._mapTickersToSecIDs = maps
tasks_ = [maps[task] for task in tasks]
db = self._dbs['EQU_M1']['self']
self._api.get_equity_M1_interMonth(db, id=1,
startYr = startYr,
endYr = endYr,
tasks = tasks_)
except AssertionError:
msg = '[MONGOD]: Cannot map tickers to secIDs; ' + \
'secID.json does not exist.'
raise VNPAST_DatabaseError(msg)
except Exception, e:
msg = '[MONGOD]: Unable to download data; ' + str(e)
raise VNPAST_DatabaseError(msg)
def download_bond_D1(self, start, end, sessionNum=30):
"""
"""
pass
def download_future_D1(self, start, end, sessionNum=30):
"""
"""
try:
db = self._dbs['FUT_D1']['self']
self._api.get_future_D1_mongod(db, start, end, sessionNum)
except Exception, e:
msg = '[MONGOD]: Unable to download data; ' + str(e)
raise VNPAST_DatabaseError(msg)
def download_option_D1(self, start, end, sessionNum=30):
"""
"""
try:
db = self._dbs['OPT_D1']['self']
self._api.get_option_D1_mongod(db, start, end, sessionNum)
except Exception, e:
msg = '[MONGOD]: Unable to download data; ' + str(e)
raise VNPAST_DatabaseError(msg)
def download_index_D1(self, start, end, sessionNum=30):
"""
"""
try:
db = self._dbs['IDX_D1']['self']
self._api.get_index_D1_mongod(db, start, end, sessionNum)
except Exception, e:
msg = '[MONGOD]: Unable to download data; ' + str(e)
raise VNPAST_DatabaseError(msg)
def download_fund_D1(self, start, end, sessionNum=30):
"""
"""
try:
db = self._dbs['FUD_D1']['self']
self._api.get_fund_D1_mongod(db, start, end, sessionNum)
except Exception, e:
msg = '[MONGOD]: Unable to download data; ' + str(e)
raise VNPAST_DatabaseError(msg)
#----------------------------------------------------------------------
# Update methods.
def __update(self, key, target1, target2, sessionNum):
"""
Basic update method.
Looks into the database specified by 'key', find the latest
record in the collection of it. Then update the collections
till last trading date.
parameters
----------
* key: string; a database alias (refer to the database config)
e.g., 'EQU_D1'.
* target1: method; pointer to the function with which controller
obtain all tickers in the database. Concretely, target1 are
self._all#Tickers methods.
* target2: method; pointer to the api overlord requesting functions
i.e. self._api.get_###_mongod methods.
* sessionNum: integer; the number of threads.
"""
try:
# get databases and tickers
db = self._dbs[key]['self']
index = self._dbs[key]['index']
allTickers = target1()
coll = db[allTickers[0]]
# find the latest timestamp in collection.
latest = coll.find_one(
sort=[(index, pymongo.DESCENDING)])[index]
start = datetime.strftime(
latest + timedelta(days=1),'%Y%m%d')
end = datetime.strftime(datetime.now(), '%Y%m%d')
# then download.
target2(db, start, end, sessionNum)
return db
except Exception, e:
msg = '[MONGOD]: Unable to update data; ' + str(e)
raise VNPAST_DatabaseError(msg)
def update_equity_D1(self, sessionNum=30):
"""
"""
db = self.__update(key = 'EQU_D1',
target1 = self._allEquTickers,
target2 = self._api.get_equity_D1_mongod,
sessionNum = sessionNum)
return db
def update_future_D1(self, sessionNum=30):
"""
"""
db = self.__update(key = 'FUT_D1',
target1 = self._allFutTickers,
target2 = self._api.get_future_D1_mongod,
sessionNum = sessionNum)
return db
def update_option_D1(self, sessionNum=30):
"""
"""
db = self.__update(key = 'OPT_D1',
target1 = self._allOptTickers,
target2 = self._api.get_option_D1_mongod,
sessionNum = sessionNum)
return db
def update_index_D1(self, sessionNum=30):
"""
"""
db = self.__update(key = 'IDX_D1',
target1 = self._allIdxTickers,
target2 = self._api.get_index_D1_mongod,
sessionNum = sessionNum)
return db
def update_fund_D1(self, sessionNum=30):
"""
"""
db = self.__update(key = 'FUD_D1',
target1 = self._allFudTickers,
target2 = self._api.get_fund_D1_mongod,
sessionNum = sessionNum)
return db
#----------------------------------------------------------------------#
# stuff that will be deprecated
def update_equity_D1_(self, sessionNum=30):
"""
"""
try:
# set databases and tickers
db = self._dbs['EQU_D1']['self']
index = self._dbs['EQU_D1']['index']
allEquTickers = self._allEquTickers()
coll = db[allEquTickers[0]]
# find the latest timestamp in collection.
latest = coll.find_one(
sort=[(index, pymongo.DESCENDING)])[index]
start = datetime.strftime(latest + timedelta(days=1),'%Y%m%d')
end = datetime.strftime(datetime.now(), '%Y%m%d')
# then download.
self._api.get_equity_D1_mongod(db, start, end, sessionNum)
except Exception, e:
msg = '[MONGOD]: Unable to update data; ' + str(e)
raise VNPAST_DatabaseError(msg)
def update_equity_M1(self):
"""
"""
pass
#----------------------------------------------------------------------
# Fetch method.
def fetch(self, dbName, ticker, start, end, output='list'):
"""
"""
# check inputs' validity.
if output not in ['df', 'list', 'json']:
raise ValueError('[MONGOD]: Unsupported output type.')
if dbName not in self._dbNames:
raise ValueError('[MONGOD]: Unable to locate database name.')
db = self._dbs[dbName]
dbSelf = db['self']
dbIndex = db['index']
try:
coll = db[ticker]
if len(start)==8 and len(end)==8:
# yyyymmdd, len()=8
start = datetime.strptime(start, '%Y%m%d')
end = datetime.strptime(end, '%Y%m%d')
elif len(start)==14 and len(end)==14:
# yyyymmdd HH:MM, len()=14
start = datetime.strptime(start, '%Y%m%d %H:%M')
end = datetime.strptime(end, '%Y%m%d %H:%M')
else:
pass
docs = []
# find in MongoDB.
for doc in coll.find(filter={dbIndex: {'$lte': end,
'$gte': start}}, projection={'_id': False}):
docs.append(doc)
if output == 'list':
return docs[::-1]
except Exception, e:
msg = '[MONGOD]: Error encountered when fetching data' + \
'from MongoDB; '+ str(e)
return -1
if __name__ == '__main__':
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.update_index_D1()

119
vn.datayes/tests.py Normal file
View File

@ -0,0 +1,119 @@
from api import *
def test_config():
cfig = Config()
print cfig.body, cfig.head, cfig.token
cfig.view()
def test_mktbar_D1():
api = PyApi(Config())
data = api.get_equity_D1()
print data.body
def test_mktbar_M1():
api = PyApi(Config())
data = api.get_equity_M1()
print data.body.tail()
def test_bond_D1():
api = PyApi(Config())
data = api.get_bond_D1()
print data.body.tail()
def test_fut_D1():
api = PyApi(Config())
data = api.get_future_D1()
print data.body
def test_fund_D1():
api = PyApi(Config())
data = api.get_fund_D1()
print data.body
def test_index_D1():
api = PyApi(Config())
data = api.get_index_D1()
print data.body
def test_option_D1():
api = PyApi(Config())
data = api.get_option_D1()
print data.body
def test_factors_D1():
api = PyApi(Config())
data = api.get_stockFactor_D1()
print data.body
def test_bs():
api = PyApi(Config())
data = api.get_balanceSheet()
print data.body
def test_cf():
api = PyApi(Config())
data = api.get_cashFlow()
print data.body
def test_is():
api = PyApi(Config())
data = api.get_incomeStatement()
print data.body
def test_output():
api = PyApi(Config())
data = api.get_equity_D1(ticker='000001',output='list')
print data
def test_mongod_get_drudgery():
c = MongoClient()
db = c['test_dy']
api = PyApi(Config())
api.get_equity_D1_drudgery(id=1, db=db,
start='20130101', end='20150801',
tasks=['000001','000002'])
def test_mongod_get_all():
c = MongoClient()
db = c['test_dy']
api = PyApi(Config())
api.get_equity_D1_mongod(db=db, start='20130101', end='20150801')
def test_mktbar_M1_get_drudgery():
c = MongoClient()
db = c['test_dy_m1']
api = PyApi(Config())
api.get_equity_M1_drudgery(id=1, db=db,
start='20150701', end='20150801',
tasks=['000001.XSHE','000002.XSHE'])
def test_mktbar_M1_get_all():
c = MongoClient()
db = c['test_dy_m1']
api = PyApi(Config())
api.get_equity_M1_mongod(db=db)
def test_mktbar_M1_get_interM():
c = MongoClient()
db = c['test_dy_m1']
api = PyApi(Config())
api.get_equity_M1_interMonth(db=db, id=0, tasks=['000001.XSHE','000002.XSHE'])
if __name__ == '__main__':
#test_config()
#test_mktbar_D1()
#test_bond_D1()
#test_fut_D1()
#test_fund_D1()
#test_index_D1()
#test_option_D1()
#test_factors_D1()
#test_mktbar_M1()
#test_bs()
#test_cf()
#test_is()
#test_output()
#test_mongod_get_all()
#test_mktbar_M1_get_drudgery()
#test_mktbar_M1_get_all()
test_mktbar_M1_get_interM()

52
vn.datayes/update.sh Executable file
View File

@ -0,0 +1,52 @@
#!/bin/bash
echo [API]: Prepare to update DATAYES_FUTURE_D1...
python - << EOF
from storage import *
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.update_future_D1()
EOF
echo [API]: DATAYES_FUTURE_D1 updated.
echo [API]: Prepare to update DATAYES_INDEX_D1...
python - << EOF
from storage import *
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.update_index_D1()
EOF
echo [API]: DATAYES_INDEX_D1 updated.
echo [API]: Prepare to update DATAYES_OPTION_D1...
python - << EOF
from storage import *
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.update_option_D1()
EOF
echo [API]: DATAYES_OPTION_D1 updated.
echo [API]: Prepare to update DATAYES_FUND_D1...
python - << EOF
from storage import *
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.update_fund_D1()
EOF
echo [API]: DATAYES_FUND_D1 updated.
echo [MONGOD]: Update finished.
echo [API]: Prepare to update DATAYES_EQUITY_D1...
python - << EOF
from storage import *
dc = DBConfig()
api = PyApi(Config())
mc = MongodController(dc, api)
mc.update_equity_D1()
EOF
echo [API]: DATAYES_EQUITY_D1 updated.