通联历史数据模块。
This commit is contained in:
zedyang
parent
1e793c15f9
commit
444269a50c
0
vn.datayes/__init__.py
Normal file
0
vn.datayes/__init__.py
Normal file
1561
vn.datayes/api.py
Normal file
1561
vn.datayes/api.py
Normal file
File diff suppressed because it is too large
Load Diff
2801
vn.datayes/config/db_EQU_D1.csv
Normal file
2801
vn.datayes/config/db_EQU_D1.csv
Normal file
File diff suppressed because it is too large
Load Diff
51
vn.datayes/download.sh
Executable file
51
vn.datayes/download.sh
Executable file
@ -0,0 +1,51 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo [API]: Prepare to construct DATAYES_FUTURE_D1, {20150101, 20150801}...
|
||||||
|
python - << EOF
|
||||||
|
from storage import *
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
mc.download_future_D1('20150101','20150801')
|
||||||
|
EOF
|
||||||
|
echo [MONGOD]: DATAYES_FUTURE_D1 constructed.
|
||||||
|
|
||||||
|
echo [API]: Prepare to construct DATAYES_OPTION_D1, {20150101, 20150801}...
|
||||||
|
python - << EOF
|
||||||
|
from storage import *
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
mc.download_option_D1('20150101','20150801')
|
||||||
|
EOF
|
||||||
|
echo [MONGOD]: DATAYES_OPTION_D1 constructed.
|
||||||
|
|
||||||
|
echo [API]: Prepare to construct DATAYES_INDEX_D1, {20150101, 20150801}...
|
||||||
|
python - << EOF
|
||||||
|
from storage import *
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
mc.download_index_D1('20150101','20150801')
|
||||||
|
EOF
|
||||||
|
echo [MONGOD]: DATAYES_INDEX_D1 constructed.
|
||||||
|
|
||||||
|
echo [API]: Prepare to construct DATAYES_FUND_D1, {20150101, 20150801}...
|
||||||
|
python - << EOF
|
||||||
|
from storage import *
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
mc.download_fund_D1('20150101','20150801')
|
||||||
|
EOF
|
||||||
|
echo [MONGOD]: DATAYES_FUND_D1 constructed.
|
||||||
|
|
||||||
|
echo [API]: Prepare to construct DATAYES_EQUITY_D1, {20130101, 20150801}...
|
||||||
|
python - << EOF
|
||||||
|
from storage import *
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
mc.download_equity_D1('20130101','20150801')
|
||||||
|
EOF
|
||||||
|
echo [MONGOD]: DATAYES_EQUITY_D1 constructed.
|
||||||
33
vn.datayes/errors.py
Normal file
33
vn.datayes/errors.py
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
|
||||||
|
class VNPAST_ConfigError(Exception):
|
||||||
|
"""
|
||||||
|
Config file error, raised when config.json or python object
|
||||||
|
is broken or invalid.
|
||||||
|
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
class VNPAST_RequestError(Exception):
|
||||||
|
"""
|
||||||
|
HTTP Request Error, raised when response is not properly gotten:
|
||||||
|
* GET response.status code != 200.
|
||||||
|
* POST response.status code != 201.
|
||||||
|
* Connection timed out.
|
||||||
|
* ...
|
||||||
|
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
class VNPAST_DatabaseError(Exception):
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
class VNPAST_DataConstructorError(Exception):
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
pass
|
||||||
58
vn.datayes/fun/fetch.R
Normal file
58
vn.datayes/fun/fetch.R
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
ensure_pkgs = function(){
|
||||||
|
if (!'data.table' %in% installed.packages()){
|
||||||
|
install.packages('data.table')
|
||||||
|
}
|
||||||
|
if (!'rmongodb' %in% installed.packages()){
|
||||||
|
install.packages('rmongodb')
|
||||||
|
}
|
||||||
|
require(data.table)
|
||||||
|
require(rmongodb)
|
||||||
|
return(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
get_connection = function(){
|
||||||
|
client = mongo.create()
|
||||||
|
return(client)
|
||||||
|
}
|
||||||
|
|
||||||
|
get_dbs = function(){
|
||||||
|
client = mongo.create()
|
||||||
|
if(mongo.is.connected(client) == TRUE) {
|
||||||
|
dbs = mongo.get.databases(client)
|
||||||
|
}
|
||||||
|
return(dbs)
|
||||||
|
}
|
||||||
|
|
||||||
|
get_colls = function(db){
|
||||||
|
client = mongo.create()
|
||||||
|
if(mongo.is.connected(client) == TRUE) {
|
||||||
|
colls = mongo.get.database.collections(client, db)
|
||||||
|
}
|
||||||
|
return(colls)
|
||||||
|
}
|
||||||
|
|
||||||
|
view_doc = function(coll, one=1){
|
||||||
|
client = mongo.create()
|
||||||
|
if(mongo.is.connected(client) == TRUE) {
|
||||||
|
if(one==1){
|
||||||
|
doc = mongo.find.one(client, coll)
|
||||||
|
}
|
||||||
|
else{
|
||||||
|
doc = mongo.find.all(client, coll)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return(doc)
|
||||||
|
}
|
||||||
|
|
||||||
|
fetch = function(coll, start, end){
|
||||||
|
client = mongo.create()
|
||||||
|
if(mongo.is.connected(client) == TRUE) {
|
||||||
|
docs = mongo.find.all(client, coll,
|
||||||
|
query = list(
|
||||||
|
'date'=list('lte'=end),
|
||||||
|
'date'=list('gte'=start)
|
||||||
|
))
|
||||||
|
}
|
||||||
|
return(docs)
|
||||||
|
}
|
||||||
|
|
||||||
1
vn.datayes/names/equTicker.json
Normal file
1
vn.datayes/names/equTicker.json
Normal file
File diff suppressed because one or more lines are too long
1
vn.datayes/names/fudTicker.json
Normal file
1
vn.datayes/names/fudTicker.json
Normal file
@ -0,0 +1 @@
|
|||||||
|
["150001", "150008", "150009", "150012", "150013", "150016", "150017", "150018", "150019", "150020", "150021", "150022", "150023", "150027", "150028", "150029", "150030", "150031", "150032", "150033", "150034", "150035", "150036", "150037", "150039", "150040", "150042", "150047", "150048", "150049", "150050", "150051", "150052", "150053", "150054", "150055", "150056", "150057", "150058", "150059", "150060", "150064", "150065", "150066", "150067", "150073", "150075", "150076", "150077", "150080", "150083", "150084", "150085", "150086", "150088", "150089", "150090", "150091", "150092", "150093", "150094", "150095", "150096", "150097", "150098", "150099", "150100", "150101", "150102", "150104", "150105", "150106", "150107", "150108", "150109", "150112", "150113", "150114", "150117", "150118", "150120", "150121", "150122", "150123", "150124", "150128", "150129", "150130", "150131", "150133", "150134", "150135", "150136", "150137", "150138", "150139", "150140", "150141", "150142", "150143", "150144", "150145", "150146", "150147", "150148", "150149", "150150", "150151", "150152", "150153", "150154", "150156", "150157", "150158", "150160", "150161", "150164", "150165", "150167", "150168", "150169", "150170", "150171", "150172", "150173", "150174", "150175", "150176", "150177", "150178", "150179", "150180", "150181", "150182", "150184", "150185", "150186", "150187", "150188", "150189", "150190", "150191", "150192", "150193", "150194", "150195", "150196", "150197", "150198", "150199", "150200", "150201", "150203", "150204", "150205", "150206", "150207", "150208", "150209", "150210", "150211", "150212", "150213", "150214", "150215", "150216", "150217", "150218", "150219", "150220", "150221", "150222", "150223", "150224", "150227", "150228", "150229", "150230", "150241", "150242", "159001", "159003", "159005", "159901", "159902", "159903", "159905", "159906", "159907", "159908", "159909", "159910", "159911", "159912", "159913", "159915", "159916", "159917", "159918", "159919", "159920", "159921", "159922", "159923", "159924", "159925", "159926", "159927", "159928", "159929", "159930", "159931", "159932", "159933", "159934", "159935", "159936", "159937", "159938", "159939", "159940", "160105", "160106", "160119", "160123", "160125", "160128", "160130", "160131", "160133", "160211", "160212", "160215", "160216", "160220", "160311", "160314", "160415", "160416", "160505", "160512", "160513", "160515", "160607", "160610", "160611", "160613", "160615", "160616", "160617", "160618", "160621", "160706", "160716", "160717", "160719", "160720", "160805", "160807", "160810", "160812", "160813", "160910", "160915", "160916", "160918", "160919", "161005", "161010", "161015", "161017", "161019", "161115", "161116", "161117", "161119", "161210", "161213", "161216", "161217", "161219", "161222", "161224", "161505", "161607", "161610", "161614", "161706", "161713", "161714", "161716", "161810", "161813", "161815", "161820", "161821", "161831", "161903", "161907", "161908", "161911", "162006", "162105", "162207", "162307", "162308", "162411", "162605", "162607", "162703", "162711", "162712", "162715", "163001", "163110", "163208", "163210", "163302", "163402", "163407", "163409", "163412", "163415", "163503", "163801", "163819", "163821", "163824", "163827", "164105", "164208", "164606", "164701", "164702", "164705", "164808", "164810", "164814", "164815", "164902", "165309", "165311", "165313", "165508", "165509", "165510", "165512", "165513", "165516", "165517", "165705", "165806", "166001", "166006", "166007", "166008", "166009", "166011", "166401", "166902", "166904", "167901", "169101", "184721", "184722", "184728", "500038", "500056", "500058", "502000", "502001", "502002", "502048", "502049", "502050", "505888", "510010", "510020", "510030", "510050", "510060", "510070", "510090", "510110", "510120", "510130", "510150", "510160", "510170", "510180", "510190", "510210", "510220", "510230", "510260", "510270", "510280", "510290", "510300", "510310", "510330", "510410", "510420", "510430", "510440", "510450", "510500", "510510", "510520", "510610", "510620", "510630", "510650", "510660", "510680", "510700", "510880", "510900", "511010", "511210", "511220", "511800", "511810", "511860", "511880", "511990", "512010", "512070", "512110", "512120", "512210", "512220", "512230", "512300", "512310", "512600", "512610", "512640", "512990", "513030", "513100", "513500", "513600", "513660", "518800", "518880"]
|
||||||
1
vn.datayes/names/futTicker.json
Normal file
1
vn.datayes/names/futTicker.json
Normal file
@ -0,0 +1 @@
|
|||||||
|
["a1505", "a1507", "a1509", "a1511", "a1601", "a1603", "a1605", "a1607", "a1609", "Ag(T+D)", "ag1505", "ag1506", "ag1507", "ag1508", "ag1509", "ag1510", "ag1511", "ag1512", "ag1601", "ag1602", "ag1603", "ag1604", "al1505", "al1506", "al1507", "al1508", "al1509", "al1510", "al1511", "al1512", "al1601", "al1602", "al1603", "al1604", "Au(T+D)", "au1505", "au1506", "au1507", "au1508", "au1510", "au1512", "au1602", "au1604", "b1505", "b1507", "b1509", "b1511", "b1601", "b1603", "bb1505", "bb1506", "bb1507", "bb1508", "bb1509", "bb1510", "bb1511", "bb1512", "bb1601", "bb1602", "bb1603", "bb1604", "bu1505", "bu1506", "bu1507", "bu1508", "bu1509", "bu1510", "bu1512", "bu1603", "bu1606", "bu1609", "bu1612", "bu1703", "c1505", "c1507", "c1509", "c1511", "c1601", "c1603", "CBOZF", "CBOZL", "CBOZN", "CBOZT", "CF505", "CF507", "CF509", "CF511", "CF601", "CF603", "COMGC", "COMHG", "COMSI", "cs1505", "cs1507", "cs1509", "cs1511", "cs1601", "cs1603", "cu1505", "cu1506", "cu1507", "cu1508", "cu1509", "cu1510", "cu1511", "cu1512", "cu1601", "cu1602", "cu1603", "cu1604", "fb1505", "fb1506", "fb1507", "fb1508", "fb1509", "fb1510", "fb1511", "fb1512", "fb1601", "fb1602", "fb1603", "fb1604", "FG505", "FG506", "FG507", "FG508", "FG509", "FG510", "FG511", "FG512", "FG601", "FG602", "FG603", "FG604", "fu1506", "fu1507", "fu1508", "fu1509", "fu1510", "fu1511", "fu1512", "fu1601", "fu1603", "fu1604", "fu1605", "hc1505", "hc1506", "hc1507", "hc1508", "hc1509", "hc1510", "hc1511", "hc1512", "hc1601", "hc1602", "hc1603", "hc1604", "i1505", "i1506", "i1507", "i1508", "i1509", "i1510", "i1511", "i1512", "i1601", "i1602", "i1603", "i1604", "IC1505", "IC1506", "IC1509", "IC1512", "IF1505", "IF1506", "IF1509", "IF1512", "IH1505", "IH1506", "IH1509", "IH1512", "j1505", "j1506", "j1507", "j1508", "j1509", "j1510", "j1511", "j1512", "j1601", "j1602", "j1603", "j1604", "jd1505", "jd1506", "jd1509", "jd1510", "jd1511", "jd1512", "jd1601", "jd1602", "jd1603", "jd1604", "jm1505", "jm1506", "jm1507", "jm1508", "jm1509", "jm1510", "jm1511", "jm1512", "jm1601", "jm1602", "jm1603", "jm1604", "JR505", "JR507", "JR509", "JR511", "JR601", "JR603", "l1505", "l1506", "l1507", "l1508", "l1509", "l1510", "l1511", "l1512", "l1601", "l1602", "l1603", "l1604", "LLG", "LR505", "LR507", "LR509", "LR511", "LR601", "LR603", "m1505", "m1507", "m1508", "m1509", "m1511", "m1512", "m1601", "m1603", "MA506", "MA507", "MA508", "MA509", "MA510", "MA511", "MA512", "MA601", "MA602", "MA603", "MA604", "ME505", "ni1507", "ni1508", "ni1509", "ni1510", "ni1511", "ni1512", "ni1601", "ni1602", "ni1603", "ni1604", "NYMBZ", "NYMNG", "OI505", "OI507", "OI509", "OI511", "OI601", "OI603", "p1505", "p1506", "p1507", "p1508", "p1509", "p1510", "p1511", "p1512", "p1601", "p1602", "p1603", "p1604", "pb1505", "pb1506", "pb1507", "pb1508", "pb1509", "pb1510", "pb1511", "pb1512", "pb1601", "pb1602", "pb1603", "pb1604", "PM505", "PM507", "PM509", "PM511", "PM601", "PM603", "pp1505", "pp1506", "pp1507", "pp1508", "pp1509", "pp1510", "pp1511", "pp1512", "pp1601", "pp1602", "pp1603", "pp1604", "rb1505", "rb1506", "rb1507", "rb1508", "rb1509", "rb1510", "rb1511", "rb1512", "rb1601", "rb1602", "rb1603", "rb1604", "RI505", "RI507", "RI509", "RI511", "RI601", "RI603", "RM505", "RM507", "RM508", "RM509", "RM511", "RM601", "RM603", "RS507", "RS508", "RS509", "RS511", "ru1505", "ru1506", "ru1507", "ru1508", "ru1509", "ru1510", "ru1511", "ru1601", "ru1603", "ru1604", "SF505", "SF506", "SF507", "SF508", "SF509", "SF510", "SF511", "SF512", "SF601", "SF602", "SF603", "SF604", "SM505", "SM506", "SM507", "SM508", "SM509", "SM510", "SM511", "SM512", "SM601", "SM602", "SM603", "SM604", "sn1507", "sn1508", "sn1509", "sn1510", "sn1511", "sn1512", "sn1601", "sn1602", "sn1603", "sn1604", "SR505", "SR507", "SR509", "SR511", "SR601", "SR603", "SR605", "SR607", "SR609", "T1509", "T1512", "T1603", "TA505", "TA506", "TA507", "TA508", "TA509", "TA510", "TA511", "TA512", "TA601", "TA602", "TA603", "TA604", "TC506", "TC507", "TC508", "TC509", "TC510", "TC511", "TC512", "TC601", "TC602", "TC603", "TC604", "TF1506", "TF1509", "TF1512", "TOCRU", "v1505", "v1506", "v1507", "v1508", "v1509", "v1510", "v1511", "v1512", "v1601", "v1602", "v1603", "v1604", "WH505", "WH507", "WH509", "WH511", "WH601", "WH603", "wr1505", "wr1506", "wr1507", "wr1508", "wr1509", "wr1510", "wr1511", "wr1512", "wr1601", "wr1602", "wr1603", "wr1604", "y1505", "y1507", "y1508", "y1509", "y1511", "y1512", "y1601", "y1603", "zn1505", "zn1506", "zn1507", "zn1508", "zn1509", "zn1510", "zn1511", "zn1512", "zn1601", "zn1602", "zn1603", "zn1604"]
|
||||||
1
vn.datayes/names/idxTicker.json
Normal file
1
vn.datayes/names/idxTicker.json
Normal file
File diff suppressed because one or more lines are too long
1
vn.datayes/names/optTicker.json
Normal file
1
vn.datayes/names/optTicker.json
Normal file
@ -0,0 +1 @@
|
|||||||
|
["510050C1512M02950", "510050P1512M03500", "510050C1512M02900", "510050P1512M02950", "510050P1512M02900", "510050C1505M03200", "510050P1505M03200", "510050C1506M03200", "510050P1506M03200", "510050C1509M03200", "510050P1509M03200", "510050C1506M03500", "510050P1505M03500", "510050C1509M03500", "510050P1506M03500", "510050P1512M03300", "510050P1512M03200", "510050C1505M03500", "510050P1512M03400", "510050P1506M02500", "510050C1509M02500", "510050P1509M02500", "510050C1512M03500", "510050P1509M03500", "510050C1506M02500", "510050C1506M03400", "510050P1505M03400", "510050C1509M03300", "510050P1506M03300", "510050C1506M03300", "510050P1505M03300", "510050C1505M03400", "510050P1509M03300", "510050C1505M03300", "510050P1509M02450", "510050C1509M02450", "510050P1506M02450", "510050C1506M02450", "510050P1509M02400", "510050P1506M03000", "510050C1509M03000", "510050P1505M03000", "510050C1506M03000", "510050C1505M03000", "510050P1509M02950", "510050C1509M02950", "510050P1509M02900", "510050P1509M02600", "510050P1506M02600", "510050C1509M02600", "510050C1506M02650", "510050P1506M02650", "510050C1509M02650", "510050P1509M03000", "510050C1505M03100", "510050C1506M03100", "510050P1505M03100", "510050C1509M03100", "510050P1506M03100", "510050P1509M03100", "510050C1506M02750", "510050P1506M02750", "510050C1509M02750", "510050P1509M02750", "510050P1509M02550", "510050C1506M02550", "510050C1509M02550", "510050P1506M02550", "510050C1506M02600", "510050P1506M02850", "510050C1509M02850", "510050P1509M02850", "510050P1509M02700", "510050C1505M02850", "510050P1505M02800", "510050P1505M02850", "510050C1506M02850", "510050P1509M02650", "510050C1509M02700", "510050P1506M02700", "510050C1506M02700", "510050P1509M02200", "510050P1509M02250", "510050C1509M02350", "510050C1509M02400", "510050C1509M02250", "510050C1509M02300", "510050P1506M02400", "510050C1509M02200", "510050P1509M02300", "510050P1509M02350", "510050P1506M02900", "510050C1506M02950", "510050C1506M02900", "510050P1505M02950", "510050P1505M02900", "510050C1505M02950", "510050C1505M02900", "510050C1509M02900", "510050P1506M02950", "510050C1505M02650", "510050C1505M02700", "510050P1506M02800", "510050C1509M02800", "510050C1506M02800", "510050C1505M02550", "510050C1505M02600", "510050P1509M02800", "510050C1505M02500", "510050P1509M03400", "510050P1506M02300", "510050C1505M02800", "510050C1505M02750", "510050P1505M02700", "510050P1505M02750", "510050P1505M02550", "510050P1505M02500", "510050P1505M02650", "510050P1505M02600", "510050C1506M02300", "510050C1506M02250", "510050P1506M02350", "510050C1512M03000", "510050P1506M03400", "510050C1509M03400", "510050C1512M03300", "510050C1512M03400", "510050C1512M03100", "510050C1512M03200", "510050C1506M02200", "510050P1512M03000", "510050P1512M03100", "510050C1506M02400", "510050C1506M02350", "510050P1506M02250", "510050P1506M02200"]
|
||||||
1
vn.datayes/names/secID.json
Normal file
1
vn.datayes/names/secID.json
Normal file
File diff suppressed because one or more lines are too long
84
vn.datayes/prepare.sh
Executable file
84
vn.datayes/prepare.sh
Executable file
@ -0,0 +1,84 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
dir_n=names
|
||||||
|
if [ ! -d $dir_n ]; then
|
||||||
|
mkdir $dir_n
|
||||||
|
fi
|
||||||
|
|
||||||
|
dir_c=config
|
||||||
|
if [ ! -d $dir_c ]; then
|
||||||
|
mkdir $dir_c
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo [vn-past]: Configuration starts.
|
||||||
|
|
||||||
|
python - << EOF
|
||||||
|
|
||||||
|
from storage import *
|
||||||
|
import pandas as pd
|
||||||
|
import os
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
|
||||||
|
mc._collNames['equTicker'] = mc._allEquTickers()
|
||||||
|
print '[MONGOD]: Equity tickers collected.'
|
||||||
|
|
||||||
|
mc._collNames['secID'] = mc._allSecIds()
|
||||||
|
print '[MONGOD]: Security IDs collected.'
|
||||||
|
|
||||||
|
mc._collNames['futTicker'] = mc._allFutTickers()
|
||||||
|
print '[MONGOD]: Future tickers collected.'
|
||||||
|
|
||||||
|
mc._collNames['optTicker'] = mc._allOptTickers()
|
||||||
|
print '[MONGOD]: Option symbols collected.'
|
||||||
|
|
||||||
|
mc._collNames['fudTicker'] = mc._allFudTickers()
|
||||||
|
print '[MONGOD]: Mutual Fund symbols collected.'
|
||||||
|
|
||||||
|
mc._collNames['idxTicker'] = mc._allIdxTickers()
|
||||||
|
print '[MONGOD]: Index symbols collected.'
|
||||||
|
|
||||||
|
mc._ensure_index()
|
||||||
|
|
||||||
|
EOF
|
||||||
|
|
||||||
|
|
||||||
|
echo [vn-past]: Configuration finished.
|
||||||
|
echo [vn-past]: Selected databases:
|
||||||
|
cd ./names
|
||||||
|
ls -l
|
||||||
|
|
||||||
|
echo [vn-past]: Prepare to construct[c]/update[u] databases...
|
||||||
|
|
||||||
|
read -r -p "[vn-past]: Waiting for orders[c/u]: " response
|
||||||
|
if [[ $response =~ ^([uU][pP][dD][aA][tT][eE]|[uU])$ ]]
|
||||||
|
then
|
||||||
|
echo [API]: Prepare to update data...
|
||||||
|
read -r -p "[API]: Confirm? [y/N] " response
|
||||||
|
if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]]
|
||||||
|
then
|
||||||
|
cd -
|
||||||
|
chmod +x update.sh
|
||||||
|
./update.sh
|
||||||
|
|
||||||
|
else
|
||||||
|
echo [vn-past]: Do not update.
|
||||||
|
:
|
||||||
|
fi
|
||||||
|
|
||||||
|
else
|
||||||
|
echo [API]: Prepare to download Bars...
|
||||||
|
read -r -p "[API]: Confirm? [y/N] " response
|
||||||
|
if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]]
|
||||||
|
then
|
||||||
|
cd -
|
||||||
|
chmod +x download.sh
|
||||||
|
./download.sh
|
||||||
|
|
||||||
|
else
|
||||||
|
echo [vn-past]: Do not download.
|
||||||
|
:
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
echo [vn-past]: Finished.
|
||||||
657
vn.datayes/storage.py
Normal file
657
vn.datayes/storage.py
Normal file
@ -0,0 +1,657 @@
|
|||||||
|
import os
|
||||||
|
import json
|
||||||
|
import pymongo
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from api import Config, PyApi
|
||||||
|
from api import BaseDataContainer, History, Bar
|
||||||
|
|
||||||
|
from errors import (VNPAST_ConfigError, VNPAST_RequestError,
|
||||||
|
VNPAST_DataConstructorError, VNPAST_DatabaseError)
|
||||||
|
|
||||||
|
class DBConfig(Config):
|
||||||
|
"""
|
||||||
|
Json-like config object; inherits from Config()
|
||||||
|
|
||||||
|
Contains all kinds of settings relating to database settings.
|
||||||
|
|
||||||
|
privates
|
||||||
|
--------
|
||||||
|
Inherited from api.Config, plus:
|
||||||
|
|
||||||
|
* client: pymongo.MongoClient object, the connection
|
||||||
|
that is to be used for this session.
|
||||||
|
* body: dictionary; the main content of config.
|
||||||
|
|
||||||
|
- client: pymongo.MongoClient(), refers to self.client.
|
||||||
|
|
||||||
|
- dbs: dictionary, is a mapping from database alias
|
||||||
|
to another dictionary, which inclues configurations
|
||||||
|
and themselves(i.e. pymongo.database entity)
|
||||||
|
Concretely, dbs has the structure like:
|
||||||
|
{
|
||||||
|
alias1 : {
|
||||||
|
'self': client[dbName1],
|
||||||
|
'index': dbIndex1,
|
||||||
|
'collNames': collectionNameType1
|
||||||
|
},
|
||||||
|
alias2 : {
|
||||||
|
'self': client[dbName2],
|
||||||
|
'index': dbIndex2,
|
||||||
|
'collNames': collectionNameType2
|
||||||
|
}, ...
|
||||||
|
}
|
||||||
|
where alias#: string;
|
||||||
|
dbs.alias#.self: pymongo.database;
|
||||||
|
dbs.alias#.index: string;
|
||||||
|
dbs.alias#.collNames: string;
|
||||||
|
|
||||||
|
- dbNames: list; a list of database alias.
|
||||||
|
|
||||||
|
"""
|
||||||
|
head = 'DB config'
|
||||||
|
|
||||||
|
client = pymongo.MongoClient()
|
||||||
|
body = {
|
||||||
|
'client': client,
|
||||||
|
'dbs': {
|
||||||
|
'EQU_M1': {
|
||||||
|
'self': client['DATAYES_EQUITY_M1'],
|
||||||
|
'index': 'dateTime',
|
||||||
|
'collNames': 'secID'
|
||||||
|
},
|
||||||
|
'EQU_D1': {
|
||||||
|
'self': client['DATAYES_EQUITY_D1'],
|
||||||
|
'index': 'date',
|
||||||
|
'collNames': 'equTicker'
|
||||||
|
},
|
||||||
|
'FUT_D1': {
|
||||||
|
'self': client['DATAYES_FUTURE_D1'],
|
||||||
|
'index': 'date',
|
||||||
|
'collNames': 'futTicker'
|
||||||
|
},
|
||||||
|
'OPT_D1': {
|
||||||
|
'self': client['DATAYES_OPTION_D1'],
|
||||||
|
'index': 'date',
|
||||||
|
'collNames': 'optTicker'
|
||||||
|
},
|
||||||
|
'FUD_D1': {
|
||||||
|
'self': client['DATAYES_FUND_D1'],
|
||||||
|
'index': 'date',
|
||||||
|
'collNames': 'fudTicker'
|
||||||
|
},
|
||||||
|
'IDX_D1': {
|
||||||
|
'self': client['DATAYES_INDEX_D1'],
|
||||||
|
'index': 'date',
|
||||||
|
'collNames': 'idxTicker'
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'dbNames': ['EQU_M1', 'EQU_D1', 'FUT_D1',
|
||||||
|
'OPT_D1', 'FUD_D1', 'IDX_D1']
|
||||||
|
}
|
||||||
|
|
||||||
|
def __init__(self, head=None, token=None, body=None):
|
||||||
|
"""
|
||||||
|
Inherited constructor.
|
||||||
|
|
||||||
|
parameters
|
||||||
|
----------
|
||||||
|
* head: string; the name of config file. Default is None.
|
||||||
|
* token: string; user's token.
|
||||||
|
* body: dictionary; the main content of config
|
||||||
|
"""
|
||||||
|
super(DBConfig, self).__init__(head, token, body)
|
||||||
|
|
||||||
|
def view(self):
|
||||||
|
""" Reloaded Prettify printing method. """
|
||||||
|
config_view = {
|
||||||
|
'dbConfig_head' : self.head,
|
||||||
|
'dbConfig_body' : str(self.body),
|
||||||
|
}
|
||||||
|
print json.dumps(config_view,
|
||||||
|
indent=4,
|
||||||
|
sort_keys=True)
|
||||||
|
|
||||||
|
#----------------------------------------------------------------------
|
||||||
|
# MongoDB Controller class
|
||||||
|
|
||||||
|
class MongodController(object):
|
||||||
|
"""
|
||||||
|
The MongoDB controller interface.
|
||||||
|
|
||||||
|
MongodController is initialized with a DBConfig configuration
|
||||||
|
object and a PyApi object, which has already been contructed with
|
||||||
|
its own Config json. The default version of constructor actually
|
||||||
|
does nothing special about the database. Yet if user executes shell
|
||||||
|
script prepare.sh to prepare the connection, MongodController will
|
||||||
|
firstly gather symbols that are going to become collection names
|
||||||
|
in corresponding databases. This process is done one database by another,
|
||||||
|
user can skip useless databases by editing the scripts.
|
||||||
|
Then, it ensures the index of each collection due to the 'index' value
|
||||||
|
in DBConfig.body.dbs. Concretely, for D1 bars, the index will be 'date',
|
||||||
|
and for intraday bars, it will be 'dateTime'; both take the form of
|
||||||
|
datetime.datetime timestamp.
|
||||||
|
|
||||||
|
download() and update() methods of controller dynamically construct
|
||||||
|
and maintain the databases, requesting data via PyApi. Once the database
|
||||||
|
is constructed, MongodController can access required data via its fetch()
|
||||||
|
method.
|
||||||
|
|
||||||
|
|
||||||
|
privates
|
||||||
|
--------
|
||||||
|
* _config: DBConfig object; a container of all useful settings for the
|
||||||
|
databases.
|
||||||
|
|
||||||
|
* _api: PyApi object; is responsible for making requests.
|
||||||
|
|
||||||
|
* _client: pymongo.MongoClient object; the connection to MongoDB.
|
||||||
|
|
||||||
|
* _dbs: dictionary; a mapping from database names to another dictionary,
|
||||||
|
which includes configurations of the database and the pymongo.database
|
||||||
|
entity. Inherited from _config.body.['dbs']. Note that keys
|
||||||
|
self._dbs are mere strings, only self._dbs[key]['self'] refers to the
|
||||||
|
pymongo.Database object.
|
||||||
|
|
||||||
|
* _dbNames: list; a list of names of databases.
|
||||||
|
|
||||||
|
* _collNames: dictionary; mapping from self._db[key]['collNames'] attribute
|
||||||
|
to the names of collections(i.e. tickers) within.
|
||||||
|
- example: _collNames['equTicker'] = ['000001', '000002', ...]
|
||||||
|
|
||||||
|
* _connected: boolean; whether the MongoClient was connected to or not.
|
||||||
|
|
||||||
|
* _mapTickersToSecIDs: dictionary; mapping from stock tickers to
|
||||||
|
its security ID.
|
||||||
|
|
||||||
|
|
||||||
|
example
|
||||||
|
-------
|
||||||
|
>> myApi = PyApi(Config())
|
||||||
|
>> mydbs = DBConfig()
|
||||||
|
>> controller = MongodController(mydbs, myApi)
|
||||||
|
>> controller._get_coll_names()
|
||||||
|
>> controller._ensure_index()
|
||||||
|
>> controller.download_equity_D1(20130101, 20150801)
|
||||||
|
>> controller.update_equity_D1()
|
||||||
|
|
||||||
|
"""
|
||||||
|
_config = DBConfig()
|
||||||
|
_api = None
|
||||||
|
|
||||||
|
_client = None
|
||||||
|
_dbs = None
|
||||||
|
_dbNames = []
|
||||||
|
_collNames = dict()
|
||||||
|
_connected = False
|
||||||
|
|
||||||
|
_mapTickersToSecIDs = dict()
|
||||||
|
|
||||||
|
def __init__(self, config, api):
|
||||||
|
"""
|
||||||
|
Constructor.
|
||||||
|
|
||||||
|
parameters
|
||||||
|
----------
|
||||||
|
* config: DBConfig object; specifies database configs.
|
||||||
|
* api: PyApi object.
|
||||||
|
|
||||||
|
"""
|
||||||
|
self._api = api # Set Datayes PyApi.
|
||||||
|
if config.body:
|
||||||
|
try:
|
||||||
|
self._config = config.body
|
||||||
|
self._client = config.body['client']
|
||||||
|
self._dbs = config.body['dbs']
|
||||||
|
self._dbNames = config.body['dbNames']
|
||||||
|
self._connected = True
|
||||||
|
except KeyError:
|
||||||
|
msg = '[MONGOD]: Unable to configure database; ' + \
|
||||||
|
'config file is incomplete.'
|
||||||
|
raise VNPAST_ConfigError(msg)
|
||||||
|
except Exception,e:
|
||||||
|
msg = '[MONGOD]: Unable to configure database; ' + str(e)
|
||||||
|
raise VNPAST_ConfigError(msg)
|
||||||
|
|
||||||
|
if self._connected:
|
||||||
|
#self._get_coll_names()
|
||||||
|
#self._ensure_index()
|
||||||
|
pass
|
||||||
|
|
||||||
|
def view(self):
|
||||||
|
"""
|
||||||
|
NOT IMPLEMENTED
|
||||||
|
"""
|
||||||
|
return
|
||||||
|
|
||||||
|
#----------------------------------------------------------------------
|
||||||
|
# Get collection names methods.
|
||||||
|
|
||||||
|
"""
|
||||||
|
Decorator;
|
||||||
|
Targeting at path dName, if exists, read data from this file;
|
||||||
|
if not, execute handle() which returns a json-like data and
|
||||||
|
stores the data at dName path.
|
||||||
|
|
||||||
|
parameters
|
||||||
|
----------
|
||||||
|
* dName: string; the specific path of file that __md looks at.
|
||||||
|
"""
|
||||||
|
def __md(dName):
|
||||||
|
def _md(get):
|
||||||
|
def handle(*args, **kwargs):
|
||||||
|
try:
|
||||||
|
if os.path.isfile(dName):
|
||||||
|
# if directory exists, read from it.
|
||||||
|
jsonFile = open(dName,'r')
|
||||||
|
data = json.loads(jsonFile.read())
|
||||||
|
jsonFile.close()
|
||||||
|
else:
|
||||||
|
# if not, get data via *get method,
|
||||||
|
# then write to the file.
|
||||||
|
data = get(*args, **kwargs)
|
||||||
|
jsonFile = open(dName, 'w+')
|
||||||
|
jsonFile.write(json.dumps(data))
|
||||||
|
jsonFile.close()
|
||||||
|
#print data
|
||||||
|
return data
|
||||||
|
except Exception,e:
|
||||||
|
raise e
|
||||||
|
return handle
|
||||||
|
return _md
|
||||||
|
|
||||||
|
@__md('names/equTicker.json')
|
||||||
|
def _allEquTickers(self):
|
||||||
|
"""get all equity tickers, decorated by @__md()."""
|
||||||
|
data = self._api.get_equity_D1()
|
||||||
|
allEquTickers = list(data.body['ticker'])
|
||||||
|
return allEquTickers
|
||||||
|
|
||||||
|
@__md('names/secID.json')
|
||||||
|
def _allSecIds(self):
|
||||||
|
"""get all security IDs, decorated by @__md()."""
|
||||||
|
data = self._api.get_equity_D1()
|
||||||
|
allTickers = list(data.body['ticker'])
|
||||||
|
exchangeCDs = list(data.body['exchangeCD'])
|
||||||
|
allSecIds = [allTickers[k]+'.'+exchangeCDs[k] for k in range(
|
||||||
|
len(allTickers))]
|
||||||
|
return allSecIds
|
||||||
|
|
||||||
|
@__md('names/futTicker.json')
|
||||||
|
def _allFutTickers(self):
|
||||||
|
"""get all future tickers, decorated by @__md()."""
|
||||||
|
data = self._api.get_future_D1()
|
||||||
|
allFutTickers = list(data.body['ticker'])
|
||||||
|
return allFutTickers
|
||||||
|
|
||||||
|
@__md('names/optTicker.json')
|
||||||
|
def _allOptTickers(self):
|
||||||
|
"""get all option tickers, decorated by @__md()."""
|
||||||
|
data = self._api.get_option_D1()
|
||||||
|
allOptTickers = list(data.body['ticker'])
|
||||||
|
return allOptTickers
|
||||||
|
|
||||||
|
@__md('names/fudTicker.json')
|
||||||
|
def _allFudTickers(self):
|
||||||
|
"""get all fund tickers, decorated by @__md()."""
|
||||||
|
data = self._api.get_fund_D1()
|
||||||
|
allFudTickers = list(data.body['ticker'])
|
||||||
|
return allFudTickers
|
||||||
|
|
||||||
|
@__md('names/idxTicker.json')
|
||||||
|
def _allIdxTickers(self):
|
||||||
|
"""get all index tickers, decorated by @__md()."""
|
||||||
|
data = self._api.get_index_D1()
|
||||||
|
allIdxTickers = list(data.body['ticker'])
|
||||||
|
return allIdxTickers
|
||||||
|
|
||||||
|
@__md('names/bndTicker.json')
|
||||||
|
def _allBndTickers(self):
|
||||||
|
"""get all bond tickers, decorated by @__md()."""
|
||||||
|
data = self._api.get_bond_D1()
|
||||||
|
allBndTickers = list(data.body['ticker'])
|
||||||
|
return allBndTickers
|
||||||
|
|
||||||
|
def _get_coll_names(self):
|
||||||
|
"""
|
||||||
|
get all instruments'names and store them in self._collNames.
|
||||||
|
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if not os.path.exists('names'):
|
||||||
|
os.makedirs('names')
|
||||||
|
|
||||||
|
self._collNames['equTicker'] = self._allEquTickers()
|
||||||
|
self._collNames['fudTicker'] = self._allFudTickers()
|
||||||
|
self._collNames['secID'] = self._allSecIds()
|
||||||
|
self._collNames['futTicker'] = self._allFutTickers()
|
||||||
|
self._collNames['optTicker'] = self._allOptTickers()
|
||||||
|
self._collNames['idxTicker'] = self._allIdxTickers()
|
||||||
|
|
||||||
|
print '[MONGOD]: Collection names gotten.'
|
||||||
|
return 1
|
||||||
|
except AssertionError:
|
||||||
|
warning = '[MONGOD]: Warning, collection names ' + \
|
||||||
|
'is an empty list.'
|
||||||
|
print warning
|
||||||
|
except Exception, e:
|
||||||
|
msg = '[MONGOD]: Unable to set collection names; ' + \
|
||||||
|
str(e)
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
|
||||||
|
#----------------------------------------------------------------------
|
||||||
|
# Ensure collection index method.
|
||||||
|
|
||||||
|
def _ensure_index(self):
|
||||||
|
"""
|
||||||
|
Ensure indices for all databases and collections.
|
||||||
|
|
||||||
|
first access self._dbs config to get index column names;
|
||||||
|
then get collection names from self._collNames and loop
|
||||||
|
over all collections.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if self._collNames and self._dbs:
|
||||||
|
try:
|
||||||
|
for dbName in self._dbs:
|
||||||
|
# Iterate over database configurations.
|
||||||
|
|
||||||
|
db = self._dbs[dbName]
|
||||||
|
dbSelf = db['self']
|
||||||
|
index = db['index']
|
||||||
|
collNames = self._collNames[db['collNames']]
|
||||||
|
# db['self'] is the pymongo.Database object.
|
||||||
|
|
||||||
|
for name in collNames:
|
||||||
|
coll = dbSelf[name]
|
||||||
|
coll.ensure_index([(index,
|
||||||
|
pymongo.DESCENDING)], unique=True)
|
||||||
|
print '[MONGOD]: MongoDB index set.'
|
||||||
|
return 1
|
||||||
|
except KeyError:
|
||||||
|
msg = '[MONGOD]: Unable to set collection indices; ' + \
|
||||||
|
'infomation in Config.body["dbs"] is incomplete.'
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
except Exception, e:
|
||||||
|
msg = '[MONGOD]: Unable to set collection indices; ' + str(e)
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
|
||||||
|
#----------------------------------------------------------------------
|
||||||
|
# Download method.
|
||||||
|
|
||||||
|
def download_equity_D1(self, start, end, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
db = self._dbs['EQU_D1']['self']
|
||||||
|
self._api.get_equity_D1_mongod(db, start, end, sessionNum)
|
||||||
|
except Exception, e:
|
||||||
|
msg = '[MONGOD]: Unable to download data; ' + str(e)
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
|
||||||
|
def download_equity_M1(self, tasks, startYr=2012, endYr=2015):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
# map equity tickers to security IDs.
|
||||||
|
if self._mapTickersToSecIDs:
|
||||||
|
maps = self._mapTickersToSecIDs
|
||||||
|
else:
|
||||||
|
assert os.isfile('./names/secID.json')
|
||||||
|
jsonFile = open(dName,'r')
|
||||||
|
allSecIds = json.loads(jsonFile.read())
|
||||||
|
jsonFile.close()
|
||||||
|
allTickers = [s.split('.')[0] for s in allSecIds]
|
||||||
|
maps = dict(zip(allTickers, allSecIds))
|
||||||
|
self._mapTickersToSecIDs = maps
|
||||||
|
tasks_ = [maps[task] for task in tasks]
|
||||||
|
|
||||||
|
db = self._dbs['EQU_M1']['self']
|
||||||
|
self._api.get_equity_M1_interMonth(db, id=1,
|
||||||
|
startYr = startYr,
|
||||||
|
endYr = endYr,
|
||||||
|
tasks = tasks_)
|
||||||
|
except AssertionError:
|
||||||
|
msg = '[MONGOD]: Cannot map tickers to secIDs; ' + \
|
||||||
|
'secID.json does not exist.'
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
except Exception, e:
|
||||||
|
msg = '[MONGOD]: Unable to download data; ' + str(e)
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
|
||||||
|
def download_bond_D1(self, start, end, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
def download_future_D1(self, start, end, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
db = self._dbs['FUT_D1']['self']
|
||||||
|
self._api.get_future_D1_mongod(db, start, end, sessionNum)
|
||||||
|
except Exception, e:
|
||||||
|
msg = '[MONGOD]: Unable to download data; ' + str(e)
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
|
||||||
|
def download_option_D1(self, start, end, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
db = self._dbs['OPT_D1']['self']
|
||||||
|
self._api.get_option_D1_mongod(db, start, end, sessionNum)
|
||||||
|
except Exception, e:
|
||||||
|
msg = '[MONGOD]: Unable to download data; ' + str(e)
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
|
||||||
|
def download_index_D1(self, start, end, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
db = self._dbs['IDX_D1']['self']
|
||||||
|
self._api.get_index_D1_mongod(db, start, end, sessionNum)
|
||||||
|
except Exception, e:
|
||||||
|
msg = '[MONGOD]: Unable to download data; ' + str(e)
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
|
||||||
|
def download_fund_D1(self, start, end, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
db = self._dbs['FUD_D1']['self']
|
||||||
|
self._api.get_fund_D1_mongod(db, start, end, sessionNum)
|
||||||
|
except Exception, e:
|
||||||
|
msg = '[MONGOD]: Unable to download data; ' + str(e)
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
|
||||||
|
#----------------------------------------------------------------------
|
||||||
|
# Update methods.
|
||||||
|
|
||||||
|
def __update(self, key, target1, target2, sessionNum):
|
||||||
|
"""
|
||||||
|
Basic update method.
|
||||||
|
Looks into the database specified by 'key', find the latest
|
||||||
|
record in the collection of it. Then update the collections
|
||||||
|
till last trading date.
|
||||||
|
|
||||||
|
parameters
|
||||||
|
----------
|
||||||
|
* key: string; a database alias (refer to the database config)
|
||||||
|
e.g., 'EQU_D1'.
|
||||||
|
* target1: method; pointer to the function with which controller
|
||||||
|
obtain all tickers in the database. Concretely, target1 are
|
||||||
|
self._all#Tickers methods.
|
||||||
|
* target2: method; pointer to the api overlord requesting functions
|
||||||
|
i.e. self._api.get_###_mongod methods.
|
||||||
|
* sessionNum: integer; the number of threads.
|
||||||
|
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# get databases and tickers
|
||||||
|
db = self._dbs[key]['self']
|
||||||
|
index = self._dbs[key]['index']
|
||||||
|
allTickers = target1()
|
||||||
|
coll = db[allTickers[0]]
|
||||||
|
|
||||||
|
# find the latest timestamp in collection.
|
||||||
|
latest = coll.find_one(
|
||||||
|
sort=[(index, pymongo.DESCENDING)])[index]
|
||||||
|
start = datetime.strftime(
|
||||||
|
latest + timedelta(days=1),'%Y%m%d')
|
||||||
|
end = datetime.strftime(datetime.now(), '%Y%m%d')
|
||||||
|
|
||||||
|
# then download.
|
||||||
|
target2(db, start, end, sessionNum)
|
||||||
|
return db
|
||||||
|
|
||||||
|
except Exception, e:
|
||||||
|
msg = '[MONGOD]: Unable to update data; ' + str(e)
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def update_equity_D1(self, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
db = self.__update(key = 'EQU_D1',
|
||||||
|
target1 = self._allEquTickers,
|
||||||
|
target2 = self._api.get_equity_D1_mongod,
|
||||||
|
sessionNum = sessionNum)
|
||||||
|
return db
|
||||||
|
|
||||||
|
def update_future_D1(self, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
db = self.__update(key = 'FUT_D1',
|
||||||
|
target1 = self._allFutTickers,
|
||||||
|
target2 = self._api.get_future_D1_mongod,
|
||||||
|
sessionNum = sessionNum)
|
||||||
|
return db
|
||||||
|
|
||||||
|
def update_option_D1(self, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
db = self.__update(key = 'OPT_D1',
|
||||||
|
target1 = self._allOptTickers,
|
||||||
|
target2 = self._api.get_option_D1_mongod,
|
||||||
|
sessionNum = sessionNum)
|
||||||
|
return db
|
||||||
|
|
||||||
|
def update_index_D1(self, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
db = self.__update(key = 'IDX_D1',
|
||||||
|
target1 = self._allIdxTickers,
|
||||||
|
target2 = self._api.get_index_D1_mongod,
|
||||||
|
sessionNum = sessionNum)
|
||||||
|
return db
|
||||||
|
|
||||||
|
def update_fund_D1(self, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
db = self.__update(key = 'FUD_D1',
|
||||||
|
target1 = self._allFudTickers,
|
||||||
|
target2 = self._api.get_fund_D1_mongod,
|
||||||
|
sessionNum = sessionNum)
|
||||||
|
return db
|
||||||
|
|
||||||
|
#----------------------------------------------------------------------#
|
||||||
|
# stuff that will be deprecated
|
||||||
|
|
||||||
|
def update_equity_D1_(self, sessionNum=30):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# set databases and tickers
|
||||||
|
db = self._dbs['EQU_D1']['self']
|
||||||
|
index = self._dbs['EQU_D1']['index']
|
||||||
|
allEquTickers = self._allEquTickers()
|
||||||
|
coll = db[allEquTickers[0]]
|
||||||
|
|
||||||
|
# find the latest timestamp in collection.
|
||||||
|
latest = coll.find_one(
|
||||||
|
sort=[(index, pymongo.DESCENDING)])[index]
|
||||||
|
start = datetime.strftime(latest + timedelta(days=1),'%Y%m%d')
|
||||||
|
end = datetime.strftime(datetime.now(), '%Y%m%d')
|
||||||
|
|
||||||
|
# then download.
|
||||||
|
self._api.get_equity_D1_mongod(db, start, end, sessionNum)
|
||||||
|
|
||||||
|
except Exception, e:
|
||||||
|
msg = '[MONGOD]: Unable to update data; ' + str(e)
|
||||||
|
raise VNPAST_DatabaseError(msg)
|
||||||
|
|
||||||
|
def update_equity_M1(self):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
#----------------------------------------------------------------------
|
||||||
|
# Fetch method.
|
||||||
|
|
||||||
|
def fetch(self, dbName, ticker, start, end, output='list'):
|
||||||
|
"""
|
||||||
|
|
||||||
|
"""
|
||||||
|
# check inputs' validity.
|
||||||
|
if output not in ['df', 'list', 'json']:
|
||||||
|
raise ValueError('[MONGOD]: Unsupported output type.')
|
||||||
|
if dbName not in self._dbNames:
|
||||||
|
raise ValueError('[MONGOD]: Unable to locate database name.')
|
||||||
|
|
||||||
|
db = self._dbs[dbName]
|
||||||
|
dbSelf = db['self']
|
||||||
|
dbIndex = db['index']
|
||||||
|
try:
|
||||||
|
coll = db[ticker]
|
||||||
|
if len(start)==8 and len(end)==8:
|
||||||
|
# yyyymmdd, len()=8
|
||||||
|
start = datetime.strptime(start, '%Y%m%d')
|
||||||
|
end = datetime.strptime(end, '%Y%m%d')
|
||||||
|
elif len(start)==14 and len(end)==14:
|
||||||
|
# yyyymmdd HH:MM, len()=14
|
||||||
|
start = datetime.strptime(start, '%Y%m%d %H:%M')
|
||||||
|
end = datetime.strptime(end, '%Y%m%d %H:%M')
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
docs = []
|
||||||
|
|
||||||
|
# find in MongoDB.
|
||||||
|
for doc in coll.find(filter={dbIndex: {'$lte': end,
|
||||||
|
'$gte': start}}, projection={'_id': False}):
|
||||||
|
docs.append(doc)
|
||||||
|
|
||||||
|
if output == 'list':
|
||||||
|
return docs[::-1]
|
||||||
|
|
||||||
|
except Exception, e:
|
||||||
|
msg = '[MONGOD]: Error encountered when fetching data' + \
|
||||||
|
'from MongoDB; '+ str(e)
|
||||||
|
return -1
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
|
||||||
|
mc.update_index_D1()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
119
vn.datayes/tests.py
Normal file
119
vn.datayes/tests.py
Normal file
@ -0,0 +1,119 @@
|
|||||||
|
from api import *
|
||||||
|
|
||||||
|
def test_config():
|
||||||
|
cfig = Config()
|
||||||
|
print cfig.body, cfig.head, cfig.token
|
||||||
|
cfig.view()
|
||||||
|
|
||||||
|
def test_mktbar_D1():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_equity_D1()
|
||||||
|
print data.body
|
||||||
|
|
||||||
|
def test_mktbar_M1():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_equity_M1()
|
||||||
|
print data.body.tail()
|
||||||
|
|
||||||
|
def test_bond_D1():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_bond_D1()
|
||||||
|
print data.body.tail()
|
||||||
|
|
||||||
|
def test_fut_D1():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_future_D1()
|
||||||
|
print data.body
|
||||||
|
|
||||||
|
def test_fund_D1():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_fund_D1()
|
||||||
|
print data.body
|
||||||
|
|
||||||
|
def test_index_D1():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_index_D1()
|
||||||
|
print data.body
|
||||||
|
|
||||||
|
def test_option_D1():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_option_D1()
|
||||||
|
print data.body
|
||||||
|
|
||||||
|
def test_factors_D1():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_stockFactor_D1()
|
||||||
|
print data.body
|
||||||
|
|
||||||
|
def test_bs():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_balanceSheet()
|
||||||
|
print data.body
|
||||||
|
|
||||||
|
def test_cf():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_cashFlow()
|
||||||
|
print data.body
|
||||||
|
|
||||||
|
def test_is():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_incomeStatement()
|
||||||
|
print data.body
|
||||||
|
|
||||||
|
def test_output():
|
||||||
|
api = PyApi(Config())
|
||||||
|
data = api.get_equity_D1(ticker='000001',output='list')
|
||||||
|
print data
|
||||||
|
|
||||||
|
def test_mongod_get_drudgery():
|
||||||
|
c = MongoClient()
|
||||||
|
db = c['test_dy']
|
||||||
|
api = PyApi(Config())
|
||||||
|
api.get_equity_D1_drudgery(id=1, db=db,
|
||||||
|
start='20130101', end='20150801',
|
||||||
|
tasks=['000001','000002'])
|
||||||
|
|
||||||
|
def test_mongod_get_all():
|
||||||
|
c = MongoClient()
|
||||||
|
db = c['test_dy']
|
||||||
|
api = PyApi(Config())
|
||||||
|
api.get_equity_D1_mongod(db=db, start='20130101', end='20150801')
|
||||||
|
|
||||||
|
def test_mktbar_M1_get_drudgery():
|
||||||
|
c = MongoClient()
|
||||||
|
db = c['test_dy_m1']
|
||||||
|
api = PyApi(Config())
|
||||||
|
api.get_equity_M1_drudgery(id=1, db=db,
|
||||||
|
start='20150701', end='20150801',
|
||||||
|
tasks=['000001.XSHE','000002.XSHE'])
|
||||||
|
|
||||||
|
def test_mktbar_M1_get_all():
|
||||||
|
c = MongoClient()
|
||||||
|
db = c['test_dy_m1']
|
||||||
|
api = PyApi(Config())
|
||||||
|
api.get_equity_M1_mongod(db=db)
|
||||||
|
|
||||||
|
def test_mktbar_M1_get_interM():
|
||||||
|
c = MongoClient()
|
||||||
|
db = c['test_dy_m1']
|
||||||
|
api = PyApi(Config())
|
||||||
|
api.get_equity_M1_interMonth(db=db, id=0, tasks=['000001.XSHE','000002.XSHE'])
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
#test_config()
|
||||||
|
#test_mktbar_D1()
|
||||||
|
#test_bond_D1()
|
||||||
|
#test_fut_D1()
|
||||||
|
#test_fund_D1()
|
||||||
|
#test_index_D1()
|
||||||
|
#test_option_D1()
|
||||||
|
#test_factors_D1()
|
||||||
|
#test_mktbar_M1()
|
||||||
|
#test_bs()
|
||||||
|
#test_cf()
|
||||||
|
#test_is()
|
||||||
|
#test_output()
|
||||||
|
#test_mongod_get_all()
|
||||||
|
#test_mktbar_M1_get_drudgery()
|
||||||
|
#test_mktbar_M1_get_all()
|
||||||
|
test_mktbar_M1_get_interM()
|
||||||
52
vn.datayes/update.sh
Executable file
52
vn.datayes/update.sh
Executable file
@ -0,0 +1,52 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo [API]: Prepare to update DATAYES_FUTURE_D1...
|
||||||
|
python - << EOF
|
||||||
|
from storage import *
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
mc.update_future_D1()
|
||||||
|
EOF
|
||||||
|
echo [API]: DATAYES_FUTURE_D1 updated.
|
||||||
|
|
||||||
|
echo [API]: Prepare to update DATAYES_INDEX_D1...
|
||||||
|
python - << EOF
|
||||||
|
from storage import *
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
mc.update_index_D1()
|
||||||
|
EOF
|
||||||
|
echo [API]: DATAYES_INDEX_D1 updated.
|
||||||
|
|
||||||
|
echo [API]: Prepare to update DATAYES_OPTION_D1...
|
||||||
|
python - << EOF
|
||||||
|
from storage import *
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
mc.update_option_D1()
|
||||||
|
EOF
|
||||||
|
echo [API]: DATAYES_OPTION_D1 updated.
|
||||||
|
|
||||||
|
echo [API]: Prepare to update DATAYES_FUND_D1...
|
||||||
|
python - << EOF
|
||||||
|
from storage import *
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
mc.update_fund_D1()
|
||||||
|
EOF
|
||||||
|
echo [API]: DATAYES_FUND_D1 updated.
|
||||||
|
echo [MONGOD]: Update finished.
|
||||||
|
|
||||||
|
echo [API]: Prepare to update DATAYES_EQUITY_D1...
|
||||||
|
python - << EOF
|
||||||
|
from storage import *
|
||||||
|
dc = DBConfig()
|
||||||
|
api = PyApi(Config())
|
||||||
|
mc = MongodController(dc, api)
|
||||||
|
mc.update_equity_D1()
|
||||||
|
EOF
|
||||||
|
echo [API]: DATAYES_EQUITY_D1 updated.
|
||||||
Loading…
Reference in New Issue
Block a user