From b6924e7fdce66ff18d0611b76d4a4941eb4b0218 Mon Sep 17 00:00:00 2001 From: msincenselee Date: Sat, 14 Sep 2019 11:07:22 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0:=E5=A2=9E=E5=8A=A0mongo?= =?UTF-8?q?=E7=8B=AC=E7=AB=8B=E6=8E=A5=E5=8F=A3=EF=BC=8C=E6=9B=B4=E6=96=B0?= =?UTF-8?q?FAQ?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Q_n_A.md | 61 ++++++++ vnpy/data/mongo/__init__.py | 0 vnpy/data/mongo/mongo_data.py | 270 ++++++++++++++++++++++++++++++++++ 3 files changed, 331 insertions(+) create mode 100644 vnpy/data/mongo/__init__.py create mode 100644 vnpy/data/mongo/mongo_data.py diff --git a/Q_n_A.md b/Q_n_A.md index 7a4fe80d..93395d9d 100644 --- a/Q_n_A.md +++ b/Q_n_A.md @@ -123,6 +123,7 @@ #5、碰到的问题：version `GLIBCXX_3.4.21' not found conda install libgcc + 若出现更高版本需求，参见第10点 #6、碰到的问题：在3.5 env下安装RqPlus时，报错:talib/common.c:242:28: fatal error: ta-lib/ta_defs.h: No such file or directory locate ta_defs.h @@ -148,3 +149,63 @@ pip install autobahn pip install twisted 若出现找不到rc.exe， 请先使用vs x86&x64界面，激活py35后，再运行 pip install pyOpenSSL + +# 10、升级gcc + 使用奇正MOM的CTP API时，提示`GLIBCXX_3.4.22' not found，当前centos最高版本是 3.4.21，通过yum不能升级，需要手工下载升级。 + + wget http://ftp.de.debian.org/debian/pool/main/g/gcc-9/libstdc++6_9.2.1-8_amd64.deb +解压 + + ar -x libstdc++6_9.2.1-8_amd64.deb + （就是 ar 命令，不是tar） + tar -xvf data.tar.xz + +安装 + + 删除： rm /usr/lib64/libstdc++.so.6 + 拷贝： cp usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28 /usr/lib64/ + 连接： ln /usr/lib64/libstdc++.so.6.0.28 /usr/lib64/libstdc++.so.6 +结果 + + strings /usr/lib64/libstdc++.so.6 | grep GLIBCXX + + GLIBCXX_3.4 + GLIBCXX_3.4.1 + GLIBCXX_3.4.2 + GLIBCXX_3.4.3 + GLIBCXX_3.4.4 + GLIBCXX_3.4.5 + GLIBCXX_3.4.6 + GLIBCXX_3.4.7 + GLIBCXX_3.4.8 + GLIBCXX_3.4.9 + GLIBCXX_3.4.10 + GLIBCXX_3.4.11 + GLIBCXX_3.4.12 + GLIBCXX_3.4.13 + GLIBCXX_3.4.14 + GLIBCXX_3.4.15 + GLIBCXX_3.4.16 + GLIBCXX_3.4.17 + GLIBCXX_3.4.18 + GLIBCXX_3.4.19 + GLIBCXX_3.4.20 + GLIBCXX_3.4.21 + GLIBCXX_3.4.22 + GLIBCXX_3.4.23 + GLIBCXX_3.4.24 + GLIBCXX_3.4.25 + GLIBCXX_DEBUG_MESSAGE_LENGTH + +# 11、升级glibc + 使用奇正MOM的CTP API时，提示`GLIBC_2.18' not found，当前centos最高版本是 3.4.21，通过yum不能升级，需要手工下载升级。 + + root 用户登录 + wget http://ftp.gnu.org/gnu/glibc/glibc-2.18.tar.gz + tar –zxvf glibc-2.18.tar.gz + cd glibc-2.18 + mkdir build + cd build + ../configure --prefix=/usr --disable-profile --enable-add-ons --with-headers=/usr/include --with-binutils=/usr/bin + make –j4 + make install diff --git a/vnpy/data/mongo/__init__.py b/vnpy/data/mongo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vnpy/data/mongo/mongo_data.py b/vnpy/data/mongo/mongo_data.py new file mode 100644 index 00000000..1494851c --- /dev/null +++ b/vnpy/data/mongo/mongo_data.py @@ -0,0 +1,270 @@ +# encoding: UTF-8 + +# Mongodb 数据服务 + + +import sys,os,traceback + +from datetime import datetime,timedelta +from pymongo import MongoClient, ASCENDING +from pymongo.errors import ConnectionFailure,AutoReconnect +from time import sleep + +from vnpy.trader.vtFunction import loadMongoSetting +from vnpy.trader.setup_logger import setup_logger + +class MongoData(object): + dbClient = None + db_has_connected = False + + def __init__(self): + self.dbConnect() + + def dbConnect(self, force=False): + if (self.dbClient is None or self.db_has_connected == False) or force: + host, port, logging = loadMongoSetting() + print('connecting to Mongo:{}:{}'.format(host, port)) + try: + # 设置MongoDB操作的超时时间为0.5秒 + self.dbClient = MongoClient(host, port, connectTimeoutMS=500) + # 调用server_info查询服务器状态，防止服务器异常并未连接成功 + self.dbClient.server_info() + + self.db_has_connected = True + print('mongo db connected') + + except Exception as ex: + self.dbClient = None + self.db_has_connected = False + self.writeError(u'connect to mongo {}:{} exception:{}'.format(host,port,str(ex))) + + def writeLog(self,content): + print(content) + + def writeError(self,content): + print(content,file=sys.stderr) + + # ---------------------------------------------------------------------- + def dbInsert(self, dbName, collectionName, d): + """向MongoDB中插入数据，d是具体数据""" + try: + if self.dbClient: + db = self.dbClient[dbName] + collection = db[collectionName] + collection.insert_one(d) + else: + self.writeError(u'db insert fail') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect(force=True) + + except AutoReconnect as ex: + self.writeLog(u'数据库连接断开重连:{}'.format(str(ex))) + sleep(1) + except ConnectionFailure: + self.dbClient = None + self.writeError(u'数据库连接断开') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect(force=True) + except Exception as ex: + self.writeError(u'dbInsert exception:{}'.format(str(ex))) + + def dbInsertMany(self,dbName, collectionName, data_list,ordered=True): + """ + 向MongoDB中插入数据，data_list是具体数据 列表 + :param dbName: + :param collectionName: + :param data_list: + :param ordered: 是否忽略insert error + :return: + """ + if not isinstance(data_list, list): + self.writeError('data_list should be a list') + return + try: + if self.dbClient: + db = self.dbClient[dbName] + collection = db[collectionName] + collection.insert_many(data_list, ordered = ordered) + else: + self.writeError('db insert many fail') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect(force=True) + + except AutoReconnect as ex: + self.writeError(u'数据库连接断开重连:{}'.format(str(ex))) + sleep(1) + except ConnectionFailure: + self.dbClient = None + self.writeError(u'数据库连接断开') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect(force=True) + except Exception as ex: + self.writeError(u'dbInsertMany exception:{}'.format(str(ex))) + + + # ---------------------------------------------------------------------- + def dbQueryOne(self, dbName, collectionName, flt): + """从MongoDB中读取数据，d是查询要求，返回的是一个结果""" + try: + if self.dbClient: + db = self.dbClient[dbName] + collection = db[collectionName] + + return collection.find_one(flt) + + else: + self.writeLog('db query fail') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect() + + except AutoReconnect as ex: + self.writeError(u'数据库连接断开重连:{}'.format(str(ex))) + sleep(1) + except ConnectionFailure: + self.dbClient = None + self.writeError(u'数据库连接断开') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect() + except Exception as ex: + self.writeError(u'dbQuery exception:{}'.format(str(ex))) + + return None + + # ---------------------------------------------------------------------- + def dbQuery(self, dbName, collectionName, d, sortKey='', sortDirection=ASCENDING): + """从MongoDB中读取数据，d是查询要求，返回的是数据库查询的指针""" + try: + if self.dbClient: + db = self.dbClient[dbName] + collection = db[collectionName] + + if sortKey: + cursor = collection.find(d).sort(sortKey, sortDirection) # 对查询出来的数据进行排序 + else: + cursor = collection.find(d) + + if cursor: + return list(cursor) + else: + return [] + else: + self.writeLog('db query fail') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect() + + except AutoReconnect as ex: + self.writeError(u'数据库连接断开重连:{}'.format(str(ex))) + sleep(1) + except ConnectionFailure: + self.dbClient = None + self.writeError(u'数据库连接断开') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect() + except Exception as ex: + self.writeError(u'dbQuery exception:{}'.format(str(ex))) + + return [] + + def dbQueryBySort(self, dbName, collectionName, d, sortName, sortType, limitNum=0): + """从MongoDB中读取数据，d是查询要求，sortName是排序的字段,sortType是排序类型 + 返回的是数据库查询的指针""" + try: + if self.dbClient: + db = self.dbClient[dbName] + collection = db[collectionName] + if limitNum > 0: + cursor = collection.find(d).sort(sortName, sortType).limit(limitNum) + else: + cursor = collection.find(d).sort(sortName, sortType) + if cursor: + return list(cursor) + else: + return [] + else: + self.writeLog('db query fail') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect() + + except AutoReconnect as ex: + self.writeError(u'数据库连接断开重连:{}'.format(str(ex))) + sleep(1) + except ConnectionFailure: + self.dbClient = None + self.writeError(u'数据库连接断开') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect() + except Exception as ex: + self.writeError(u'dbQueryBySort exception:{}'.format(str(ex))) + + return [] + + # ---------------------------------------------------------------------- + def dbUpdate(self, dbName, collectionName, d, flt, upsert=False,replace=True): + """向MongoDB中更新数据，d是具体数据，flt是过滤条件，upsert代表若无是否要插入""" + try: + if self.dbClient: + db = self.dbClient[dbName] + collection = db[collectionName] + if replace: + rtn =collection.replace_one(flt, d, upsert) + else: + rtn = collection.update_one(flt, {'$set':d}, upsert) + + return rtn + + else: + self.writeLog('db update fail') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect() + + except AutoReconnect as ex: + self.writeError(u'数据库连接断开重连:{}'.format(str(ex))) + sleep(1) + except ConnectionFailure: + self.dbClient = None + self.writeError(u'数据库连接断开') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect() + except Exception as ex: + self.writeError(u'dbUpdate exception:{}'.format(str(ex))) + + return None + + def dbDelete(self,dbName, collectionName, flt): + """ + 向mongodb中，删除数据，flt是过滤条件 + :param dbName: + :param collectionName: + :param flt: + :return: + """ + try: + if self.dbClient: + db = self.dbClient[dbName] + collection = db[collectionName] + collection.delete_many(flt) + else: + self.writeLog('db delete fail') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect() + except ConnectionFailure: + self.dbClient = None + self.writeError(u'数据库连接断开') + if self.db_has_connected: + self.writeLog(u'重新尝试连接数据库') + self.dbConnect() + except Exception as ex: + self.writeError(u'dbDelete exception:{}'.format(str(ex)))