From 66cdf33b2343dd2d48b7224c449547363fdc4c41 Mon Sep 17 00:00:00 2001 From: "vn.py" Date: Wed, 8 May 2019 11:19:21 +0800 Subject: [PATCH 1/5] [Add] sync request function of RestClient --- vnpy/api/rest/rest_client.py | 40 +++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/vnpy/api/rest/rest_client.py b/vnpy/api/rest/rest_client.py index 7ae2c386..255ba35f 100644 --- a/vnpy/api/rest/rest_client.py +++ b/vnpy/api/rest/rest_client.py @@ -30,7 +30,7 @@ class Request(object): params: dict, data: dict, headers: dict, - callback: Callable, + callback: Callable = None, on_failed: Callable = None, on_error: Callable = None, extra: Any = None, @@ -284,3 +284,41 @@ class RestClient(object): """ url = self.url_base + path return url + + def request( + self, + method: str, + path: str, + params: dict = None, + data: dict = None, + headers: dict = None, + ): + """ + Add a new request. + :param method: GET, POST, PUT, DELETE, QUERY + :param path: + :param params: dict for query string + :param data: dict for body + :param headers: dict for headers + :return: requests.Response + """ + request = Request( + method, + path, + params, + data, + headers + ) + request = self.sign(request) + + url = self.make_full_url(request.path) + + response = requests.request( + request.method, + url, + headers=request.headers, + params=request.params, + data=request.data, + proxies=self.proxies, + ) + return response From 9932045b5964bd4f05fa5d956bde2c363416c194 Mon Sep 17 00:00:00 2001 From: "vn.py" Date: Wed, 8 May 2019 11:26:42 +0800 Subject: [PATCH 2/5] [Add] HistoryRequest and related query functions in BaseGateway --- vnpy/trader/gateway.py | 18 ++++++++++++++---- vnpy/trader/object.py | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/vnpy/trader/gateway.py b/vnpy/trader/gateway.py index edd2c6ab..0583f125 100644 --- a/vnpy/trader/gateway.py +++ b/vnpy/trader/gateway.py @@ -27,12 +27,13 @@ from .object import ( OrderRequest, CancelRequest, SubscribeRequest, + HistoryRequest ) class BaseGateway(ABC): """ - Abstract gateway class for creating gateways connection + Abstract gateway class for creating gateways connection to different trading systems. # How to implement a gateway: @@ -206,8 +207,6 @@ class BaseGateway(ABC): Cancel an existing order. implementation should finish the tasks blow: * send request to server - - """ pass @@ -215,7 +214,6 @@ class BaseGateway(ABC): def query_account(self): """ Query account balance. - """ pass @@ -226,6 +224,18 @@ class BaseGateway(ABC): """ pass + def query_bar_history(self, req: HistoryRequest): + """ + Query bar history data. + """ + pass + + def query_tick_history(self, req: HistoryRequest): + """ + Query tick history data. + """ + pass + def get_default_setting(self): """ Return default setting dict. diff --git a/vnpy/trader/object.py b/vnpy/trader/object.py index 734093cb..7a551a85 100644 --- a/vnpy/trader/object.py +++ b/vnpy/trader/object.py @@ -236,6 +236,8 @@ class ContractData(BaseData): min_volume: float = 1 # minimum trading volume of the contract stop_supported: bool = False # whether server supports stop order net_position: bool = False # whether gateway uses net position volume + bar_history: bool = False # whether gateway provides bar history data + tick_history: bool = False # whether gateway provides tick history data option_strike: float = 0 option_underlying: str = "" # vt_symbol of underlying contract @@ -310,3 +312,20 @@ class CancelRequest: def __post_init__(self): """""" self.vt_symbol = f"{self.symbol}.{self.exchange.value}" + + +@dataclass +class HistoryRequest: + """ + Request sending to specific gateway for querying history data. + """ + + symbol: str + exchange: Exchange + start: datetime + end: datetime = None + interval: Interval = None + + def __post_init__(self): + """""" + self.vt_symbol = f"{self.symbol}.{self.exchange.value}" From e1c824307ea6c1c3a8b3a2c882e59b8969a50fd2 Mon Sep 17 00:00:00 2001 From: "vn.py" Date: Wed, 8 May 2019 14:21:08 +0800 Subject: [PATCH 3/5] [Del] remove query tick history function --- vnpy/gateway/bitmex/bitmex_gateway.py | 10 ++++++++++ vnpy/trader/engine.py | 18 +++++++++++++++++- vnpy/trader/gateway.py | 8 +------- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/vnpy/gateway/bitmex/bitmex_gateway.py b/vnpy/gateway/bitmex/bitmex_gateway.py index 678f09c9..9127c64a 100644 --- a/vnpy/gateway/bitmex/bitmex_gateway.py +++ b/vnpy/gateway/bitmex/bitmex_gateway.py @@ -34,6 +34,7 @@ from vnpy.trader.object import ( OrderRequest, CancelRequest, SubscribeRequest, + HistoryRequest ) REST_HOST = "https://www.bitmex.com/api/v1" @@ -124,6 +125,10 @@ class BitmexGateway(BaseGateway): """""" pass + def query_history(self, req: HistoryRequest): + """""" + return self.rest_api.query_history(req) + def close(self): """""" self.rest_api.stop() @@ -279,6 +284,10 @@ class BitmexRestApi(RestClient): on_error=self.on_cancel_order_error, ) + def query_history(self, req: HistoryRequest): + """""" + pass + def on_send_order_failed(self, status_code: str, request: Request): """ Callback when sending order failed on server. @@ -615,6 +624,7 @@ class BitmexWebsocketApi(WebsocketClient): size=d["lotSize"], stop_supported=True, net_position=True, + bar_history=True, gateway_name=self.gateway_name, ) diff --git a/vnpy/trader/engine.py b/vnpy/trader/engine.py index 12e3c3d3..96246487 100644 --- a/vnpy/trader/engine.py +++ b/vnpy/trader/engine.py @@ -22,7 +22,13 @@ from .event import ( EVENT_LOG ) from .gateway import BaseGateway -from .object import CancelRequest, LogData, OrderRequest, SubscribeRequest +from .object import ( + CancelRequest, + LogData, + OrderRequest, + SubscribeRequest, + HistoryRequest +) from .setting import SETTINGS from .utility import get_folder_path @@ -174,6 +180,16 @@ class MainEngine: if gateway: gateway.cancel_order(req) + def query_history(self, req: HistoryRequest, gateway_name: str): + """ + Send cancel order request to a specific gateway. + """ + gateway = self.get_gateway(gateway_name) + if gateway: + return gateway.query_history(req) + else: + return None + def close(self): """ Make sure every gateway and app is closed properly before diff --git a/vnpy/trader/gateway.py b/vnpy/trader/gateway.py index 0583f125..ac2cd704 100644 --- a/vnpy/trader/gateway.py +++ b/vnpy/trader/gateway.py @@ -224,18 +224,12 @@ class BaseGateway(ABC): """ pass - def query_bar_history(self, req: HistoryRequest): + def query_history(self, req: HistoryRequest): """ Query bar history data. """ pass - def query_tick_history(self, req: HistoryRequest): - """ - Query tick history data. - """ - pass - def get_default_setting(self): """ Return default setting dict. From 8c707435e88b81d27ea7e5ed660f4fbcd6e0ff58 Mon Sep 17 00:00:00 2001 From: "vn.py" Date: Wed, 8 May 2019 15:39:15 +0800 Subject: [PATCH 4/5] [Add] download bar data from gateway in CtaBacktester --- vnpy/api/rest/rest_client.py | 2 +- vnpy/app/cta_backtester/engine.py | 24 ++++++-- vnpy/app/cta_backtester/ui/widget.py | 4 +- vnpy/gateway/bitmex/bitmex_gateway.py | 89 +++++++++++++++++++++++++-- vnpy/trader/object.py | 3 +- 5 files changed, 108 insertions(+), 14 deletions(-) diff --git a/vnpy/api/rest/rest_client.py b/vnpy/api/rest/rest_client.py index 255ba35f..7bd4568e 100644 --- a/vnpy/api/rest/rest_client.py +++ b/vnpy/api/rest/rest_client.py @@ -258,7 +258,7 @@ class RestClient(object): request.response = response status_code = response.status_code - if status_code / 100 == 2: # 2xx都算成功，尽管交易所都用200 + if status_code // 100 == 2: # 2xx都算成功，尽管交易所都用200 jsonBody = response.json() request.callback(jsonBody, request) request.status = RequestStatus.success diff --git a/vnpy/app/cta_backtester/engine.py b/vnpy/app/cta_backtester/engine.py index 573feefe..40b3fb03 100644 --- a/vnpy/app/cta_backtester/engine.py +++ b/vnpy/app/cta_backtester/engine.py @@ -9,6 +9,7 @@ from vnpy.event import Event, EventEngine from vnpy.trader.engine import BaseEngine, MainEngine from vnpy.trader.constant import Interval from vnpy.trader.utility import extract_vt_symbol +from vnpy.trader.object import HistoryRequest from vnpy.trader.rqdata import rqdata_client from vnpy.trader.database import database_manager from vnpy.app.cta_strategy import ( @@ -337,10 +338,25 @@ class BacktesterEngine(BaseEngine): """ self.write_log(f"{vt_symbol}-{interval}开始下载历史数据") - symbol, exchange = extract_vt_symbol(vt_symbol) - data = rqdata_client.query_bar( - symbol, exchange, Interval(interval), start, end - ) + contract = self.main_engine.get_contract(vt_symbol) + + # If history data provided in gateway, then query + if contract and contract.history_data: + req = HistoryRequest( + symbol=contract.symbol, + exchange=contract.exchange, + interval=Interval(interval), + start=start, + end=end + ) + data = self.main_engine.query_history(req, contract.gateway_name) + # Otherwise use RQData to query data + else: + symbol, exchange = extract_vt_symbol(vt_symbol) + + data = rqdata_client.query_bar( + symbol, exchange, Interval(interval), start, end + ) if data: database_manager.save_bar_data(data) diff --git a/vnpy/app/cta_backtester/ui/widget.py b/vnpy/app/cta_backtester/ui/widget.py index 2198c11d..6be8f167 100644 --- a/vnpy/app/cta_backtester/ui/widget.py +++ b/vnpy/app/cta_backtester/ui/widget.py @@ -56,14 +56,14 @@ class BacktesterManager(QtWidgets.QWidget): self.class_combo = QtWidgets.QComboBox() self.class_combo.addItems(self.class_names) - self.symbol_line = QtWidgets.QLineEdit("IF88.CFFEX") + self.symbol_line = QtWidgets.QLineEdit("XBTUSD.BITMEX") self.interval_combo = QtWidgets.QComboBox() for inteval in Interval: self.interval_combo.addItem(inteval.value) end_dt = datetime.now() - start_dt = end_dt - timedelta(days=3 * 365) + start_dt = end_dt - timedelta(days=30)# * 365) self.start_date_edit = QtWidgets.QDateEdit( QtCore.QDate( diff --git a/vnpy/gateway/bitmex/bitmex_gateway.py b/vnpy/gateway/bitmex/bitmex_gateway.py index 9127c64a..338ebfad 100644 --- a/vnpy/gateway/bitmex/bitmex_gateway.py +++ b/vnpy/gateway/bitmex/bitmex_gateway.py @@ -7,7 +7,7 @@ import hmac import sys import time from copy import copy -from datetime import datetime +from datetime import datetime, timedelta from threading import Lock from urllib.parse import urlencode @@ -21,7 +21,8 @@ from vnpy.trader.constant import ( OrderType, Product, Status, - Offset + Offset, + Interval ) from vnpy.trader.gateway import BaseGateway from vnpy.trader.object import ( @@ -31,6 +32,7 @@ from vnpy.trader.object import ( PositionData, AccountData, ContractData, + BarData, OrderRequest, CancelRequest, SubscribeRequest, @@ -61,6 +63,18 @@ ORDERTYPE_VT2BITMEX = { } ORDERTYPE_BITMEX2VT = {v: k for k, v in ORDERTYPE_VT2BITMEX.items()} +INTERVAL_VT2BITMEX = { + Interval.MINUTE: "1m", + Interval.HOUR: "1h", + Interval.DAILY: "1d", +} + +TIMEDELTA_MAP = { + Interval.MINUTE: timedelta(minutes=1), + Interval.HOUR: timedelta(hours=1), + Interval.DAILY: timedelta(days=1), +} + class BitmexGateway(BaseGateway): """ @@ -160,7 +174,7 @@ class BitmexRestApi(RestClient): Generate BitMEX signature. """ # Sign - expires = int(time.time() + 5) + expires = int(time.time() + 30) if request.params: query = urlencode(request.params) @@ -286,7 +300,72 @@ class BitmexRestApi(RestClient): def query_history(self, req: HistoryRequest): """""" - pass + history = [] + count = 750 + start_time = req.start.isoformat() + + while True: + # Create query params + params = { + "binSize": INTERVAL_VT2BITMEX[req.interval], + "symbol": req.symbol, + "count": count, + "startTime": start_time + } + + # Add end time if specified + if req.end: + params["endTime"] = req.end.isoformat() + + # Get response from server + resp = self.request( + "GET", + "/trade/bucketed", + params=params + ) + + # Break if request failed with other status code + if resp.status_code // 100 != 2: + msg = f"获取历史数据失败，状态码：{resp.status_code}，信息：{resp.text}" + self.gateway.write_log(msg) + break + else: + data = resp.json() + + for d in data: + dt = datetime.strptime(d["timestamp"], "%Y-%m-%dT%H:%M:%S.%fZ") + bar = BarData( + symbol=req.symbol, + exchange=req.exchange, + datetime=dt, + interval=req.interval, + volume=d["volume"], + open_price=d["open"], + high_price=d["high"], + low_price=d["low"], + close_price=d["close"], + gateway_name=self.gateway_name + ) + history.append(bar) + + begin = data[0]["timestamp"] + end = data[-1]["timestamp"] + msg = f"获取历史数据成功，{req.symbol} - {req.interval.value}，{begin} - {end}" + self.gateway.write_log(msg) + + # Break if total data count less than 750 (latest date collected) + if len(data) < 750: + break + + # Update start time + start_time = bar.datetime + TIMEDELTA_MAP[req.interval] + + return history + + + + + def on_send_order_failed(self, status_code: str, request: Request): """ @@ -624,7 +703,7 @@ class BitmexWebsocketApi(WebsocketClient): size=d["lotSize"], stop_supported=True, net_position=True, - bar_history=True, + history_data=True, gateway_name=self.gateway_name, ) diff --git a/vnpy/trader/object.py b/vnpy/trader/object.py index 7a551a85..952c0360 100644 --- a/vnpy/trader/object.py +++ b/vnpy/trader/object.py @@ -236,8 +236,7 @@ class ContractData(BaseData): min_volume: float = 1 # minimum trading volume of the contract stop_supported: bool = False # whether server supports stop order net_position: bool = False # whether gateway uses net position volume - bar_history: bool = False # whether gateway provides bar history data - tick_history: bool = False # whether gateway provides tick history data + history_data: bool = False # whether gateway provides bar history data option_strike: float = 0 option_underlying: str = "" # vt_symbol of underlying contract From a68773fec96fdd19c7441555f009b30e3a82222b Mon Sep 17 00:00:00 2001 From: "vn.py" Date: Wed, 8 May 2019 15:57:35 +0800 Subject: [PATCH 5/5] [Mod] change query_bar -> query_history for rqdata_client --- vnpy/app/cta_backtester/engine.py | 23 +++++++++++------------ vnpy/app/cta_backtester/ui/widget.py | 4 ++-- vnpy/app/cta_strategy/engine.py | 10 ++++++++-- vnpy/gateway/bitmex/bitmex_gateway.py | 7 +------ vnpy/trader/rqdata.py | 19 +++++++++---------- 5 files changed, 31 insertions(+), 32 deletions(-) diff --git a/vnpy/app/cta_backtester/engine.py b/vnpy/app/cta_backtester/engine.py index 40b3fb03..7ed94fbf 100644 --- a/vnpy/app/cta_backtester/engine.py +++ b/vnpy/app/cta_backtester/engine.py @@ -338,25 +338,24 @@ class BacktesterEngine(BaseEngine): """ self.write_log(f"{vt_symbol}-{interval}开始下载历史数据") + symbol, exchange = extract_vt_symbol(vt_symbol) + + req = HistoryRequest( + symbol=symbol, + exchange=exchange, + interval=Interval(interval), + start=start, + end=end + ) + contract = self.main_engine.get_contract(vt_symbol) # If history data provided in gateway, then query if contract and contract.history_data: - req = HistoryRequest( - symbol=contract.symbol, - exchange=contract.exchange, - interval=Interval(interval), - start=start, - end=end - ) data = self.main_engine.query_history(req, contract.gateway_name) # Otherwise use RQData to query data else: - symbol, exchange = extract_vt_symbol(vt_symbol) - - data = rqdata_client.query_bar( - symbol, exchange, Interval(interval), start, end - ) + data = rqdata_client.query_history(req) if data: database_manager.save_bar_data(data) diff --git a/vnpy/app/cta_backtester/ui/widget.py b/vnpy/app/cta_backtester/ui/widget.py index 6be8f167..2198c11d 100644 --- a/vnpy/app/cta_backtester/ui/widget.py +++ b/vnpy/app/cta_backtester/ui/widget.py @@ -56,14 +56,14 @@ class BacktesterManager(QtWidgets.QWidget): self.class_combo = QtWidgets.QComboBox() self.class_combo.addItems(self.class_names) - self.symbol_line = QtWidgets.QLineEdit("XBTUSD.BITMEX") + self.symbol_line = QtWidgets.QLineEdit("IF88.CFFEX") self.interval_combo = QtWidgets.QComboBox() for inteval in Interval: self.interval_combo.addItem(inteval.value) end_dt = datetime.now() - start_dt = end_dt - timedelta(days=30)# * 365) + start_dt = end_dt - timedelta(days=3 * 365) self.start_date_edit = QtWidgets.QDateEdit( QtCore.QDate( diff --git a/vnpy/app/cta_strategy/engine.py b/vnpy/app/cta_strategy/engine.py index a28b4878..e17f49ec 100644 --- a/vnpy/app/cta_strategy/engine.py +++ b/vnpy/app/cta_strategy/engine.py @@ -16,6 +16,7 @@ from vnpy.trader.engine import BaseEngine, MainEngine from vnpy.trader.object import ( OrderRequest, SubscribeRequest, + HistoryRequest, LogData, TickData, BarData, @@ -136,9 +137,14 @@ class CtaEngine(BaseEngine): """ Query bar data from RQData. """ - data = rqdata_client.query_bar( - symbol, exchange, interval, start, end + req = HistoryRequest( + symbol=symbol, + exchange=exchange, + interval=interval, + start=start, + end=end ) + data = rqdata_client.query_history(req) return data def process_tick_event(self, event: Event): diff --git a/vnpy/gateway/bitmex/bitmex_gateway.py b/vnpy/gateway/bitmex/bitmex_gateway.py index 338ebfad..cb815936 100644 --- a/vnpy/gateway/bitmex/bitmex_gateway.py +++ b/vnpy/gateway/bitmex/bitmex_gateway.py @@ -360,12 +360,7 @@ class BitmexRestApi(RestClient): # Update start time start_time = bar.datetime + TIMEDELTA_MAP[req.interval] - return history - - - - - + return history def on_send_order_failed(self, status_code: str, request: Request): """ diff --git a/vnpy/trader/rqdata.py b/vnpy/trader/rqdata.py index 965d32c7..f3e650bc 100644 --- a/vnpy/trader/rqdata.py +++ b/vnpy/trader/rqdata.py @@ -7,7 +7,7 @@ from rqdatac.services.get_price import get_price as rqdata_get_price from .setting import SETTINGS from .constant import Exchange, Interval -from .object import BarData +from .object import BarData, HistoryRequest INTERVAL_VT2RQ = { @@ -89,17 +89,16 @@ class RqdataClient: return rq_symbol - def query_bar( - self, - symbol: str, - exchange: Exchange, - interval: Interval, - start: datetime, - end: datetime - ): + def query_history(self, req: HistoryRequest): """ - Query bar data from RQData. + Query history bar data from RQData. """ + symbol = req.symbol + exchange = req.exchange + interval = req.interval + start = req.start + end = req.end + rq_symbol = self.to_rq_symbol(symbol, exchange) if rq_symbol not in self.symbols: return None