From 1074a26b77966f1f91a6c185584ef37c8c4a7153 Mon Sep 17 00:00:00 2001 From: nanoric Date: Fri, 12 Apr 2019 04:41:56 -0400 Subject: [PATCH 01/62] [Add] added support for PostgreSQL --- tests/app/test_csv_loader.py | 90 ++++++++++++++++++++++ tests/backtesting/getdata.py | 12 +-- vnpy/app/csv_loader/engine.py | 134 +++++++++++++++++++-------------- vnpy/trader/database.py | 137 ++++++++++++++++++++++++---------- vnpy/trader/setting.py | 15 ++-- 5 files changed, 278 insertions(+), 110 deletions(-) create mode 100644 tests/app/test_csv_loader.py diff --git a/tests/app/test_csv_loader.py b/tests/app/test_csv_loader.py new file mode 100644 index 00000000..b52a2183 --- /dev/null +++ b/tests/app/test_csv_loader.py @@ -0,0 +1,90 @@ +""" +Test if csv loader works fine +""" +import tempfile +import unittest + +from vnpy.app.csv_loader import CsvLoaderEngine +from vnpy.trader.constant import Exchange, Interval + + +class TestCsvLoader(unittest.TestCase): + + def setUp(self) -> None: + self.engine = CsvLoaderEngine(None, None) # no engine is necessary for CsvLoader + + def test_load(self): + data = """"Datetime","Open","High","Low","Close","Volume" +2010-04-16 09:16:00,3450.0,3488.0,3450.0,3468.0,489 +2010-04-16 09:17:00,3468.0,3473.8,3467.0,3467.0,302 +2010-04-16 09:18:00,3467.0,3471.0,3466.0,3467.0,203 +2010-04-16 09:19:00,3467.0,3468.2,3448.0,3448.0,280 +2010-04-16 09:20:00,3448.0,3459.0,3448.0,3454.0,250 +2010-04-16 09:21:00,3454.0,3456.8,3454.0,3456.8,109 +""" + with tempfile.TemporaryFile("w+t") as f: + f.write(data) + f.seek(0) + + self.engine.load_by_handle( + f, + symbol="1", + exchange=Exchange.BITMEX, + interval=Interval.MINUTE, + datetime_head="Datetime", + open_head="Open", + close_head="Close", + low_head="Low", + high_head="High", + volume_head="Volume", + datetime_format="%Y-%m-%d %H:%M:%S", + ) + + def test_load_duplicated(self): + data = """"Datetime","Open","High","Low","Close","Volume" +2010-04-16 09:16:00,3450.0,3488.0,3450.0,3468.0,489 +2010-04-16 09:17:00,3468.0,3473.8,3467.0,3467.0,302 +2010-04-16 09:18:00,3467.0,3471.0,3466.0,3467.0,203 +2010-04-16 09:19:00,3467.0,3468.2,3448.0,3448.0,280 +2010-04-16 09:20:00,3448.0,3459.0,3448.0,3454.0,250 +2010-04-16 09:21:00,3454.0,3456.8,3454.0,3456.8,109 +""" + with tempfile.TemporaryFile("w+t") as f: + f.write(data) + f.seek(0) + + self.engine.load_by_handle( + f, + symbol="1", + exchange=Exchange.BITMEX, + interval=Interval.MINUTE, + datetime_head="Datetime", + open_head="Open", + close_head="Close", + low_head="Low", + high_head="High", + volume_head="Volume", + datetime_format="%Y-%m-%d %H:%M:%S", + ) + + with tempfile.TemporaryFile("w+t") as f: + f.write(data) + f.seek(0) + + self.engine.load_by_handle( + f, + symbol="1", + exchange=Exchange.BITMEX, + interval=Interval.MINUTE, + datetime_head="Datetime", + open_head="Open", + close_head="Close", + low_head="Low", + high_head="High", + volume_head="Volume", + datetime_format="%Y-%m-%d %H:%M:%S", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/backtesting/getdata.py b/tests/backtesting/getdata.py index 77436bcc..62d02465 100644 --- a/tests/backtesting/getdata.py +++ b/tests/backtesting/getdata.py @@ -2,7 +2,7 @@ from time import time import rqdatac as rq -from vnpy.trader.database import DbBarData, DB +from vnpy.trader.database import DbBarData USERNAME = "" PASSWORD = "" @@ -39,11 +39,11 @@ def download_minute_bar(vt_symbol): df = rq.get_price(symbol, frequency="1m", fields=FIELDS) - with DB.atomic(): - for ix, row in df.iterrows(): - print(row.name) - bar = generate_bar_from_row(row, symbol, exchange) - DbBarData.replace(bar.__data__).execute() + bars = [] + for ix, row in df.iterrows(): + bar = generate_bar_from_row(row, symbol, exchange) + bars.append(bar) + DbBarData.save_all(bars) end = time() cost = (end - start) * 1000 diff --git a/vnpy/app/csv_loader/engine.py b/vnpy/app/csv_loader/engine.py index fa88afe7..d9d29385 100644 --- a/vnpy/app/csv_loader/engine.py +++ b/vnpy/app/csv_loader/engine.py @@ -22,15 +22,13 @@ Sample csv file: import csv from datetime import datetime - -from peewee import chunked +from typing import TextIO from vnpy.event import EventEngine from vnpy.trader.constant import Exchange, Interval -from vnpy.trader.database import DbBarData, DB +from vnpy.trader.database import DbBarData from vnpy.trader.engine import BaseEngine, MainEngine - APP_NAME = "CsvLoader" @@ -41,17 +39,70 @@ class CsvLoaderEngine(BaseEngine): """""" super().__init__(main_engine, event_engine, APP_NAME) - self.file_path: str = "" + self.file_path: str = '' self.symbol: str = "" self.exchange: Exchange = Exchange.SSE self.interval: Interval = Interval.MINUTE - self.datetime_head: str = "" - self.open_head: str = "" - self.close_head: str = "" - self.low_head: str = "" - self.high_head: str = "" - self.volume_head: str = "" + self.datetime_head: str = '' + self.open_head: str = '' + self.close_head: str = '' + self.low_head: str = '' + self.high_head: str = '' + self.volume_head: str = '' + + def load_by_handle( + self, + f: TextIO, + symbol: str, + exchange: Exchange, + interval: Interval, + datetime_head: str, + open_head: str, + close_head: str, + low_head: str, + high_head: str, + volume_head: str, + datetime_format: str + ): + """ + load by text mode file handle + """ + reader = csv.DictReader(f) + + db_bars = [] + start = None + count = 0 + for item in reader: + if datetime_format: + dt = datetime.strptime(item[datetime_head], datetime_format) + else: + dt = datetime.fromisoformat(item[datetime_head]) + + db_bar = DbBarData( + symbol=symbol, + exchange=exchange.value, + datetime=dt, + interval=interval.value, + volume=item[volume_head], + open_price=item[open_head], + high_price=item[high_head], + low_price=item[low_head], + close_price=item[close_head], + ) + + db_bars.append(db_bar) + + # do some statistics + count += 1 + if not start: + start = db_bar.datetime + end = db_bar.datetime + + # insert into database + DbBarData.save_all(db_bars) + + return start, end, count def load( self, @@ -67,47 +118,20 @@ class CsvLoaderEngine(BaseEngine): volume_head: str, datetime_format: str ): - """""" - vt_symbol = f"{symbol}.{exchange.value}" - - start = None - end = None - count = 0 - - with open(file_path, "rt") as f: - reader = csv.DictReader(f) - - db_bars = [] - - for item in reader: - dt = datetime.strptime(item[datetime_head], datetime_format) - - db_bar = { - "symbol": symbol, - "exchange": exchange.value, - "datetime": dt, - "interval": interval.value, - "volume": item[volume_head], - "open_price": item[open_head], - "high_price": item[high_head], - "low_price": item[low_head], - "close_price": item[close_head], - "vt_symbol": vt_symbol, - "gateway_name": "DB" - } - - db_bars.append(db_bar) - - # do some statistics - count += 1 - if not start: - start = db_bar["datetime"] - - end = db_bar["datetime"] - - # Insert into DB - with DB.atomic(): - for batch in chunked(db_bars, 50): - DbBarData.insert_many(batch).on_conflict_replace().execute() - - return start, end, count + """ + load by filename + """ + with open(file_path, 'rt') as f: + return self.load_by_handle( + f, + symbol=symbol, + exchange=exchange, + interval=interval, + datetime_head=datetime_head, + open_head=open_head, + close_head=close_head, + low_head=low_head, + high_head=high_head, + volume_head=volume_head, + datetime_format=datetime_format, + ) diff --git a/vnpy/trader/database.py b/vnpy/trader/database.py index 1e90dcbc..ef0c9cc2 100644 --- a/vnpy/trader/database.py +++ b/vnpy/trader/database.py @@ -1,56 +1,85 @@ """""" +from enum import Enum +from typing import List -from peewee import CharField, DateTimeField, FloatField, Model, MySQLDatabase, PostgresqlDatabase, \ - SqliteDatabase +from peewee import ( + AutoField, + CharField, + Database, + DateTimeField, + FloatField, + Model, + MySQLDatabase, + PostgresqlDatabase, + SqliteDatabase, + chunked, +) from .constant import Exchange, Interval from .object import BarData, TickData from .setting import SETTINGS -from .utility import resolve_path +from .utility import get_file_path + + +class Driver(Enum): + SQLITE = "sqlite" + MYSQL = "mysql" + POSTGRESQL = "postgresql" + + +_db: Database +_driver: Driver def init(): - db_settings = SETTINGS['database'] - driver = db_settings["driver"] + global _driver + db_settings = {k[9:]: v for k, v in SETTINGS.items() if k.startswith("database.")} + _driver = Driver(db_settings["driver"]) init_funcs = { - "sqlite": init_sqlite, - "mysql": init_mysql, - "postgresql": init_postgresql, + Driver.SQLITE: init_sqlite, + Driver.MYSQL: init_mysql, + Driver.POSTGRESQL: init_postgresql, } - assert driver in init_funcs - del db_settings['driver'] - return init_funcs[driver](db_settings) + assert _driver in init_funcs + del db_settings["driver"] + return init_funcs[_driver](db_settings) def init_sqlite(settings: dict): - global DB - database = settings['database'] + global _db + database = settings["database"] - DB = SqliteDatabase(str(resolve_path(database))) + _db = SqliteDatabase(str(get_file_path(database))) def init_mysql(settings: dict): - global DB - DB = MySQLDatabase(**settings) + global _db + _db = MySQLDatabase(**settings) def init_postgresql(settings: dict): - global DB - DB = PostgresqlDatabase(**settings) + global _db + _db = PostgresqlDatabase(**settings) init() -class DbBarData(Model): +class ModelBase(Model): + def to_dict(self): + return self.__data__ + + +class DbBarData(ModelBase): """ Candlestick bar data for database storage. - Index is defined unique with vt_symbol, interval and datetime. + Index is defined unique with datetime, interval, symbol """ + id = AutoField() symbol = CharField() exchange = CharField() datetime = DateTimeField() @@ -62,12 +91,9 @@ class DbBarData(Model): low_price = FloatField() close_price = FloatField() - vt_symbol = CharField() - gateway_name = CharField() - class Meta: - database = DB - indexes = ((("vt_symbol", "interval", "datetime"), True),) + database = _db + indexes = ((("datetime", "interval", "symbol"), True),) @staticmethod def from_bar(bar: BarData): @@ -85,8 +111,6 @@ class DbBarData(Model): db_bar.high_price = bar.high_price db_bar.low_price = bar.low_price db_bar.close_price = bar.close_price - db_bar.vt_symbol = bar.vt_symbol - db_bar.gateway_name = "DB" return db_bar @@ -104,18 +128,40 @@ class DbBarData(Model): high_price=self.high_price, low_price=self.low_price, close_price=self.close_price, - gateway_name=self.gateway_name, + gateway_name="DB", ) return bar + @staticmethod + def save_all(objs: List["DbBarData"]): + """ + save a list of objects, update if exists. + """ + with _db.atomic(): + if _driver is Driver.POSTGRESQL: + for bar in objs: + DbBarData.insert(bar.to_dict()).on_conflict( + update=bar.to_dict(), + conflict_target=( + DbBarData.datetime, + DbBarData.interval, + DbBarData.symbol, + ), + ).execute() + else: + for c in chunked(objs, 50): + DbBarData.insert_many(c).on_conflict_replace() -class DbTickData(Model): + +class DbTickData(ModelBase): """ Tick data for database storage. - Index is defined unique with vt_symbol, interval and datetime. + Index is defined unique with (datetime, symbol) """ + id = AutoField() + symbol = CharField() exchange = CharField() datetime = DateTimeField() @@ -131,6 +177,7 @@ class DbTickData(Model): high_price = FloatField() low_price = FloatField() close_price = FloatField() + pre_close = FloatField() bid_price_1 = FloatField() bid_price_2 = FloatField() @@ -156,12 +203,9 @@ class DbTickData(Model): ask_volume_4 = FloatField() ask_volume_5 = FloatField() - vt_symbol = CharField() - gateway_name = CharField() - class Meta: - database = DB - indexes = ((("vt_symbol", "datetime"), True),) + database = _db + indexes = ((("datetime", "symbol"), True),) @staticmethod def from_tick(tick: TickData): @@ -210,9 +254,6 @@ class DbTickData(Model): db_tick.ask_volume_4 = tick.ask_volume_4 db_tick.ask_volume_5 = tick.ask_volume_5 - db_tick.vt_symbol = tick.vt_symbol - db_tick.gateway_name = "DB" - return tick def to_tick(self): @@ -237,7 +278,7 @@ class DbTickData(Model): ask_price_1=self.ask_price_1, bid_volume_1=self.bid_volume_1, ask_volume_1=self.ask_volume_1, - gateway_name=self.gateway_name, + gateway_name="DB", ) if self.bid_price_2: @@ -263,6 +304,20 @@ class DbTickData(Model): return tick + @staticmethod + def save_all(objs: List["DbTickData"]): + with _db.atomic(): + if _driver is Driver.POSTGRESQL: + for bar in objs: + DbTickData.insert(bar.to_dict()).on_conflict( + update=bar.to_dict(), + preserve=(DbTickData.id), + conflict_target=(DbTickData.datetime, DbTickData.symbol), + ).execute() + else: + for c in chunked(objs, 50): + DbBarData.insert_many(c).on_conflict_replace() -DB.connect() -DB.create_tables([DbBarData, DbTickData]) + +_db.connect() +_db.create_tables([DbBarData, DbTickData]) diff --git a/vnpy/trader/setting.py b/vnpy/trader/setting.py index b27dbf96..778b147c 100644 --- a/vnpy/trader/setting.py +++ b/vnpy/trader/setting.py @@ -24,14 +24,13 @@ SETTINGS = { "rqdata.username": "", "rqdata.password": "", - "database": { - "driver": "sqlite", # sqlite, mysql, postgresql - "database": "{VNPY_TEMP}/database.db", # for sqlite, use this as filepath - "host": "localhost", - "port": 3306, - "user": "root", - "password": "" - } + + "database.driver": "sqlite", # see database.Driver + "database.database": "database.db", # for sqlite, use this as filepath + "database.host": "localhost", + "database.port": 3306, + "database.user": "root", + "database.password": "" } # Load global setting from json file. From f797152bd5ca8d808080e83089dbb71762d5b67b Mon Sep 17 00:00:00 2001 From: nanoric Date: Fri, 12 Apr 2019 07:15:55 -0400 Subject: [PATCH 02/62] [Mod] make flake8 happy --- vnpy/app/csv_loader/engine.py | 40 ++++++++++++++-------------- vnpy/app/cta_strategy/backtesting.py | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/vnpy/app/csv_loader/engine.py b/vnpy/app/csv_loader/engine.py index d9d29385..7125e3ad 100644 --- a/vnpy/app/csv_loader/engine.py +++ b/vnpy/app/csv_loader/engine.py @@ -39,17 +39,17 @@ class CsvLoaderEngine(BaseEngine): """""" super().__init__(main_engine, event_engine, APP_NAME) - self.file_path: str = '' + self.file_path: str = "" self.symbol: str = "" self.exchange: Exchange = Exchange.SSE self.interval: Interval = Interval.MINUTE - self.datetime_head: str = '' - self.open_head: str = '' - self.close_head: str = '' - self.low_head: str = '' - self.high_head: str = '' - self.volume_head: str = '' + self.datetime_head: str = "" + self.open_head: str = "" + self.close_head: str = "" + self.low_head: str = "" + self.high_head: str = "" + self.volume_head: str = "" def load_by_handle( self, @@ -63,7 +63,7 @@ class CsvLoaderEngine(BaseEngine): low_head: str, high_head: str, volume_head: str, - datetime_format: str + datetime_format: str, ): """ load by text mode file handle @@ -78,17 +78,17 @@ class CsvLoaderEngine(BaseEngine): dt = datetime.strptime(item[datetime_head], datetime_format) else: dt = datetime.fromisoformat(item[datetime_head]) - + db_bar = DbBarData( - symbol=symbol, - exchange=exchange.value, - datetime=dt, - interval=interval.value, - volume=item[volume_head], - open_price=item[open_head], - high_price=item[high_head], - low_price=item[low_head], - close_price=item[close_head], + symbol=symbol, + exchange=exchange.value, + datetime=dt, + interval=interval.value, + volume=item[volume_head], + open_price=item[open_head], + high_price=item[high_head], + low_price=item[low_head], + close_price=item[close_head], ) db_bars.append(db_bar) @@ -116,12 +116,12 @@ class CsvLoaderEngine(BaseEngine): low_head: str, high_head: str, volume_head: str, - datetime_format: str + datetime_format: str, ): """ load by filename """ - with open(file_path, 'rt') as f: + with open(file_path, "rt") as f: return self.load_by_handle( f, symbol=symbol, diff --git a/vnpy/app/cta_strategy/backtesting.py b/vnpy/app/cta_strategy/backtesting.py index c1ba8e67..f1458fe7 100644 --- a/vnpy/app/cta_strategy/backtesting.py +++ b/vnpy/app/cta_strategy/backtesting.py @@ -1002,4 +1002,4 @@ def load_tick_data( .order_by(DbTickData.datetime) ) data = [db_tick.db_tick() for db_tick in s] - return data \ No newline at end of file + return data From 5d0bf006c65200a933fd25a43a27cd4e48909592 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sat, 13 Apr 2019 23:02:30 -0400 Subject: [PATCH 03/62] [Add] added support for mongodb [Add] added tests for database and CsvLoaderEngine [Mod] changed .travis.yaml --- .travis.yml | 8 +- tests/app/__init__.py | 1 + tests/test_all.py | 20 + tests/{load_all.py => test_import_all.py} | 18 + tests/trader/__init__.py | 2 + tests/trader/test_database.py | 125 ++++++ tests/trader/test_settings.py | 24 ++ tests/travis_env.sh | 16 + vnpy/app/csv_loader/engine.py | 21 +- vnpy/app/cta_strategy/backtesting.py | 58 ++- vnpy/app/cta_strategy/engine.py | 68 ++-- vnpy/trader/database/__init__.py | 12 + vnpy/trader/database/database.py | 53 +++ .../database_mongo.py} | 289 +++++++------- vnpy/trader/database/database_sql.py | 370 ++++++++++++++++++ vnpy/trader/database/initialize.py | 24 ++ vnpy/trader/setting.py | 8 +- vnpy/trader/utility.py | 20 +- 18 files changed, 892 insertions(+), 245 deletions(-) create mode 100644 tests/app/__init__.py create mode 100644 tests/test_all.py rename tests/{load_all.py => test_import_all.py} (63%) create mode 100644 tests/trader/__init__.py create mode 100644 tests/trader/test_database.py create mode 100644 tests/trader/test_settings.py create mode 100644 tests/travis_env.sh create mode 100644 vnpy/trader/database/__init__.py create mode 100644 vnpy/trader/database/database.py rename vnpy/trader/{database.py => database/database_mongo.py} (51%) create mode 100644 vnpy/trader/database/database_sql.py create mode 100644 vnpy/trader/database/initialize.py diff --git a/.travis.yml b/.travis.yml index 8622616d..d38b8fed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,19 @@ language: python +cache: pip + dist: xenial # required for Python >= 3.7 (travis-ci/travis-ci#9069) python: - "3.7" +services: + - mongodb + - mysql + - postgresql script: # todo: use python unittest - - mkdir run; cd run; python ../tests/load_all.py + - cd tests; source travis_env.sh; python test_all.py matrix: include: diff --git a/tests/app/__init__.py b/tests/app/__init__.py new file mode 100644 index 00000000..9aafa67d --- /dev/null +++ b/tests/app/__init__.py @@ -0,0 +1 @@ +from .test_csv_loader import * diff --git a/tests/test_all.py b/tests/test_all.py new file mode 100644 index 00000000..e824ff21 --- /dev/null +++ b/tests/test_all.py @@ -0,0 +1,20 @@ +# tests/runner.py +import unittest + +# import your test modules +import test_import_all +import trader +import app + +# initialize the test suite +loader = unittest.TestLoader() +suite = unittest.TestSuite() + +# add tests to the test suite +suite.addTests(loader.loadTestsFromModule(test_import_all)) +suite.addTests(loader.loadTestsFromModule(trader)) +suite.addTests(loader.loadTestsFromModule(app)) + +# initialize a runner, pass it your suite and run it +runner = unittest.TextTestRunner(verbosity=3) +result = runner.run(suite) diff --git a/tests/load_all.py b/tests/test_import_all.py similarity index 63% rename from tests/load_all.py rename to tests/test_import_all.py index feae9c12..df930c89 100644 --- a/tests/load_all.py +++ b/tests/test_import_all.py @@ -2,23 +2,41 @@ import unittest +# noinspection PyUnresolvedReferences class ImportTest(unittest.TestCase): # noinspection PyUnresolvedReferences def test_import_all(self): from vnpy.event import EventEngine + def test_import_main_engine(self): from vnpy.trader.engine import MainEngine + + def test_import_ui(self): from vnpy.trader.ui import MainWindow, create_qapp + def test_import_bitmex_gateway(self): from vnpy.gateway.bitmex import BitmexGateway + + def test_import_futu_gateway(self): from vnpy.gateway.futu import FutuGateway + + def test_import_ib_gateway(self): from vnpy.gateway.ib import IbGateway + + def test_import_ctp_gateway(self): from vnpy.gateway.ctp import CtpGateway + + def test_import_tiger_gateway(self): from vnpy.gateway.tiger import TigerGateway + + def test_import_oes_gateway(self): from vnpy.gateway.oes import OesGateway + def test_import_cta_strategy_app(self): from vnpy.app.cta_strategy import CtaStrategyApp + + def test_import_csv_loader_app(self): from vnpy.app.csv_loader import CsvLoaderApp diff --git a/tests/trader/__init__.py b/tests/trader/__init__.py new file mode 100644 index 00000000..c5776fc1 --- /dev/null +++ b/tests/trader/__init__.py @@ -0,0 +1,2 @@ +from .test_database import * +from .test_settings import * diff --git a/tests/trader/test_database.py b/tests/trader/test_database.py new file mode 100644 index 00000000..59f1dcfc --- /dev/null +++ b/tests/trader/test_database.py @@ -0,0 +1,125 @@ +""" +Test if database works fine +""" +import os +import unittest +from datetime import datetime, timedelta + +from vnpy.trader.constant import Exchange, Interval +from vnpy.trader.database.database import Driver +from vnpy.trader.object import BarData, TickData + +os.environ['VNPY_TESTING'] = '1' + +profiles = { + Driver.SQLITE: { + "driver": "sqlite", + "database": "test_db.db", + }, + Driver.MYSQL: { + "driver": "mysql", + "database": os.environ['VNPY_TEST_MYSQL_DATABASE'], + "host": os.environ['VNPY_TEST_MYSQL_HOST'], + "port": int(os.environ['VNPY_TEST_MYSQL_PORT']), + "user": os.environ["VNPY_TEST_MYSQL_USER"], + "password": os.environ['VNPY_TEST_MYSQL_PASSWORD'], + }, + Driver.POSTGRESQL: { + "driver": "postgresql", + "database": os.environ['VNPY_TEST_POSTGRESQL_DATABASE'], + "host": os.environ['VNPY_TEST_POSTGRESQL_HOST'], + "port": int(os.environ['VNPY_TEST_POSTGRESQL_PORT']), + "user": os.environ["VNPY_TEST_POSTGRESQL_USER"], + "password": os.environ['VNPY_TEST_POSTGRESQL_PASSWORD'], + }, + Driver.MONGODB: { + "driver": "mongodb", + "database": os.environ['VNPY_TEST_MONGODB_DATABASE'], + "host": os.environ['VNPY_TEST_MONGODB_HOST'], + "port": int(os.environ['VNPY_TEST_MONGODB_PORT']), + "user": "", + "password": "", + "authentication_source": "", + }, +} + + +def now(): + return datetime.utcnow() + + +bar = BarData( + gateway_name="DB", + symbol="test_symbol", + exchange=Exchange.BITMEX, + datetime=now(), + interval=Interval.MINUTE, +) + +tick = TickData( + gateway_name="DB", + symbol="test_symbol", + exchange=Exchange.BITMEX, + datetime=now(), + name="DB_test_symbol", +) + + +class TestDatabase(unittest.TestCase): + + def connect(self, settings: dict): + from vnpy.trader.database.initialize import init # noqa + self.manager = init(settings) + + def test_upsert_bar(self): + for driver, settings in profiles.items(): + with self.subTest(driver=driver, settings=settings): + self.connect(settings) + self.manager.save_bar_data([bar]) + self.manager.save_bar_data([bar]) + + def test_save_load_bar(self): + for driver, settings in profiles.items(): + with self.subTest(driver=driver, settings=settings): + self.connect(settings) + # save first + self.manager.save_bar_data([bar]) + + # and load + results = self.manager.load_bar_data( + symbol=bar.symbol, + exchange=bar.exchange, + interval=bar.interval, + start=bar.datetime - timedelta(seconds=1), # time is not accuracy + end=now() + timedelta(seconds=1), # time is not accuracy + ) + count = len(results) + self.assertNotEqual(count, 0) + + def test_upsert_tick(self): + for driver, settings in profiles.items(): + with self.subTest(driver=driver, settings=settings): + self.connect(settings) + self.manager.save_tick_data([tick]) + self.manager.save_tick_data([tick]) + + def test_save_load_tick(self): + for driver, settings in profiles.items(): + with self.subTest(driver=driver, settings=settings): + self.connect(settings) + # save first + self.manager.save_tick_data([tick]) + + # and load + results = self.manager.load_tick_data( + symbol=bar.symbol, + exchange=bar.exchange, + start=bar.datetime - timedelta(seconds=1), # time is not accuracy + end=now() + timedelta(seconds=1), # time is not accuracy + ) + count = len(results) + self.assertNotEqual(count, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/trader/test_settings.py b/tests/trader/test_settings.py new file mode 100644 index 00000000..c8e86132 --- /dev/null +++ b/tests/trader/test_settings.py @@ -0,0 +1,24 @@ +""" +Test if database works fine +""" +import unittest +from vnpy.trader.setting import SETTINGS, get_settings + + +class TestSettings(unittest.TestCase): + + def test_get_settings(self): + SETTINGS['a'] = 1 + got = get_settings() + self.assertIn('a', got) + self.assertEqual(got['a'], 1) + + def test_get_settings_with_prefix(self): + SETTINGS['a.a'] = 1 + got = get_settings() + self.assertIn('a', got) + self.assertEqual(got['a'], 1) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/travis_env.sh b/tests/travis_env.sh new file mode 100644 index 00000000..030d79c5 --- /dev/null +++ b/tests/travis_env.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +export VNPY_TEST_MYSQL_DATABASE=vnpy +export VNPY_TEST_MYSQL_HOST=127.0.0.1 +export VNPY_TEST_MYSQL_PORT=3306 +export VNPY_TEST_MYSQL_USER=root +export VNPY_TEST_MYSQL_PASSWORD= +export VNPY_TEST_POSTGRESQL_DATABASE=vnpy +export VNPY_TEST_POSTGRESQL_HOST=127.0.0.1 +export VNPY_TEST_POSTGRESQL_PORT=5432 +export VNPY_TEST_POSTGRESQL_USER=postgres +export VNPY_TEST_POSTGRESQL_PASSWORD= +export VNPY_TEST_MONGODB_DATABASE=vnpy +export VNPY_TEST_MONGODB_HOST=127.0.0.1 +export VNPY_TEST_MONGODB_PORT=27017 + diff --git a/vnpy/app/csv_loader/engine.py b/vnpy/app/csv_loader/engine.py index 7125e3ad..a9ff1936 100644 --- a/vnpy/app/csv_loader/engine.py +++ b/vnpy/app/csv_loader/engine.py @@ -26,8 +26,9 @@ from typing import TextIO from vnpy.event import EventEngine from vnpy.trader.constant import Exchange, Interval -from vnpy.trader.database import DbBarData +from vnpy.trader.database import database_manager from vnpy.trader.engine import BaseEngine, MainEngine +from vnpy.trader.object import BarData APP_NAME = "CsvLoader" @@ -70,7 +71,7 @@ class CsvLoaderEngine(BaseEngine): """ reader = csv.DictReader(f) - db_bars = [] + bars = [] start = None count = 0 for item in reader: @@ -79,29 +80,29 @@ class CsvLoaderEngine(BaseEngine): else: dt = datetime.fromisoformat(item[datetime_head]) - db_bar = DbBarData( + bar = BarData( symbol=symbol, - exchange=exchange.value, + exchange=exchange, datetime=dt, - interval=interval.value, + interval=interval, volume=item[volume_head], open_price=item[open_head], high_price=item[high_head], low_price=item[low_head], close_price=item[close_head], + gateway_name="DB", ) - db_bars.append(db_bar) + bars.append(bar) # do some statistics count += 1 if not start: - start = db_bar.datetime - end = db_bar.datetime + start = bar.datetime + end = bar.datetime # insert into database - DbBarData.save_all(db_bars) - + database_manager.save_bar_data(bars) return start, end, count def load( diff --git a/vnpy/app/cta_strategy/backtesting.py b/vnpy/app/cta_strategy/backtesting.py index f1458fe7..e4661107 100644 --- a/vnpy/app/cta_strategy/backtesting.py +++ b/vnpy/app/cta_strategy/backtesting.py @@ -12,9 +12,9 @@ from pandas import DataFrame from vnpy.trader.constant import (Direction, Offset, Exchange, Interval, Status) -from vnpy.trader.database import DbBarData, DbTickData -from vnpy.trader.object import OrderData, TradeData -from vnpy.trader.utility import round_to_pricetick +from vnpy.trader.database import database_manager +from vnpy.trader.object import OrderData, TradeData, BarData, TickData +from vnpy.trader.utility import round_to_pricetick, extract_vt_symbol from .base import ( BacktestingMode, @@ -103,8 +103,8 @@ class BacktestingEngine: self.strategy_class = None self.strategy = None - self.tick = None - self.bar = None + self.tick: TickData + self.bar: BarData self.datetime = None self.interval = None @@ -199,14 +199,16 @@ class BacktestingEngine: if self.mode == BacktestingMode.BAR: self.history_data = load_bar_data( - self.vt_symbol, + self.symbol, + self.exchange, self.interval, self.start, self.end ) else: self.history_data = load_tick_data( - self.vt_symbol, + self.symbol, + self.exchange, self.start, self.end ) @@ -519,7 +521,7 @@ class BacktestingEngine: else: self.daily_results[d] = DailyResult(d, price) - def new_bar(self, bar: DbBarData): + def new_bar(self, bar: BarData): """""" self.bar = bar self.datetime = bar.datetime @@ -530,7 +532,7 @@ class BacktestingEngine: self.update_daily_close(bar.close_price) - def new_tick(self, tick: DbTickData): + def new_tick(self, tick: TickData): """""" self.tick = tick self.datetime = tick.datetime @@ -965,41 +967,27 @@ def optimize( @lru_cache(maxsize=10) def load_bar_data( - vt_symbol: str, - interval: str, - start: datetime, + symbol: str, + exchange: Exchange, + interval: Interval, + start: datetime, end: datetime ): """""" - s = ( - DbBarData.select() - .where( - (DbBarData.vt_symbol == vt_symbol) - & (DbBarData.interval == interval) - & (DbBarData.datetime >= start) - & (DbBarData.datetime <= end) - ) - .order_by(DbBarData.datetime) + return database_manager.load_bar_data( + symbol, exchange, interval, start, end ) - data = [db_bar.to_bar() for db_bar in s] - return data @lru_cache(maxsize=10) def load_tick_data( - vt_symbol: str, - start: datetime, + symbol: str, + exchange: Exchange, + start: datetime, end: datetime ): """""" - s = ( - DbTickData.select() - .where( - (DbTickData.vt_symbol == vt_symbol) - & (DbTickData.datetime >= start) - & (DbTickData.datetime <= end) - ) - .order_by(DbTickData.datetime) + return database_manager.load_tick_data( + symbol, exchange, start, end ) - data = [db_tick.db_tick() for db_tick in s] - return data + diff --git a/vnpy/app/cta_strategy/engine.py b/vnpy/app/cta_strategy/engine.py index 37e5973a..7824ff2d 100644 --- a/vnpy/app/cta_strategy/engine.py +++ b/vnpy/app/cta_strategy/engine.py @@ -5,7 +5,7 @@ import os import traceback from collections import defaultdict from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, List from datetime import datetime, timedelta from threading import Thread from queue import Queue @@ -36,7 +36,7 @@ from vnpy.trader.constant import ( Status ) from vnpy.trader.utility import load_json, save_json -from vnpy.trader.database import DbTickData, DbBarData +from vnpy.trader.database import database_manager from vnpy.trader.setting import SETTINGS from .base import ( @@ -146,13 +146,12 @@ class CtaEngine(BaseEngine): self.write_log("RQData数据接口初始化成功") def query_bar_from_rq( - self, vt_symbol: str, interval: Interval, start: datetime, end: datetime + self, symbol: str, exchange: Exchange, interval: Interval, start: datetime, end: datetime ): """ Query bar data from RQData. """ - symbol, exchange_str = vt_symbol.split(".") - rq_symbol = to_rq_symbol(vt_symbol) + rq_symbol = to_rq_symbol(symbol, exchange) if rq_symbol not in self.rq_symbols: return None @@ -166,11 +165,11 @@ class CtaEngine(BaseEngine): end_date=end ) - data = [] + data: List[BarData] = [] for ix, row in df.iterrows(): bar = BarData( symbol=symbol, - exchange=Exchange(exchange_str), + exchange=exchange, interval=interval, datetime=row.name.to_pydatetime(), open_price=row["open"], @@ -529,46 +528,41 @@ class CtaEngine(BaseEngine): return self.engine_type def load_bar( - self, vt_symbol: str, days: int, interval: Interval, callback: Callable + self, symbol: str, exchange: Exchange, days: int, interval: Interval, + callback: Callable[[BarData], None] ): """""" end = datetime.now() start = end - timedelta(days) - # Query data from RQData by default, if not found, load from database. - data = self.query_bar_from_rq(vt_symbol, interval, start, end) - if not data: - s = ( - DbBarData.select() - .where( - (DbBarData.vt_symbol == vt_symbol) - & (DbBarData.interval == interval.value) - & (DbBarData.datetime >= start) - & (DbBarData.datetime <= end) - ) - .order_by(DbBarData.datetime) + # Query bars from RQData by default, if not found, load from database. + bars = self.query_bar_from_rq(symbol, exchange, interval, start, end) + if not bars: + bars = database_manager.load_bar_data( + symbol=symbol, + exchange=exchange, + interval=interval, + start=start, + end=end, ) - data = [db_bar.to_bar() for db_bar in s] - for bar in data: + for bar in bars: callback(bar) - def load_tick(self, vt_symbol: str, days: int, callback: Callable): + def load_tick(self, symbol: str, exchange: Exchange, days: int, + callback: Callable[[TickData], None]): """""" end = datetime.now() start = end - timedelta(days) - s = ( - DbTickData.select() - .where( - (DbBarData.vt_symbol == vt_symbol) - & (DbBarData.datetime >= start) - & (DbBarData.datetime <= end) - ) - .order_by(DbBarData.datetime) + ticks = database_manager.load_tick_data( + symbol=symbol, + exchange=exchange, + start=start, + end=end, ) - for tick in s: + for tick in ticks: callback(tick) def call_strategy_func( @@ -757,7 +751,7 @@ class CtaEngine(BaseEngine): """ Load strategy class from certain folder. """ - for dirpath, dirnames, filenames in os.walk(path): + for dirpath, dirnames, filenames in os.walk(str(path)): for filename in filenames: if filename.endswith(".py"): strategy_module_name = ".".join( @@ -914,19 +908,19 @@ class CtaEngine(BaseEngine): self.main_engine.send_email(subject, msg) -def to_rq_symbol(vt_symbol: str): +def to_rq_symbol(symbol: str, exchange: Exchange): """ CZCE product of RQData has symbol like "TA1905" while vt symbol is "TA905.CZCE" so need to add "1" in symbol. """ - symbol, exchange_str = vt_symbol.split(".") - if exchange_str != "CZCE": + if exchange is not Exchange.CZCE: return symbol.upper() for count, word in enumerate(symbol): if word.isdigit(): break - + + # noinspection PyUnboundLocalVariable product = symbol[:count] year = symbol[count] month = symbol[count + 1:] diff --git a/vnpy/trader/database/__init__.py b/vnpy/trader/database/__init__.py new file mode 100644 index 00000000..16cb96b8 --- /dev/null +++ b/vnpy/trader/database/__init__.py @@ -0,0 +1,12 @@ +import os +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from vnpy.trader.database.database import BaseDatabaseManager + +if "VNPY_TESTING" not in os.environ: + from vnpy.trader.setting import get_settings + from .initialize import init + + settings = get_settings("database.") + database_manager: "BaseDatabaseManager" = init(settings=settings) diff --git a/vnpy/trader/database/database.py b/vnpy/trader/database/database.py new file mode 100644 index 00000000..06077383 --- /dev/null +++ b/vnpy/trader/database/database.py @@ -0,0 +1,53 @@ +from abc import ABC, abstractmethod +from datetime import datetime +from enum import Enum +from typing import Sequence, TYPE_CHECKING + +if TYPE_CHECKING: + from vnpy.trader.constant import Interval, Exchange + from vnpy.trader.object import BarData, TickData + + +class Driver(Enum): + SQLITE = "sqlite" + MYSQL = "mysql" + POSTGRESQL = "postgresql" + MONGODB = "mongodb" + + +class BaseDatabaseManager(ABC): + + @abstractmethod + def load_bar_data( + self, + symbol: str, + exchange: "Exchange", + interval: "Interval", + start: datetime, + end: datetime + ) -> Sequence["BarData"]: + pass + + @abstractmethod + def load_tick_data( + self, + symbol: str, + exchange: "Exchange", + start: datetime, + end: datetime + ) -> Sequence["TickData"]: + pass + + @abstractmethod + def save_bar_data( + self, + datas: Sequence["BarData"], + ): + pass + + @abstractmethod + def save_tick_data( + self, + datas: Sequence["TickData"], + ): + pass diff --git a/vnpy/trader/database.py b/vnpy/trader/database/database_mongo.py similarity index 51% rename from vnpy/trader/database.py rename to vnpy/trader/database/database_mongo.py index ef0c9cc2..ef5038e1 100644 --- a/vnpy/trader/database.py +++ b/vnpy/trader/database/database_mongo.py @@ -1,99 +1,59 @@ -"""""" +from datetime import datetime from enum import Enum -from typing import List +from typing import Sequence -from peewee import ( - AutoField, - CharField, - Database, - DateTimeField, - FloatField, - Model, - MySQLDatabase, - PostgresqlDatabase, - SqliteDatabase, - chunked, -) +from mongoengine import DateTimeField, Document, FloatField, StringField, connect -from .constant import Exchange, Interval -from .object import BarData, TickData -from .setting import SETTINGS -from .utility import get_file_path +from vnpy.trader.constant import Exchange, Interval +from vnpy.trader.object import BarData, TickData +from .database import BaseDatabaseManager, Driver -class Driver(Enum): - SQLITE = "sqlite" - MYSQL = "mysql" - POSTGRESQL = "postgresql" - - -_db: Database -_driver: Driver - - -def init(): - global _driver - db_settings = {k[9:]: v for k, v in SETTINGS.items() if k.startswith("database.")} - _driver = Driver(db_settings["driver"]) - - init_funcs = { - Driver.SQLITE: init_sqlite, - Driver.MYSQL: init_mysql, - Driver.POSTGRESQL: init_postgresql, - } - - assert _driver in init_funcs - del db_settings["driver"] - return init_funcs[_driver](db_settings) - - -def init_sqlite(settings: dict): - global _db +def init(_: Driver, settings: dict): database = settings["database"] - - _db = SqliteDatabase(str(get_file_path(database))) + host = settings["host"] + port = settings["port"] + username = settings["user"] + password = settings["password"] + authentication_source = settings["authentication_source"] + if not username: # if username == '' or None, skip username + username = None + password = None + authentication_source = None + connect( + db=database, + host=host, + port=port, + username=username, + password=password, + authentication_source=authentication_source, + ) + return MongoManager() -def init_mysql(settings: dict): - global _db - _db = MySQLDatabase(**settings) - - -def init_postgresql(settings: dict): - global _db - _db = PostgresqlDatabase(**settings) - - -init() - - -class ModelBase(Model): - def to_dict(self): - return self.__data__ - - -class DbBarData(ModelBase): +class DbBarData(Document): """ Candlestick bar data for database storage. Index is defined unique with datetime, interval, symbol """ - id = AutoField() - symbol = CharField() - exchange = CharField() - datetime = DateTimeField() - interval = CharField() + symbol: str = StringField() + exchange: str = StringField() + datetime: datetime = DateTimeField() + interval: str = StringField() - volume = FloatField() - open_price = FloatField() - high_price = FloatField() - low_price = FloatField() - close_price = FloatField() + volume: float = FloatField() + open_price: float = FloatField() + high_price: float = FloatField() + low_price: float = FloatField() + close_price: float = FloatField() - class Meta: - database = _db - indexes = ((("datetime", "interval", "symbol"), True),) + meta = { + "indexes": [ + {"fields": ("datetime", "interval", "symbol", "exchange"), "unique": True} + ] + } @staticmethod def from_bar(bar: BarData): @@ -132,80 +92,56 @@ class DbBarData(ModelBase): ) return bar - @staticmethod - def save_all(objs: List["DbBarData"]): - """ - save a list of objects, update if exists. - """ - with _db.atomic(): - if _driver is Driver.POSTGRESQL: - for bar in objs: - DbBarData.insert(bar.to_dict()).on_conflict( - update=bar.to_dict(), - conflict_target=( - DbBarData.datetime, - DbBarData.interval, - DbBarData.symbol, - ), - ).execute() - else: - for c in chunked(objs, 50): - DbBarData.insert_many(c).on_conflict_replace() - -class DbTickData(ModelBase): +class DbTickData(Document): """ Tick data for database storage. Index is defined unique with (datetime, symbol) """ - id = AutoField() + symbol: str = StringField() + exchange: str = StringField() + datetime: datetime = DateTimeField() - symbol = CharField() - exchange = CharField() - datetime = DateTimeField() + name: str = StringField() + volume: float = FloatField() + last_price: float = FloatField() + last_volume: float = FloatField() + limit_up: float = FloatField() + limit_down: float = FloatField() - name = CharField() - volume = FloatField() - last_price = FloatField() - last_volume = FloatField() - limit_up = FloatField() - limit_down = FloatField() + open_price: float = FloatField() + high_price: float = FloatField() + low_price: float = FloatField() + close_price: float = FloatField() + pre_close: float = FloatField() - open_price = FloatField() - high_price = FloatField() - low_price = FloatField() - close_price = FloatField() - pre_close = FloatField() + bid_price_1: float = FloatField() + bid_price_2: float = FloatField() + bid_price_3: float = FloatField() + bid_price_4: float = FloatField() + bid_price_5: float = FloatField() - bid_price_1 = FloatField() - bid_price_2 = FloatField() - bid_price_3 = FloatField() - bid_price_4 = FloatField() - bid_price_5 = FloatField() + ask_price_1: float = FloatField() + ask_price_2: float = FloatField() + ask_price_3: float = FloatField() + ask_price_4: float = FloatField() + ask_price_5: float = FloatField() - ask_price_1 = FloatField() - ask_price_2 = FloatField() - ask_price_3 = FloatField() - ask_price_4 = FloatField() - ask_price_5 = FloatField() + bid_volume_1: float = FloatField() + bid_volume_2: float = FloatField() + bid_volume_3: float = FloatField() + bid_volume_4: float = FloatField() + bid_volume_5: float = FloatField() - bid_volume_1 = FloatField() - bid_volume_2 = FloatField() - bid_volume_3 = FloatField() - bid_volume_4 = FloatField() - bid_volume_5 = FloatField() + ask_volume_1: float = FloatField() + ask_volume_2: float = FloatField() + ask_volume_3: float = FloatField() + ask_volume_4: float = FloatField() + ask_volume_5: float = FloatField() - ask_volume_1 = FloatField() - ask_volume_2 = FloatField() - ask_volume_3 = FloatField() - ask_volume_4 = FloatField() - ask_volume_5 = FloatField() - - class Meta: - database = _db - indexes = ((("datetime", "symbol"), True),) + meta = {"indexes": [{"fields": ("datetime", "symbol", "exchange"), "unique": True}]} @staticmethod def from_tick(tick: TickData): @@ -254,7 +190,7 @@ class DbTickData(ModelBase): db_tick.ask_volume_4 = tick.ask_volume_4 db_tick.ask_volume_5 = tick.ask_volume_5 - return tick + return db_tick def to_tick(self): """ @@ -304,20 +240,63 @@ class DbTickData(ModelBase): return tick + +class MongoManager(BaseDatabaseManager): + def load_bar_data( + self, + symbol: str, + exchange: Exchange, + interval: Interval, + start: datetime, + end: datetime, + ) -> Sequence[BarData]: + s = DbBarData.objects( + symbol=symbol, + exchange=exchange.value, + interval=interval.value, + datetime__gte=start, + datetime__lte=end, + ) + data = [db_bar.to_bar() for db_bar in s] + return data + + def load_tick_data( + self, symbol: str, exchange: Exchange, start: datetime, end: datetime + ) -> Sequence[TickData]: + s = DbTickData.objects( + symbol=symbol, + exchange=exchange.value, + datetime__gte=start, + datetime__lte=end, + ) + data = [db_tick.to_tick() for db_tick in s] + return data + @staticmethod - def save_all(objs: List["DbTickData"]): - with _db.atomic(): - if _driver is Driver.POSTGRESQL: - for bar in objs: - DbTickData.insert(bar.to_dict()).on_conflict( - update=bar.to_dict(), - preserve=(DbTickData.id), - conflict_target=(DbTickData.datetime, DbTickData.symbol), - ).execute() - else: - for c in chunked(objs, 50): - DbBarData.insert_many(c).on_conflict_replace() + def to_update_param(d): + return { + "set__" + k: v.value if isinstance(v, Enum) else v + for k, v in d.__dict__.items() + } + def save_bar_data(self, datas: Sequence[BarData]): + for d in datas: + updates = self.to_update_param(d) + updates.pop("set__gateway_name") + updates.pop("set__vt_symbol") + ( + DbBarData.objects( + symbol=d.symbol, interval=d.interval.value, datetime=d.datetime + ).update_one(upsert=True, **updates) + ) -_db.connect() -_db.create_tables([DbBarData, DbTickData]) + def save_tick_data(self, datas: Sequence[TickData]): + for d in datas: + updates = self.to_update_param(d) + updates.pop("set__gateway_name") + updates.pop("set__vt_symbol") + ( + DbTickData.objects( + symbol=d.symbol, exchange=d.exchange.value, datetime=d.datetime + ).update_one(upsert=True, **updates) + ) diff --git a/vnpy/trader/database/database_sql.py b/vnpy/trader/database/database_sql.py new file mode 100644 index 00000000..a82802ad --- /dev/null +++ b/vnpy/trader/database/database_sql.py @@ -0,0 +1,370 @@ +"""""" +from datetime import datetime +from typing import List, Sequence, Type + +from peewee import ( + AutoField, + CharField, + Database, + DateTimeField, + FloatField, + Model, + MySQLDatabase, + PostgresqlDatabase, + SqliteDatabase, + chunked, +) + +from vnpy.trader.constant import Exchange, Interval +from vnpy.trader.object import BarData, TickData +from vnpy.trader.utility import get_file_path +from .database import BaseDatabaseManager, Driver + + +def init(driver: Driver, settings: dict): + init_funcs = { + Driver.SQLITE: init_sqlite, + Driver.MYSQL: init_mysql, + Driver.POSTGRESQL: init_postgresql, + } + assert driver in init_funcs + + db = init_funcs[driver](settings) + bar, tick = init_models(db, driver) + return SqlManager(bar, tick) + + +def init_sqlite(settings: dict): + database = settings["database"] + path = str(get_file_path(database)) + db = SqliteDatabase(path) + return db + + +def init_mysql(settings: dict): + keys = {"database", "user", "password", "host", "port"} + settings = {k: v for k, v in settings.items() if k in keys} + db = MySQLDatabase(**settings) + return db + + +def init_postgresql(settings: dict): + keys = {"database", "user", "password", "host", "port"} + settings = {k: v for k, v in settings.items() if k in keys} + db = PostgresqlDatabase(**settings) + return db + + +class ModelBase(Model): + + def to_dict(self): + return self.__data__ + + +def init_models(db: Database, driver: Driver): + class DbBarData(ModelBase): + """ + Candlestick bar data for database storage. + + Index is defined unique with datetime, interval, symbol + """ + + id = AutoField() + symbol: str = CharField() + exchange: str = CharField() + datetime: datetime = DateTimeField() + interval: str = CharField() + + volume: float = FloatField() + open_price: float = FloatField() + high_price: float = FloatField() + low_price: float = FloatField() + close_price: float = FloatField() + + class Meta: + database = db + indexes = ((("datetime", "interval", "symbol", "exchange"), True),) + + @staticmethod + def from_bar(bar: BarData): + """ + Generate DbBarData object from BarData. + """ + db_bar = DbBarData() + + db_bar.symbol = bar.symbol + db_bar.exchange = bar.exchange.value + db_bar.datetime = bar.datetime + db_bar.interval = bar.interval.value + db_bar.volume = bar.volume + db_bar.open_price = bar.open_price + db_bar.high_price = bar.high_price + db_bar.low_price = bar.low_price + db_bar.close_price = bar.close_price + + return db_bar + + def to_bar(self): + """ + Generate BarData object from DbBarData. + """ + bar = BarData( + symbol=self.symbol, + exchange=Exchange(self.exchange), + datetime=self.datetime, + interval=Interval(self.interval), + volume=self.volume, + open_price=self.open_price, + high_price=self.high_price, + low_price=self.low_price, + close_price=self.close_price, + gateway_name="DB", + ) + return bar + + @staticmethod + def save_all(objs: List["DbBarData"]): + """ + save a list of objects, update if exists. + """ + dicts = [i.to_dict() for i in objs] + with db.atomic(): + if driver is Driver.POSTGRESQL: + for bar in dicts: + DbBarData.insert(bar).on_conflict( + update=bar, + conflict_target=( + DbBarData.datetime, + DbBarData.interval, + DbBarData.symbol, + DbBarData.exchange, + ), + ).execute() + else: + for c in chunked(dicts, 50): + DbBarData.insert_many(c).on_conflict_replace().execute() + + class DbTickData(ModelBase): + """ + Tick data for database storage. + + Index is defined unique with (datetime, symbol) + """ + + id = AutoField() + + symbol: str = CharField() + exchange: str = CharField() + datetime: datetime = DateTimeField() + + name: str = CharField() + volume: float = FloatField() + last_price: float = FloatField() + last_volume: float = FloatField() + limit_up: float = FloatField() + limit_down: float = FloatField() + + open_price: float = FloatField() + high_price: float = FloatField() + low_price: float = FloatField() + pre_close: float = FloatField() + + bid_price_1: float = FloatField() + bid_price_2: float = FloatField(null=True) + bid_price_3: float = FloatField(null=True) + bid_price_4: float = FloatField(null=True) + bid_price_5: float = FloatField(null=True) + + ask_price_1: float = FloatField() + ask_price_2: float = FloatField(null=True) + ask_price_3: float = FloatField(null=True) + ask_price_4: float = FloatField(null=True) + ask_price_5: float = FloatField(null=True) + + bid_volume_1: float = FloatField() + bid_volume_2: float = FloatField(null=True) + bid_volume_3: float = FloatField(null=True) + bid_volume_4: float = FloatField(null=True) + bid_volume_5: float = FloatField(null=True) + + ask_volume_1: float = FloatField() + ask_volume_2: float = FloatField(null=True) + ask_volume_3: float = FloatField(null=True) + ask_volume_4: float = FloatField(null=True) + ask_volume_5: float = FloatField(null=True) + + class Meta: + database = db + indexes = ((("datetime", "symbol", "exchange"), True),) + + @staticmethod + def from_tick(tick: TickData): + """ + Generate DbTickData object from TickData. + """ + db_tick = DbTickData() + + db_tick.symbol = tick.symbol + db_tick.exchange = tick.exchange.value + db_tick.datetime = tick.datetime + db_tick.name = tick.name + db_tick.volume = tick.volume + db_tick.last_price = tick.last_price + db_tick.last_volume = tick.last_volume + db_tick.limit_up = tick.limit_up + db_tick.limit_down = tick.limit_down + db_tick.open_price = tick.open_price + db_tick.high_price = tick.high_price + db_tick.low_price = tick.low_price + db_tick.pre_close = tick.pre_close + + db_tick.bid_price_1 = tick.bid_price_1 + db_tick.ask_price_1 = tick.ask_price_1 + db_tick.bid_volume_1 = tick.bid_volume_1 + db_tick.ask_volume_1 = tick.ask_volume_1 + + if tick.bid_price_2: + db_tick.bid_price_2 = tick.bid_price_2 + db_tick.bid_price_3 = tick.bid_price_3 + db_tick.bid_price_4 = tick.bid_price_4 + db_tick.bid_price_5 = tick.bid_price_5 + + db_tick.ask_price_2 = tick.ask_price_2 + db_tick.ask_price_3 = tick.ask_price_3 + db_tick.ask_price_4 = tick.ask_price_4 + db_tick.ask_price_5 = tick.ask_price_5 + + db_tick.bid_volume_2 = tick.bid_volume_2 + db_tick.bid_volume_3 = tick.bid_volume_3 + db_tick.bid_volume_4 = tick.bid_volume_4 + db_tick.bid_volume_5 = tick.bid_volume_5 + + db_tick.ask_volume_2 = tick.ask_volume_2 + db_tick.ask_volume_3 = tick.ask_volume_3 + db_tick.ask_volume_4 = tick.ask_volume_4 + db_tick.ask_volume_5 = tick.ask_volume_5 + + return db_tick + + def to_tick(self): + """ + Generate TickData object from DbTickData. + """ + tick = TickData( + symbol=self.symbol, + exchange=Exchange(self.exchange), + datetime=self.datetime, + name=self.name, + volume=self.volume, + last_price=self.last_price, + last_volume=self.last_volume, + limit_up=self.limit_up, + limit_down=self.limit_down, + open_price=self.open_price, + high_price=self.high_price, + low_price=self.low_price, + pre_close=self.pre_close, + bid_price_1=self.bid_price_1, + ask_price_1=self.ask_price_1, + bid_volume_1=self.bid_volume_1, + ask_volume_1=self.ask_volume_1, + gateway_name="DB", + ) + + if self.bid_price_2: + tick.bid_price_2 = self.bid_price_2 + tick.bid_price_3 = self.bid_price_3 + tick.bid_price_4 = self.bid_price_4 + tick.bid_price_5 = self.bid_price_5 + + tick.ask_price_2 = self.ask_price_2 + tick.ask_price_3 = self.ask_price_3 + tick.ask_price_4 = self.ask_price_4 + tick.ask_price_5 = self.ask_price_5 + + tick.bid_volume_2 = self.bid_volume_2 + tick.bid_volume_3 = self.bid_volume_3 + tick.bid_volume_4 = self.bid_volume_4 + tick.bid_volume_5 = self.bid_volume_5 + + tick.ask_volume_2 = self.ask_volume_2 + tick.ask_volume_3 = self.ask_volume_3 + tick.ask_volume_4 = self.ask_volume_4 + tick.ask_volume_5 = self.ask_volume_5 + + return tick + + @staticmethod + def save_all(objs: List["DbTickData"]): + dicts = [i.to_dict() for i in objs] + with db.atomic(): + if driver is Driver.POSTGRESQL: + for tick in dicts: + DbTickData.insert(tick).on_conflict( + update=tick, + conflict_target=( + DbTickData.datetime, + DbTickData.symbol, + DbTickData.exchange, + ), + ).execute() + else: + for c in chunked(dicts, 50): + DbTickData.insert_many(c).on_conflict_replace().execute() + + db.connect() + db.create_tables([DbBarData, DbTickData]) + return DbBarData, DbTickData + + +class SqlManager(BaseDatabaseManager): + + def __init__(self, class_bar: Type[Model], class_tick: Type[Model]): + self.class_bar = class_bar + self.class_tick = class_tick + + def load_bar_data( + self, + symbol: str, + exchange: Exchange, + interval: Interval, + start: datetime, + end: datetime, + ) -> Sequence[BarData]: + s = ( + self.class_bar.select() + .where( + (self.class_bar.symbol == symbol) + & (self.class_bar.exchange == exchange.value) + & (self.class_bar.interval == interval.value) + & (self.class_bar.datetime >= start) + & (self.class_bar.datetime <= end) + ) + .order_by(self.class_bar.datetime) + ) + data = [db_bar.to_bar() for db_bar in s] + return data + + def load_tick_data( + self, symbol: str, exchange: Exchange, start: datetime, end: datetime + ) -> Sequence[TickData]: + s = ( + self.class_tick.select() + .where( + (self.class_tick.symbol == symbol) + & (self.class_tick.exchange == exchange.value) + & (self.class_tick.datetime >= start) + & (self.class_tick.datetime <= end) + ) + .order_by(self.class_tick.datetime) + ) + data = [db_tick.to_tick() for db_tick in s] + return data + + def save_bar_data(self, datas: Sequence[BarData]): + ds = [self.class_bar.from_bar(i) for i in datas] + self.class_bar.save_all(ds) + + def save_tick_data(self, datas: Sequence[TickData]): + ds = [self.class_tick.from_tick(i) for i in datas] + self.class_tick.save_all(ds) diff --git a/vnpy/trader/database/initialize.py b/vnpy/trader/database/initialize.py new file mode 100644 index 00000000..a15b484c --- /dev/null +++ b/vnpy/trader/database/initialize.py @@ -0,0 +1,24 @@ +"""""" +from .database import BaseDatabaseManager, Driver + + +def init(settings: dict) -> BaseDatabaseManager: + driver = Driver(settings["driver"]) + if driver is Driver.MONGODB: + return init_nosql(driver=driver, settings=settings) + else: + return init_sql(driver=driver, settings=settings) + + +def init_sql(driver: Driver, settings: dict): + from .database_sql import init + keys = {'database', "host", "port", "user", "password"} + settings = {k: v for k, v in settings.items() if k in keys} + _database_manager = init(driver, settings) + return _database_manager + + +def init_nosql(driver: Driver, settings: dict): + from .database_mongo import init + _database_manager = init(driver, settings=settings) + return _database_manager diff --git a/vnpy/trader/setting.py b/vnpy/trader/setting.py index 778b147c..1dd8b0ff 100644 --- a/vnpy/trader/setting.py +++ b/vnpy/trader/setting.py @@ -30,9 +30,15 @@ SETTINGS = { "database.host": "localhost", "database.port": 3306, "database.user": "root", - "database.password": "" + "database.password": "", + "database.authentication_source": "admin", # for mongodb } # Load global setting from json file. SETTING_FILENAME = "vt_setting.json" SETTINGS.update(load_json(SETTING_FILENAME)) + + +def get_settings(prefix: str = ""): + prefix_length = len(prefix) + return {k[prefix_length:]: v for k, v in SETTINGS.items() if k.startswith(prefix)} diff --git a/vnpy/trader/utility.py b/vnpy/trader/utility.py index 216dcbae..69fa6991 100644 --- a/vnpy/trader/utility.py +++ b/vnpy/trader/utility.py @@ -3,20 +3,28 @@ General utility functions. """ import json -import os from pathlib import Path -from typing import Callable +from typing import Callable, TYPE_CHECKING import numpy as np import talib from .object import BarData, TickData +if TYPE_CHECKING: + from vnpy.trader.constant import Exchange -def resolve_path(pattern: str): - env = dict(os.environ) - env.update({"VNPY_TEMP": str(TEMP_DIR)}) - return pattern.format(**env) + +def extract_vt_symbol(vt_symbol: str): + """ + :return: (symbol, exchange) + """ + symbol, exchange = vt_symbol.split('.') + return symbol, exchange + + +def generate_vt_symbol(symbol: str, exchange: "Exchange"): + return f'{symbol}.{exchange.value}' def _get_trader_dir(temp_name: str): From 6a7b9e139d979c1be941851e29bf6ad7fa632283 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 05:58:18 -0400 Subject: [PATCH 04/62] =?UTF-8?q?[Mod]=20=E4=BF=AE=E6=AD=A3=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 1 + tests/test_all.py | 17 ++++++++++++++--- vnpy/app/cta_strategy/backtesting.py | 3 +-- vnpy/trader/database/database.py | 4 ++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index d38b8fed..32672af3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ services: script: # todo: use python unittest + - pip install psycopg2 mongoengine pymysql # we should support all database in test environment - cd tests; source travis_env.sh; python test_all.py matrix: diff --git a/tests/test_all.py b/tests/test_all.py index e824ff21..e5f8b6c2 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -1,10 +1,10 @@ # tests/runner.py import unittest +import app # import your test modules import test_import_all import trader -import app # initialize the test suite loader = unittest.TestLoader() @@ -15,6 +15,17 @@ suite.addTests(loader.loadTestsFromModule(test_import_all)) suite.addTests(loader.loadTestsFromModule(trader)) suite.addTests(loader.loadTestsFromModule(app)) + # initialize a runner, pass it your suite and run it -runner = unittest.TextTestRunner(verbosity=3) -result = runner.run(suite) +def main(): + runner = unittest.TextTestRunner(verbosity=3) + result = runner.run(suite) + return result + + +if __name__ == '__main__': + result = main() + if result.failures: + exit(1) + else: + exit(0) diff --git a/vnpy/app/cta_strategy/backtesting.py b/vnpy/app/cta_strategy/backtesting.py index e4661107..9263ed2a 100644 --- a/vnpy/app/cta_strategy/backtesting.py +++ b/vnpy/app/cta_strategy/backtesting.py @@ -14,7 +14,7 @@ from vnpy.trader.constant import (Direction, Offset, Exchange, Interval, Status) from vnpy.trader.database import database_manager from vnpy.trader.object import OrderData, TradeData, BarData, TickData -from vnpy.trader.utility import round_to_pricetick, extract_vt_symbol +from vnpy.trader.utility import round_to_pricetick from .base import ( BacktestingMode, @@ -990,4 +990,3 @@ def load_tick_data( return database_manager.load_tick_data( symbol, exchange, start, end ) - diff --git a/vnpy/trader/database/database.py b/vnpy/trader/database/database.py index 06077383..ec765e8d 100644 --- a/vnpy/trader/database/database.py +++ b/vnpy/trader/database/database.py @@ -4,8 +4,8 @@ from enum import Enum from typing import Sequence, TYPE_CHECKING if TYPE_CHECKING: - from vnpy.trader.constant import Interval, Exchange - from vnpy.trader.object import BarData, TickData + from vnpy.trader.constant import Interval, Exchange # noqa + from vnpy.trader.object import BarData, TickData # noqa class Driver(Enum): From 509cb7c94378aabccf8ca400ab3e3f055c2388c6 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 06:02:06 -0400 Subject: [PATCH 05/62] [Mod] make flake8 happy --- vnpy/trader/database/database_sql.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/vnpy/trader/database/database_sql.py b/vnpy/trader/database/database_sql.py index a82802ad..61d6e2ff 100644 --- a/vnpy/trader/database/database_sql.py +++ b/vnpy/trader/database/database_sql.py @@ -56,7 +56,6 @@ def init_postgresql(settings: dict): class ModelBase(Model): - def to_dict(self): return self.__data__ @@ -318,7 +317,6 @@ def init_models(db: Database, driver: Driver): class SqlManager(BaseDatabaseManager): - def __init__(self, class_bar: Type[Model], class_tick: Type[Model]): self.class_bar = class_bar self.class_tick = class_tick @@ -333,14 +331,14 @@ class SqlManager(BaseDatabaseManager): ) -> Sequence[BarData]: s = ( self.class_bar.select() - .where( + .where( (self.class_bar.symbol == symbol) & (self.class_bar.exchange == exchange.value) & (self.class_bar.interval == interval.value) & (self.class_bar.datetime >= start) & (self.class_bar.datetime <= end) ) - .order_by(self.class_bar.datetime) + .order_by(self.class_bar.datetime) ) data = [db_bar.to_bar() for db_bar in s] return data @@ -350,14 +348,15 @@ class SqlManager(BaseDatabaseManager): ) -> Sequence[TickData]: s = ( self.class_tick.select() - .where( + .where( (self.class_tick.symbol == symbol) & (self.class_tick.exchange == exchange.value) & (self.class_tick.datetime >= start) & (self.class_tick.datetime <= end) ) - .order_by(self.class_tick.datetime) + .order_by(self.class_tick.datetime) ) + data = [db_tick.to_tick() for db_tick in s] return data From 9f5c2124b29746fa26f69c1b937960d5fc5b6dfb Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 06:07:20 -0400 Subject: [PATCH 06/62] [Mod] make travis happy --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 32672af3..6b4ba6e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ script: matrix: include: - name: "code quality analysis: flake8" + services: [] # service is not need before_install: - pip install flake8 install: @@ -28,6 +29,7 @@ matrix: - name: "pip install under Windows" os: "windows" + services: [] # service is unavailable under windows # language : cpp is necessary for windows language: "cpp" env: From a2679d706dcd37a91968b306e46b063144be7343 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 06:10:53 -0400 Subject: [PATCH 07/62] [Add] travis: add osx build --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6b4ba6e7..3ee3b4cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,6 +77,13 @@ matrix: - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh + - name: "pip install under osx" + language: shell # osx supports only shell + install: + - python -m pip install --upgrade pip wheel setuptools + - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl + - bash ./install.sh + - name: "sdist install under Windows" os: "windows" # language : cpp is necessary for windows From 024d5b7bb7ea7e03f1c007229da00f5ed49f865d Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 06:19:10 -0400 Subject: [PATCH 08/62] [Add] travis.yml: create database for tests --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3ee3b4cc..89c989b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,10 @@ services: - mysql - postgresql +before_script: + - mysql -e 'CREATE DATABASE vnpy;' + - psql -c 'create database vnpy;' -U postgres + script: # todo: use python unittest - pip install psycopg2 mongoengine pymysql # we should support all database in test environment From 62bdf19b3da868342dc1ccbf1ef555332a1aabe5 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 06:42:16 -0400 Subject: [PATCH 09/62] [Fix] fix travis.yml --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 89c989b5..bcd34f9c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ matrix: include: - name: "code quality analysis: flake8" services: [] # service is not need + before_script: [] before_install: - pip install flake8 install: @@ -39,7 +40,9 @@ matrix: env: - PATH=/c/Python37:/c/Python37/Scripts:$PATH before_install: - - choco install python3 --version 3.7.2 + - choco install -yf mysql mongodb + - choco install -yf postgresql --params '/Password:' + - choco install -yf python3 --version 3.7.2 install: - python -m pip install --upgrade pip wheel setuptools - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl @@ -84,7 +87,7 @@ matrix: - name: "pip install under osx" language: shell # osx supports only shell install: - - python -m pip install --upgrade pip wheel setuptools + - python -m pip install --upgrade pip wheel setuptools --user python - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh From 21ba27aa87c9155f3dc230e6b4190211627d0265 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 06:52:02 -0400 Subject: [PATCH 10/62] [mod] make flake8 happy --- tests/trader/test_settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/trader/test_settings.py b/tests/trader/test_settings.py index c8e86132..2d46e73e 100644 --- a/tests/trader/test_settings.py +++ b/tests/trader/test_settings.py @@ -2,6 +2,7 @@ Test if database works fine """ import unittest + from vnpy.trader.setting import SETTINGS, get_settings @@ -21,4 +22,4 @@ class TestSettings(unittest.TestCase): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From ce752b9d749df158be46948f9059565999c52145 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 07:03:38 -0400 Subject: [PATCH 11/62] [Fix] added alias for osx --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index bcd34f9c..a3afdb51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,6 +87,8 @@ matrix: - name: "pip install under osx" language: shell # osx supports only shell install: + - alias python=python3 + - alias pip=pip3 - python -m pip install --upgrade pip wheel setuptools --user python - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh From 3cd60e0dcc4cf2f1e4b4e7a635dc2ceac2164a3a Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 07:08:55 -0400 Subject: [PATCH 12/62] [Mod] use postgresql10 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a3afdb51..d5c9ca76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ matrix: - PATH=/c/Python37:/c/Python37/Scripts:$PATH before_install: - choco install -yf mysql mongodb - - choco install -yf postgresql --params '/Password:' + - choco install -yf postgresql10 --params '/Password:' - choco install -yf python3 --version 3.7.2 install: - python -m pip install --upgrade pip wheel setuptools From 69aae4371ed30177193c6923455ecf30c98b4296 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 11:06:21 -0400 Subject: [PATCH 13/62] [Fix] fixed windows sdist build --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d5c9ca76..8538ac20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -95,12 +95,16 @@ matrix: - name: "sdist install under Windows" os: "windows" + services: [] # service is not need + before_script: [] # language : cpp is necessary for windows language: "cpp" env: - PATH=/c/Python37:/c/Python37/Scripts:$PATH before_install: - - choco install python3 --version 3.7.2 + - choco install -yf mysql mongodb + - choco install -yf postgresql10 --params '/Password:' + - choco install -yf python3 --version 3.7.2 install: - python -m pip install --upgrade pip wheel setuptools - python setup.py sdist From 2c7443a7a93bfd89ac293957c4c68bb512cea653 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 11:10:26 -0400 Subject: [PATCH 14/62] [Fix] fixed osx build --- .travis.yml | 8 +++----- install.sh | 16 +++++++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8538ac20..9d0546ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,11 +87,9 @@ matrix: - name: "pip install under osx" language: shell # osx supports only shell install: - - alias python=python3 - - alias pip=pip3 - - python -m pip install --upgrade pip wheel setuptools --user python - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - bash ./install.sh + - python3 -m pip install --upgrade pip wheel setuptools --user python + - pip3 install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl + - bash ./install.sh python3 pip3 - name: "sdist install under Windows" os: "windows" diff --git a/install.sh b/install.sh index 2cf2043c..12a08096 100644 --- a/install.sh +++ b/install.sh @@ -1,5 +1,11 @@ #!/usr/bin/env bash +python=$1 +pip=$2 + +[[ -z $python ]] && python=python +[[ -z $pip ]] && pip=pip + # Get and build ta-lib pushd /tmp wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz @@ -11,17 +17,17 @@ sudo make install popd # old versions of ta-lib imports numpy in setup.py -pip install numpy +$pip install numpy # Install extra packages -pip install ta-lib -pip install https://vnpy-pip.oss-cn-shanghai.aliyuncs.com/colletion/ibapi-9.75.1-py3-none-any.whl +$pip install ta-lib +$pip install https://vnpy-pip.oss-cn-shanghai.aliyuncs.com/colletion/ibapi-9.75.1-py3-none-any.whl # Install Python Modules -pip install -r requirements.txt +$pip install -r requirements.txt # Install local Chinese language environment sudo locale-gen zh_CN.GB18030 # Install vn.py -pip install . \ No newline at end of file +$pip install . \ No newline at end of file From 3e37dfd1d76983058f6d9b503ae352b13646aca9 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 13:23:02 -0400 Subject: [PATCH 15/62] [Fix] windows build --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9d0546ed..fdc00bbd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,8 @@ before_script: script: # todo: use python unittest - pip install psycopg2 mongoengine pymysql # we should support all database in test environment - - cd tests; source travis_env.sh; python test_all.py + - cd tests; source travis_env.sh; + - python test_all.py matrix: include: @@ -38,7 +39,7 @@ matrix: # language : cpp is necessary for windows language: "cpp" env: - - PATH=/c/Python37:/c/Python37/Scripts:$PATH + - PATH=/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH before_install: - choco install -yf mysql mongodb - choco install -yf postgresql10 --params '/Password:' @@ -98,7 +99,7 @@ matrix: # language : cpp is necessary for windows language: "cpp" env: - - PATH=/c/Python37:/c/Python37/Scripts:$PATH + - PATH=/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH before_install: - choco install -yf mysql mongodb - choco install -yf postgresql10 --params '/Password:' From dc9cf41e1dcdf8a2f18ef8448ee004270fac83f9 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 13:26:32 -0400 Subject: [PATCH 16/62] [Mod] change build sequence --- .travis.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index fdc00bbd..5e1ce9ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,23 +51,6 @@ matrix: - pip install -r requirements.txt - pip install . - - name: "pip install under Ubuntu: gcc-8" - before_install: - # C++17 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -y - install: - # C++17 - - sudo apt-get install -y gcc-8 g++-8 - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 - # update pip & setuptools - - python -m pip install --upgrade pip wheel setuptools - # Linux install script - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - bash ./install.sh - - name: "pip install under Ubuntu: gcc-7" before_install: # C++17 @@ -92,6 +75,23 @@ matrix: - pip3 install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh python3 pip3 + - name: "pip install under Ubuntu: gcc-8" + before_install: + # C++17 + - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + - sudo apt-get update -y + install: + # C++17 + - sudo apt-get install -y gcc-8 g++-8 + - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 + - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 + - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 + # update pip & setuptools + - python -m pip install --upgrade pip wheel setuptools + # Linux install script + - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl + - bash ./install.sh + - name: "sdist install under Windows" os: "windows" services: [] # service is not need From 752905b5078cfdd500f4589a91251069d05d7bd3 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 13:28:28 -0400 Subject: [PATCH 17/62] [Fix] fixed spaces in windows path --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5e1ce9ad..9b0eb4fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: # language : cpp is necessary for windows language: "cpp" env: - - PATH=/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH + - PATH=/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program\ Files/PostgreSQL/10/bin:$PATH before_install: - choco install -yf mysql mongodb - choco install -yf postgresql10 --params '/Password:' @@ -99,7 +99,7 @@ matrix: # language : cpp is necessary for windows language: "cpp" env: - - PATH=/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH + - PATH=/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program\ Files/PostgreSQL/10/bin:$PATH before_install: - choco install -yf mysql mongodb - choco install -yf postgresql10 --params '/Password:' From 5dbca46a32f32908b9e7b56199921766389fe368 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 13:31:16 -0400 Subject: [PATCH 18/62] [Fix] try to fix windows path --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9b0eb4fd..0119c5e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: # language : cpp is necessary for windows language: "cpp" env: - - PATH=/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program\ Files/PostgreSQL/10/bin:$PATH + - PATH='/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:'/c/Program Files/PostgreSQL/10/bin':$PATH' before_install: - choco install -yf mysql mongodb - choco install -yf postgresql10 --params '/Password:' @@ -99,7 +99,7 @@ matrix: # language : cpp is necessary for windows language: "cpp" env: - - PATH=/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program\ Files/PostgreSQL/10/bin:$PATH + - PATH='/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH' before_install: - choco install -yf mysql mongodb - choco install -yf postgresql10 --params '/Password:' From 807d5f4e7d24684f0702ddd55be4941eb32ced38 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 23:30:19 -0400 Subject: [PATCH 19/62] [Fix] try to fix path --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0119c5e3..0d4b7765 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: # language : cpp is necessary for windows language: "cpp" env: - - PATH='/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:'/c/Program Files/PostgreSQL/10/bin':$PATH' + - PATH="/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH" before_install: - choco install -yf mysql mongodb - choco install -yf postgresql10 --params '/Password:' @@ -99,7 +99,7 @@ matrix: # language : cpp is necessary for windows language: "cpp" env: - - PATH='/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH' + - PATH="/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH" before_install: - choco install -yf mysql mongodb - choco install -yf postgresql10 --params '/Password:' From 5eec6d0092765658d08f6c8f6fd9a2ace48f211b Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 23:31:30 -0400 Subject: [PATCH 20/62] [Fix] not to upgrade pip & setuptools in osx --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0d4b7765..09e86a8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,7 +71,6 @@ matrix: - name: "pip install under osx" language: shell # osx supports only shell install: - - python3 -m pip install --upgrade pip wheel setuptools --user python - pip3 install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh python3 pip3 From 7d2692cbb6a4800b3551abefe7813e46fd7360a4 Mon Sep 17 00:00:00 2001 From: nanoric Date: Sun, 14 Apr 2019 23:32:00 -0400 Subject: [PATCH 21/62] [Del] remove osx build --- .travis.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 09e86a8e..8b455829 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,12 +68,6 @@ matrix: - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh - - name: "pip install under osx" - language: shell # osx supports only shell - install: - - pip3 install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - bash ./install.sh python3 pip3 - - name: "pip install under Ubuntu: gcc-8" before_install: # C++17 From 080f0cc1311c06af1e9a1b928d719bcb1df5d83c Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 00:22:21 -0400 Subject: [PATCH 22/62] [Mod] psql: use connection string [Mod] test_all.py: return 1 if errors --- .travis.yml | 2 +- tests/test_all.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8b455829..6224b4a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,8 @@ services: - postgresql before_script: + - psql -d postgresql://postgres:@localhost -c "create database vnpy;" - mysql -e 'CREATE DATABASE vnpy;' - - psql -c 'create database vnpy;' -U postgres script: # todo: use python unittest diff --git a/tests/test_all.py b/tests/test_all.py index e5f8b6c2..59b7683e 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -25,7 +25,7 @@ def main(): if __name__ == '__main__': result = main() - if result.failures: + if result.failures or result.errors: exit(1) else: exit(0) From a7a80e0e9ad8bd5afa6f85f41b6405d5dc763208 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 01:26:45 -0400 Subject: [PATCH 23/62] [Fix] try to fix errors --- .travis.yml | 86 +++++++-------------------------------------- install.sh | 2 +- tests/travis_env.sh | 40 ++++++++++++++------- 3 files changed, 41 insertions(+), 87 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6224b4a2..d9a7538f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,8 @@ services: - postgresql before_script: - - psql -d postgresql://postgres:@localhost -c "create database vnpy;" - - mysql -e 'CREATE DATABASE vnpy;' + - psql -d postgresql://postgres:${VNPY_TEST_POSTGRESQL_PASSWORD}@localhost -c "create database vnpy;" + - mysql -u root --password=${VNPY_TEST_MYSQL_PASSWORD} -e 'CREATE DATABASE vnpy;' script: # todo: use python unittest @@ -23,16 +23,6 @@ script: matrix: include: - - name: "code quality analysis: flake8" - services: [] # service is not need - before_script: [] - before_install: - - pip install flake8 - install: - - "" # prevent running "pip install -r requirements.txt" - script: - - flake8 - - name: "pip install under Windows" os: "windows" services: [] # service is unavailable under windows @@ -42,7 +32,9 @@ matrix: - PATH="/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH" before_install: - choco install -yf mysql mongodb - - choco install -yf postgresql10 --params '/Password:' + # I don't know how enable a "trust" auth method in choco installed psql + - choco install -yf postgresql10 --params '/Password:1234' + - export VNPY_TEST_POSTGRESQL_PASSWORD=1234 - choco install -yf python3 --version 3.7.2 install: - python -m pip install --upgrade pip wheel setuptools @@ -51,22 +43,15 @@ matrix: - pip install -r requirements.txt - pip install . - - name: "pip install under Ubuntu: gcc-7" - before_install: - # C++17 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -y + - name: "pip install under osx" + os: osx + services: [] + before_script: [] + language: shell # osx supports only shell + before_install: [] install: - # C++17 - - sudo apt-get install -y gcc-7 g++-7 - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-7 90 - # update pip & setuptools - - python -m pip install --upgrade pip wheel setuptools - # Linux install script - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - bash ./install.sh + - pip3 install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl + - bash ./install.sh python3 pip3 - name: "pip install under Ubuntu: gcc-8" before_install: @@ -84,48 +69,3 @@ matrix: # Linux install script - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh - - - name: "sdist install under Windows" - os: "windows" - services: [] # service is not need - before_script: [] - # language : cpp is necessary for windows - language: "cpp" - env: - - PATH="/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH" - before_install: - - choco install -yf mysql mongodb - - choco install -yf postgresql10 --params '/Password:' - - choco install -yf python3 --version 3.7.2 - install: - - python -m pip install --upgrade pip wheel setuptools - - python setup.py sdist - - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - pip install dist/`ls dist` - - - name: "sdist install under Ubuntu: gcc-8" - before_install: - # C++17 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -y - install: - # C++17 - - sudo apt-get install -y gcc-8 g++-8 - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 - # Linux install script - - python -m pip install --upgrade pip wheel setuptools - - pushd /tmp - - wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz - - tar -xf ta-lib-0.4.0-src.tar.gz - - cd ta-lib - - ./configure --prefix=/usr - - make - - sudo make install - - popd - - pip install numpy - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - python setup.py sdist - - pip install dist/`ls dist` diff --git a/install.sh b/install.sh index 12a08096..3f445045 100644 --- a/install.sh +++ b/install.sh @@ -11,7 +11,7 @@ pushd /tmp wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar -xf ta-lib-0.4.0-src.tar.gz cd ta-lib -./configure --prefix=/usr +./configure --prefix=/usr/local make sudo make install popd diff --git a/tests/travis_env.sh b/tests/travis_env.sh index 030d79c5..e2e59613 100644 --- a/tests/travis_env.sh +++ b/tests/travis_env.sh @@ -1,16 +1,30 @@ #!/usr/bin/env bash -export VNPY_TEST_MYSQL_DATABASE=vnpy -export VNPY_TEST_MYSQL_HOST=127.0.0.1 -export VNPY_TEST_MYSQL_PORT=3306 -export VNPY_TEST_MYSQL_USER=root -export VNPY_TEST_MYSQL_PASSWORD= -export VNPY_TEST_POSTGRESQL_DATABASE=vnpy -export VNPY_TEST_POSTGRESQL_HOST=127.0.0.1 -export VNPY_TEST_POSTGRESQL_PORT=5432 -export VNPY_TEST_POSTGRESQL_USER=postgres -export VNPY_TEST_POSTGRESQL_PASSWORD= -export VNPY_TEST_MONGODB_DATABASE=vnpy -export VNPY_TEST_MONGODB_HOST=127.0.0.1 -export VNPY_TEST_MONGODB_PORT=27017 +[[ -z ${VNPY_TEST_MYSQL_DATABASE} ]] && VNPY_TEST_MYSQL_DATABASE=vnpy +[[ -z ${VNPY_TEST_MYSQL_HOST} ]] && VNPY_TEST_MYSQL_HOST=127.0.0.1 +[[ -z ${VNPY_TEST_MYSQL_PORT} ]] && VNPY_TEST_MYSQL_PORT=3306 +[[ -z ${VNPY_TEST_MYSQL_USER} ]] && VNPY_TEST_MYSQL_USER=root +[[ -z ${VNPY_TEST_MYSQL_PASSWORD} ]] && VNPY_TEST_MYSQL_PASSWORD= +[[ -z ${VNPY_TEST_POSTGRESQL_DATABASE} ]] && VNPY_TEST_POSTGRESQL_DATABASE=vnpy +[[ -z ${VNPY_TEST_POSTGRESQL_HOST} ]] && VNPY_TEST_POSTGRESQL_HOST=127.0.0.1 +[[ -z ${VNPY_TEST_POSTGRESQL_PORT} ]] && VNPY_TEST_POSTGRESQL_PORT=5432 +[[ -z ${VNPY_TEST_POSTGRESQL_USER} ]] && VNPY_TEST_POSTGRESQL_USER=postgres +[[ -z ${VNPY_TEST_POSTGRESQL_PASSWORD} ]] && VNPY_TEST_POSTGRESQL_PASSWORD= +[[ -z ${VNPY_TEST_MONGODB_DATABASE} ]] && VNPY_TEST_MONGODB_DATABASE=vnpy +[[ -z ${VNPY_TEST_MONGODB_HOST} ]] && VNPY_TEST_MONGODB_HOST=127.0.0.1 +[[ -z ${VNPY_TEST_MONGODB_PORT} ]] && VNPY_TEST_MONGODB_PORT=27017 + +export VNPY_TEST_MYSQL_DATABASE +export VNPY_TEST_MYSQL_HOST +export VNPY_TEST_MYSQL_PORT +export VNPY_TEST_MYSQL_USER +export VNPY_TEST_MYSQL_PASSWORD +export VNPY_TEST_POSTGRESQL_DATABASE +export VNPY_TEST_POSTGRESQL_HOST +export VNPY_TEST_POSTGRESQL_PORT +export VNPY_TEST_POSTGRESQL_USER +export VNPY_TEST_POSTGRESQL_PASSWORD +export VNPY_TEST_MONGODB_DATABASE +export VNPY_TEST_MONGODB_HOST +export VNPY_TEST_MONGODB_PORT From 37a2aa754f3576a51e558646fae684ec402fb81a Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 01:41:02 -0400 Subject: [PATCH 24/62] [Fix] try fix osx --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d9a7538f..b890e1ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,6 @@ before_script: - mysql -u root --password=${VNPY_TEST_MYSQL_PASSWORD} -e 'CREATE DATABASE vnpy;' script: - # todo: use python unittest - pip install psycopg2 mongoengine pymysql # we should support all database in test environment - cd tests; source travis_env.sh; - python test_all.py @@ -53,6 +52,11 @@ matrix: - pip3 install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh python3 pip3 + script: + - pip3 install psycopg2 mongoengine pymysql # we should support all database in test environment + - cd tests; source travis_env.sh; + - python3 test_all.py + - name: "pip install under Ubuntu: gcc-8" before_install: # C++17 From 39930e9db378faae9d4c5da4915cba31ebb9a4cd Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 02:08:08 -0400 Subject: [PATCH 25/62] [Fix] try fix --- .travis.yml | 2 +- install.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b890e1ca..a23d2fae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ matrix: before_install: [] install: - pip3 install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - bash ./install.sh python3 pip3 + - bash ./install.sh python3 pip3 /usr/local script: - pip3 install psycopg2 mongoengine pymysql # we should support all database in test environment diff --git a/install.sh b/install.sh index 3f445045..3dc936ac 100644 --- a/install.sh +++ b/install.sh @@ -2,16 +2,18 @@ python=$1 pip=$2 +prefix=$3 [[ -z $python ]] && python=python [[ -z $pip ]] && pip=pip +[[ -z $prefix ]] && prefix=/usr # Get and build ta-lib pushd /tmp wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar -xf ta-lib-0.4.0-src.tar.gz cd ta-lib -./configure --prefix=/usr/local +./configure --prefix=$prefix make sudo make install popd From 36f66d4724a54af8929c18d236ab79b459d53fd8 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 03:17:30 -0400 Subject: [PATCH 26/62] [Mod] fixed travis --- .travis.yml | 48 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8622616d..2620d6c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,9 +5,19 @@ dist: xenial # required for Python >= 3.7 (travis-ci/travis-ci#9069) python: - "3.7" +services: + - mongodb + - mysql + - postgresql + +before_script: + - psql -d postgresql://postgres:${VNPY_TEST_POSTGRESQL_PASSWORD}@localhost -c "create database vnpy;" + - mysql -u root --password=${VNPY_TEST_MYSQL_PASSWORD} -e 'CREATE DATABASE vnpy;' + script: - # todo: use python unittest - - mkdir run; cd run; python ../tests/load_all.py + - pip install psycopg2 mongoengine pymysql # we should support all database in test environment + - cd tests; source travis_env.sh; + - python test_all.py matrix: include: @@ -21,12 +31,17 @@ matrix: - name: "pip install under Windows" os: "windows" + services: [] # service is unavailable under windows # language : cpp is necessary for windows language: "cpp" env: - - PATH=/c/Python37:/c/Python37/Scripts:$PATH + - PATH="/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH" before_install: - - choco install python3 --version 3.7.2 + - choco install -yf mysql mongodb + # I don't know how enable a "trust" auth method in choco installed psql + - choco install -yf postgresql10 --params '/Password:1234' + - export VNPY_TEST_POSTGRESQL_PASSWORD=1234 + - choco install -yf python3 --version 3.7.2 install: - python -m pip install --upgrade pip wheel setuptools - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl @@ -68,14 +83,34 @@ matrix: - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh + - name: "pip install under osx" + os: osx + language: shell # osx supports only shell + services: [] + before_install: [] + install: + - pip3 install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl + - bash ./install_osx.sh + before_script: [] + script: + - pip3 install psycopg2 mongoengine pymysql # we should support all database in test environment + - cd tests; source travis_env.sh; + # osx is complicated for me, skip the tests + # python3 test_all.py + - name: "sdist install under Windows" os: "windows" + services: [] # service is unavailable under windows # language : cpp is necessary for windows language: "cpp" env: - - PATH=/c/Python37:/c/Python37/Scripts:$PATH + - PATH="/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH" before_install: - - choco install python3 --version 3.7.2 + - choco install -yf mysql mongodb + # I don't know how enable a "trust" auth method in choco installed psql + - choco install -yf postgresql10 --params '/Password:1234' + - export VNPY_TEST_POSTGRESQL_PASSWORD=1234 + - choco install -yf python3 --version 3.7.2 install: - python -m pip install --upgrade pip wheel setuptools - python setup.py sdist @@ -108,3 +143,4 @@ matrix: - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - python setup.py sdist - pip install dist/`ls dist` + From f17d2d8b5dea3a9bd6d886040c1282cf808936e3 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 03:17:49 -0400 Subject: [PATCH 27/62] [Add] added install_osx.sh --- install_osx.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 install_osx.sh diff --git a/install_osx.sh b/install_osx.sh new file mode 100644 index 00000000..eb96cc53 --- /dev/null +++ b/install_osx.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +bash ./install.sh python3 pip3 /usr/local \ No newline at end of file From 41534abdbc6641f63988de4798cbd0211cf7ba09 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 05:24:09 -0400 Subject: [PATCH 28/62] appveyor --- .travis.yml | 146 --------------------------------------------------- appveyor.yml | 37 +++++++++++++ 2 files changed, 37 insertions(+), 146 deletions(-) delete mode 100644 .travis.yml create mode 100644 appveyor.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2620d6c6..00000000 --- a/.travis.yml +++ /dev/null @@ -1,146 +0,0 @@ -language: python - -dist: xenial # required for Python >= 3.7 (travis-ci/travis-ci#9069) - -python: - - "3.7" - -services: - - mongodb - - mysql - - postgresql - -before_script: - - psql -d postgresql://postgres:${VNPY_TEST_POSTGRESQL_PASSWORD}@localhost -c "create database vnpy;" - - mysql -u root --password=${VNPY_TEST_MYSQL_PASSWORD} -e 'CREATE DATABASE vnpy;' - -script: - - pip install psycopg2 mongoengine pymysql # we should support all database in test environment - - cd tests; source travis_env.sh; - - python test_all.py - -matrix: - include: - - name: "code quality analysis: flake8" - before_install: - - pip install flake8 - install: - - "" # prevent running "pip install -r requirements.txt" - script: - - flake8 - - - name: "pip install under Windows" - os: "windows" - services: [] # service is unavailable under windows - # language : cpp is necessary for windows - language: "cpp" - env: - - PATH="/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH" - before_install: - - choco install -yf mysql mongodb - # I don't know how enable a "trust" auth method in choco installed psql - - choco install -yf postgresql10 --params '/Password:1234' - - export VNPY_TEST_POSTGRESQL_PASSWORD=1234 - - choco install -yf python3 --version 3.7.2 - install: - - python -m pip install --upgrade pip wheel setuptools - - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - pip install -r requirements.txt - - pip install . - - - name: "pip install under Ubuntu: gcc-8" - before_install: - # C++17 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -y - install: - # C++17 - - sudo apt-get install -y gcc-8 g++-8 - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 - # update pip & setuptools - - python -m pip install --upgrade pip wheel setuptools - # Linux install script - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - bash ./install.sh - - - name: "pip install under Ubuntu: gcc-7" - before_install: - # C++17 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -y - install: - # C++17 - - sudo apt-get install -y gcc-7 g++-7 - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-7 90 - # update pip & setuptools - - python -m pip install --upgrade pip wheel setuptools - # Linux install script - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - bash ./install.sh - - - name: "pip install under osx" - os: osx - language: shell # osx supports only shell - services: [] - before_install: [] - install: - - pip3 install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - bash ./install_osx.sh - before_script: [] - script: - - pip3 install psycopg2 mongoengine pymysql # we should support all database in test environment - - cd tests; source travis_env.sh; - # osx is complicated for me, skip the tests - # python3 test_all.py - - - name: "sdist install under Windows" - os: "windows" - services: [] # service is unavailable under windows - # language : cpp is necessary for windows - language: "cpp" - env: - - PATH="/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH" - before_install: - - choco install -yf mysql mongodb - # I don't know how enable a "trust" auth method in choco installed psql - - choco install -yf postgresql10 --params '/Password:1234' - - export VNPY_TEST_POSTGRESQL_PASSWORD=1234 - - choco install -yf python3 --version 3.7.2 - install: - - python -m pip install --upgrade pip wheel setuptools - - python setup.py sdist - - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - pip install dist/`ls dist` - - - name: "sdist install under Ubuntu: gcc-8" - before_install: - # C++17 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -y - install: - # C++17 - - sudo apt-get install -y gcc-8 g++-8 - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 - # Linux install script - - python -m pip install --upgrade pip wheel setuptools - - pushd /tmp - - wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz - - tar -xf ta-lib-0.4.0-src.tar.gz - - cd ta-lib - - ./configure --prefix=/usr - - make - - sudo make install - - popd - - pip install numpy - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - python setup.py sdist - - pip install dist/`ls dist` - diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..1e418c04 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,37 @@ +image: + - Visual Studio 2017 + - Ubuntu1804 + +services: + - mysql + - mongodb + - postgresql + +environment: + PATH: C:\Python37-x64;C:\Python37-x64\Scripts;C:\Program Files\PostgreSQL\9.6\bin\;C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql;%PATH% + VNPY_TEST_MYSQL_USER: root + VNPY_TEST_MYSQL_PASSWORD: Password12! + VNPY_TEST_POSTGRESQL_USER: postgres + VNPY_TEST_POSTGRESQL_PASSWORD: Password12! + + matrix: + - aaafb: a + - bbb: fa + +before_build: + - psql -d postgresql://%VNPY_TEST_POSTGRESQL_USER%:%VNPY_TEST_POSTGRESQL_PASSWORD%@localhost -c "create database vnpy;" + - mysql -u %VNPY_TEST_MYSQL_USER% --password=%VNPY_TEST_MYSQL_PASSWORD% -e 'CREATE DATABASE vnpy;' + - python -m pip install --upgrade pip wheel setuptools + + +install: + - pip install psycopg2 mongoengine pymysql # we should support all database in test environment + - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl + - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl + - pip install -r requirements.txt + - pip install . + +test_script: + - cd tests; source travis_env.sh; + - python test_all.py + From 872244edbcec1062b506ed70166fa9702886f5af Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 05:33:08 -0400 Subject: [PATCH 29/62] [Del] remove Ubuntu1804 --- appveyor.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 1e418c04..c6b64f30 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,5 @@ image: - Visual Studio 2017 - - Ubuntu1804 services: - mysql @@ -14,16 +13,14 @@ environment: VNPY_TEST_POSTGRESQL_USER: postgres VNPY_TEST_POSTGRESQL_PASSWORD: Password12! - matrix: - - aaafb: a - - bbb: fa - before_build: - psql -d postgresql://%VNPY_TEST_POSTGRESQL_USER%:%VNPY_TEST_POSTGRESQL_PASSWORD%@localhost -c "create database vnpy;" - - mysql -u %VNPY_TEST_MYSQL_USER% --password=%VNPY_TEST_MYSQL_PASSWORD% -e 'CREATE DATABASE vnpy;' + - ls "C:\Program Files" + - ls "C:\Program Files\MySQL\" + - ps: ls -Rec "C:\Program Files\MySQL\" + - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u %VNPY_TEST_MYSQL_USER% --password=%VNPY_TEST_MYSQL_PASSWORD% -e 'CREATE DATABASE vnpy;' - python -m pip install --upgrade pip wheel setuptools - install: - pip install psycopg2 mongoengine pymysql # we should support all database in test environment - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl From ccb57bcf9ab5063b0c1ffad5c2b5f77fe2d31286 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 05:41:16 -0400 Subject: [PATCH 30/62] use ps --- appveyor.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index c6b64f30..e021ae93 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,8 +15,6 @@ environment: before_build: - psql -d postgresql://%VNPY_TEST_POSTGRESQL_USER%:%VNPY_TEST_POSTGRESQL_PASSWORD%@localhost -c "create database vnpy;" - - ls "C:\Program Files" - - ls "C:\Program Files\MySQL\" - ps: ls -Rec "C:\Program Files\MySQL\" - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u %VNPY_TEST_MYSQL_USER% --password=%VNPY_TEST_MYSQL_PASSWORD% -e 'CREATE DATABASE vnpy;' - python -m pip install --upgrade pip wheel setuptools From b54652cd79e07db54cd7bdd642196698e08bcd17 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 05:45:47 -0400 Subject: [PATCH 31/62] [Mod] ps var --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e021ae93..2347c2c5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,16 +7,16 @@ services: - postgresql environment: - PATH: C:\Python37-x64;C:\Python37-x64\Scripts;C:\Program Files\PostgreSQL\9.6\bin\;C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql;%PATH% + PATH: C:\Python37-x64;C:\Python37-x64\Scripts;C:\Program Files\PostgreSQL\9.6\bin\;%PATH% VNPY_TEST_MYSQL_USER: root VNPY_TEST_MYSQL_PASSWORD: Password12! VNPY_TEST_POSTGRESQL_USER: postgres VNPY_TEST_POSTGRESQL_PASSWORD: Password12! + MYSQL_PWD: Password12! before_build: - psql -d postgresql://%VNPY_TEST_POSTGRESQL_USER%:%VNPY_TEST_POSTGRESQL_PASSWORD%@localhost -c "create database vnpy;" - - ps: ls -Rec "C:\Program Files\MySQL\" - - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u %VNPY_TEST_MYSQL_USER% --password=%VNPY_TEST_MYSQL_PASSWORD% -e 'CREATE DATABASE vnpy;' + - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $VNPY_TEST_MYSQL_USER " -e 'CREATE DATABASE vnpy;' - python -m pip install --upgrade pip wheel setuptools install: From 807e8ab69b089032102f3c962c125e887b584831 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 05:53:56 -0400 Subject: [PATCH 32/62] all ps --- appveyor.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 2347c2c5..4c2f9385 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,3 +1,5 @@ +clone_depth: 1 + image: - Visual Studio 2017 @@ -15,8 +17,8 @@ environment: MYSQL_PWD: Password12! before_build: - - psql -d postgresql://%VNPY_TEST_POSTGRESQL_USER%:%VNPY_TEST_POSTGRESQL_PASSWORD%@localhost -c "create database vnpy;" - - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $VNPY_TEST_MYSQL_USER " -e 'CREATE DATABASE vnpy;' + - ps: psql -d "postgresql://$VNPY_TEST_POSTGRESQL_USER:$VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" + - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $VNPY_TEST_MYSQL_USER -e 'CREATE DATABASE vnpy;' - python -m pip install --upgrade pip wheel setuptools install: From fecc1c87e46438aad368bb546427c764b3bc5b31 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 05:57:32 -0400 Subject: [PATCH 33/62] [Mod] formal travis gcc upgrade --- .travis.yml | 51 ++++++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2620d6c6..55cf5718 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,16 +50,15 @@ matrix: - pip install . - name: "pip install under Ubuntu: gcc-8" - before_install: - # C++17 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -y + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-8 + env: + - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" install: - # C++17 - - sudo apt-get install -y gcc-8 g++-8 - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 # update pip & setuptools - python -m pip install --upgrade pip wheel setuptools # Linux install script @@ -67,16 +66,15 @@ matrix: - bash ./install.sh - name: "pip install under Ubuntu: gcc-7" - before_install: - # C++17 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -y + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-7 + env: + - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" install: - # C++17 - - sudo apt-get install -y gcc-7 g++-7 - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-7 90 # update pip & setuptools - python -m pip install --upgrade pip wheel setuptools # Linux install script @@ -119,16 +117,15 @@ matrix: - pip install dist/`ls dist` - name: "sdist install under Ubuntu: gcc-8" - before_install: - # C++17 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -y + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-7 + env: + - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" install: - # C++17 - - sudo apt-get install -y gcc-8 g++-8 - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 # Linux install script - python -m pip install --upgrade pip wheel setuptools - pushd /tmp From 7e6207efb8f27d9000d2cc5c4ff5bd7a722ac433 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 05:59:34 -0400 Subject: [PATCH 34/62] [Add] env prefix --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 4c2f9385..2a09eb10 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,8 +17,8 @@ environment: MYSQL_PWD: Password12! before_build: - - ps: psql -d "postgresql://$VNPY_TEST_POSTGRESQL_USER:$VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" - - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $VNPY_TEST_MYSQL_USER -e 'CREATE DATABASE vnpy;' + - ps: psql -d "postgresql://$ENV:VNPY_TEST_POSTGRESQL_USER:$ENV:VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" + - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $ENV:VNPY_TEST_MYSQL_USER -e 'CREATE DATABASE vnpy;' - python -m pip install --upgrade pip wheel setuptools install: From c4b29254c5363b198f674860aee8dfa0c8fab7c5 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 06:03:07 -0400 Subject: [PATCH 35/62] [Mod] new sequence --- appveyor.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 2a09eb10..61df6ea2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,18 +16,20 @@ environment: VNPY_TEST_POSTGRESQL_PASSWORD: Password12! MYSQL_PWD: Password12! -before_build: - - ps: psql -d "postgresql://$ENV:VNPY_TEST_POSTGRESQL_USER:$ENV:VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" - - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $ENV:VNPY_TEST_MYSQL_USER -e 'CREATE DATABASE vnpy;' +install: - python -m pip install --upgrade pip wheel setuptools -install: +build_script: - pip install psycopg2 mongoengine pymysql # we should support all database in test environment - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - pip install -r requirements.txt - pip install . +before_test: + - ps: psql -d "postgresql://$ENV:VNPY_TEST_POSTGRESQL_USER:$ENV:VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" + - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $ENV:VNPY_TEST_MYSQL_USER -e 'CREATE DATABASE vnpy;' + test_script: - cd tests; source travis_env.sh; - python test_all.py From 4ae728288323f7176d607416a4af3e5f0c697796 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 06:05:20 -0400 Subject: [PATCH 36/62] echo --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 61df6ea2..14f12227 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,6 +27,7 @@ build_script: - pip install . before_test: + - ps: echo psql -d "postgresql://$ENV:VNPY_TEST_POSTGRESQL_USER:$ENV:VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" - ps: psql -d "postgresql://$ENV:VNPY_TEST_POSTGRESQL_USER:$ENV:VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $ENV:VNPY_TEST_MYSQL_USER -e 'CREATE DATABASE vnpy;' From b06868509283aa4d57319149352a1dfc4e8dd394 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 06:10:29 -0400 Subject: [PATCH 37/62] mod seq --- appveyor.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 14f12227..5511872e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,6 +17,8 @@ environment: MYSQL_PWD: Password12! install: + - ps: psql -d "postgresql://$ENV:VNPY_TEST_POSTGRESQL_USER:$ENV:VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" + - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $ENV:VNPY_TEST_MYSQL_USER -e 'CREATE DATABASE vnpy;' - python -m pip install --upgrade pip wheel setuptools build_script: @@ -26,11 +28,6 @@ build_script: - pip install -r requirements.txt - pip install . -before_test: - - ps: echo psql -d "postgresql://$ENV:VNPY_TEST_POSTGRESQL_USER:$ENV:VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" - - ps: psql -d "postgresql://$ENV:VNPY_TEST_POSTGRESQL_USER:$ENV:VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" - - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $ENV:VNPY_TEST_MYSQL_USER -e 'CREATE DATABASE vnpy;' - test_script: - cd tests; source travis_env.sh; - python test_all.py From 46b5c89f85321f232f14161898bc54a1e32c17dc Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 06:11:28 -0400 Subject: [PATCH 38/62] [Mod] new seq --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 5511872e..b67258e2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,9 +17,10 @@ environment: MYSQL_PWD: Password12! install: + - python -m pip install --upgrade pip wheel setuptools +before_build: - ps: psql -d "postgresql://$ENV:VNPY_TEST_POSTGRESQL_USER:$ENV:VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $ENV:VNPY_TEST_MYSQL_USER -e 'CREATE DATABASE vnpy;' - - python -m pip install --upgrade pip wheel setuptools build_script: - pip install psycopg2 mongoengine pymysql # we should support all database in test environment From a5c0dbcaa18885f238ad763b47b13694c396d8b7 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 06:14:27 -0400 Subject: [PATCH 39/62] [Mod] added{} --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index b67258e2..f864c5a9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,7 +19,7 @@ environment: install: - python -m pip install --upgrade pip wheel setuptools before_build: - - ps: psql -d "postgresql://$ENV:VNPY_TEST_POSTGRESQL_USER:$ENV:VNPY_TEST_POSTGRESQL_PASSWORD@localhost" -c "create database vnpy;" + - ps: psql -d "postgresql://${ENV:VNPY_TEST_POSTGRESQL_USER}:${ENV:VNPY_TEST_POSTGRESQL_PASSWORD}@localhost" -c "create database vnpy;" - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $ENV:VNPY_TEST_MYSQL_USER -e 'CREATE DATABASE vnpy;' build_script: From d3167a8bdf49730341fe119d8d11dd054a8b6b1c Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 06:24:29 -0400 Subject: [PATCH 40/62] [Add] sdist build --- appveyor.yml | 50 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index f864c5a9..d3194b9c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,5 +1,9 @@ clone_depth: 1 +configuration: + - pip + - sdist + image: - Visual Studio 2017 @@ -10,26 +14,58 @@ services: environment: PATH: C:\Python37-x64;C:\Python37-x64\Scripts;C:\Program Files\PostgreSQL\9.6\bin\;%PATH% + VNPY_TEST_MYSQL_DATABASE: vnpy + VNPY_TEST_MYSQL_HOST: localhost + VNPY_TEST_MYSQL_PORT: 3306 VNPY_TEST_MYSQL_USER: root VNPY_TEST_MYSQL_PASSWORD: Password12! + + VNPY_TEST_POSTGRESQL_DATABASE: + VNPY_TEST_POSTGRESQL_HOST: localhost + VNPY_TEST_POSTGRESQL_PORT: 5432 VNPY_TEST_POSTGRESQL_USER: postgres VNPY_TEST_POSTGRESQL_PASSWORD: Password12! + + VNPY_TEST_MONGODB_DATABASE: vnpy + VNPY_TEST_MONGODB_HOST: localhost + VNPY_TEST_MONGODB_PORT: 27017 + MYSQL_PWD: Password12! + matrix: + - BUILD: pip + - BUILD: sdist + install: - python -m pip install --upgrade pip wheel setuptools before_build: - ps: psql -d "postgresql://${ENV:VNPY_TEST_POSTGRESQL_USER}:${ENV:VNPY_TEST_POSTGRESQL_PASSWORD}@localhost" -c "create database vnpy;" - ps: . "C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -u $ENV:VNPY_TEST_MYSQL_USER -e 'CREATE DATABASE vnpy;' -build_script: - - pip install psycopg2 mongoengine pymysql # we should support all database in test environment - - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - pip install -r requirements.txt - - pip install . +for: + - matrix: + only: + configuration: + pip + build_script: + - pip install psycopg2 mongoengine pymysql # we should support all database in test environment + - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl + - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl + - pip install -r requirements.txt + - pip install . + + - matrix: + only: + configuration: + sdist + build_script: + - python setup.py sdist + - pip install psycopg2 mongoengine pymysql # we should support all database in test environment + - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl + - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl + - pip install dist/`ls dist` test_script: - - cd tests; source travis_env.sh; + - cd tests; - python test_all.py From 6d6a592dd379aabf2a7fbccd316ca46000ed210c Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 06:25:14 -0400 Subject: [PATCH 41/62] [Fix] fix only --- appveyor.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d3194b9c..d38d1915 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -45,8 +45,7 @@ before_build: for: - matrix: only: - configuration: - pip + - configuration: pip build_script: - pip install psycopg2 mongoengine pymysql # we should support all database in test environment - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl @@ -56,8 +55,7 @@ for: - matrix: only: - configuration: - sdist + - configuration: sdist build_script: - python setup.py sdist - pip install psycopg2 mongoengine pymysql # we should support all database in test environment From b782e18a0e20b21db972cab6369c6b33e081ab3c Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 06:26:24 -0400 Subject: [PATCH 42/62] [Del] remove matrix in env --- appveyor.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d38d1915..1ee8bc46 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -32,10 +32,6 @@ environment: MYSQL_PWD: Password12! - matrix: - - BUILD: pip - - BUILD: sdist - install: - python -m pip install --upgrade pip wheel setuptools before_build: From 100aa8d41744baec327d1bceab417a9becad69d4 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 06:30:13 -0400 Subject: [PATCH 43/62] [Mod] use ps --- appveyor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 1ee8bc46..e2f73236 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -60,6 +60,5 @@ for: - pip install dist/`ls dist` test_script: - - cd tests; - - python test_all.py + - ps: cd tests; python test_all.py From 5b9b081a63e032b24c722306ddc4559c7e363ee2 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 06:35:22 -0400 Subject: [PATCH 44/62] [Mod] added missed key --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index e2f73236..842a1ab0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,7 +20,7 @@ environment: VNPY_TEST_MYSQL_USER: root VNPY_TEST_MYSQL_PASSWORD: Password12! - VNPY_TEST_POSTGRESQL_DATABASE: + VNPY_TEST_POSTGRESQL_DATABASE: vnpy VNPY_TEST_POSTGRESQL_HOST: localhost VNPY_TEST_POSTGRESQL_PORT: 5432 VNPY_TEST_POSTGRESQL_USER: postgres From 8ec4a1f78aca521cc59f0f5699da4d24d7285eae Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 06:38:46 -0400 Subject: [PATCH 45/62] [Del] removed windows build from travis --- .travis.yml | 52 ++++++++++++---------------------------------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/.travis.yml b/.travis.yml index 55cf5718..d06c6d1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,26 +29,6 @@ matrix: script: - flake8 - - name: "pip install under Windows" - os: "windows" - services: [] # service is unavailable under windows - # language : cpp is necessary for windows - language: "cpp" - env: - - PATH="/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH" - before_install: - - choco install -yf mysql mongodb - # I don't know how enable a "trust" auth method in choco installed psql - - choco install -yf postgresql10 --params '/Password:1234' - - export VNPY_TEST_POSTGRESQL_PASSWORD=1234 - - choco install -yf python3 --version 3.7.2 - install: - - python -m pip install --upgrade pip wheel setuptools - - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - pip install -r requirements.txt - - pip install . - - name: "pip install under Ubuntu: gcc-8" addons: apt: @@ -58,6 +38,10 @@ matrix: - g++-8 env: - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" + before_install: + - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 + - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 + - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 install: # update pip & setuptools - python -m pip install --upgrade pip wheel setuptools @@ -74,6 +58,10 @@ matrix: - g++-7 env: - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" + before_install: + - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 + - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 + - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 install: # update pip & setuptools - python -m pip install --upgrade pip wheel setuptools @@ -96,26 +84,6 @@ matrix: # osx is complicated for me, skip the tests # python3 test_all.py - - name: "sdist install under Windows" - os: "windows" - services: [] # service is unavailable under windows - # language : cpp is necessary for windows - language: "cpp" - env: - - PATH="/c/Python37:/c/Python37/Scripts:/c/tools/mysql/current/bin:/c/Program Files/PostgreSQL/10/bin:$PATH" - before_install: - - choco install -yf mysql mongodb - # I don't know how enable a "trust" auth method in choco installed psql - - choco install -yf postgresql10 --params '/Password:1234' - - export VNPY_TEST_POSTGRESQL_PASSWORD=1234 - - choco install -yf python3 --version 3.7.2 - install: - - python -m pip install --upgrade pip wheel setuptools - - python setup.py sdist - - pip install https://pip.vnpy.com/colletion/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - pip install dist/`ls dist` - - name: "sdist install under Ubuntu: gcc-8" addons: apt: @@ -125,6 +93,10 @@ matrix: - g++-7 env: - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" + before_install: + - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 + - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 + - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 install: # Linux install script - python -m pip install --upgrade pip wheel setuptools From 8ebcf984e3e0d0df120a9162f29696718d76f7fb Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 07:01:18 -0400 Subject: [PATCH 46/62] [Fix] fixed gcc-8 links --- .travis.yml | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index d06c6d1e..0324742b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,26 +49,6 @@ matrix: - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh - - name: "pip install under Ubuntu: gcc-7" - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-7 - env: - - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" - before_install: - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 - install: - # update pip & setuptools - - python -m pip install --upgrade pip wheel setuptools - # Linux install script - - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - bash ./install.sh - - name: "pip install under osx" os: osx language: shell # osx supports only shell @@ -94,9 +74,9 @@ matrix: env: - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" before_install: - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 + - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 + - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 90 + - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-7 90 install: # Linux install script - python -m pip install --upgrade pip wheel setuptools From 98c677118aecf41e83a706353041384857f1db67 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 08:54:28 -0400 Subject: [PATCH 47/62] [Add] run tests under osx --- .travis.yml | 28 +++++++++--------- tests/trader/test_database.py | 55 ++++++++++++++++++----------------- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0324742b..b1268f55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,21 +49,6 @@ matrix: - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh - - name: "pip install under osx" - os: osx - language: shell # osx supports only shell - services: [] - before_install: [] - install: - - pip3 install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - bash ./install_osx.sh - before_script: [] - script: - - pip3 install psycopg2 mongoengine pymysql # we should support all database in test environment - - cd tests; source travis_env.sh; - # osx is complicated for me, skip the tests - # python3 test_all.py - - name: "sdist install under Ubuntu: gcc-8" addons: apt: @@ -93,3 +78,16 @@ matrix: - python setup.py sdist - pip install dist/`ls dist` + - name: "pip install under osx" + os: osx + language: shell # osx supports only shell + services: [] + before_install: [] + install: + - pip3 install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl + - bash ./install_osx.sh + before_script: [] + script: + - pip3 install psycopg2 mongoengine pymysql # we should support all database in test environment + - cd tests; source travis_env.sh; + - VNPY_TEST_ONLY_SQLITE=1 python3 test_all.py diff --git a/tests/trader/test_database.py b/tests/trader/test_database.py index 59f1dcfc..df009baf 100644 --- a/tests/trader/test_database.py +++ b/tests/trader/test_database.py @@ -15,33 +15,36 @@ profiles = { Driver.SQLITE: { "driver": "sqlite", "database": "test_db.db", - }, - Driver.MYSQL: { - "driver": "mysql", - "database": os.environ['VNPY_TEST_MYSQL_DATABASE'], - "host": os.environ['VNPY_TEST_MYSQL_HOST'], - "port": int(os.environ['VNPY_TEST_MYSQL_PORT']), - "user": os.environ["VNPY_TEST_MYSQL_USER"], - "password": os.environ['VNPY_TEST_MYSQL_PASSWORD'], - }, - Driver.POSTGRESQL: { - "driver": "postgresql", - "database": os.environ['VNPY_TEST_POSTGRESQL_DATABASE'], - "host": os.environ['VNPY_TEST_POSTGRESQL_HOST'], - "port": int(os.environ['VNPY_TEST_POSTGRESQL_PORT']), - "user": os.environ["VNPY_TEST_POSTGRESQL_USER"], - "password": os.environ['VNPY_TEST_POSTGRESQL_PASSWORD'], - }, - Driver.MONGODB: { - "driver": "mongodb", - "database": os.environ['VNPY_TEST_MONGODB_DATABASE'], - "host": os.environ['VNPY_TEST_MONGODB_HOST'], - "port": int(os.environ['VNPY_TEST_MONGODB_PORT']), - "user": "", - "password": "", - "authentication_source": "", - }, + } } +if 'VNPY_TEST_ONLY_SQLITE' not in os.environ: + profiles.update({ + Driver.MYSQL: { + "driver": "mysql", + "database": os.environ['VNPY_TEST_MYSQL_DATABASE'], + "host": os.environ['VNPY_TEST_MYSQL_HOST'], + "port": int(os.environ['VNPY_TEST_MYSQL_PORT']), + "user": os.environ["VNPY_TEST_MYSQL_USER"], + "password": os.environ['VNPY_TEST_MYSQL_PASSWORD'], + }, + Driver.POSTGRESQL: { + "driver": "postgresql", + "database": os.environ['VNPY_TEST_POSTGRESQL_DATABASE'], + "host": os.environ['VNPY_TEST_POSTGRESQL_HOST'], + "port": int(os.environ['VNPY_TEST_POSTGRESQL_PORT']), + "user": os.environ["VNPY_TEST_POSTGRESQL_USER"], + "password": os.environ['VNPY_TEST_POSTGRESQL_PASSWORD'], + }, + Driver.MONGODB: { + "driver": "mongodb", + "database": os.environ['VNPY_TEST_MONGODB_DATABASE'], + "host": os.environ['VNPY_TEST_MONGODB_HOST'], + "port": int(os.environ['VNPY_TEST_MONGODB_PORT']), + "user": "", + "password": "", + "authentication_source": "", + }, + }) def now(): From bb73601705318258519436558a63ec5e38e0de58 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 08:59:57 -0400 Subject: [PATCH 48/62] [Mod] use CC and CXX (specific in MATRIX_EVAL) --- .travis.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index b1268f55..68f911b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,9 @@ services: - mysql - postgresql +before_install: + - eval "${MATRIX_EVAL}" + before_script: - psql -d postgresql://postgres:${VNPY_TEST_POSTGRESQL_PASSWORD}@localhost -c "create database vnpy;" - mysql -u root --password=${VNPY_TEST_MYSQL_PASSWORD} -e 'CREATE DATABASE vnpy;' @@ -38,10 +41,6 @@ matrix: - g++-8 env: - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" - before_install: - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 install: # update pip & setuptools - python -m pip install --upgrade pip wheel setuptools @@ -58,10 +57,6 @@ matrix: - g++-7 env: - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" - before_install: - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-7 90 install: # Linux install script - python -m pip install --upgrade pip wheel setuptools From a87f990815d5111ed1715f6ecae965d8c0913852 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 09:06:23 -0400 Subject: [PATCH 49/62] [Add] enable travis cache for pip --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 68f911b2..fdd635b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,11 @@ language: python dist: xenial # required for Python >= 3.7 (travis-ci/travis-ci#9069) +cache: pip + +git: + depth: 1 + python: - "3.7" From c4f0e1ba2ae13426640d2a164efa72a414877589 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 09:10:41 -0400 Subject: [PATCH 50/62] [Mod] echo test --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fdd635b8..321aacfb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ services: - postgresql before_install: + - echo eval "${MATRIX_EVAL}" - eval "${MATRIX_EVAL}" before_script: From 4c1fcb91171f1d819e5c250192d0208ff6c17e48 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 09:14:23 -0400 Subject: [PATCH 51/62] [Add] added cache for appveyor --- appveyor.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index f60e91c3..437edefc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,5 +1,8 @@ clone_depth: 1 +cache: + - '%LOCALAPPDATA%\pip\Cache' + configuration: - pip - sdist From 874b01d437557c611200e63a6ee274c21e479708 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 09:25:54 -0400 Subject: [PATCH 52/62] [Mod] use update-alternatives again --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 321aacfb..47e3f3db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,10 +15,6 @@ services: - mysql - postgresql -before_install: - - echo eval "${MATRIX_EVAL}" - - eval "${MATRIX_EVAL}" - before_script: - psql -d postgresql://postgres:${VNPY_TEST_POSTGRESQL_PASSWORD}@localhost -c "create database vnpy;" - mysql -u root --password=${VNPY_TEST_MYSQL_PASSWORD} -e 'CREATE DATABASE vnpy;' @@ -45,8 +41,10 @@ matrix: - ubuntu-toolchain-r-test packages: - g++-8 - env: - - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8" + before_install: + - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 + - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 90 + - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-7 90 install: # update pip & setuptools - python -m pip install --upgrade pip wheel setuptools @@ -54,15 +52,17 @@ matrix: - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - bash ./install.sh - - name: "sdist install under Ubuntu: gcc-8" + - name: "sdist install under Ubuntu: gcc-7" addons: apt: sources: - ubuntu-toolchain-r-test packages: - g++-7 - env: - - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" + before_install: + - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 + - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 90 + - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-7 90 install: # Linux install script - python -m pip install --upgrade pip wheel setuptools From 4c4dac1ae9ff302c9254a534a3ecaf19ee09facb Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 09:33:18 -0400 Subject: [PATCH 53/62] [Fix] fix gcc 8 link error --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 47e3f3db..0ef835b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,9 +42,9 @@ matrix: packages: - g++-8 before_install: - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 - - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 90 - - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-7 90 + - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 + - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 + - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 install: # update pip & setuptools - python -m pip install --upgrade pip wheel setuptools From 818146f0265e677baf5d3ae71cb02e04d068d628 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 09:35:25 -0400 Subject: [PATCH 54/62] [Add] use make -j2 for travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0ef835b3..3583f550 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,6 +50,7 @@ matrix: - python -m pip install --upgrade pip wheel setuptools # Linux install script - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl + - alias make=make -j2 - bash ./install.sh - name: "sdist install under Ubuntu: gcc-7" @@ -77,6 +78,7 @@ matrix: - pip install numpy - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - python setup.py sdist + - alias make=make -j2 - pip install dist/`ls dist` - name: "pip install under osx" From 48590fdafd4307552f154008f8c2efabe242fddd Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 09:48:10 -0400 Subject: [PATCH 55/62] [Add] quotes make -j2 --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3583f550..840dd228 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ services: before_script: - psql -d postgresql://postgres:${VNPY_TEST_POSTGRESQL_PASSWORD}@localhost -c "create database vnpy;" - mysql -u root --password=${VNPY_TEST_MYSQL_PASSWORD} -e 'CREATE DATABASE vnpy;' + - alias make="make -j2" script: - pip install psycopg2 mongoengine pymysql # we should support all database in test environment @@ -50,7 +51,6 @@ matrix: - python -m pip install --upgrade pip wheel setuptools # Linux install script - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - alias make=make -j2 - bash ./install.sh - name: "sdist install under Ubuntu: gcc-7" @@ -78,7 +78,6 @@ matrix: - pip install numpy - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - python setup.py sdist - - alias make=make -j2 - pip install dist/`ls dist` - name: "pip install under osx" From 430220959de47c186bcd4dde0032564e555946ca Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 10:11:14 -0400 Subject: [PATCH 56/62] [Add] use ccompiler from numpy if there is one --- .travis.yml | 7 ++++--- setup.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 840dd228..89d73a45 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,6 @@ services: before_script: - psql -d postgresql://postgres:${VNPY_TEST_POSTGRESQL_PASSWORD}@localhost -c "create database vnpy;" - mysql -u root --password=${VNPY_TEST_MYSQL_PASSWORD} -e 'CREATE DATABASE vnpy;' - - alias make="make -j2" script: - pip install psycopg2 mongoengine pymysql # we should support all database in test environment @@ -43,6 +42,7 @@ matrix: packages: - g++-8 before_install: + - alias make="make -j2" - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 @@ -51,7 +51,7 @@ matrix: - python -m pip install --upgrade pip wheel setuptools # Linux install script - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - bash ./install.sh + - NPY_NUM_BUILD_JOBS=2 bash ./install.sh - name: "sdist install under Ubuntu: gcc-7" addons: @@ -61,6 +61,7 @@ matrix: packages: - g++-7 before_install: + - alias make="make -j2" - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 90 - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-7 90 @@ -78,7 +79,7 @@ matrix: - pip install numpy - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - python setup.py sdist - - pip install dist/`ls dist` + - NPY_NUM_BUILD_JOBS=2 pip install dist/`ls dist` - name: "pip install under osx" os: osx diff --git a/setup.py b/setup.py index 28250618..4540f70e 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,23 @@ import sys from setuptools import Extension, find_packages, setup + +def hook_compiler(): + """ + if numpy is installed, use compiler from that instead of setuptools, + which run faster in multi-core env + """ + try: + from numpy.distutils.ccompiler import CCompiler_compile + import distutils.ccompiler + + distutils.ccompiler.CCompiler.compile = CCompiler_compile + except ImportError: + pass + + +hook_compiler() + with open("vnpy/__init__.py", "rb") as f: version_line = re.search( r"__version__\s+=\s+(.*)", f.read().decode("utf-8") From c3546a8bf755e1852e2a5ad2e92f574e541234c5 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 11:12:08 -0400 Subject: [PATCH 57/62] [Del] remove numpy ccompiler --- .travis.yml | 6 ++---- setup.py | 17 ----------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index 89d73a45..0ef835b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,6 @@ matrix: packages: - g++-8 before_install: - - alias make="make -j2" - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-8 90 - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-8 90 @@ -51,7 +50,7 @@ matrix: - python -m pip install --upgrade pip wheel setuptools # Linux install script - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - - NPY_NUM_BUILD_JOBS=2 bash ./install.sh + - bash ./install.sh - name: "sdist install under Ubuntu: gcc-7" addons: @@ -61,7 +60,6 @@ matrix: packages: - g++-7 before_install: - - alias make="make -j2" - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90 - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 90 - sudo update-alternatives --install /usr/bin/gcc cc /usr/bin/gcc-7 90 @@ -79,7 +77,7 @@ matrix: - pip install numpy - pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl - python setup.py sdist - - NPY_NUM_BUILD_JOBS=2 pip install dist/`ls dist` + - pip install dist/`ls dist` - name: "pip install under osx" os: osx diff --git a/setup.py b/setup.py index 4540f70e..28250618 100644 --- a/setup.py +++ b/setup.py @@ -22,23 +22,6 @@ import sys from setuptools import Extension, find_packages, setup - -def hook_compiler(): - """ - if numpy is installed, use compiler from that instead of setuptools, - which run faster in multi-core env - """ - try: - from numpy.distutils.ccompiler import CCompiler_compile - import distutils.ccompiler - - distutils.ccompiler.CCompiler.compile = CCompiler_compile - except ImportError: - pass - - -hook_compiler() - with open("vnpy/__init__.py", "rb") as f: version_line = re.search( r"__version__\s+=\s+(.*)", f.read().decode("utf-8") From a99b8eb8159ff649396d37ef19536cd8dcc2936e Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 11:16:24 -0400 Subject: [PATCH 58/62] [Add] custom monkey path for CCompiler --- setup.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/setup.py b/setup.py index 28250618..05155ce3 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,35 @@ import sys from setuptools import Extension, find_packages, setup + +# https://stackoverflow.com/a/13176803 +# monkey-patch for parallel compilation +def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, + extra_preargs=None, extra_postargs=None, depends=None): + # those lines are copied from distutils.ccompiler.CCompiler directly + macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, + include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + # parallel code + N = 2 # number of parallel compilations + import multiprocessing.pool + def _single_compile(obj): + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # convert to list, imap is evaluated on-demand + list(multiprocessing.pool.ThreadPool(N).imap(_single_compile, objects)) + return objects + + +import distutils.ccompiler + +distutils.ccompiler.CCompiler.compile = parallelCCompile + with open("vnpy/__init__.py", "rb") as f: version_line = re.search( r"__version__\s+=\s+(.*)", f.read().decode("utf-8") From 73e135b685262609da7e5ed3b22d191d35518c35 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 12:20:54 -0400 Subject: [PATCH 59/62] [Add] install.sh: parallel make for ta-lib --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 3dc936ac..dc748ee5 100644 --- a/install.sh +++ b/install.sh @@ -14,7 +14,7 @@ wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar -xf ta-lib-0.4.0-src.tar.gz cd ta-lib ./configure --prefix=$prefix -make +make -j sudo make install popd From 2e341bffc90c5a657a55d59d7212bac78fe345aa Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 12:26:49 -0400 Subject: [PATCH 60/62] [Mod] enable optimize under Linux build --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 05155ce3..086576db 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ if platform.uname().system == "Windows": extra_link_args = [] else: compiler_flags = ["-std=c++17", + "-O3", # Optimization "-Wno-delete-incomplete", "-Wno-sign-compare", ] extra_link_args = ["-lstdc++"] From 2f618d57966fcbd9f0f45237cf75ace9bd4370a5 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 12:56:08 -0400 Subject: [PATCH 61/62] [Del] remove parallel compile patch --- setup.py | 148 +++++++++++++++++++++++-------------------------------- 1 file changed, 62 insertions(+), 86 deletions(-) diff --git a/setup.py b/setup.py index 086576db..e0745716 100644 --- a/setup.py +++ b/setup.py @@ -22,35 +22,6 @@ import sys from setuptools import Extension, find_packages, setup - -# https://stackoverflow.com/a/13176803 -# monkey-patch for parallel compilation -def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - # those lines are copied from distutils.ccompiler.CCompiler directly - macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, - include_dirs, sources, - depends, extra_postargs) - cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) - # parallel code - N = 2 # number of parallel compilations - import multiprocessing.pool - def _single_compile(obj): - try: - src, ext = build[obj] - except KeyError: - return - self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) - - # convert to list, imap is evaluated on-demand - list(multiprocessing.pool.ThreadPool(N).imap(_single_compile, objects)) - return objects - - -import distutils.ccompiler - -distutils.ccompiler.CCompiler.compile = parallelCCompile - with open("vnpy/__init__.py", "rb") as f: version_line = re.search( r"__version__\s+=\s+(.*)", f.read().decode("utf-8") @@ -58,67 +29,72 @@ with open("vnpy/__init__.py", "rb") as f: version = str(ast.literal_eval(version_line)) if platform.uname().system == "Windows": - compiler_flags = ["/MP", "/std:c++17", # standard - "/O2", "/Ob2", "/Oi", "/Ot", "/Oy", "/GL", # Optimization - "/wd4819" # 936 code page - ] + compiler_flags = [ + "/MP", "/std:c++17", # standard + "/O2", "/Ob2", "/Oi", "/Ot", "/Oy", "/GL", # Optimization + "/wd4819" # 936 code page + ] extra_link_args = [] else: - compiler_flags = ["-std=c++17", - "-O3", # Optimization - "-Wno-delete-incomplete", "-Wno-sign-compare", - ] + compiler_flags = [ + "-std=c++17", # standard + "-O3", # Optimization + "-Wno-delete-incomplete", "-Wno-sign-compare", + ] extra_link_args = ["-lstdc++"] -vnctpmd = Extension("vnpy.api.ctp.vnctpmd", - [ - "vnpy/api/ctp/vnctp/vnctpmd/vnctpmd.cpp", - ], - include_dirs=["vnpy/api/ctp/include", - "vnpy/api/ctp/vnctp", ], - define_macros=[], - undef_macros=[], - library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"], - libraries=["thostmduserapi", "thosttraderapi", ], - extra_compile_args=compiler_flags, - extra_link_args=extra_link_args, - depends=[], - runtime_library_dirs=["$ORIGIN"], - language="cpp", - ) -vnctptd = Extension("vnpy.api.ctp.vnctptd", - [ - "vnpy/api/ctp/vnctp/vnctptd/vnctptd.cpp", - ], - include_dirs=["vnpy/api/ctp/include", - "vnpy/api/ctp/vnctp", ], - define_macros=[], - undef_macros=[], - library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"], - libraries=["thostmduserapi", "thosttraderapi", ], - extra_compile_args=compiler_flags, - extra_link_args=extra_link_args, - runtime_library_dirs=["$ORIGIN"], - depends=[], - language="cpp", - ) -vnoes = Extension("vnpy.api.oes.vnoes", - [ - "vnpy/api/oes/vnoes/generated_files/classes_1.cpp", - "vnpy/api/oes/vnoes/generated_files/classes_2.cpp", - "vnpy/api/oes/vnoes/generated_files/module.cpp", - ], - include_dirs=["vnpy/api/oes/include", - "vnpy/api/oes/vnoes", ], - define_macros=[("BRIGAND_NO_BOOST_SUPPORT", "1")], - undef_macros=[], - library_dirs=["vnpy/api/oes/libs"], - libraries=["oes_api"], - extra_compile_args=compiler_flags, - extra_link_args=extra_link_args, - depends=[], - language="cpp", - ) +vnctpmd = Extension( + "vnpy.api.ctp.vnctpmd", + [ + "vnpy/api/ctp/vnctp/vnctpmd/vnctpmd.cpp", + ], + include_dirs=["vnpy/api/ctp/include", + "vnpy/api/ctp/vnctp", ], + define_macros=[], + undef_macros=[], + library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"], + libraries=["thostmduserapi", "thosttraderapi", ], + extra_compile_args=compiler_flags, + extra_link_args=extra_link_args, + depends=[], + runtime_library_dirs=["$ORIGIN"], + language="cpp", +) +vnctptd = Extension( + "vnpy.api.ctp.vnctptd", + [ + "vnpy/api/ctp/vnctp/vnctptd/vnctptd.cpp", + ], + include_dirs=["vnpy/api/ctp/include", + "vnpy/api/ctp/vnctp", ], + define_macros=[], + undef_macros=[], + library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"], + libraries=["thostmduserapi", "thosttraderapi", ], + extra_compile_args=compiler_flags, + extra_link_args=extra_link_args, + runtime_library_dirs=["$ORIGIN"], + depends=[], + language="cpp", +) +vnoes = Extension( + "vnpy.api.oes.vnoes", + [ + "vnpy/api/oes/vnoes/generated_files/classes_1.cpp", + "vnpy/api/oes/vnoes/generated_files/classes_2.cpp", + "vnpy/api/oes/vnoes/generated_files/module.cpp", + ], + include_dirs=["vnpy/api/oes/include", + "vnpy/api/oes/vnoes", ], + define_macros=[("BRIGAND_NO_BOOST_SUPPORT", "1")], + undef_macros=[], + library_dirs=["vnpy/api/oes/libs"], + libraries=["oes_api"], + extra_compile_args=compiler_flags, + extra_link_args=extra_link_args, + depends=[], + language="cpp", +) if platform.uname().system == "Windows": # use pre-built pyd for windows ( support python 3.7 only ) From dd99609109aa5af49170589c20a55f9c4275eec9 Mon Sep 17 00:00:00 2001 From: nanoric Date: Mon, 15 Apr 2019 13:40:03 -0400 Subject: [PATCH 62/62] [Add] skip some tests under osx --- setup.py | 5 ++++- tests/test_import_all.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e0745716..3b0d1ca7 100644 --- a/setup.py +++ b/setup.py @@ -92,13 +92,16 @@ vnoes = Extension( libraries=["oes_api"], extra_compile_args=compiler_flags, extra_link_args=extra_link_args, + runtime_library_dirs=["$ORIGIN"], depends=[], language="cpp", ) -if platform.uname().system == "Windows": +if platform.system() == "Windows": # use pre-built pyd for windows ( support python 3.7 only ) ext_modules = [] +elif platform.system() == "Darwin": + ext_modules = [] else: ext_modules = [vnctptd, vnctpmd, vnoes] diff --git a/tests/test_import_all.py b/tests/test_import_all.py index df930c89..38d240f8 100644 --- a/tests/test_import_all.py +++ b/tests/test_import_all.py @@ -1,5 +1,6 @@ # flake8: noqa import unittest +import platform # noinspection PyUnresolvedReferences @@ -24,12 +25,14 @@ class ImportTest(unittest.TestCase): def test_import_ib_gateway(self): from vnpy.gateway.ib import IbGateway + @unittest.skipIf(platform.system() == "Darwin", "Not supported yet under osx") def test_import_ctp_gateway(self): from vnpy.gateway.ctp import CtpGateway def test_import_tiger_gateway(self): from vnpy.gateway.tiger import TigerGateway + @unittest.skipIf(platform.system() == "Darwin", "Not supported yet under osx") def test_import_oes_gateway(self): from vnpy.gateway.oes import OesGateway