diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..d12c40ca --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,142 @@ +# This file is a template, and might need editing before it works on your project. +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/python/tags/ +image: registry.cn-shanghai.aliyuncs.com/vnpy-ci/gcc-7-python-3.7:latest + +.services: + services: &services + - postgres:latest + - mysql:latest + - mongo:latest + +# Change pip's cache directory to be inside the project directory since we can +# only cache local items. +variables: &variables + GIT_DEPTH: "1" + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + POSTGRES_DB: &db_name "vnpy" + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: &db_password "1234" + VNPY_TEST_POSTGRESQL_PASSWORD: *db_password + MYSQL_DATABASE: *db_name + MYSQL_ROOT_PASSWORD: *db_password + VNPY_TEST_MYSQL_PASSWORD: *db_password + VNPY_BUILD_PARALLEL: "auto" + +# Pip's cache doesn't store the python packages +# https://pip.pypa.io/en/stable/reference/pip_install/#caching +# +# If you want to also cache the installed packages, you have to install +# them in a virtualenv and cache it as well. +.default_cache: + cache: + <<: &cache + key: "pip_and_venv" + untracked: false + policy: pull + paths: + - .cache/pip + - venv/ + + + +before_script: + - echo $PWD + - python -V + - gcc --version + - free + - date + + # venv + - pip install virtualenv + - virtualenv venv + - source venv/bin/activate + + # some envs + - source ci/env.sh + +.scripts: + script: + - &install_scripts | + date + python -m pip --version + python -m pip install --upgrade pip wheel setuptools + python -m pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl + bash ci/gitlab_pre_install.sh + + date + bash ./install.sh + date + + - &test_scripts | + date + cd tests + python test_all.py + date +################################## +# stages + +stages: # I use anchors for IDE hints only + - &single_module single_module + - &build_all build_all + + +########################################### +## jobs: +flake8: + stage: *single_module + image: python:3.7 + cache: + key: 'flake8' + paths: + - .cache/pip + - venv/ + script: + - pip install flake8 + - flake8 + +ctp: + <<: &test_single_module + stage: *single_module + image: registry.cn-shanghai.aliyuncs.com/vnpy-ci/gcc-8-python-3.7:latest + services: *services + cache: + <<: *cache + script: + - *install_scripts + - *test_scripts + variables: + <<: *variables + VNPY_BUILD_CTP: 1 + +oes: + <<: *test_single_module + variables: + <<: *variables + VNPY_BUILD_OES: 1 + +no_building: + <<: *test_single_module + cache: + <<: *cache + policy: pull-push + variables: + <<: *variables + VNPY_BUILD_OES: 0 + VNPY_BUILD_CTP: 0 + +build-all-gcc8: + stage: *build_all + variables: + <<: *variables + image: registry.cn-shanghai.aliyuncs.com/vnpy-ci/gcc-8-python-3.7:latest + services: *services + cache: + key: "build-all" + paths: [] + script: + - unset VNPY_BUILD_CTP + - unset VNPY_BUILD_OES + - *install_scripts + - *test_scripts + diff --git a/tests/travis_env.sh b/ci/env.sh similarity index 82% rename from tests/travis_env.sh rename to ci/env.sh index e2e59613..5c4441f0 100644 --- a/tests/travis_env.sh +++ b/ci/env.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +# databases [[ -z ${VNPY_TEST_MYSQL_DATABASE} ]] && VNPY_TEST_MYSQL_DATABASE=vnpy [[ -z ${VNPY_TEST_MYSQL_HOST} ]] && VNPY_TEST_MYSQL_HOST=127.0.0.1 [[ -z ${VNPY_TEST_MYSQL_PORT} ]] && VNPY_TEST_MYSQL_PORT=3306 @@ -28,3 +29,14 @@ export VNPY_TEST_POSTGRESQL_PASSWORD export VNPY_TEST_MONGODB_DATABASE export VNPY_TEST_MONGODB_HOST export VNPY_TEST_MONGODB_PORT + +# ext_modules: +# disable all building first +[[ -z ${VNPY_BUILD_CTP} ]] && VNPY_BUILD_CTP=0 +[[ -z ${VNPY_BUILD_OES} ]] && VNPY_BUILD_OES=0 +export VNPY_BUILD_CTP +export VNPY_BUILD_OES + +# build option +[[ -z ${VNPY_BUILD_PARALLEL} ]] && VNPY_BUILD_PARALLEL=auto +export VNPY_BUILD_PARALLEL diff --git a/ci/gitlab_pre_install.sh b/ci/gitlab_pre_install.sh new file mode 100644 index 00000000..97e5528f --- /dev/null +++ b/ci/gitlab_pre_install.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +pip install cryptography diff --git a/ci/parallel_build_distutils.py b/ci/parallel_build_distutils.py new file mode 100644 index 00000000..f1686444 --- /dev/null +++ b/ci/parallel_build_distutils.py @@ -0,0 +1,46 @@ +""" +monkey patch distutils to use multi-core to build single extension. +these code is based on: +https://stackoverflow.com/a/13176803/4797109 +""" + +import multiprocessing.pool + +parallel: int = 1 + + +# monkey-patch for parallel compilation +def parallel_compile(self, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None, + ): + # those lines are copied from distutils.ccompiler.CCompiler directly + macros, objects, extra_postargs, pp_opts, build = self._setup_compile( + output_dir, macros, include_dirs, sources, depends, extra_postargs + ) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + def _single_compile(obj): + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # convert to list, imap is evaluated on-demand + list(multiprocessing.pool.ThreadPool(parallel).imap(_single_compile, objects)) + return objects + + +def patch_distutils(jobs: int = None): + if jobs: + global parallel + parallel = jobs + import distutils.ccompiler + distutils.ccompiler.CCompiler.compile = parallel_compile diff --git a/install.sh b/install.sh index 117df7fb..255d2426 100644 --- a/install.sh +++ b/install.sh @@ -2,6 +2,7 @@ python=$1 prefix=$2 +shift 2 [[ -z $python ]] && python=python [[ -z $prefix ]] && prefix=/usr @@ -9,14 +10,22 @@ prefix=$2 $python -m pip install --upgrade pip setuptools wheel # Get and build ta-lib -pushd /tmp -wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz -tar -xf ta-lib-0.4.0-src.tar.gz -cd ta-lib -./configure --prefix=$prefix -make -j -sudo make install -popd +function install-ta-lib() +{ + pushd /tmp + wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz + tar -xf ta-lib-0.4.0-src.tar.gz + cd ta-lib + ./configure --prefix=$prefix + make -j + make install + popd +} +function ta-lib-exists() +{ + ta-lib-config --libs > /dev/null +} +ta-lib-exists || install-ta-lib # old versions of ta-lib imports numpy in setup.py $python -m pip install numpy @@ -30,7 +39,7 @@ $python -m pip install https://vnpy-pip.oss-cn-shanghai.aliyuncs.com/colletion/i $python -m pip install -r requirements.txt # Install local Chinese language environment -sudo locale-gen zh_CN.GB18030 +locale-gen zh_CN.GB18030 # Install vn.py -$python -m pip install . \ No newline at end of file +$python -m pip install . $@ \ No newline at end of file diff --git a/setup.py b/setup.py index ab20f5fb..b2c52b23 100644 --- a/setup.py +++ b/setup.py @@ -23,31 +23,6 @@ import sys from setuptools import Extension, find_packages, setup -with open("vnpy/__init__.py", "rb") as f: - version_line = re.search( - r"__version__\s+=\s+(.*)", f.read().decode("utf-8") - ).group(1) - version = str(ast.literal_eval(version_line)) - -if platform.uname().system == "Windows": - compiler_flags = [ - "/MP", "/std:c++17", # standard - "/O2", "/Ob2", "/Oi", "/Ot", "/Oy", "/GL", # Optimization - "/bigobj", # Better compatibility - "/wd4819", # 936 code page - "/D_CRT_SECURE_NO_WARNINGS", # suppress warning of unsafe functions like fopen, strcpy, etc - ] - extra_link_args = [] - runtime_library_dirs = None -else: - compiler_flags = [ - "-std=c++17", # standard - "-O3", # Optimization - "-Wno-delete-incomplete", "-Wno-sign-compare", - ] - extra_link_args = ["-lstdc++"] - runtime_library_dirs = ["$ORIGIN"] - def gather_autocxxpy_generated_files(root: str): fs = [os.path.join(root, "module.cpp")] @@ -60,83 +35,16 @@ def gather_autocxxpy_generated_files(root: str): return fs -vnctpmd = Extension( - "vnpy.api.ctp.vnctpmd", - [ - "vnpy/api/ctp/vnctp/vnctpmd/vnctpmd.cpp", - ], - include_dirs=["vnpy/api/ctp/include", - "vnpy/api/ctp/vnctp", ], - define_macros=[], - undef_macros=[], - library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"], - libraries=["thostmduserapi_se", "thosttraderapi_se", ], - extra_compile_args=compiler_flags, - extra_link_args=extra_link_args, - runtime_library_dirs=runtime_library_dirs, - depends=[], - language="cpp", -) -vnctptd = Extension( - "vnpy.api.ctp.vnctptd", - [ - "vnpy/api/ctp/vnctp/vnctptd/vnctptd.cpp", - ], - include_dirs=["vnpy/api/ctp/include", - "vnpy/api/ctp/vnctp", ], - define_macros=[], - undef_macros=[], - library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"], - libraries=["thostmduserapi_se", "thosttraderapi_se", ], - extra_compile_args=compiler_flags, - extra_link_args=extra_link_args, - runtime_library_dirs=runtime_library_dirs, - depends=[], - language="cpp", -) -vnoes = Extension( - name="vnpy.api.oes.vnoes", - sources=gather_autocxxpy_generated_files( - "vnpy/api/oes/vnoes/generated_files/", - ), - include_dirs=["vnpy/api/oes/vnoes/include", - "vnpy/api/oes/vnoes/include/oes", ], - define_macros=[("BRIGAND_NO_BOOST_SUPPORT", "1")], - undef_macros=[], - library_dirs=["vnpy/api/oes/vnoes/libs"], - libraries=["oes_api"], - extra_compile_args=compiler_flags, - extra_link_args=extra_link_args, - runtime_library_dirs=runtime_library_dirs, - depends=[], - language="cpp", -) - -if platform.system() == "Windows": - # use pre-built pyd for windows ( support python 3.7 only ) - ext_modules = [] -elif platform.system() == "Darwin": - ext_modules = [] -else: - ext_modules = [vnctptd, vnctpmd] - - -def check_extension_build_flag(key: str, module: Extension): +def check_extension_build_flag(ext_modules, key: str, module: Extension): value = os.environ.get(key, None) if value is not None: - global ext_modules if value == '1': ext_modules = list(set(ext_modules) | {module}) elif value == '0': ext_modules = list(set(ext_modules) - {module}) else: raise ValueError(f"Flag {key} should be '0' or '1', but {repr(value)} got.") - - -check_extension_build_flag("VNPY_BUILD_OES", vnoes) - - -pkgs = find_packages() + return ext_modules def is_psycopg2_exists(): @@ -147,34 +55,140 @@ def is_psycopg2_exists(): return False -install_requires = [ - "PyQt5", - "qdarkstyle", - "requests", - "websocket-client", - "peewee", - "pymysql", - "mongoengine", - "numpy", - "pandas", - "matplotlib", - "seaborn", - "futu-api", - "tigeropen", - "rqdatac", - "ta-lib", - "ibapi", - "deap" -] -if not is_psycopg2_exists(): - install_requires.append("psycopg2-binary") +def get_install_requires(): + install_requires = [ + "PyQt5", + "qdarkstyle", + "requests", + "websocket-client", + "peewee", + "pymysql", + "mongoengine", + "numpy", + "pandas", + "matplotlib", + "seaborn", + "futu-api", + "tigeropen", + "rqdatac", + "ta-lib", + "ibapi", + "deap" + ] + if not is_psycopg2_exists(): + install_requires.append("psycopg2-binary") -if sys.version_info.minor < 7: - install_requires.append("dataclasses") + if sys.version_info.minor < 7: + install_requires.append("dataclasses") + return install_requires + + +def get_version_string(): + global version + with open("vnpy/__init__.py", "rb") as f: + version_line = re.search( + r"__version__\s+=\s+(.*)", f.read().decode("utf-8") + ).group(1) + return str(ast.literal_eval(version_line)) + + +def get_ext_modules(): + if platform.uname().system == "Windows": + compiler_flags = [ + "/MP", "/std:c++17", # standard + "/O2", "/Ob2", "/Oi", "/Ot", "/Oy", "/GL", # Optimization + "/bigobj", # Better compatibility + "/wd4819", # 936 code page + "/D_CRT_SECURE_NO_WARNINGS", + # suppress warning of unsafe functions like fopen, strcpy, etc + ] + extra_link_args = [] + runtime_library_dirs = None + else: + compiler_flags = [ + "-std=c++17", # standard + "-O3", # Optimization + "-Wno-delete-incomplete", "-Wno-sign-compare", + ] + extra_link_args = ["-lstdc++"] + runtime_library_dirs = ["$ORIGIN"] + vnctpmd = Extension( + "vnpy.api.ctp.vnctpmd", + [ + "vnpy/api/ctp/vnctp/vnctpmd/vnctpmd.cpp", + ], + include_dirs=["vnpy/api/ctp/include", + "vnpy/api/ctp/vnctp", ], + define_macros=[], + undef_macros=[], + library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"], + libraries=["thostmduserapi_se", "thosttraderapi_se", ], + extra_compile_args=compiler_flags, + extra_link_args=extra_link_args, + runtime_library_dirs=runtime_library_dirs, + depends=[], + language="cpp", + ) + vnctptd = Extension( + "vnpy.api.ctp.vnctptd", + [ + "vnpy/api/ctp/vnctp/vnctptd/vnctptd.cpp", + ], + include_dirs=["vnpy/api/ctp/include", + "vnpy/api/ctp/vnctp", ], + define_macros=[], + undef_macros=[], + library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"], + libraries=["thostmduserapi_se", "thosttraderapi_se", ], + extra_compile_args=compiler_flags, + extra_link_args=extra_link_args, + runtime_library_dirs=runtime_library_dirs, + depends=[], + language="cpp", + ) + vnoes = Extension( + name="vnpy.api.oes.vnoes", + sources=gather_autocxxpy_generated_files( + "vnpy/api/oes/vnoes/generated_files/", + ), + include_dirs=["vnpy/api/oes/vnoes/include", + "vnpy/api/oes/vnoes/include/oes", ], + define_macros=[("BRIGAND_NO_BOOST_SUPPORT", "1")], + undef_macros=[], + library_dirs=["vnpy/api/oes/vnoes/libs"], + libraries=["oes_api"], + extra_compile_args=compiler_flags, + extra_link_args=extra_link_args, + runtime_library_dirs=runtime_library_dirs, + depends=[], + language="cpp", + ) + if platform.system() == "Windows": + # use pre-built pyd for windows ( support python 3.7 only ) + ext_modules = [] + elif platform.system() == "Darwin": + ext_modules = [] + else: + ext_modules = [vnctptd, vnctpmd, vnoes] + + ext_modules = check_extension_build_flag(ext_modules, "VNPY_BUILD_OES", vnoes) + ext_modules = check_extension_build_flag(ext_modules, "VNPY_BUILD_CTP", vnctptd) + ext_modules = check_extension_build_flag(ext_modules, "VNPY_BUILD_CTP", vnctpmd) + + return ext_modules + + +parallel = os.environ.get('VNPY_BUILD_PARALLEL', None) +if parallel: + if parallel == 'auto': + parallel = os.cpu_count() + from ci.parallel_build_distutils import patch_distutils + + patch_distutils(int(parallel)) setup( name="vnpy", - version=version, + version=get_version_string(), author="vn.py team", author_email="vn.py@foxmail.com", license="MIT", @@ -183,7 +197,7 @@ setup( long_description=__doc__, keywords='quant quantitative investment trading algotrading', include_package_data=True, - packages=pkgs, + packages=find_packages(exclude=["tests", "ci", "tests.*"]), package_data={"": [ "*.ico", "*.ini", @@ -191,7 +205,7 @@ setup( "*.so", "*.pyd", ]}, - install_requires=install_requires, + install_requires=get_install_requires(), classifiers=[ "Development Status :: 5 - Production/Stable", "Operating System :: Microsoft :: Windows :: Windows 7", @@ -209,5 +223,5 @@ setup( "Natural Language :: Chinese (Simplified)", "Natural Language :: Chinese (Simplified)" ], - ext_modules=ext_modules + ext_modules=get_ext_modules(), )