[Add] use gitlab-ci

.gitlab-ci.yml

@@ -0,0 +1,142 @@
+# This file is a template, and might need editing before it works on your project.
+# Official language image. Look for the different tagged releases at:
+# https://hub.docker.com/r/library/python/tags/
+image: registry.cn-shanghai.aliyuncs.com/vnpy-ci/gcc-7-python-3.7:latest
+
+.services:
+  services: &services
+    - postgres:latest
+    - mysql:latest
+    - mongo:latest
+
+# Change pip's cache directory to be inside the project directory since we can
+# only cache local items.
+variables: &variables
+  GIT_DEPTH: "1"
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  POSTGRES_DB: &db_name "vnpy"
+  POSTGRES_USER: "postgres"
+  POSTGRES_PASSWORD: &db_password "1234"
+  VNPY_TEST_POSTGRESQL_PASSWORD: *db_password
+  MYSQL_DATABASE: *db_name
+  MYSQL_ROOT_PASSWORD: *db_password
+  VNPY_TEST_MYSQL_PASSWORD: *db_password
+  VNPY_BUILD_PARALLEL: "auto"
+
+# Pip's cache doesn't store the python packages
+# https://pip.pypa.io/en/stable/reference/pip_install/#caching
+#
+# If you want to also cache the installed packages, you have to install
+# them in a virtualenv and cache it as well.
+.default_cache:
+  cache:
+    <<: &cache
+      key: "pip_and_venv"
+      untracked: false
+      policy: pull
+      paths:
+        - .cache/pip
+        - venv/
+
+
+
+before_script:
+  - echo $PWD
+  - python -V
+  - gcc --version
+  - free
+  - date
+  
+  # venv
+  - pip install virtualenv
+  - virtualenv venv
+  - source venv/bin/activate
+
+  # some envs
+  - source ci/env.sh
+
+.scripts:
+  script:
+    - &install_scripts |
+      date
+      python -m pip --version
+      python -m pip install --upgrade pip wheel setuptools
+      python -m pip install https://pip.vnpy.com/colletion/ibapi-9.75.1-001-py3-none-any.whl
+      bash ci/gitlab_pre_install.sh
+
+      date
+      bash ./install.sh
+      date
+
+    - &test_scripts |
+      date
+      cd tests
+      python test_all.py
+      date
+##################################
+# stages
+
+stages:  # I use anchors for IDE hints only
+  - &single_module single_module
+  - &build_all build_all
+
+
+###########################################
+## jobs:
+flake8:
+  stage: *single_module
+  image: python:3.7
+  cache:
+    key: 'flake8'
+    paths:
+      - .cache/pip
+      - venv/
+  script:
+    - pip install flake8
+    - flake8
+
+ctp:
+  <<: &test_single_module
+    stage: *single_module
+    image: registry.cn-shanghai.aliyuncs.com/vnpy-ci/gcc-8-python-3.7:latest
+    services: *services
+    cache:
+      <<: *cache
+    script:
+      - *install_scripts
+      - *test_scripts
+  variables:
+    <<: *variables
+    VNPY_BUILD_CTP: 1
+
+oes:
+  <<: *test_single_module
+  variables:
+    <<: *variables
+    VNPY_BUILD_OES: 1
+
+no_building:
+  <<: *test_single_module
+  cache:
+    <<: *cache
+    policy: pull-push
+  variables:
+    <<: *variables
+    VNPY_BUILD_OES: 0
+    VNPY_BUILD_CTP: 0
+
+build-all-gcc8:
+  stage: *build_all
+  variables:
+    <<: *variables
+  image: registry.cn-shanghai.aliyuncs.com/vnpy-ci/gcc-8-python-3.7:latest
+  services: *services
+  cache:
+    key: "build-all"
+    paths: []
+  script:
+    - unset VNPY_BUILD_CTP
+    - unset VNPY_BUILD_OES
+    - *install_scripts
+    - *test_scripts
+

ci/env.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+# databases
 [[ -z ${VNPY_TEST_MYSQL_DATABASE} ]] && VNPY_TEST_MYSQL_DATABASE=vnpy
 [[ -z ${VNPY_TEST_MYSQL_HOST} ]] && VNPY_TEST_MYSQL_HOST=127.0.0.1
 [[ -z ${VNPY_TEST_MYSQL_PORT} ]] && VNPY_TEST_MYSQL_PORT=3306
@@ -28,3 +29,14 @@ export VNPY_TEST_POSTGRESQL_PASSWORD
 export VNPY_TEST_MONGODB_DATABASE
 export VNPY_TEST_MONGODB_HOST
 export VNPY_TEST_MONGODB_PORT
+
+# ext_modules:
+# disable all building first
+[[ -z ${VNPY_BUILD_CTP} ]] && VNPY_BUILD_CTP=0
+[[ -z ${VNPY_BUILD_OES} ]] && VNPY_BUILD_OES=0
+export VNPY_BUILD_CTP
+export VNPY_BUILD_OES
+
+# build option
+[[ -z ${VNPY_BUILD_PARALLEL} ]] && VNPY_BUILD_PARALLEL=auto
+export VNPY_BUILD_PARALLEL

ci/gitlab_pre_install.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+pip install cryptography

ci/parallel_build_distutils.py

@@ -0,0 +1,46 @@
+"""
+monkey patch distutils to use multi-core to build single extension.
+these code is based on:
+https://stackoverflow.com/a/13176803/4797109
+"""
+
+import multiprocessing.pool
+
+parallel: int = 1
+
+
+# monkey-patch for parallel compilation
+def parallel_compile(self,
+                     sources,
+                     output_dir=None,
+                     macros=None,
+                     include_dirs=None,
+                     debug=0,
+                     extra_preargs=None,
+                     extra_postargs=None,
+                     depends=None,
+                     ):
+    # those lines are copied from distutils.ccompiler.CCompiler directly
+    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
+        output_dir, macros, include_dirs, sources, depends, extra_postargs
+    )
+    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+
+    def _single_compile(obj):
+        try:
+            src, ext = build[obj]
+        except KeyError:
+            return
+        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+
+    # convert to list, imap is evaluated on-demand
+    list(multiprocessing.pool.ThreadPool(parallel).imap(_single_compile, objects))
+    return objects
+
+
+def patch_distutils(jobs: int = None):
+    if jobs:
+        global parallel
+        parallel = jobs
+    import distutils.ccompiler
+    distutils.ccompiler.CCompiler.compile = parallel_compile

install.sh

@@ -2,6 +2,7 @@
 
 python=$1
 prefix=$2
+shift 2
 
 [[ -z $python ]] && python=python
 [[ -z $prefix ]] && prefix=/usr
@@ -9,14 +10,22 @@ prefix=$2
 $python -m pip install --upgrade pip setuptools wheel
 
 # Get and build ta-lib
-pushd /tmp
+function install-ta-lib()
-wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
+{
-tar -xf ta-lib-0.4.0-src.tar.gz
+    pushd /tmp
-cd ta-lib
+    wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
-./configure --prefix=$prefix
+    tar -xf ta-lib-0.4.0-src.tar.gz
-make -j
+    cd ta-lib
-sudo make install
+    ./configure --prefix=$prefix
-popd
+    make -j
+    make install
+    popd
+}
+function ta-lib-exists()
+{
+    ta-lib-config --libs > /dev/null
+}
+ta-lib-exists || install-ta-lib
 
 # old versions of ta-lib imports numpy in setup.py
 $python -m pip install numpy
@@ -30,7 +39,7 @@ $python -m pip install https://vnpy-pip.oss-cn-shanghai.aliyuncs.com/colletion/i
 $python -m pip install -r requirements.txt
 
 # Install local Chinese language environment
-sudo locale-gen zh_CN.GB18030
+locale-gen zh_CN.GB18030
 
 # Install vn.py
-$python -m pip install .
+$python -m pip install . $@

setup.py

@@ -23,31 +23,6 @@ import sys
 
 from setuptools import Extension, find_packages, setup
 
-with open("vnpy/__init__.py", "rb") as f:
-    version_line = re.search(
-        r"__version__\s+=\s+(.*)", f.read().decode("utf-8")
-    ).group(1)
-    version = str(ast.literal_eval(version_line))
-
-if platform.uname().system == "Windows":
-    compiler_flags = [
-        "/MP", "/std:c++17",  # standard
-        "/O2", "/Ob2", "/Oi", "/Ot", "/Oy", "/GL",  # Optimization
-        "/bigobj",  # Better compatibility
-        "/wd4819",  # 936 code page
-        "/D_CRT_SECURE_NO_WARNINGS",  # suppress warning of unsafe functions like fopen, strcpy, etc
-    ]
-    extra_link_args = []
-    runtime_library_dirs = None
-else:
-    compiler_flags = [
-        "-std=c++17",  # standard
-        "-O3",  # Optimization
-        "-Wno-delete-incomplete", "-Wno-sign-compare",
-    ]
-    extra_link_args = ["-lstdc++"]
-    runtime_library_dirs = ["$ORIGIN"]
-
 
 def gather_autocxxpy_generated_files(root: str):
     fs = [os.path.join(root, "module.cpp")]
@@ -60,83 +35,16 @@ def gather_autocxxpy_generated_files(root: str):
     return fs
 
 
-vnctpmd = Extension(
+def check_extension_build_flag(ext_modules, key: str, module: Extension):
-    "vnpy.api.ctp.vnctpmd",
-    [
-        "vnpy/api/ctp/vnctp/vnctpmd/vnctpmd.cpp",
-    ],
-    include_dirs=["vnpy/api/ctp/include",
-                  "vnpy/api/ctp/vnctp", ],
-    define_macros=[],
-    undef_macros=[],
-    library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"],
-    libraries=["thostmduserapi_se", "thosttraderapi_se", ],
-    extra_compile_args=compiler_flags,
-    extra_link_args=extra_link_args,
-    runtime_library_dirs=runtime_library_dirs,
-    depends=[],
-    language="cpp",
-)
-vnctptd = Extension(
-    "vnpy.api.ctp.vnctptd",
-    [
-        "vnpy/api/ctp/vnctp/vnctptd/vnctptd.cpp",
-    ],
-    include_dirs=["vnpy/api/ctp/include",
-                  "vnpy/api/ctp/vnctp", ],
-    define_macros=[],
-    undef_macros=[],
-    library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"],
-    libraries=["thostmduserapi_se", "thosttraderapi_se", ],
-    extra_compile_args=compiler_flags,
-    extra_link_args=extra_link_args,
-    runtime_library_dirs=runtime_library_dirs,
-    depends=[],
-    language="cpp",
-)
-vnoes = Extension(
-    name="vnpy.api.oes.vnoes",
-    sources=gather_autocxxpy_generated_files(
-        "vnpy/api/oes/vnoes/generated_files/",
-    ),
-    include_dirs=["vnpy/api/oes/vnoes/include",
-                  "vnpy/api/oes/vnoes/include/oes", ],
-    define_macros=[("BRIGAND_NO_BOOST_SUPPORT", "1")],
-    undef_macros=[],
-    library_dirs=["vnpy/api/oes/vnoes/libs"],
-    libraries=["oes_api"],
-    extra_compile_args=compiler_flags,
-    extra_link_args=extra_link_args,
-    runtime_library_dirs=runtime_library_dirs,
-    depends=[],
-    language="cpp",
-)
-
-if platform.system() == "Windows":
-    # use pre-built pyd for windows ( support python 3.7 only )
-    ext_modules = []
-elif platform.system() == "Darwin":
-    ext_modules = []
-else:
-    ext_modules = [vnctptd, vnctpmd]
-
-
-def check_extension_build_flag(key: str, module: Extension):
     value = os.environ.get(key, None)
     if value is not None:
-        global ext_modules
         if value == '1':
             ext_modules = list(set(ext_modules) | {module})
         elif value == '0':
             ext_modules = list(set(ext_modules) - {module})
         else:
             raise ValueError(f"Flag {key} should be '0' or '1', but {repr(value)} got.")
-
+    return ext_modules
-
-check_extension_build_flag("VNPY_BUILD_OES", vnoes)
-
-
-pkgs = find_packages()
 
 
 def is_psycopg2_exists():
@@ -147,34 +55,140 @@ def is_psycopg2_exists():
         return False
 
 
-install_requires = [
+def get_install_requires():
-    "PyQt5",
+    install_requires = [
-    "qdarkstyle",
+        "PyQt5",
-    "requests",
+        "qdarkstyle",
-    "websocket-client",
+        "requests",
-    "peewee",
+        "websocket-client",
-    "pymysql",
+        "peewee",
-    "mongoengine",
+        "pymysql",
-    "numpy",
+        "mongoengine",
-    "pandas",
+        "numpy",
-    "matplotlib",
+        "pandas",
-    "seaborn",
+        "matplotlib",
-    "futu-api",
+        "seaborn",
-    "tigeropen",
+        "futu-api",
-    "rqdatac",
+        "tigeropen",
-    "ta-lib",
+        "rqdatac",
-    "ibapi",
+        "ta-lib",
-    "deap"
+        "ibapi",
-]
+        "deap"
-if not is_psycopg2_exists():
+    ]
-    install_requires.append("psycopg2-binary")
+    if not is_psycopg2_exists():
+        install_requires.append("psycopg2-binary")
 
-if sys.version_info.minor < 7:
+    if sys.version_info.minor < 7:
-    install_requires.append("dataclasses")
+        install_requires.append("dataclasses")
+    return install_requires
+
+
+def get_version_string():
+    global version
+    with open("vnpy/__init__.py", "rb") as f:
+        version_line = re.search(
+            r"__version__\s+=\s+(.*)", f.read().decode("utf-8")
+        ).group(1)
+        return str(ast.literal_eval(version_line))
+
+
+def get_ext_modules():
+    if platform.uname().system == "Windows":
+        compiler_flags = [
+            "/MP", "/std:c++17",  # standard
+            "/O2", "/Ob2", "/Oi", "/Ot", "/Oy", "/GL",  # Optimization
+            "/bigobj",  # Better compatibility
+            "/wd4819",  # 936 code page
+            "/D_CRT_SECURE_NO_WARNINGS",
+            # suppress warning of unsafe functions like fopen, strcpy, etc
+        ]
+        extra_link_args = []
+        runtime_library_dirs = None
+    else:
+        compiler_flags = [
+            "-std=c++17",  # standard
+            "-O3",  # Optimization
+            "-Wno-delete-incomplete", "-Wno-sign-compare",
+        ]
+        extra_link_args = ["-lstdc++"]
+        runtime_library_dirs = ["$ORIGIN"]
+    vnctpmd = Extension(
+        "vnpy.api.ctp.vnctpmd",
+        [
+            "vnpy/api/ctp/vnctp/vnctpmd/vnctpmd.cpp",
+        ],
+        include_dirs=["vnpy/api/ctp/include",
+                      "vnpy/api/ctp/vnctp", ],
+        define_macros=[],
+        undef_macros=[],
+        library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"],
+        libraries=["thostmduserapi_se", "thosttraderapi_se", ],
+        extra_compile_args=compiler_flags,
+        extra_link_args=extra_link_args,
+        runtime_library_dirs=runtime_library_dirs,
+        depends=[],
+        language="cpp",
+    )
+    vnctptd = Extension(
+        "vnpy.api.ctp.vnctptd",
+        [
+            "vnpy/api/ctp/vnctp/vnctptd/vnctptd.cpp",
+        ],
+        include_dirs=["vnpy/api/ctp/include",
+                      "vnpy/api/ctp/vnctp", ],
+        define_macros=[],
+        undef_macros=[],
+        library_dirs=["vnpy/api/ctp/libs", "vnpy/api/ctp"],
+        libraries=["thostmduserapi_se", "thosttraderapi_se", ],
+        extra_compile_args=compiler_flags,
+        extra_link_args=extra_link_args,
+        runtime_library_dirs=runtime_library_dirs,
+        depends=[],
+        language="cpp",
+    )
+    vnoes = Extension(
+        name="vnpy.api.oes.vnoes",
+        sources=gather_autocxxpy_generated_files(
+            "vnpy/api/oes/vnoes/generated_files/",
+        ),
+        include_dirs=["vnpy/api/oes/vnoes/include",
+                      "vnpy/api/oes/vnoes/include/oes", ],
+        define_macros=[("BRIGAND_NO_BOOST_SUPPORT", "1")],
+        undef_macros=[],
+        library_dirs=["vnpy/api/oes/vnoes/libs"],
+        libraries=["oes_api"],
+        extra_compile_args=compiler_flags,
+        extra_link_args=extra_link_args,
+        runtime_library_dirs=runtime_library_dirs,
+        depends=[],
+        language="cpp",
+    )
+    if platform.system() == "Windows":
+        # use pre-built pyd for windows ( support python 3.7 only )
+        ext_modules = []
+    elif platform.system() == "Darwin":
+        ext_modules = []
+    else:
+        ext_modules = [vnctptd, vnctpmd, vnoes]
+
+    ext_modules = check_extension_build_flag(ext_modules, "VNPY_BUILD_OES", vnoes)
+    ext_modules = check_extension_build_flag(ext_modules, "VNPY_BUILD_CTP", vnctptd)
+    ext_modules = check_extension_build_flag(ext_modules, "VNPY_BUILD_CTP", vnctpmd)
+
+    return ext_modules
+
+
+parallel = os.environ.get('VNPY_BUILD_PARALLEL', None)
+if parallel:
+    if parallel == 'auto':
+        parallel = os.cpu_count()
+    from ci.parallel_build_distutils import patch_distutils
+
+    patch_distutils(int(parallel))
 
 setup(
     name="vnpy",
-    version=version,
+    version=get_version_string(),
     author="vn.py team",
     author_email="vn.py@foxmail.com",
     license="MIT",
@@ -183,7 +197,7 @@ setup(
     long_description=__doc__,
     keywords='quant quantitative investment trading algotrading',
     include_package_data=True,
-    packages=pkgs,
+    packages=find_packages(exclude=["tests", "ci", "tests.*"]),
     package_data={"": [
         "*.ico",
         "*.ini",
@@ -191,7 +205,7 @@ setup(
         "*.so",
         "*.pyd",
     ]},
-    install_requires=install_requires,
+    install_requires=get_install_requires(),
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Operating System :: Microsoft :: Windows :: Windows 7",
@@ -209,5 +223,5 @@ setup(
         "Natural Language :: Chinese (Simplified)",
         "Natural Language :: Chinese (Simplified)"
     ],
-    ext_modules=ext_modules
+    ext_modules=get_ext_modules(),
 )