From 04143b9fc6e1c3fb0c7f9695cdd1a2bd7ddb1d32 Mon Sep 17 00:00:00 2001
From: Grzegorz Bokota <bokota+github@gmail.com>
Date: Tue, 17 May 2022 23:21:47 +0200
Subject: [PATCH] Setup build wheels using cibuildwheel for all systems (#559)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* initial setup of cibuildwheel
* skip windows test
* revert obsolete change
* skip pypy on windows
* clean old wheel build configuration
* bump cibuildwheel version
* fix flake test
* limit wheel build run
* undo skiping/failing tests
* comment win32 steps in build-ocl-windows
* use stable ICD on macos
* Update scripts/build-ocl-windows.sh
Co-authored-by: Andreas Klöckner <inform@tiker.net>
* rename file for wheel build
* add conda-forge explanation
* remove mako from build-system requires
* disable mako check for build under cibuildwheel
* remove obsolete check of mako
* add license information
add missed SCRIPT_DIR
* try use configure script
* musllinux build
* Apply suggestions from code review
Co-authored-by: Andreas Klöckner <inform@tiker.net>
Co-authored-by: Andreas Klöckner <inform@tiker.net>
---
.github/workflows/ci.yml | 37 ---------------
.github/workflows/wheels.yml | 65 ++++++++++++++++++++++++++
.gitignore | 2 +
aksetup_helper.py | 3 +-
pyproject.toml | 35 ++++++++++++++
scripts/build-ocl-macos.sh | 18 ++++++++
scripts/build-ocl-windows.sh | 22 +++++++++
scripts/build-ocl.sh | 23 +++++++++
scripts/build-wheels.sh | 90 ------------------------------------
setup.py | 28 ++---------
test/test_wrapper.py | 10 ++--
11 files changed, 174 insertions(+), 159 deletions(-)
create mode 100644 .github/workflows/wheels.yml
create mode 100644 pyproject.toml
create mode 100644 scripts/build-ocl-macos.sh
create mode 100644 scripts/build-ocl-windows.sh
create mode 100644 scripts/build-ocl.sh
delete mode 100755 scripts/build-wheels.sh
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b0bfffcf..eda59215 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -120,43 +120,6 @@ jobs:
(cd examples; rm -f gl_*)
run_examples --no-require-main
- wheels:
- name: Build and upload wheels
- runs-on: ubuntu-latest
- strategy:
- matrix:
- DOCKER_IMAGE:
- - quay.io/pypa/manylinux2014_x86_64
- # Disable i686 builds for now: no binary wheels for cryptography,
- # source build fails, e.g. https://github.com/inducer/pyopencl/pull/421/checks?check_run_id=1781071632
- # - quay.io/pypa/manylinux2014_i686
- steps:
- - uses: actions/checkout@v2
- - name: "Main Script"
- env:
- TWINE_USERNAME: __token__
- TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
- DOCKER_IMAGE: ${{ matrix.DOCKER_IMAGE }}
-
- run: |
- pwd
- ls -la
-
- # Only perform upload for tag builds, otherwise unset TWINE_USERNAME to prevent
- if ! [[ $GITHUB_REF == refs/tags* ]]; then
- echo "Not a tag build, GITHUB_REF is '$GITHUB_REF'. Unsetting TWINE_USERNAME"
- unset TWINE_USERNAME
- fi
-
- if [[ $DOCKER_IMAGE == *i686* ]]; then
- PRE_CMD=linux32
- else
- PRE_CMD=""
- fi
-
- docker run --rm -v `pwd`:/io -e TWINE_USERNAME -e TWINE_PASSWORD $DOCKER_IMAGE $PRE_CMD /io/scripts/build-wheels.sh
- ls wheelhouse/
-
downstream_tests:
strategy:
matrix:
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
new file mode 100644
index 00000000..37faac22
--- /dev/null
+++ b/.github/workflows/wheels.yml
@@ -0,0 +1,65 @@
+name: Build and upload to PyPI
+
+# Build on every branch push, tag push, and pull request change:
+on:
+ push:
+ branches:
+ - main
+ tags:
+ - v*
+ pull_request:
+ schedule:
+ - cron: '17 3 * * 0'
+
+jobs:
+ build_wheels:
+ name: Build wheels on ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-20.04, windows-2019, macos-10.15]
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ submodules: 'true'
+
+ - name: Build wheels
+ uses: pypa/cibuildwheel@v2.5.0
+
+ - uses: actions/upload-artifact@v2
+ with:
+ path: ./wheelhouse/*.whl
+
+ build_sdist:
+ name: Build source distribution
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Build sdist
+ run: pipx run build --sdist
+
+ - uses: actions/upload-artifact@v2
+ with:
+ path: dist/*.tar.gz
+
+ upload_pypi:
+ needs: [build_wheels, build_sdist]
+ runs-on: ubuntu-latest
+ # upload to PyPI on every tag starting with 'v'
+ if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+ # alternatively, to publish when a GitHub Release is created, use the following rule:
+ # if: github.event_name == 'release' && github.event.action == 'published'
+ steps:
+ - uses: actions/download-artifact@v2
+ with:
+ name: artifact
+ path: dist
+
+ - uses: pypa/gh-action-pypi-publish@v1.4.2
+ with:
+ user: __token__
+ password: ${{ secrets.TWINE_PASSWORD }}
+ # To test: repository_url: https://test.pypi.org/legacy/
diff --git a/.gitignore b/.gitignore
index 103ff507..3700f0a8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -66,3 +66,5 @@ cffi_build.py
.cache
.pytest_cache
.idea
+
+wheelhouse
diff --git a/aksetup_helper.py b/aksetup_helper.py
index de3ae069..89e66da3 100644
--- a/aksetup_helper.py
+++ b/aksetup_helper.py
@@ -1,3 +1,4 @@
+import os
import sys
try:
from setuptools import Extension
@@ -256,7 +257,7 @@ def expand_options(options):
class ConfigSchema:
- def __init__(self, options, conf_file="siteconf.py", conf_dir="."):
+ def __init__(self, options, conf_file="siteconf.py", conf_dir=os.path.dirname(__file__)):
self.optdict = dict((opt.name, opt) for opt in options)
self.options = options
self.conf_dir = conf_dir
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 00000000..8733abce
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,35 @@
+[build-system]
+# Minimum requirements for the build system to execute.
+requires = ["setuptools>=42.0.0", "wheel>=0.34.2", "Cython", "oldest-supported-numpy", "pybind11>=2.5.0"] # PEP 508 specifications.
+build-backend = "setuptools.build_meta"
+
+
+[tool.cibuildwheel]
+test-command = "pytest {project}/test"
+test-extras = ["test"]
+
+[tool.cibuildwheel.linux]
+test-command = ""
+before-all = [
+ "yum install -y git openssl-devel ruby",
+ "bash {package}/scripts/build-ocl.sh",
+]
+
+[[tool.cibuildwheel.overrides]]
+select = "*-musllinux*"
+before-all = [
+ "apk add ruby git openssl-dev",
+ "bash {package}/scripts/build-ocl.sh",
+]
+
+[tool.cibuildwheel.macos]
+skip = "pp*"
+before-all = "bash {package}/scripts/build-ocl-macos.sh"
+test-command = "pytest {project}/test/test_array.py" # same limitation as conda-forge
+# https://github.com/conda-forge/pyopencl-feedstock/blob/6f3c5de59b18c9518abba3cb94f6ae92964553f8/recipe/meta.yaml#L62-L63
+
+[tool.cibuildwheel.windows]
+skip = ["*-win32", "pp*"]
+test-command = ""
+before-all = "bash {package}/scripts/build-ocl-windows.sh"
+before-build = "python configure.py --cxxflags=-ID:/a/pyopencl/pyopencl/OpenCL-Headers/install/include --ldflags=\"/LIBPATH:C:/Program Files/OpenCL-ICD-Loader/lib\""
diff --git a/scripts/build-ocl-macos.sh b/scripts/build-ocl-macos.sh
new file mode 100644
index 00000000..9277c164
--- /dev/null
+++ b/scripts/build-ocl-macos.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+set -o xtrace
+
+git clone --branch v2022.01.04 https://github.com/KhronosGroup/OpenCL-ICD-Loader
+git clone --branch v2022.01.04 https://github.com/KhronosGroup/OpenCL-Headers
+
+
+
+cmake -D CMAKE_INSTALL_PREFIX=./OpenCL-Headers/install -S ./OpenCL-Headers -B ./OpenCL-Headers/build
+cmake --build ./OpenCL-Headers/build --target install
+
+cmake -D CMAKE_PREFIX_PATH=${PWD}/OpenCL-Headers/install -D OPENCL_ICD_LOADER_HEADERS_DIR=${PWD}/OpenCL-Headers/install/include -D CMAKE_INSTALL_PREFIX=./OpenCL-ICD-Loader/install -S ./OpenCL-ICD-Loader -B ./OpenCL-ICD-Loader/build
+cmake --build ./OpenCL-ICD-Loader/build --target install --config Release
+
+echo "PyOpenCL wheel includes Khronos Group OpenCL-ICD-Loader which is licensed as below" >> ${SCRIPT_DIR}/../LICENSE
+cat ./OpenCL-ICD-Loader/LICENSE >> ${SCRIPT_DIR}/../LICENSE
\ No newline at end of file
diff --git a/scripts/build-ocl-windows.sh b/scripts/build-ocl-windows.sh
new file mode 100644
index 00000000..01b7ef31
--- /dev/null
+++ b/scripts/build-ocl-windows.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+set -o xtrace
+
+git clone --branch v2022.01.04 https://github.com/KhronosGroup/OpenCL-ICD-Loader
+
+git clone --branch v2022.01.04 https://github.com/KhronosGroup/OpenCL-Headers
+
+
+cmake -D CMAKE_INSTALL_PREFIX=./OpenCL-Headers/install -S ./OpenCL-Headers -B ./OpenCL-Headers/build
+cmake --build ./OpenCL-Headers/build --target install
+
+# if someone would like to try to create win32 wheels bellow lines may be useful
+# cmake -D CMAKE_PREFIX_PATH=${PWD}/OpenCL-Headers/install -DOPENCL_ICD_LOADER_HEADERS_DIR=${PWD}/OpenCL-Headers/install/include -S ./OpenCL-ICD-Loader -B ./OpenCL-ICD-Loader/build
+# cmake --build ./OpenCL-ICD-Loader/build --target install --config Release
+
+cmake -D CMAKE_PREFIX_PATH=${PWD}/OpenCL-Headers/install -D OPENCL_ICD_LOADER_HEADERS_DIR=${PWD}/OpenCL-Headers/install/include -S ./OpenCL-ICD-Loader -B ./OpenCL-ICD-Loader/build2 -A x64
+cmake --build ./OpenCL-ICD-Loader/build2 --target install --config Release
+
+echo "PyOpenCL wheel includes Khronos Group OpenCL-ICD-Loader which is licensed as below" >> ${SCRIPT_DIR}/../LICENSE
+cat ./OpenCL-ICD-Loader/LICENSE >> ${SCRIPT_DIR}/../LICENSE
\ No newline at end of file
diff --git a/scripts/build-ocl.sh b/scripts/build-ocl.sh
new file mode 100644
index 00000000..c1d97bad
--- /dev/null
+++ b/scripts/build-ocl.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+set -e -x
+
+mkdir -p ~/deps
+cd ~/deps
+
+git clone --branch v2.3.1 https://github.com/OCL-dev/ocl-icd
+cd ocl-icd
+curl -L -O https://raw.githubusercontent.com/conda-forge/ocl-icd-feedstock/e2c03e3ddb1ff86630ccf80dc7b87a81640025ea/recipe/install-headers.patch
+git apply install-headers.patch
+curl -L -O https://github.com/isuruf/ocl-icd/commit/3862386b51930f95d9ad1089f7157a98165d5a6b.patch
+git apply 3862386b51930f95d9ad1089f7157a98165d5a6b.patch
+autoreconf -i
+chmod +x configure
+./configure --prefix=/usr
+make -j4
+make install
+
+# Bundle license files
+echo "PyOpenCL wheel includes ocl-icd which is licensed as below" >> ${SCRIPT_DIR}/../LICENSE
+cat ~/deps/ocl-icd/COPYING >> ${SCRIPT_DIR}/../LICENSE
\ No newline at end of file
diff --git a/scripts/build-wheels.sh b/scripts/build-wheels.sh
deleted file mode 100755
index 34c8cd71..00000000
--- a/scripts/build-wheels.sh
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/bin/bash
-set -e -x
-
-mkdir -p /deps
-cd /deps
-
-function start_spinner {
- if [ -n "$SPINNER_PID" ]; then
- return
- fi
-
- >&2 echo "Building libraries..."
- # Start a process that runs as a keep-alive
- # to avoid travis quitting if there is no output
- (while true; do
- sleep 60
- >&2 echo "Still building..."
- done) &
- SPINNER_PID=$!
- disown
-}
-
-function stop_spinner {
- if [ ! -n "$SPINNER_PID" ]; then
- return
- fi
-
- kill $SPINNER_PID
- unset SPINNER_PID
-
- >&2 echo "Building libraries finished."
-}
-
-#start_spinner
-
-git config --global --add safe.directory /io
-
-curl https://tiker.net/tmp/.tmux.conf
-yum install -y git yum openssl-devel ruby
-
-git clone --branch v2.3.0 https://github.com/OCL-dev/ocl-icd
-cd ocl-icd
-curl -L -O https://raw.githubusercontent.com/conda-forge/ocl-icd-feedstock/e2c03e3ddb1ff86630ccf80dc7b87a81640025ea/recipe/install-headers.patch
-git apply install-headers.patch
-curl -L -O https://github.com/isuruf/ocl-icd/commit/3862386b51930f95d9ad1089f7157a98165d5a6b.patch
-git apply 3862386b51930f95d9ad1089f7157a98165d5a6b.patch
-autoreconf -i
-chmod +x configure
-./configure --prefix=/usr
-make -j4
-make install
-cd ..
-
-# Bundle license files
-echo "PyOpenCL wheel includes ocl-icd which is licensed as below" >> /io/LICENSE
-cat /deps/ocl-icd/COPYING >> /io/LICENSE
-
-# Compile wheels
-for PYBIN in /opt/python/*/bin; do
- # Build with the oldest numpy available to be compatible with newer ones
- "${PYBIN}/pip" install oldest-supported-numpy pybind11 mako
- "${PYBIN}/pip" wheel /io/ -w wheelhouse/ --no-deps
-done
-
-# Bundle external shared libraries into the wheels
-for whl in wheelhouse/pyopencl*.whl; do
- auditwheel repair "$whl" -w /io/wheelhouse/ --lib-sdir=/.libs
-done
-
-if [[ "${TWINE_USERNAME}" == "" ]]; then
- echo "TWINE_USERNAME not set. Skipping uploading wheels"
- exit 0
-fi
-
-/opt/python/cp39-cp39/bin/pip install twine
-for WHEEL in /io/wheelhouse/pyopencl*.whl; do
- # dev
- # /opt/python/cp39-cp39/bin/twine upload \
- # --skip-existing \
- # --repository-url https://test.pypi.org/legacy/ \
- # -u "${TWINE_USERNAME}" -p "${TWINE_PASSWORD}" \
- # "${WHEEL}"
- # prod
- /opt/python/cp39-cp39/bin/twine upload \
- --skip-existing \
- -u "${TWINE_USERNAME}" -p "${TWINE_PASSWORD}" \
- "${WHEEL}"
-done
-
-#stop_spinner
diff --git a/setup.py b/setup.py
index 31c096c3..00ddcb34 100644
--- a/setup.py
+++ b/setup.py
@@ -31,6 +31,8 @@ import os
import sys
from os.path import exists
+sys.path.append(os.path.dirname(__file__))
+
def get_config_schema():
from aksetup_helper import (ConfigSchema, Option,
@@ -161,31 +163,6 @@ def main():
exec(compile(version_file_contents, "pyopencl/version.py", "exec"), ver_dic)
- try:
- import mako # noqa
- except ImportError:
- print(SEPARATOR)
- print("Mako is not installed.")
- print(SEPARATOR)
- print("That is not a problem, as most of PyOpenCL will be just fine ")
- print("without it. Some higher-level parts of pyopencl (such as ")
- print("pyopencl.reduction) will not function without the templating engine ")
- print("Mako [1] being installed. If you would like this functionality to ")
- print("work, you might want to install Mako after you finish ")
- print("installing PyOpenCL.")
- print("")
- print("Simply type")
- print("python -m pip install mako")
- print("either now or after the installation completes to fix this.")
- print("")
- print("[1] http://www.makotemplates.org/")
- print(SEPARATOR)
- print("Hit Ctrl-C now if you'd like to think about the situation.")
- print(SEPARATOR)
-
- from aksetup_helper import count_down_delay
- count_down_delay(delay=5)
-
if not exists("pyopencl/compyte/dtypes.py"):
print(75 * "-")
print("You are missing important files from the pyopencl distribution.")
@@ -263,6 +240,7 @@ def main():
extras_require={
"pocl": ["pocl_binary_distribution>=1.2"],
"oclgrind": ["oclgrind_binary_distribution>=18.3"],
+ "test": ["pytest>=7.0.0", "Mako"],
},
include_package_data=True,
package_data={
diff --git a/test/test_wrapper.py b/test/test_wrapper.py
index ff4ee0cd..1f94f640 100644
--- a/test/test_wrapper.py
+++ b/test/test_wrapper.py
@@ -637,8 +637,8 @@ def test_vector_args(ctx_factory):
def test_header_dep_handling(ctx_factory):
context = ctx_factory()
- from os.path import exists
- assert exists("empty-header.h") # if this fails, change dir to pyopencl/test
+ from os.path import exists, dirname, join
+ assert exists(join(dirname(__file__), "empty-header.h"))
kernel_src = """
#include <empty-header.h>
@@ -648,10 +648,8 @@ def test_header_dep_handling(ctx_factory):
}
"""
- import os
-
- cl.Program(context, kernel_src).build(["-I", os.getcwd()])
- cl.Program(context, kernel_src).build(["-I", os.getcwd()])
+ cl.Program(context, kernel_src).build(["-I", dirname(__file__)])
+ cl.Program(context, kernel_src).build(["-I", dirname(__file__)])
def test_context_dep_memoize(ctx_factory):
--
GitLab