diff --git a/aksetup_helper.py b/aksetup_helper.py
index f90d085284f00e6a6cd2a44e6b9bd63a857485f3..73c0e4feea717db170f2d0607bfc961c09d97968 100644
--- a/aksetup_helper.py
+++ b/aksetup_helper.py
@@ -36,10 +36,12 @@ def setup(*args, **kwargs):
 
 def get_numpy_incpath():
     from imp import find_module
-    # avoid actually importing numpy, it screws up distutils
-    file, pathname, descr = find_module("numpy")
-    from os.path import join
-    return join(pathname, "core", "include")
+    # importing numpy will fail if numpy was installed by pip because of
+    # setup_requires. Doing the following should avoid that
+    # http://stackoverflow.com/questions/19919905/how-to-bootstrap-numpy-installation-in-setup-py
+    __builtins__.__NUMPY_SETUP__ = False
+    import numpy
+    return numpy.get_include()
 
 
 class NumpyExtension(Extension):
diff --git a/pyproject.toml b/pyproject.toml
deleted file mode 100644
index 948bc1dba517d2a9080b47e67253fad1fdf09801..0000000000000000000000000000000000000000
--- a/pyproject.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-[build-system]
-requires = ["setuptools", "wheel", "numpy", "pybind11"]
diff --git a/setup.py b/setup.py
index aa8d70e16363b6a3fb969a235b3e18ede31645b4..4beca19a8f0cd20fe685a89e2e085388d43e8018 100644
--- a/setup.py
+++ b/setup.py
@@ -232,7 +232,6 @@ def main():
                         ],
                     include_dirs=INCLUDE_DIRS + [
                         get_pybind_include(),
-                        get_pybind_include(user=True)
                         ],
                     library_dirs=conf["CL_LIB_DIR"],
                     libraries=conf["CL_LIBNAME"],
@@ -244,7 +243,7 @@ def main():
                 ],
 
             setup_requires=[
-                "pybind11",
+                "pybind11>=2.5.0",
                 "numpy",
                 ],