diff --git a/.travis.yml b/.travis.yml
index 937754b78eddc99a3adeddb7cd3778ee2bf454a9..bdd24bc894484dfdc63ee8b34eea77b02fb43439 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,18 +13,6 @@ matrix:
     env:
       - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686
       - PRE_CMD=linux32
-  - sudo: required
-    services:
-      - docker
-    arch: ppc64le
-    env:
-      - DOCKER_IMAGE=quay.io/pypa/manylinux2014_ppc64le
-  - sudo: required
-    services:
-      - docker
-    arch: arm64
-    env:
-      - DOCKER_IMAGE=quay.io/pypa/manylinux2014_aarch64
   - sudo: required
     services:
       - docker
diff --git a/aksetup_helper.py b/aksetup_helper.py
index a8ba81183da72f76a5fd7fcdd25c71f540384991..a168adf7a381ca8c33d005493030bdf7fab9addf 100644
--- a/aksetup_helper.py
+++ b/aksetup_helper.py
@@ -36,15 +36,10 @@ def setup(*args, **kwargs):
 
 def get_numpy_incpath():
     from imp import find_module
-    # importing numpy will fail if numpy was installed by pip because of
-    # setup_requires. Doing the following should avoid that
-    # http://stackoverflow.com/questions/19919905/how-to-bootstrap-numpy-installation-in-setup-py
-    if isinstance(__builtins__, dict):
-        __builtins__["__NUMPY_SETUP__"] = False
-    else:
-        __builtins__.__NUMPY_SETUP__ = False
-    import numpy
-    return numpy.get_include()
+    # avoid actually importing numpy, it screws up distutils
+    file, pathname, descr = find_module("numpy")
+    from os.path import join
+    return join(pathname, "core", "include")
 
 
 class NumpyExtension(Extension):
@@ -167,10 +162,12 @@ def hack_distutils(debug=False, fast_link=True, what_opt=3):
         from distutils import sysconfig
 
         cvars = sysconfig.get_config_vars()
-        cflags = cvars.get('OPT')
+
+        bad_prefixes = ['-g', '-O', '-Wstrict-prototypes', '-DNDEBUG']
+
+        cflags = cvars.get("OPT")
         if cflags:
-            cflags = remove_prefixes(cflags.split(),
-                    ['-g', '-O', '-Wstrict-prototypes', '-DNDEBUG'])
+            cflags = remove_prefixes(cflags.split(), bad_prefixes)
             if debug:
                 cflags.append("-g")
             else:
@@ -180,11 +177,17 @@ def hack_distutils(debug=False, fast_link=True, what_opt=3):
                     cflags.append("-O%s" % what_opt)
                     cflags.append("-DNDEBUG")
 
-            cvars['OPT'] = str.join(' ', cflags)
-            if "BASECFLAGS" in cvars:
-                cvars["CFLAGS"] = cvars["BASECFLAGS"] + " " + cvars["OPT"]
-            else:
-                assert "CFLAGS" in cvars
+            cvars["OPT"] = str.join(' ', cflags)
+
+        cflags = cvars.get("CONFIGURE_CFLAGS")
+        if cflags:
+            cflags = remove_prefixes(cflags.split(), bad_prefixes)
+            cvars["CONFIGURE_CFLAGS"] = str.join(' ', cflags)
+
+        if "BASECFLAGS" in cvars:
+            cvars["CFLAGS"] = cvars["BASECFLAGS"] + " " + cvars.get("OPT", "")
+        else:
+            assert "CFLAGS" in cvars
 
         if fast_link:
             for varname in ["LDSHARED", "BLDSHARED"]:
@@ -940,23 +943,42 @@ class PybindBuildExtCommand(NumpyBuildExtCommand):
         'unix': [],
     }
 
-    if sys.platform == 'darwin':
-        c_opts['unix'] += ['-stdlib=libc++', '-mmacosx-version-min=10.7']
-
     def build_extensions(self):
         ct = self.compiler.compiler_type
         opts = self.c_opts.get(ct, [])
+        cxx_opts = []
+
         if ct in ['unix', 'mingw32']:
             opts.append('-DVERSION_INFO="%s"' % self.distribution.get_version())
-            opts.append(cpp_flag(self.compiler))
+            cxx_opts.append(cpp_flag(self.compiler))
             if has_flag(self.compiler, '-fvisibility=hidden'):
                 opts.append('-fvisibility=hidden')
+            if sys.platform == 'darwin':
+                if has_flag(self.compiler, '-stdlib=libc++'):
+                    opts.append('-stdlib=libc++')
+                if has_flag(self.compiler, '-mmacosx-version-min=10.7'):
+                    opts.append('-mmacosx-version-min=10.7')
         elif ct == 'msvc':
             opts.append('/DVERSION_INFO=\\"%s\\"' % self.distribution.get_version())
         for ext in self.extensions:
             ext.extra_compile_args = ext.extra_compile_args + opts
 
-        NumpyBuildExtCommand.build_extensions(self)
+        prev__compile = self.compiler._compile
+
+        # -std=... used on C files causes an error on Apple LLVM
+        # https://gitlab.tiker.net/inducer/pymetis/-/jobs/102421
+        def _compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
+            if ext == ".cpp":
+                cc_args = cc_args + cxx_opts
+
+            return prev__compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+
+        self.compiler._compile = _compile
+
+        try:
+            NumpyBuildExtCommand.build_extensions(self)
+        finally:
+            self.compiler._compile = prev__compile
 
 # }}}