diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e1579ae818bfedb952d960b231627eeadf413695..2b81f42666a61e9b026ece1f839a889d3ca63766 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -105,8 +105,7 @@ Flake8:
 
 Benchmarks:
   script:
-  - sed 's/python=3/python=3.6/' .test-conda-env-py3.yml > .test-conda-env.yml
-  - CONDA_ENVIRONMENT=.test-conda-env.yml
+  - CONDA_ENVIRONMENT=.test-conda-env-py3.yml
   - PROJECT=sumpy
   - PYOPENCL_TEST=portable:pthread
   - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-benchmark-py-project.sh
diff --git a/asv.conf.json b/asv.conf.json
index 6d6fd675c48cbdf02e0bc8448f22890dde2a3e26..90133beb7af7d1a4b5bb23745b9cc543b1191d7b 100644
--- a/asv.conf.json
+++ b/asv.conf.json
@@ -75,6 +75,8 @@
         "pip+git+https://github.com/inducer/pymbolic#egg=pymbolic": [""],
         "pip+git+https://gitlab.tiker.net/inducer/boxtree#egg=boxtree": [""],
         "pip+git+https://github.com/inducer/loopy#egg=loopy": [""],
+        // https://github.com/pypa/pip/issues/8667
+        "pip" : ["<20.2"],
     },
 
     // Combinations of libraries/python versions can be excluded/included