From 5531b8f76cf3eb58da042e17092635f9a8ff205f Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Sun, 21 Jun 2020 02:39:59 -0500 Subject: [PATCH] Add a benchmark --- .gitignore | 2 + .gitlab-ci.yml | 15 +++ .test-conda-env.yml | 18 ++++ asv.conf.json | 158 ++++++++++++++++++++++++++++++++ benchmarks/__init__.py | 0 benchmarks/run_sumpy_kernels.py | 62 +++++++++++++ 6 files changed, 255 insertions(+) create mode 100644 .test-conda-env.yml create mode 100644 asv.conf.json create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/run_sumpy_kernels.py diff --git a/.gitignore b/.gitignore index 7be271c37..e7ea1299a 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ loopy/_git_rev.py virtualenv-[0-9]*[0-9] *.so + +.asv diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 07a049239..c975b4d5f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -141,3 +141,18 @@ Flake8: - python3 except: - tags + +Benchmarks: + script: + - export CONDA_ENVIRONMENT=.test-conda-env.yml + - export PROJECT=loopy + - export PYOPENCL_TEST=portable:pthread + - export LOOPY_NO_CACHE=1 + - export ASV_FACTOR=1.5 + - curl -L -O -k https://gitlab.tiker.net/inducer/ci-support/raw/master/build-and-benchmark-py-project.sh + - ". ./build-and-benchmark-py-project.sh" + tags: + - linux + - benchmark + except: + - tags diff --git a/.test-conda-env.yml b/.test-conda-env.yml new file mode 100644 index 000000000..6f7a4d2af --- /dev/null +++ b/.test-conda-env.yml @@ -0,0 +1,18 @@ +name: test-conda-env +channels: +- conda-forge +- nodefaults + +dependencies: +- git +- numpy +- pocl +- pocl-cuda +- islpy +- pyopencl +- python=3 + +- pip +- pip: + - git+https://github.com/inducer/pytools + - git+https://github.com/inducer/pymbolic diff --git a/asv.conf.json b/asv.conf.json new file mode 100644 index 000000000..5ef608174 --- /dev/null +++ b/asv.conf.json @@ -0,0 +1,158 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "loopy", + + // The project's homepage + "project_url": "https://documen.tician.de/loopy", + + // The URL or local path of the source code repository for the + // project being benchmarked + "repo": ".", + + // The Python project's subdirectory in your repo. If missing or + // the empty string, the project is assumed to be located at the root + // of the repository. + // "repo_subdir": "", + + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "default" (for mercurial). + "branches": ["master"], // for git + + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + // "dvcs": "git", + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "conda", + + // timeout in seconds for installing any dependencies in environment + // defaults to 10 min + //"install_timeout": 600, + + // the base URL to show a commit for the project. + "show_commit_url": "http://gitlab.tiker.net/inducer/loopy/commits/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + // "pythons": ["2.7", "3.6"], + + // The list of conda channel names to be searched for benchmark + // dependency packages in the specified order + "conda_channels": ["conda-forge", "defaults"], + + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list or empty string indicates to just test against the default + // (latest) version. null indicates that the package is to not be + // installed. If the package to be tested is only available from + // PyPi, and the 'environment_type' is conda, then you can preface + // the package name by 'pip+', and the package will be installed via + // pip (with all the conda available packages installed first, + // followed by the pip installed packages). + // + // "matrix": { + // "numpy": ["1.6", "1.7"], + // "six": ["", null], // test with and without six installed + // "pip+emcee": [""], // emcee is only available for install with pip. + // }, + "matrix": { + "numpy" : [""], + "pyopencl" : [""], + "islpy" : [""], + "pocl" : [""], + "pip+git+https://github.com/inducer/pymbolic#egg=pymbolic": [""], + "pip+git+https://gitlab.tiker.net/inducer/boxtree#egg=boxtree": [""], + "pip+git+https://github.com/inducer/loopy#egg=loopy": [""], + }, + + // Combinations of libraries/python versions can be excluded/included + // from the set to test. Each entry is a dictionary containing additional + // key-value pairs to include/exclude. + // + // An exclude entry excludes entries where all values match. The + // values are regexps that should match the whole string. + // + // An include entry adds an environment. Only the packages listed + // are installed. The 'python' key is required. The exclude rules + // do not apply to includes. + // + // In addition to package names, the following keys are available: + // + // - python + // Python version, as in the *pythons* variable above. + // - environment_type + // Environment type, as above. + // - sys_platform + // Platform, as in sys.platform. Possible values for the common + // cases: 'linux2', 'win32', 'cygwin', 'darwin'. + // + // "exclude": [ + // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows + // {"environment_type": "conda", "six": null}, // don't run without six on conda + // ], + // + // "include": [ + // // additional env for python2.7 + // {"python": "2.7", "numpy": "1.8"}, + // // additional env if run on windows+conda + // {"platform": "win32", "environment_type": "conda", "python": "2.7", "libpython": ""}, + // ], + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + // "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + "env_dir": ".asv/env", + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": ".asv/results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": ".asv/html", + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache wheels of the recent builds in each + // environment, making them faster to install next time. This is + // number of builds to keep, per environment. + // "wheel_cache_size": 0 + + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // } + + // The thresholds for relative change in results, after which `asv + // publish` starts reporting regressions. Dictionary of the same + // form as in ``regressions_first_commits``, with values + // indicating the thresholds. If multiple entries match, the + // maximum is taken. If no entry matches, the default is 5%. + // + // "regressions_thresholds": { + // "some_benchmark": 0.01, // Threshold of 1% + // "another_benchmark": 0.5, // Threshold of 50% + // } +} diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/run_sumpy_kernels.py b/benchmarks/run_sumpy_kernels.py new file mode 100644 index 000000000..7550d578a --- /dev/null +++ b/benchmarks/run_sumpy_kernels.py @@ -0,0 +1,62 @@ +import numpy as np + +from pyopencl.tools import ( # noqa + pytest_generate_tests_for_pyopencl as pytest_generate_tests) + +import logging +logger = logging.getLogger(__name__) +from .sumpy_kernels import m2l_3d_order_6 + +import loopy as lp + +class SumpyBenchmarkSuite: + + params = [ + "m2l_3d_order_6" + ] + + param_names = ['test_name'] + + version = 1 + + def setup_cache(self): + data = {} + for param in self.params: + self.setup(data, param) + data[param] = {} + knl = globals()[param]() + data[param]["instantiated"] = knl + preprocessed = lp.preprocess_kernel(knl) + data[param]["preprocessed"] = preprocessed + scheduled = lp.get_one_scheduled_kernel(preprocessed) + data[param]["scheduled"] = scheduled + return data + + def setup(self, data, param): + logging.basicConfig(level=logging.INFO) + np.random.seed(17) + + def time_instantiate(self, data, param): + create_knl = globals()[param] + create_knl() + + def time_preprocess(self, data, param): + lp.preprocess_kernel(data[param]["instantiated"]) + + def time_schedule(self, data, param): + lp.get_one_scheduled_kernel(data[param]["preprocessed"]) + + def time_generate_code(self, data, param): + lp.generate_code_v2(data[param]["scheduled"]) + + time_instantiate.timeout = 600.0 + time_preprocess.timeout = 600.0 + time_schedule.timeout = 600.0 + time_generate_code.timeout = 600.0 + + # Run memory benchmarks as well + peakmem_instantiate = time_instantiate + peakmem_preprocess = time_preprocess + peakmem_schedule = time_schedule + peakmem_generate_code = time_generate_code + -- GitLab