From 399f9fde643e7a95f1b753fd9f22becfed2ed438 Mon Sep 17 00:00:00 2001 From: xywei Date: Mon, 26 Oct 2020 22:43:01 -0500 Subject: [PATCH 1/5] Automate docker image fixes --- .gitlab-ci.yml | 1 + docker/Dockerfile.firedrake | 13 +++++++++---- docker/to_loopyy.sh | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 docker/to_loopyy.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 429231e..e291668 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -160,6 +160,7 @@ Documentation Sphinx: Docker Image w/t Firedrake: script: - docker info + - docker pull firedrakeproject/firedrake:latest - docker build --no-cache -t xywei/volumential-firedrake:latest -f docker/Dockerfile.firedrake . - bash -c 'if [ "$BRANCH_PROTECTED" = 1 ]; then exit 0; else exit 1; fi' - docker login --username "${DOCKERHUB_USERNAME}" --password "${DOCKERHUB_ACCESS_TOKEN}" diff --git a/docker/Dockerfile.firedrake b/docker/Dockerfile.firedrake index ccc06e8..3c6164c 100644 --- a/docker/Dockerfile.firedrake +++ b/docker/Dockerfile.firedrake @@ -19,12 +19,17 @@ RUN ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /usr/lib/libOpenCL.so ########################################## USER firedrake -WORKDIR /home/firedrake +WORKDIR /home/firedrake/firedrake/src RUN git clone --recurse-submodules "https://gitlab.tiker.net/xywei/volumential.git" # Code generation in pypvfm's setup.py needs mako -RUN bash -c "source firedrake/bin/activate; pip install mako" +RUN bash -c "source ~/firedrake/bin/activate; pip install mako" -RUN bash -c "source firedrake/bin/activate; cd volumential/contrib/meshgen11_dealii; ./build.sh" -RUN bash -c "source firedrake/bin/activate; cd volumential; pip install -r requirements.txt" +RUN bash -c "source ~/firedrake/bin/activate; cd volumential/contrib/meshgen11_dealii; ./build.sh" +RUN bash -c "source ~/firedrake/bin/activate; cd volumential; pip install -r requirements.txt" +COPY docker/to_loopyy.sh . +RUN bash -c "source ~/firedrake/bin/activate; sh to_loopyy.sh" + +# Some convenience utils +RUN bash -c "source ~/firedrake/bin/activate; pip install tqdm jupyterlab" diff --git a/docker/to_loopyy.sh b/docker/to_loopyy.sh new file mode 100644 index 0000000..2f9c4ff --- /dev/null +++ b/docker/to_loopyy.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +cd /home/firedrake/firedrake/src/ +mv loo-py/ loopyy/ + +# under loopyy +cd loopyy/ +find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/loopy/loopyy/g' +find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/loo.py/loo.pyy/g' +mv loopy loopyy +mv bin/loopy bin/loopyy +cd .. + +# under meshmode +cd meshmode/ +find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/loopy/loopyy/g' +find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/loo.py/loo.pyy/g' +cd .. + +# under pytential +cd pytential/ +find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/loopy/loopyy/g' +find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/loo.py/loo.pyy/g' +find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/from __future__.*//g' # to use py3.6 +cd .. + +# under volumential +cd volumential/ +find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/loopy/loopyy/g' +find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/loo.py/loo.pyy/g' +find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/from __future__.*//g' # to use py3.6 +cd .. + +rm /home/firedrake/firedrake/lib/python3.6/site-packages/loo.py.egg-link +pip install -e ./loopyy +pip install -e ./loopy -- GitLab From ca17859dd1bbfa415dd212349dedcf6f6f9e9c7e Mon Sep 17 00:00:00 2001 From: xywei Date: Tue, 27 Oct 2020 11:36:54 -0500 Subject: [PATCH 2/5] Update work dir, and make sure loopyy is installed in the virtualenv --- docker/Dockerfile.firedrake | 2 ++ docker/to_loopyy.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docker/Dockerfile.firedrake b/docker/Dockerfile.firedrake index 3c6164c..c059847 100644 --- a/docker/Dockerfile.firedrake +++ b/docker/Dockerfile.firedrake @@ -31,5 +31,7 @@ RUN bash -c "source ~/firedrake/bin/activate; cd volumential; pip install -r req COPY docker/to_loopyy.sh . RUN bash -c "source ~/firedrake/bin/activate; sh to_loopyy.sh" +########################################## # Some convenience utils +WORKDIR /home/firedrake/ RUN bash -c "source ~/firedrake/bin/activate; pip install tqdm jupyterlab" diff --git a/docker/to_loopyy.sh b/docker/to_loopyy.sh index 2f9c4ff..7ed6ef0 100644 --- a/docker/to_loopyy.sh +++ b/docker/to_loopyy.sh @@ -31,6 +31,8 @@ find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/l find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/from __future__.*//g' # to use py3.6 cd .. +# fix egg links and make sure both loopy and loopyy are installed rm /home/firedrake/firedrake/lib/python3.6/site-packages/loo.py.egg-link pip install -e ./loopyy pip install -e ./loopy +pip install -e ./loopyy -- GitLab From 86c53664fc84775314904795162bb4015fac26fa Mon Sep 17 00:00:00 2001 From: xywei Date: Tue, 27 Oct 2020 12:30:39 -0500 Subject: [PATCH 3/5] Add two convenient images that launch jupyterlab on start --- .gitlab-ci.yml | 4 ++++ docker/Dockerfile.firedrake | 5 ----- docker/Dockerfile.firedrake_jupyterlab | 19 +++++++++++++++++++ docker/Dockerfile.firedrake_jupyterlab_vim | 7 +++++++ docker/start_jupyterlab.sh | 4 ++++ 5 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 docker/Dockerfile.firedrake_jupyterlab create mode 100644 docker/Dockerfile.firedrake_jupyterlab_vim create mode 100644 docker/start_jupyterlab.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e291668..ff6a091 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -162,9 +162,13 @@ Docker Image w/t Firedrake: - docker info - docker pull firedrakeproject/firedrake:latest - docker build --no-cache -t xywei/volumential-firedrake:latest -f docker/Dockerfile.firedrake . + - docker build --no-cache -t xywei/volumential-firedrake-jupyterlab:latest -f docker/Dockerfile.firedrake_jupyterlab . + - docker build --no-cache -t xywei/volumential-firedrake-jupyterlab-vim:latest -f docker/Dockerfile.firedrake_jupyterlab_vim . - bash -c 'if [ "$BRANCH_PROTECTED" = 1 ]; then exit 0; else exit 1; fi' - docker login --username "${DOCKERHUB_USERNAME}" --password "${DOCKERHUB_ACCESS_TOKEN}" - docker push xywei/volumential-firedrake:latest + - docker push xywei/volumential-firedrake-jupyterlab:latest + - docker push xywei/volumential-firedrake-jupyterlab-vim:latest tags: - docker except: diff --git a/docker/Dockerfile.firedrake b/docker/Dockerfile.firedrake index c059847..cd5252e 100644 --- a/docker/Dockerfile.firedrake +++ b/docker/Dockerfile.firedrake @@ -30,8 +30,3 @@ RUN bash -c "source ~/firedrake/bin/activate; cd volumential/contrib/meshgen11_d RUN bash -c "source ~/firedrake/bin/activate; cd volumential; pip install -r requirements.txt" COPY docker/to_loopyy.sh . RUN bash -c "source ~/firedrake/bin/activate; sh to_loopyy.sh" - -########################################## -# Some convenience utils -WORKDIR /home/firedrake/ -RUN bash -c "source ~/firedrake/bin/activate; pip install tqdm jupyterlab" diff --git a/docker/Dockerfile.firedrake_jupyterlab b/docker/Dockerfile.firedrake_jupyterlab new file mode 100644 index 0000000..7d99b80 --- /dev/null +++ b/docker/Dockerfile.firedrake_jupyterlab @@ -0,0 +1,19 @@ +# Working environment that also starts a jupyterlab server on start +FROM xywei/volumential-firedrake:latest + +# This DockerFile is looked after by +MAINTAINER Xiaoyu Wei + +########################################## +# install nodejs for jupyter lab extensions +USER root +RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - +RUN apt-get install -y nodejs && rm -rf /var/lib/apt/lists/* + +########################################## +USER firedrake +COPY docker/start_jupyterlab.sh . +RUN bash -c "source ~/firedrake/bin/activate; pip install tqdm jupyterlab" +RUN bash -c "source ~/firedrake/bin/activate; jupyter labextension install @jupyter-widgets/jupyterlab-manager" +RUN bash -c "source ~/firedrake/bin/activate; jupyter labextension install @jupyterlab/toc" +CMD ["sh", "start_jupyterlab.sh"] diff --git a/docker/Dockerfile.firedrake_jupyterlab_vim b/docker/Dockerfile.firedrake_jupyterlab_vim new file mode 100644 index 0000000..25e2e2b --- /dev/null +++ b/docker/Dockerfile.firedrake_jupyterlab_vim @@ -0,0 +1,7 @@ +# Jupyter lab working envioronment with vim mode +FROM xywei/volumential-firedrake-jupyterlab:latest + +# This DockerFile is looked after by +MAINTAINER Xiaoyu Wei + +RUN bash -c "source ~/firedrake/bin/activate; jupyter labextension install @axlair/jupyterlab_vim" diff --git a/docker/start_jupyterlab.sh b/docker/start_jupyterlab.sh new file mode 100644 index 0000000..127ba22 --- /dev/null +++ b/docker/start_jupyterlab.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +. ~/firedrake/bin/activate +jupyter lab --ip=0.0.0.0 -- GitLab From 5cda43550692a91b290ee6d3976bfca8549a9279 Mon Sep 17 00:00:00 2001 From: xywei Date: Wed, 28 Oct 2020 15:29:49 -0500 Subject: [PATCH 4/5] Allow CI to run without dealii --- .gitlab-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ff6a091..211b499 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,6 @@ Python 3 POCL: tags: - python3 - pocl - - dealii - large-node except: - tags @@ -61,7 +60,6 @@ Python 3 Intel: - ". ./build-and-test-py-project.sh" tags: - python3 - - dealii - intel-cl-cpu - large-node except: @@ -120,7 +118,6 @@ Python 3 POCL Examples: tags: - python3 - pocl - - dealii - large-node except: - tags -- GitLab From e47ce12ddda71a2200e3342e2f327f6088ef5baa Mon Sep 17 00:00:00 2001 From: xywei Date: Wed, 28 Oct 2020 16:23:43 -0500 Subject: [PATCH 5/5] Doc build does not require pocl and dealii --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 211b499..3038472 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,8 +148,6 @@ Documentation Sphinx: - git push tags: - python3 - - pocl - - dealii except: - tags allow_failure: true -- GitLab