change-flow-page-type

 #! /usr/bin/env python
 from __future__ import print_function
+
 import os
+
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "relate.settings")
 
 import django
+
 django.setup()
 
-from course.models import FlowPageData
 import sys
 
+from course.models import FlowPageData
+
 course_identifier = sys.argv[1]
 flow_id = sys.argv[2]
 group_id = sys.argv[3]
@@ -32,6 +36,7 @@ print(fpages.count(), "pages total")
 raw_input("[Enter to continue]")
 
 from django.db import transaction
+
 with transaction.atomic():
     for fpd in fpages:
         assert fpd.page_type == old_type

collectstatic.sh

+#! /bin/bash
+
+find node_modules/mathjax/es5 -name '*.js' -exec sed -i /sourceMappingURL=/d '{}' \;
+poetry run python manage.py collectstatic

contrib/course-material-tree/make-tree-html

@@ -2,12 +2,124 @@
 
 from __future__ import print_function
 from os.path import join, basename
+from urllib.parse import quote_plus
 from glob import glob
 import re
 
 
 section_nr = [1]
 
+
+def rewrite_org_element(args, org_el):
+    el_type = org_el[0]
+    org_props = org_el[1]
+    assert org_props is None or isinstance(org_props, dict)
+    contents = org_el[2:]
+
+    had_relate_properties = False
+    promote_to_parent_level = False
+    node_props = {}
+    if el_type == "org-data":
+        title, = [
+                subel[1]["key"]
+                for el in contents
+                if el[0] == "section"
+                for subel in el[2:]
+                if subel[0] == "keyword"
+                if subel[1]["key"] == "TITLE"]
+        node_props["text"] = title
+
+    elif el_type == "headline":
+        node_props["text"] = org_props["raw-value"]
+
+        prop_container = [
+                subel[2:]
+                for el in contents
+                if el[0] == "section"
+                for subel in el[2:]
+                if subel[0] == "property-drawer"]
+
+        if prop_container:
+            prop_kws = prop_container[0]
+        else:
+            prop_kws = []
+
+        properties = {
+                ch[1]["key"]: ch[1]["value"]
+                for ch in prop_kws
+                if ch[0] == "node-property"
+                }
+
+        had_relate_properties = had_relate_properties or any(
+                key.startswith("RELATE_") for key in properties)
+
+        if "RELATE_TREE_SECTION_NAME" in properties:
+            node_props["section"] = properties["RELATE_TREE_SECTION_NAME"]
+        if "RELATE_TREE_SECTION_OPENED" in properties:
+            node_props["opened"] = properties["RELATE_TREE_SECTION_OPENED"]
+        if "RELATE_TREE_ICON" in properties:
+            node_props["icon"] = properties["RELATE_TREE_ICON"]
+        if "RELATE_TREE_LINK" in properties:
+            node_props["link"] = properties["RELATE_TREE_LINK"]
+        if "RELATE_PROMOTE_TO_PARENT_LEVEL" in properties:
+            promote_to_parent_level = True
+
+    if el_type == "org-data":
+        level = 0
+    else:
+        level = org_props.get("level")
+
+    if ("text" in node_props
+            and level is not None
+            and level < len(args.org_level_icon)):
+        node_props["icon"] = args.org_level_icon[level]
+
+    siblings = []
+    children = []
+    node_props["nodes"] = children
+
+    for ch in contents:
+        if isinstance(ch, list):
+            child_props, child_siblings, promote_child_to_parent_level = \
+                    rewrite_org_element(args, ch)
+            if "text" in child_props:
+                if promote_child_to_parent_level:
+                    siblings.append(child_props)
+                else:
+                    children.append(child_props)
+
+            children.extend(child_siblings)
+
+    if args.org_stop_level:
+        if (level is not None
+                and level > args.org_stop_level
+                and not had_relate_properties):
+            node_props = {}
+
+    if el_type == "headline" and org_props["todo-keyword"] is not None:
+        node_props = {}
+    if (el_type == "headline"
+            and org_props["tags"] is not None
+            and "noexport" in org_props["tags"]):
+        node_props = {}
+
+    return node_props, siblings, promote_to_parent_level
+
+
+def load_org_json(args, infile_name):
+    # https://github.com/ludios/org-to-json
+
+    from json import loads
+    with open(infile_name, "rb") as inf:
+        json = loads(inf.read())
+
+    node_props, siblings, promote_to_parent_level = \
+            rewrite_org_element(args, json)
+    assert not siblings
+    assert not promote_to_parent_level
+    return node_props
+
+
 def normalize_nodes(nodes, text_icon):
     for i in range(len(nodes)):
         node = nodes[i]
@@ -45,9 +157,16 @@ def find_section_nodes(section_dict, node):
 
 
 class RenderSettings:
-    def __init__(self, default_icon, number_sections):
+    def __init__(self, default_icon, number_sections, tree_replacements):
         self.default_icon = default_icon
         self.number_sections = number_sections
+        self.tree_replacements = tree_replacements
+
+    def apply_replacements(self, s):
+        for key, val in self.tree_replacements.items():
+            s = s.replace(key, val)
+
+        return s
 
 
 def render(settings, outf, node, indent=0, skip=1):
@@ -79,12 +198,14 @@ def render(settings, outf, node, indent=0, skip=1):
         if "link" in node:
             print(
                 indent * " ",
-                "<a href=\"%s\">%s</a>" % (
-                    node["link"],
-                    text),
+                '<a href="%s">%s</a>' % (
+                    settings.apply_replacements(node["link"]),
+                    settings.apply_replacements(text)),
                 file=outf, sep="")
         else:
-            print(indent * " ", text, file=outf, sep="")
+            print(
+                    indent * " ",
+                    settings.apply_replacements(text), file=outf, sep="")
 
     subnodes = node.get("nodes", [])
     if subnodes:
@@ -99,6 +220,7 @@ def render(settings, outf, node, indent=0, skip=1):
         indent -= 2
         print(indent * " ", "</li>", file=outf, sep="")
 
+
 FN_REGEX = re.compile(r"^([0-9]+)-(.*)(\.[a-z]+)$")
 
 
@@ -127,12 +249,19 @@ def get_section_id_and_display_name(trunk, include_extension):
     return section_id, display_name
 
 
+def blacklisted_glob(basedir, pattern, blacklist_regexps):
+    return sorted(
+            name
+            for name in glob(join(basedir, pattern))
+            if not any(bl_re.match(name[len(basedir)+1:])
+                for bl_re in blacklist_regexps))
+
 
 def main():
     import argparse
 
     parser = argparse.ArgumentParser(
-            description='Turn a YAML file into a jsTree-compatible data file')
+            description="Turn a YAML file into a jsTree-compatible data file")
 
     parser.add_argument("-o", "--output-file", metavar="FILE", required=True)
 
@@ -142,37 +271,74 @@ def main():
     parser.add_argument("--ipynb-as-py", action="store_true")
     parser.add_argument("--ipynb-as-ipynb", action="store_true")
 
-    parser.add_argument("--py-dir", metavar="DIRECTORY")
-    parser.add_argument("--py-urlroot", metavar="URL",
+    parser.add_argument("--interactive-nb-urlroot", metavar="URL",
+            help="(without the trailing slash)")
+    parser.add_argument("--interactive-nb-cleared-urlroot", metavar="URL",
+            help="(without the trailing slash)")
+    parser.add_argument("--ipynb-main-link",
+            choices=["static", "interactive", "interactive-cleared"],
+            default="static")
+
+    parser.add_argument("--source-dir", metavar="DIRECTORY")
+    parser.add_argument("--source-urlroot", metavar="URL",
             help="(without the trailing slash)")
+    parser.add_argument("--source-wildcard", metavar="WILDCARD", nargs="*")
 
     parser.add_argument("--pdf-dir", metavar="DIRECTORY")
     parser.add_argument("--pdf-urlroot", metavar="URL",
             help="(without the trailing slash)")
 
     parser.add_argument("--default-icon", metavar="ICON_STR",
-            default="fa fa-file-o")
+            default="bi bi-file-earmark")
     parser.add_argument("--text-icon", metavar="ICON_STR",
-            default="fa fa-file-o")
+            default="bi bi-file-earmark")
     parser.add_argument("--number-sections", action="store_true")
+    parser.add_argument("--blacklist-file", metavar="PATTERN_FILE")
+
+    parser.add_argument("--org-stop-level", metavar="INT", type=int)
+    parser.add_argument("--org-level-icon", metavar="CSS_CLASSES", nargs="*")
+
+    parser.add_argument("--tree-replacement", metavar="KEY=VALUE", nargs="*")
 
     parser.add_argument("input_file", metavar="FILE")
 
     args = parser.parse_args()
 
-    from yaml import load
-    with open(args.input_file, "rb") as inf:
-        root_node = load(inf)
+    blacklist_regexps = []
+    if args.blacklist_file is not None:
+        with open(args.blacklist_file, "rt") as bl_file:
+            import fnmatch
+            for pattern in bl_file:
+                blacklist_regexps.append(
+                        re.compile(fnmatch.translate(pattern.strip())))
+
+    if args.input_file.endswith(".yml") or args.input_file.endswith(".yml"):
+        from yaml import safe_load
+        with open(args.input_file, "rb") as inf:
+            root_node = safe_load(inf)
+    elif args.input_file.endswith(".org.json"):
+        root_node = load_org_json(args, args.input_file)
+    else:
+        raise ValueError("unknown extension of input file: %s" % args.input_file)
 
     normalize_nodes([root_node], args.text_icon)
 
+    tree_replacements = {}
+    if args.tree_replacement is not None:
+        for tr in args.tree_replacement:
+            eq_ind = tr.find("=")
+            if eq_ind < 0:
+                raise ValueError(f"tree replacement '{tr}' contains no equal sign")
+            tree_replacements[tr[:eq_ind]] = tr[eq_ind+1:]
+
     section_dict = {}
     find_section_nodes(section_dict, root_node)
 
     # {{{ demos
 
     if args.ipynb_dir is not None:
-        for fn in sorted(glob(join(args.ipynb_dir, "*", "*.ipynb"))):
+        for fn in blacklisted_glob(args.ipynb_dir, join("*", "*.ipynb"),
+                blacklist_regexps):
             trunk = fn[len(args.ipynb_dir)+1:]
 
             section_id, display_name = get_section_id_and_display_name(
@@ -180,60 +346,89 @@ def main():
 
             link_ipynb = args.ipynb_urlroot + "/" + trunk
             link_html = link_ipynb.replace(".ipynb", ".html")
+            main_link = link_html
 
             sub_nodes = [{
                         "text": "View on the web",
                         "link": link_html,
-                        "icon": "fa fa-newspaper-o",
+                        "icon": "bi bi-newspaper",
                         }]
 
+            if args.interactive_nb_urlroot:
+                interactive_nb_url = args.interactive_nb_urlroot + quote_plus(trunk)
+
+                sub_nodes.append({
+                    "text": "Run interactively",
+                    "link": interactive_nb_url,
+                    "icon": "bi bi-keyboard",
+                    })
+                if args.ipynb_main_link == "interactive":
+                    main_link = interactive_nb_url
+
+            if args.interactive_nb_cleared_urlroot:
+                interactive_nb_url = (args.interactive_nb_cleared_urlroot
+                                      + quote_plus(trunk))
+
+                sub_nodes.append({
+                    "text": "Run interactively with cleared input",
+                    "link": interactive_nb_url,
+                    "icon": "bi bi-keyboard",
+                    })
+                if args.ipynb_main_link == "interactive-cleared":
+                    main_link = interactive_nb_url
+
             if args.ipynb_as_py:
                 link_py = link_ipynb.replace(".ipynb", ".py")
                 sub_nodes.append({
                     "text": "Download Python script",
                     "link": link_py,
-                    "icon": "fa fa-terminal",
+                    "icon": "bi bi-terminal",
                     })
 
             if args.ipynb_as_ipynb:
                 sub_nodes.append({
                     "text": "Download Jupyter notebook",
                     "link": link_ipynb,
-                    "icon": "fa fa-download",
+                    "icon": "bi bi-download",
                     })
 
             demo_node = {
                 "text": "Demo: " + display_name,
-                "link": link_html,
-                "icon": "fa fa-keyboard-o",
+                "link": main_link,
+                "icon": "bi bi-keyboard",
                 "nodes": sub_nodes,
                 }
 
-            section_dict[section_id]["nodes"].append(demo_node)
+            if section_id in section_dict:
+                section_dict[section_id]["nodes"].append(demo_node)
 
     # }}}
 
-    # {{{ python source
+    # {{{ general source files
 
-    if args.py_dir is not None:
-        for fn in sorted(glob(join(args.py_dir, "*", "*.py"))):
-            trunk = fn[len(args.py_dir)+1:]
-            section_id, display_name = get_section_id_and_display_name(
-                    trunk, include_extension=True)
+    if args.source_dir is not None:
+        for source_wildcard in args.source_wildcard:
+            for fn in blacklisted_glob(args.source_dir, join("*", source_wildcard),
+                    blacklist_regexps):
+                trunk = fn[len(args.source_dir)+1:]
+                section_id, display_name = get_section_id_and_display_name(
+                        trunk, include_extension=True)
 
-            src_node = {
-                "text": "Code: " + display_name,
-                "link": args.py_urlroot + "/" + trunk,
-                "icon": "fa fa-file-text-o",
-            }
-            section_dict[section_id]["nodes"].append(src_node)
+                src_node = {
+                    "text": "Code: " + display_name,
+                    "link": args.source_urlroot + "/" + trunk,
+                    "icon": "bi bi-file-earmark-text",
+                }
+                if section_id in section_dict:
+                    section_dict[section_id]["nodes"].append(src_node)
 
     # }}}
 
     # {{{ notes
 
     if args.pdf_dir is not None:
-        for fn in sorted(glob(join(args.pdf_dir, "*.pdf"))):
+        for fn in blacklisted_glob(args.pdf_dir, join("*.pdf"),
+                blacklist_regexps):
             if "autosave" in fn:
                 continue
 
@@ -244,17 +439,20 @@ def main():
             notes_node = {
                 "text": "PDF: " + basename(display_name),
                 "link": args.pdf_urlroot + "/" + trunk,
-                "icon": "fa fa-book",
+                "icon": "bi bi-book",
                 }
-            section_dict[section_id]["nodes"].insert(0, notes_node)
+
+            if section_id in section_dict:
+                section_dict[section_id]["nodes"].insert(0, notes_node)
 
     # }}}
 
-    with open(args.output_file, "wt") as outf:
+    with open(args.output_file, "wt", encoding="utf-8") as outf:
         render(
                 RenderSettings(
                     default_icon=args.default_icon,
-                    number_sections=args.number_sections),
+                    number_sections=args.number_sections,
+                    tree_replacements=tree_replacements),
                 outf, root_node)
 
 

contrib/course-material-tree/outline-tree.yml

@@ -4,25 +4,25 @@ text: "CS 123"
 nodes:
 
 -   text: "Introduction"
-    icon: "fa fa-cube"
+    icon: "bi bi-box"
     opened: true
     section: 0
     nodes:
     -   text: "Notes"
-        icon: "fa fa-book"  # see http://fontawesome.io/icons/
+        icon: "bi bi-book"  # see http://fontawesome.io/icons/
         link: "http://andreask.cs.illinois.edu/cs598apk-f15/notes/notes.pdf"
 
 -   text: "Dense Matrices and Computation"
     section: 1
-    icon: "fa fa-cube"
+    icon: "bi bi-box"
     opened: true
     nodes:
     -   text: "Notes"
-        icon: "fa fa-book"
+        icon: "bi bi-book"
         link: "http://andreask.cs.illinois.edu/cs598apk-f15/notes/notes.pdf#page=10"
 
     -   text: "Finding structure with randomness: Probabilistic algorithms for constructing approximate matrix decompositions by Halko/Martinsson/Tropp"
-        icon: "fa fa-paperclip"
+        icon: "bi bi-paperclip"
         link: http://arxiv.org/abs/0909.4061
 
     -   text: "Sources and Targets"

contrib/flow-to-worksheet

@@ -22,6 +22,7 @@ TEMPLATE = Template(r"""
 \usepackage{examtron}
 \pagestyle{empty}
 
+\usepackage{longtable}
 \usepackage{titlesec}
 \usepackage{tikz}
 \titleformat{\section}
@@ -42,14 +43,22 @@ TEMPLATE = Template(r"""
 
 INCLUDE_GRAPHICS_MEDIA_RE = re.compile(r"\\includegraphics\{media:(.*)\}")
 INCLUDE_GRAPHICS_REPO_RE = re.compile(r"\\includegraphics\{repo:(.*)\}")
+FORMULA_ALIGN_RE = re.compile(
+        r"\\\["
+        r"\s*(\\begin\{align\*?\})"
+        r"(.*?)"
+        r"(\\end\{align\*?\})\s*"
+        r"\\\]", re.DOTALL)
 
 
 def convert_markup(s):
     result = pypandoc.convert(s, 'latex', format='markdown')
     result, _ = INCLUDE_GRAPHICS_MEDIA_RE.subn(
-            r"\includegraphics[height=4cm]{media/\1}", result)
+            r"\\includegraphics[height=4cm]{media/\1}", result)
     result, _ = INCLUDE_GRAPHICS_REPO_RE.subn(
-            r"\includegraphics[height=4cm]{\1}", result)
+            r"\\includegraphics[height=4cm]{\1}", result)
+    result, _ = FORMULA_ALIGN_RE.subn(
+            r"\1\2\3", result)
     return result
 
 
@@ -88,6 +97,9 @@ def convert_page_inner(page):
             "SurveyChoiceQuestion"]:
         prompt = convert_markup(page.prompt)
 
+        if page.type == "MultipleChoiceQuestion":
+            prompt += "\n\n(Select all that apply.)"
+
         choices = [
             "\item "
             +

contrib/jupyterlite/.gitignore

+files
+env
+pack
+_output
+.cache
+.jupyterlite.doit.db
+mamba-root
+micromamba

contrib/jupyterlite/build-environment.yml

+name: build-env
+channels:
+  - conda-forge
+dependencies:
+  - python
+  - pip
+  - jupyter_server
+  - jupyterlite-core >=0.1.0,<0.2.0
+  - jupyterlite-xeus-python >=0.9.2,<0.10.0

contrib/jupyterlite/build.sh

+#! /bin/bash
+
+set -eo pipefail
+
+EXAM=0
+if test "$1" == "--exam"; then
+    echo "BUILDING IN EXAM MODE"
+    EXAM=1
+fi
+
+MATHJAX_VER=2.7.7
+
+./cleanup.sh
+
+EXTRA_BUILD_FLAGS=()
+
+if false; then
+    python3 -m venv env
+
+    source env/bin/activate
+    pip install jupyterlite-core
+    # update the pyodide kernel and pyodide in lockstep
+    pip install 'jupyterlite-pyodide-kernel==0.1.2'
+
+    EXTRA_BUILD_FLAGS=(--pyodide https://github.com/pyodide/pyodide/releases/download/0.24.0/pyodide-0.24.0.tar.bz2)
+else
+    curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj --strip=1 bin/micromamba
+    ./micromamba create -y -f build-environment.yml -p ./mamba-root
+    eval "$(./micromamba shell hook --shell bash)"
+    micromamba activate ./mamba-root
+fi
+
+# jupyter-server appears to be needed for indexing contents
+pip install jupyter-server libarchive-c
+
+mkdir -p pack
+
+if [[ "$EXAM" = 0 ]]; then
+    mkdir -p files/{cs450,cs555,cs598apk}-kloeckner
+
+    git clone https://github.com/inducer/numerics-notes pack/numerics-notes
+    git clone https://github.com/inducer/numpde-notes pack/numpde-notes
+    git clone https://github.com/inducer/fast-alg-ie-notes pack/fast-alg-ie-notes
+
+    cp -R pack/numerics-notes/demos files/cs450-kloeckner/demos
+    cp -R pack/numerics-notes/cleared-demos files/cs450-kloeckner/cleared
+    cp -R pack/numpde-notes/demos files/cs555-kloeckner/demos
+    cp -R pack/numpde-notes/cleared-demos files/cs555-kloeckner/cleared
+    cp -R pack/fast-alg-ie-notes/demos files/cs598apk-kloeckner/demos
+    cp -R pack/fast-alg-ie-notes/cleared-demos files/cs598apk-kloeckner/cleared
+fi
+
+curl -L "https://github.com/mathjax/MathJax/archive/$MATHJAX_VER.zip" \
+        -o "pack/mathjax-$MATHJAX_VER.zip"
+
+(cd pack; unzip -q mathjax-$MATHJAX_VER.zip)
+
+jupyter lite init
+jupyter lite build \
+        --mathjax-dir "pack/MathJax-$MATHJAX_VER" \
+        "${EXTRA_BUILD_FLAGS[@]}"
+
+# vim: sw=4

contrib/jupyterlite/cleanup.sh

+#! /bin/bash
+
+set -e
+
+rm -Rf .cache env .jupyterlite.doit.db _output pack micromamba mamba-root files

contrib/jupyterlite/environment.yml

+name: xeus-python-kernel
+channels:
+- https://repo.mamba.pm/emscripten-forge
+- https://repo.mamba.pm/conda-forge
+dependencies:
+- numpy
+- scipy
+- matplotlib
+- sympy

contrib/jupyterlite/upload.sh

+#! /bin/bash
+
+if test -d files/cs450-kloeckner; then
+    echo "Uploading in normal mode..."
+    rsync --archive --delete -v _output/ rl:/web/jupyterlite/main/
+else
+    echo "Uploading in exam mode..."
+    rsync --archive --delete -v _output/ rl:/web/jupyterlite/exam/
+fi

course/__init__.py

-default_app_config = 'course.apps.CourseConfig'
\ No newline at end of file
+# (empty)

course/admin.py

-# -*- coding: utf-8 -*-
+from __future__ import annotations
+
 
 __copyright__ = "Copyright (C) 2014 Andreas Kloeckner"
 
@@ -22,63 +23,80 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 """
 
-import six
-
-from typing import Any  # noqa
+from typing import TYPE_CHECKING, Any
 
-from django.utils.translation import (
-        ugettext_lazy as _, string_concat, pgettext)
+from django import forms
 from django.contrib import admin
+from django.db.models import QuerySet
+from django.utils.translation import gettext_lazy as _, pgettext
 
+from course.constants import exam_ticket_states, participation_permission as pperm
+from course.enrollment import approve_enrollment, deny_enrollment
 from course.models import (
-        Course, Event,
-        ParticipationTag,
-        Participation, ParticipationPermission,
-        ParticipationRole, ParticipationRolePermission,
-        ParticipationPreapproval,
-        InstantFlowRequest,
-        FlowSession, FlowPageData,
-        FlowPageVisit, FlowPageVisitGrade,
-        FlowRuleException,
-        GradingOpportunity, GradeChange, InstantMessage,
-        Exam, ExamTicket)
-from django import forms
-from course.enrollment import (approve_enrollment, deny_enrollment)
-from course.constants import (
-        participation_permission as pperm,
-        exam_ticket_states
-        )
+    AuthenticationToken,
+    Course,
+    Event,
+    Exam,
+    ExamTicket,
+    FlowPageData,
+    FlowPageVisit,
+    FlowPageVisitGrade,
+    FlowRuleException,
+    FlowSession,
+    GradeChange,
+    GradingOpportunity,
+    InstantFlowRequest,
+    InstantMessage,
+    Participation,
+    ParticipationPermission,
+    ParticipationPreapproval,
+    ParticipationRole,
+    ParticipationRolePermission,
+    ParticipationTag,
+)
+from relate.utils import string_concat
+
+
+if TYPE_CHECKING:
+    from accounts.models import User
 
 
 # {{{ permission helpers
 
-def _filter_courses_for_user(queryset, user):
+def _filter_courses_for_user(queryset: QuerySet, user: User) -> QuerySet:
     if user.is_superuser:
         return queryset
     z = queryset.filter(
             participations__user=user,
             participations__roles__permissions__permission=pperm.use_admin_interface)
-    print(z.query)
     return z
 
 
-def _filter_course_linked_obj_for_user(queryset, user):
+def _filter_course_linked_obj_for_user(queryset: QuerySet, user: User) -> QuerySet:
     if user.is_superuser:
         return queryset
     return queryset.filter(
             course__participations__user=user,
-            course__participations__roles__permissions__permission  # noqa
-            =pperm.use_admin_interface
+            course__participations__roles__permissions__permission=pperm.use_admin_interface
             )
 
 
-def _filter_participation_linked_obj_for_user(queryset, user):
+def _filter_participation_linked_obj_for_user(
+            queryset: QuerySet, user: User
+        ) -> QuerySet:
     if user.is_superuser:
         return queryset
     return queryset.filter(
         participation__course__participations__user=user,
-        participation__course__participations__roles__permissions__permission  # noqa
-        =pperm.use_admin_interface)
+        participation__course__participations__roles__permissions__permission=pperm.use_admin_interface)
+
+# }}}
+
+
+# {{{ list filter helper
+
+def _filter_related_only(filter_arg: str) -> tuple[str, Any]:
+    return (filter_arg, admin.RelatedOnlyFieldListFilter)
 
 # }}}
 
@@ -88,7 +106,7 @@ def _filter_participation_linked_obj_for_user(queryset, user):
 class UnsafePasswordInput(forms.TextInput):
     # This sends passwords back to the user--not ideal, but OK for the XMPP
     # password.
-    input_type = 'password'
+    input_type = "password"
 
 
 class CourseAdminForm(forms.ModelForm):
@@ -100,6 +118,7 @@ class CourseAdminForm(forms.ModelForm):
         exclude = ()
 
 
+@admin.register(Course)
 class CourseAdmin(admin.ModelAdmin):
     list_display = (
             "identifier",
@@ -134,6 +153,8 @@ class CourseAdmin(admin.ModelAdmin):
             "name",
             "time_period")
 
+    readonly_fields = ("identifier",)
+
     form = CourseAdminForm
 
     save_on_top = True
@@ -145,18 +166,17 @@ class CourseAdmin(admin.ModelAdmin):
         return False
 
     def get_queryset(self, request):
-        qs = super(CourseAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         return _filter_courses_for_user(qs, request.user)
 
     # }}}
 
-admin.site.register(Course, CourseAdmin)
-
 # }}}
 
 
 # {{{ events
 
+@admin.register(Event)
 class EventAdmin(admin.ModelAdmin):
     list_display = (
             "course",
@@ -165,7 +185,7 @@ class EventAdmin(admin.ModelAdmin):
             "time",
             "end_time",
             "shown_in_calendar")
-    list_filter = ("course", "kind", "shown_in_calendar")
+    list_filter = (_filter_related_only("course"), "kind", "shown_in_calendar")
 
     date_hierarchy = "time"
 
@@ -174,56 +194,53 @@ class EventAdmin(admin.ModelAdmin):
             "kind",
             )
 
-    def __unicode__(self):
-        return u"%s %d in %s" % (self.kind, self.ordinal, self.course)
-
-    if six.PY3:
-        __str__ = __unicode__
+    def __str__(self):  # pragma: no cover  # not used
+        return "{}{} in {}".format(
+            self.kind,
+            f" ({self.ordinal!s})" if self.ordinal is not None else "",
+            self.course)
 
-    list_editable = ("ordinal", "time", "end_time", "shown_in_calendar")
+    list_editable = ("kind", "ordinal", "time", "end_time", "shown_in_calendar")
 
     # {{{ permissions
 
     def get_queryset(self, request):
-        qs = super(EventAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         return _filter_course_linked_obj_for_user(qs, request.user)
 
     def formfield_for_foreignkey(self, db_field, request, **kwargs):
         if db_field.name == "course":
             kwargs["queryset"] = _filter_courses_for_user(
                     Course.objects, request.user)
-        return super(EventAdmin, self).formfield_for_foreignkey(
+        return super().formfield_for_foreignkey(
                 db_field, request, **kwargs)
 
     # }}}
 
-admin.site.register(Event, EventAdmin)
-
 # }}}
 
 
 # {{{ participation tags
 
+@admin.register(ParticipationTag)
 class ParticipationTagAdmin(admin.ModelAdmin):
-    list_filter = ("course",)
+    list_filter = (_filter_related_only("course"),)
 
     # {{{ permissions
 
     def get_queryset(self, request):
-        qs = super(ParticipationTagAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         return _filter_course_linked_obj_for_user(qs, request.user)
 
     def formfield_for_foreignkey(self, db_field, request, **kwargs):
         if db_field.name == "course":
             kwargs["queryset"] = _filter_courses_for_user(
                     Course.objects, request.user)
-        return super(ParticipationTagAdmin, self).formfield_for_foreignkey(
+        return super().formfield_for_foreignkey(
                 db_field, request, **kwargs)
 
     # }}}
 
-admin.site.register(ParticipationTag, ParticipationTagAdmin)
-
 # }}}
 
 
@@ -234,12 +251,17 @@ class ParticipationRolePermissionInline(admin.TabularInline):
     extra = 3
 
 
+@admin.register(ParticipationRole)
 class ParticipationRoleAdmin(admin.ModelAdmin):
     inlines = (ParticipationRolePermissionInline,)
 
-    list_filter = ("course", "identifier")
+    list_filter = (_filter_related_only("course"), "identifier")
 
-admin.site.register(ParticipationRole, ParticipationRoleAdmin)
+    def get_queryset(self, request):
+        qs = super().get_queryset(request)
+        if request.user.is_superuser:
+            return qs
+        return _filter_course_linked_obj_for_user(qs, request.user)
 
 
 class ParticipationPermissionInline(admin.TabularInline):
@@ -253,7 +275,7 @@ class ParticipationForm(forms.ModelForm):
         exclude = ("role",)
 
     def clean(self):
-        super(ParticipationForm, self).clean()
+        super().clean()
 
         for tag in self.cleaned_data.get("tags", []):
             if tag.course != self.cleaned_data.get("course"):
@@ -270,34 +292,44 @@ class ParticipationForm(forms.ModelForm):
                                "participation.")})
 
 
+@admin.register(Participation)
 class ParticipationAdmin(admin.ModelAdmin):
     form = ParticipationForm
 
+    @admin.display(
+        description=_("Roles")
+    )
     def get_roles(self, obj):
-        return ", ".join(six.text_type(role.name) for role in obj.roles.all())
-
-    get_roles.short_description = _("Roles")  # type: ignore
-
+        return ", ".join(str(role.name) for role in obj.roles.all())
+
+    @admin.display(
+        description=_("Tags")
+    )
+    def get_tags(self, obj):
+        return ", ".join(str(tag.name) for tag in obj.tags.all())
+
+    # Fixme: This can be misleading when Non-superuser click on the
+    # link of a user who also attend other courses.
+    @admin.display(
+        description=pgettext("real name of a user", "Name"),
+        ordering="user__last_name",
+    )
     def get_user(self, obj):
-        from django.urls import reverse
         from django.conf import settings
+        from django.urls import reverse
+        from django.utils.html import mark_safe
 
-        return string_concat(
+        return mark_safe(string_concat(
                 "<a href='%(link)s'>", "%(user_fullname)s",
                 "</a>"
                 ) % {
                     "link": reverse(
-                        "admin:%s_change"
-                        % settings.AUTH_USER_MODEL.replace(".", "_")
-                        .lower(),
+                        "admin:{}_change".format(
+                            settings.AUTH_USER_MODEL.replace(".", "_").lower()),
                         args=(obj.user.id,)),
                     "user_fullname": obj.user.get_full_name(
                         force_verbose_blank=True),
-                    }
-
-    get_user.short_description = pgettext("real name of a user", "Name")  # type:ignore  # noqa
-    get_user.admin_order_field = "user__last_name"  # type: ignore
-    get_user.allow_tags = True  # type: ignore
+                    })
 
     list_display = (
             "user",
@@ -305,8 +337,19 @@ class ParticipationAdmin(admin.ModelAdmin):
             "course",
             "get_roles",
             "status",
+            "get_tags",
             )
-    list_filter = ("course", "roles__name", "status", "tags")
+
+    def get_list_filter(self, request):
+        if request is not None and request.user.is_superuser:
+            return ("course",
+                    "roles__name",
+                    "status",
+                    "tags")
+        return (_filter_related_only("course"),
+                   _filter_related_only("roles"),
+                   "status",
+                   _filter_related_only("tags"))
 
     raw_id_fields = ("user",)
 
@@ -328,33 +371,35 @@ class ParticipationAdmin(admin.ModelAdmin):
     # {{{ permissions
 
     def get_queryset(self, request):
-        qs = super(ParticipationAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         return _filter_course_linked_obj_for_user(qs, request.user)
 
     def formfield_for_foreignkey(self, db_field, request, **kwargs):
         if db_field.name == "course":
             kwargs["queryset"] = _filter_courses_for_user(
                     Course.objects, request.user)
+
+        # Fixme: This seems not to be not reachable
         if db_field.name == "tags":
             kwargs["queryset"] = _filter_course_linked_obj_for_user(
                     ParticipationTag.objects, request.user)
-        return super(ParticipationAdmin, self).formfield_for_foreignkey(
+        return super().formfield_for_foreignkey(
                 db_field, request, **kwargs)
 
     # }}}
 
-admin.site.register(Participation, ParticipationAdmin)
-
 
+@admin.register(ParticipationPreapproval)
 class ParticipationPreapprovalAdmin(admin.ModelAdmin):
+    @admin.display(
+        description=_("Roles")
+    )
     def get_roles(self, obj):
-        return ", ".join(six.text_type(role.name) for role in obj.roles.all())
-
-    get_roles.short_description = _("Roles")  # type: ignore
+        return ", ".join(str(role.name) for role in obj.roles.all())
 
     list_display = ("email", "institutional_id", "course", "get_roles",
             "creation_time", "creator")
-    list_filter = ("course", "roles")
+    list_filter = (_filter_related_only("course"), _filter_related_only("roles"))
 
     search_fields = (
             "email", "institutional_id",
@@ -363,7 +408,7 @@ class ParticipationPreapprovalAdmin(admin.ModelAdmin):
     # {{{ permissions
 
     def get_queryset(self, request):
-        qs = super(ParticipationPreapprovalAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         if request.user.is_superuser:
             return qs
         return _filter_course_linked_obj_for_user(qs, request.user)
@@ -378,19 +423,30 @@ class ParticipationPreapprovalAdmin(admin.ModelAdmin):
         if db_field.name == "course":
             kwargs["queryset"] = _filter_courses_for_user(
                     Course.objects, request.user)
-        return super(ParticipationPreapprovalAdmin, self).formfield_for_foreignkey(
+        return super().formfield_for_foreignkey(
                 db_field, request, **kwargs)
 
     # }}}
 
-admin.site.register(ParticipationPreapproval, ParticipationPreapprovalAdmin)
-
 # }}}
 
 
+@admin.register(AuthenticationToken)
+class AuthenticationTokenAdmin(admin.ModelAdmin):
+    list_display = ("id", "participation", "restrict_to_participation_role",
+            "description", "valid_until", "revocation_time")
+
+    date_hierarchy = "creation_time"
+
+    search_fields = (
+            "id", "description", "participation__user__username"
+            )
+
+
+@admin.register(InstantFlowRequest)
 class InstantFlowRequestAdmin(admin.ModelAdmin):
     list_display = ("course", "flow_id", "start_time", "end_time", "cancelled")
-    list_filter = ("course",)
+    list_filter = (_filter_related_only("course"),)
 
     date_hierarchy = "start_time"
 
@@ -398,8 +454,6 @@ class InstantFlowRequestAdmin(admin.ModelAdmin):
             "email",
             )
 
-admin.site.register(InstantFlowRequest, InstantFlowRequestAdmin)
-
 
 # {{{ flow sessions
 
@@ -408,16 +462,18 @@ class FlowPageDataInline(admin.TabularInline):
     extra = 0
 
 
+@admin.register(FlowSession)
 class FlowSessionAdmin(admin.ModelAdmin):
+    @admin.display(
+        description=_("Participant"),
+        ordering="participation__user",
+    )
     def get_participant(self, obj):
         if obj.participation is None:
             return None
 
         return obj.participation.user
 
-    get_participant.short_description = _("Participant")  # type: ignore
-    get_participant.admin_order_field = "participation__user"  # type: ignore
-
     search_fields = (
             "=id",
             "flow_id",
@@ -439,7 +495,7 @@ class FlowSessionAdmin(admin.ModelAdmin):
             "completion_time",
             "access_rules_tag",
             "in_progress",
-            #"expiration_mode",
+            # "expiration_mode",
             )
     list_display_links = (
             "flow_id",
@@ -449,7 +505,7 @@ class FlowSessionAdmin(admin.ModelAdmin):
     date_hierarchy = "start_time"
 
     list_filter = (
-            "course",
+            _filter_related_only("course"),
             "flow_id",
             "in_progress",
             "access_rules_tag",
@@ -469,20 +525,18 @@ class FlowSessionAdmin(admin.ModelAdmin):
         return False
 
     def get_queryset(self, request):
-        qs = super(FlowSessionAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         return _filter_course_linked_obj_for_user(qs, request.user)
 
     def formfield_for_foreignkey(self, db_field, request, **kwargs):
         if db_field.name == "course":
             kwargs["queryset"] = _filter_courses_for_user(
                     Course.objects, request.user)
-        return super(FlowSessionAdmin, self).formfield_for_foreignkey(
+        return super().formfield_for_foreignkey(
                 db_field, request, **kwargs)
 
     # }}}
 
-admin.site.register(FlowSession, FlowSessionAdmin)
-
 # }}}
 
 
@@ -494,14 +548,14 @@ class FlowPageVisitGradeInline(admin.TabularInline):
 
 
 class HasAnswerListFilter(admin.SimpleListFilter):
-    title = 'has answer'
+    title = "has answer"
 
-    parameter_name = 'has_answer'
+    parameter_name = "has_answer"
 
     def lookups(self, request, model_admin):
         return (
-            ('y', 'Yes'),
-            ('n', 'No'),
+            ("y", _("Yes")),
+            ("n", _("No")),
         )
 
     def queryset(self, request, queryset):
@@ -510,56 +564,91 @@ class HasAnswerListFilter(admin.SimpleListFilter):
         return queryset.filter(answer__isnull=self.value() != "y")
 
 
+class FlowIdListFilter(admin.SimpleListFilter):
+    """
+    This is only necessary when flow_id is only accessible by FlowSession, which is
+    a ForeignKey in the model
+    """
+    title = _("Flow ID")
+    parameter_name = "flow_id"
+
+    def lookups(self, request, model_admin):
+        qs = model_admin.get_queryset(request)
+        if not request.user.is_superuser:
+            qs = qs.filter(
+                flow_session__course__participations__user=request.user,
+                flow_session__course__participations__roles__permissions__permission=pperm.use_admin_interface)
+
+        flow_ids = qs.values_list("flow_session__flow_id", flat=True).distinct()
+        return zip(flow_ids, flow_ids, strict=True)
+
+    def queryset(self, request, queryset):
+        if self.value():
+            return queryset.filter(flow_session__flow_id=self.value())
+        else:
+            return queryset
+
+
+@admin.register(FlowPageVisit)
 class FlowPageVisitAdmin(admin.ModelAdmin):
+    @admin.display(
+        description=_("Course"),
+        ordering="flow_session__course",
+    )
     def get_course(self, obj):
         return obj.flow_session.course
-    get_course.short_description = _("Course")  # type: ignore
-    get_course.admin_order_field = "flow_session__course"  # type: ignore
 
+    @admin.display(
+        description=_("Flow ID"),
+        ordering="flow_session__flow_id",
+    )
     def get_flow_id(self, obj):
         return obj.flow_session.flow_id
-    get_flow_id.short_description = _("Flow ID")  # type: ignore
-    get_flow_id.admin_order_field = "flow_session__flow_id"  # type: ignore
 
+    @admin.display(
+        description=_("Page ID"),
+        ordering="page_data__page_id",
+    )
     def get_page_id(self, obj):
-        if obj.page_data.ordinal is None:
+        if obj.page_data.page_ordinal is None:
             return string_concat("%s/%s (", _("not in use"), ")") % (
                     obj.page_data.group_id,
                     obj.page_data.page_id)
         else:
-            return "%s/%s (%s)" % (
-                    obj.page_data.group_id,
-                    obj.page_data.page_id,
-                    obj.page_data.ordinal)
-
-    get_page_id.short_description = _("Page ID")  # type: ignore
-    get_page_id.admin_order_field = "page_data__page_id"  # type: ignore
-
+            return (
+                f"{obj.page_data.group_id}/{obj.page_data.page_id} "
+                f"({obj.page_data.page_ordinal})")
+
+    @admin.display(
+        description=_("Owner"),
+        ordering="flow_session__participation",
+    )
     def get_participant(self, obj):
         if obj.flow_session.participation:
             return obj.flow_session.participation.user
         else:
             return string_concat("(", _("anonymous"), ")")
 
-    get_participant.short_description = _("Owner")  # type: ignore
-    get_participant.admin_order_field = "flow_session__participation"  # type: ignore
-
+    @admin.display(
+        description=_("Has answer"),
+        boolean=True,
+    )
     def get_answer_is_null(self, obj):
         return obj.answer is not None
-    get_answer_is_null.short_description = _("Has answer")  # type: ignore
-    get_answer_is_null.boolean = True  # type: ignore
 
+    @admin.display(
+        description=_("Flow Session ID"),
+        ordering="flow_session__id",
+    )
     def get_flow_session_id(self, obj):
         return obj.flow_session.id
-    get_flow_session_id.short_description = _("Flow Session ID")  # type: ignore
-    get_flow_session_id.admin_order_field = "flow_session__id"  # type: ignore
 
     list_filter = (
             HasAnswerListFilter,
             "is_submitted_answer",
             "is_synthetic",
-            "flow_session__participation__course",
-            "flow_session__flow_id",
+            _filter_related_only("flow_session__participation__course"),
+            FlowIdListFilter,
             )
     date_hierarchy = "visit_time"
     list_display = (
@@ -604,33 +693,37 @@ class FlowPageVisitAdmin(admin.ModelAdmin):
         return False
 
     def get_queryset(self, request):
-        qs = super(FlowPageVisitAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         if request.user.is_superuser:
             return qs
         return qs.filter(
             flow_session__course__participations__user=request.user,
-            flow_session__course__participations__roles__permissions__identifier  # noqa
-            =pperm.use_admin_interface)
+            flow_session__course__participations__roles__permissions__permission=pperm.use_admin_interface)
 
     # }}}
 
-admin.site.register(FlowPageVisit, FlowPageVisitAdmin)
-
 # }}}
 
 
 # {{{ flow access
 
+@admin.register(FlowRuleException)
 class FlowRuleExceptionAdmin(admin.ModelAdmin):
+    @admin.display(
+        description=_("Course"),
+        ordering="participation__course",
+    )
     def get_course(self, obj):
         return obj.participation.course
-    get_course.short_description = _("Course")  # type: ignore
-    get_course.admin_order_field = "participation__course"  # type: ignore
 
+    @admin.display(
+        description=_("Participant"),
+        ordering="participation__user",
+    )
     def get_participant(self, obj):
         return obj.participation.user
-    get_participant.short_description = _("Participant")  # type: ignore
-    get_participant.admin_order_field = "participation__user"  # type: ignore
+
+    ordering = ("-creation_time",)
 
     search_fields = (
             "flow_id",
@@ -653,7 +746,7 @@ class FlowRuleExceptionAdmin(admin.ModelAdmin):
             "flow_id",
             )
     list_filter = (
-            "participation__course",
+            _filter_related_only("participation__course"),
             "flow_id",
             "kind",
             )
@@ -669,25 +762,24 @@ class FlowRuleExceptionAdmin(admin.ModelAdmin):
         return False
 
     def get_queryset(self, request):
-        qs = super(FlowRuleExceptionAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         return _filter_participation_linked_obj_for_user(qs, request.user)
 
     exclude = ("creator", "creation_time")
 
-    def save_model(self, request, obj, form, change):
+    def save_model(self, request, obj, form, change):  # pragma: no cover
+        # This won't work since it's not allowed to add
         obj.creator = request.user
         obj.save()
 
     # }}}
 
-
-admin.site.register(FlowRuleException, FlowRuleExceptionAdmin)
-
 # }}}
 
 
 # {{{ grading
 
+@admin.register(GradingOpportunity)
 class GradingOpportunityAdmin(admin.ModelAdmin):
     list_display = (
             "name",
@@ -698,7 +790,7 @@ class GradingOpportunityAdmin(admin.ModelAdmin):
             "shown_in_participant_grade_book",
             )
     list_filter = (
-            "course",
+            _filter_related_only("course"),
             "shown_in_grade_book",
             "shown_in_participant_grade_book",
             )
@@ -712,45 +804,51 @@ class GradingOpportunityAdmin(admin.ModelAdmin):
     exclude = ("creation_time",)
 
     def get_queryset(self, request):
-        qs = super(GradingOpportunityAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         return _filter_course_linked_obj_for_user(qs, request.user)
 
     def formfield_for_foreignkey(self, db_field, request, **kwargs):
         if db_field.name == "course":
             kwargs["queryset"] = _filter_courses_for_user(
                     Course.objects, request.user)
-        return super(GradingOpportunityAdmin, self).formfield_for_foreignkey(
+        return super().formfield_for_foreignkey(
                 db_field, request, **kwargs)
 
     # }}}
 
-admin.site.register(GradingOpportunity, GradingOpportunityAdmin)
-
 
+@admin.register(GradeChange)
 class GradeChangeAdmin(admin.ModelAdmin):
+    @admin.display(
+        description=_("Course"),
+        ordering="participation__course",
+    )
     def get_course(self, obj):
         return obj.participation.course
-    get_course.short_description = _("Course")  # type: ignore
-    get_course.admin_order_field = "participation__course"  # type: ignore
 
+    @admin.display(
+        description=_("Opportunity"),
+        ordering="opportunity",
+    )
     def get_opportunity(self, obj):
         return obj.opportunity.name
-    get_opportunity.short_description = _("Opportunity")  # type: ignore
-    get_opportunity.admin_order_field = "opportunity"  # type: ignore
 
+    @admin.display(
+        description=_("Participant"),
+        ordering="participation__user",
+    )
     def get_participant(self, obj):
         return obj.participation.user
-    get_participant.short_description = _("Participant")  # type: ignore
-    get_participant.admin_order_field = "participation__user"  # type: ignore
 
+    @admin.display(
+        description="%"
+    )
     def get_percentage(self, obj):
-        if obj.points is None or obj.max_points is None:
+        if obj.points is None or not obj.max_points:
             return None
         else:
             return round(100*obj.points/obj.max_points)
 
-    get_percentage.short_description = "%"  # type: ignore
-
     list_display = (
             "get_opportunity",
             "get_participant",
@@ -779,8 +877,8 @@ class GradeChangeAdmin(admin.ModelAdmin):
             )
 
     list_filter = (
-            "opportunity__course",
-            "opportunity",
+            _filter_related_only("opportunity__course"),
+            _filter_related_only("opportunity"),
             "state",
             )
 
@@ -789,7 +887,7 @@ class GradeChangeAdmin(admin.ModelAdmin):
     # {{{ permission
 
     def get_queryset(self, request):
-        qs = super(GradeChangeAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         return _filter_participation_linked_obj_for_user(qs, request.user)
 
     exclude = ("creator", "grade_time")
@@ -800,25 +898,28 @@ class GradeChangeAdmin(admin.ModelAdmin):
 
     # }}}
 
-admin.site.register(GradeChange, GradeChangeAdmin)
-
 # }}}
 
 
 # {{{ instant message
 
+@admin.register(InstantMessage)
 class InstantMessageAdmin(admin.ModelAdmin):
+    @admin.display(
+        description=_("Course"),
+        ordering="participation__course",
+    )
     def get_course(self, obj):
         return obj.participation.course
-    get_course.short_description = _("Course")  # type: ignore
-    get_course.admin_order_field = "participation__course"  # type: ignore
 
+    @admin.display(
+        description=_("Participant"),
+        ordering="participation__user",
+    )
     def get_participant(self, obj):
         return obj.participation.user
-    get_participant.short_description = _("Participant")  # type: ignore
-    get_participant.admin_order_field = "participation__user"  # type: ignore
 
-    list_filter = ("participation__course",)
+    list_filter = (_filter_related_only("participation__course"),)
     list_display = (
             "get_course",
             "get_participant",
@@ -844,28 +945,29 @@ class InstantMessageAdmin(admin.ModelAdmin):
         return False
 
     def get_queryset(self, request):
-        qs = super(InstantMessageAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         return _filter_participation_linked_obj_for_user(qs, request.user)
 
     # }}}
 
-admin.site.register(InstantMessage, InstantMessageAdmin)
-
 # }}}
 
 
 # {{{ exam tickets
 
+@admin.register(Exam)
 class ExamAdmin(admin.ModelAdmin):
     list_filter = (
-            "course",
+            _filter_related_only("course"),
             "active",
+            "listed",
             )
 
     list_display = (
             "course",
             "flow_id",
             "active",
+            "listed",
             "no_exams_before",
             )
 
@@ -878,37 +980,40 @@ class ExamAdmin(admin.ModelAdmin):
     # {{{ permissions
 
     def get_queryset(self, request):
-        qs = super(ExamAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         return _filter_course_linked_obj_for_user(qs, request.user)
 
     def formfield_for_foreignkey(self, db_field, request, **kwargs):
         if db_field.name == "course":
             kwargs["queryset"] = _filter_courses_for_user(
                     Course.objects, request.user)
-        return super(ExamAdmin, self).formfield_for_foreignkey(
+        return super().formfield_for_foreignkey(
                 db_field, request, **kwargs)
 
     # }}}
 
-admin.site.register(Exam, ExamAdmin)
-
 
+@admin.register(ExamTicket)
 class ExamTicketAdmin(admin.ModelAdmin):
+    @admin.display(
+        description=_("Course"),
+        ordering="participation__course",
+    )
     def get_course(self, obj):
         return obj.participation.course
 
-    get_course.short_description = _("Participant")  # type: ignore
-    get_course.admin_order_field = "participation__course"  # type: ignore
-
     list_filter = (
-            "participation__course",
+            _filter_related_only("participation__course"),
             "state",
             )
 
+    raw_id_fields = ("participation",)
+
     list_display = (
             "get_course",
             "exam",
             "participation",
+            "require_login",
             "state",
             "creation_time",
             "usage_time",
@@ -928,7 +1033,7 @@ class ExamTicketAdmin(admin.ModelAdmin):
     # {{{ permissions
 
     def get_queryset(self, request):
-        qs = super(ExamTicketAdmin, self).get_queryset(request)
+        qs = super().get_queryset(request)
         return _filter_participation_linked_obj_for_user(qs, request.user)
 
     exclude = ("creator",)
@@ -939,17 +1044,16 @@ class ExamTicketAdmin(admin.ModelAdmin):
 
     # }}}
 
-    def revoke_exam_tickets(self, request, queryset):  # noqa
+    @admin.action(
+        description=_("Revoke Exam Tickets")
+    )
+    def revoke_exam_tickets(self, request, queryset):
         queryset \
                 .filter(state=exam_ticket_states.valid) \
                 .update(state=exam_ticket_states.revoked)
 
-    revoke_exam_tickets.short_description = _("Revoke Exam Tickets")  # type: ignore
-
     actions = [revoke_exam_tickets]
 
-admin.site.register(ExamTicket, ExamTicketAdmin)
-
 # }}}
 
 # vim: foldmethod=marker

course/analytics.py

-# -*- coding: utf-8 -*-
+from __future__ import annotations
 
-from __future__ import division
 
 __copyright__ = "Copyright (C) 2014 Andreas Kloeckner"
 
@@ -25,29 +24,19 @@ THE SOFTWARE.
 """
 
 
-import six
-from django.utils.translation import ugettext as _, pgettext, string_concat
-from django.shortcuts import (  # noqa
-        render, get_object_or_404, redirect)
+from django import http
+from django.contrib import messages
 from django.contrib.auth.decorators import login_required
-from django.core.exceptions import PermissionDenied
+from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
 from django.db import connection
+from django.shortcuts import get_object_or_404, redirect, render  # noqa
 from django.urls import reverse
-from django.core.exceptions import ObjectDoesNotExist
-from django import http
-from django.contrib import messages
-
-from course.utils import course_view, render_course_page, PageInstanceCache
-from course.models import (
-        FlowSession,
-        FlowPageVisit,
-        flow_permission)
-
-from course.constants import (
-        participation_permission as pperm,
-        )
+from django.utils.translation import gettext as _, pgettext
 
+from course.constants import participation_permission as pperm
 from course.content import get_flow_desc
+from course.models import FlowPageVisit, FlowSession, flow_permission
+from course.utils import PageInstanceCache, course_view, render_course_page
 
 
 # {{{ flow list
@@ -74,7 +63,7 @@ def flow_list(pctx):
 
 # {{{ histogram tool
 
-class BinInfo(object):
+class BinInfo:
     def __init__(self, title, raw_weight, percentage, url=None):
         self.title = title
         self.raw_weight = raw_weight
@@ -82,7 +71,7 @@ class BinInfo(object):
         self.url = url
 
 
-class Histogram(object):
+class Histogram:
     def __init__(self, num_bin_count=10, num_bin_starts=None,
             num_min_value=None, num_max_value=None,
             num_enforce_bounds=False, num_log_bins=False,
@@ -97,7 +86,7 @@ class Histogram(object):
         self.num_bin_title_formatter = num_bin_title_formatter
 
     def add_data_point(self, value, weight=1):
-        if isinstance(value, six.string_types):
+        if isinstance(value, str):
             self.string_weights[value] = \
                     self.string_weights.get(value, 0) + weight
         elif value is None:
@@ -130,7 +119,7 @@ class Histogram(object):
     def total_weight(self):
         return (
                 sum(weight for val, weight in self.num_values)
-                + sum(six.itervalues(self.string_weights)))
+                + sum(self.string_weights.values()))
 
     def get_bin_info_list(self):
         min_value = self.num_min_value
@@ -151,11 +140,17 @@ class Histogram(object):
                     max_value = 1
 
             if self.num_log_bins:
-                from math import log, exp
+                min_value = max(min_value, 1e-15)
+                max_value = max(max_value, 1.01*min_value)
+
+                from math import exp, log
                 bin_width = (log(max_value) - log(min_value))/self.num_bin_count
                 num_bin_starts = [
                         exp(log(min_value)+bin_width*i)
                         for i in range(self.num_bin_count)]
+                # Rounding error means exp(log(min_value)) may be greater
+                # than min_value, so set start of first bin to min_value
+                num_bin_starts[0] = min_value
             else:
                 bin_width = (max_value - min_value)/self.num_bin_count
                 num_bin_starts = [
@@ -189,14 +184,14 @@ class Histogram(object):
                         100*weight/total_weight
                         if total_weight
                         else None))
-                for start, weight in zip(num_bin_starts, bins)]
+                for start, weight in zip(num_bin_starts, bins, strict=True)]
 
         str_bin_info = [
                 BinInfo(
                     title=key,
                     raw_weight=temp_string_weights[key],
                     percentage=100*temp_string_weights[key]/total_weight)
-                for key in sorted(six.iterkeys(temp_string_weights))]
+                for key in sorted(temp_string_weights)]
 
         return num_bin_info + str_bin_info
 
@@ -207,12 +202,12 @@ class Histogram(object):
         if max_len < 20:
             from django.template.loader import render_to_string
             return render_to_string("course/histogram-wide.html", {
-                "bin_info_list": self.get_bin_info_list(),
+                "bin_info_list": bin_info_list,
                 })
         else:
             from django.template.loader import render_to_string
             return render_to_string("course/histogram.html", {
-                "bin_info_list": self.get_bin_info_list(),
+                "bin_info_list": bin_info_list,
                 })
 
 # }}}
@@ -276,7 +271,7 @@ def make_grade_histogram(pctx, flow_id):
     return hist
 
 
-class PageAnswerStats(object):
+class PageAnswerStats:
     def __init__(self, group_id, page_id, title, average_correctness,
             average_emptiness, answer_count, total_count, url=None):
         self.group_id = group_id
@@ -405,6 +400,7 @@ def make_time_histogram(pctx, flow_id):
             course=pctx.course,
             flow_id=flow_id)
 
+    from relate.utils import string_concat
     hist = Histogram(
             num_log_bins=True,
             num_bin_title_formatter=(
@@ -472,7 +468,7 @@ def flow_analytics(pctx, flow_id):
 
 # {{{ page analytics
 
-class AnswerStats(object):
+class AnswerStats:
     def __init__(self, normalized_answer, correctness, count,
             percentage):
         self.normalized_answer = normalized_answer
@@ -493,8 +489,6 @@ def page_analytics(pctx, flow_id, group_id, page_id):
     restrict_to_first_attempt = int(
             bool(pctx.request.GET.get("restrict_to_first_attempt") == "1"))
 
-    is_multiple_submit = is_flow_multiple_submit(flow_desc)
-
     page_cache = PageInstanceCache(pctx.repo, pctx.course, flow_id)
 
     visits = (FlowPageVisit.objects
@@ -509,6 +503,9 @@ def page_analytics(pctx, flow_id, group_id, page_id):
                 ))
 
     if connection.features.can_distinct_on_fields:
+
+        is_multiple_submit = is_flow_multiple_submit(flow_desc)
+
         if restrict_to_first_attempt:
             visits = (visits
                     .distinct("flow_session__participation__id")
@@ -540,9 +537,15 @@ def page_analytics(pctx, flow_id, group_id, page_id):
                 flow_session=visit.flow_session)
 
         title = page.title(grading_page_context, visit.page_data.data)
-        body = page.body(grading_page_context, visit.page_data.data)
+        body = page.analytic_view_body(grading_page_context, visit.page_data.data)
+
         normalized_answer = page.normalized_answer(
-                grading_page_context, visit.page_data.data, visit.answer)
+                    grading_page_context, visit.page_data.data, visit.answer)
+        if normalized_answer is None:
+            normalized_answer = _("(No answer)")
+        else:
+            import bleach
+            normalized_answer = bleach.clean(normalized_answer)
 
         answer_feedback = visit.get_most_recent_feedback()
 
@@ -560,7 +563,7 @@ def page_analytics(pctx, flow_id, group_id, page_id):
 
     answer_stats = []
     for (normalized_answer, correctness), count in \
-            six.iteritems(normalized_answer_and_correctness_to_count):
+            normalized_answer_and_correctness_to_count.items():
         answer_stats.append(
                 AnswerStats(
                     normalized_answer=normalized_answer,

course/api.py

+from __future__ import annotations
+
+
+__copyright__ = "Copyright (C) 2017 Andreas Kloeckner"
+
+__license__ = """
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+"""
+
+from typing import TYPE_CHECKING, Any
+
+from django import http
+from django.core.exceptions import PermissionDenied
+from django.shortcuts import get_object_or_404
+
+from course.auth import APIError, with_course_api_auth
+from course.constants import participation_permission as pperm
+from course.models import FlowSession
+
+
+# {{{ mypy
+
+if TYPE_CHECKING:
+    from course.auth import APIContext
+
+# }}}
+
+
+def flow_session_to_json(sess: FlowSession) -> Any:
+    last_activity = sess.last_activity()
+    return {
+            "id": sess.id,
+            "participation_username": (
+                sess.participation.user.username
+                if sess.participation is not None
+                else None),
+            "participation_institutional_id": (
+                sess.participation.user.institutional_id
+                if sess.participation is not None
+                else None),
+            "active_git_commit_sha": sess.active_git_commit_sha,
+            "flow_id": sess.flow_id,
+
+            "start_time": sess.start_time.isoformat(),
+            "completion_time": sess.completion_time,
+            "last_activity_time": (
+                last_activity.isoformat()
+                if last_activity is not None
+                else None),
+            "page_count": sess.page_count,
+
+            "in_progress": sess.in_progress,
+            "access_rules_tag": sess.access_rules_tag,
+            "expiration_mode": sess.expiration_mode,
+            "points": sess.points,
+            "max_points": sess.max_points,
+            "result_comment": sess.result_comment,
+            }
+
+
+@with_course_api_auth("Token")
+def get_flow_sessions(
+        api_ctx: APIContext, course_identifier: str) -> http.HttpResponse:
+    if not api_ctx.has_permission(pperm.view_gradebook):
+        raise PermissionDenied("token role does not have required permissions")
+
+    try:
+        flow_id = api_ctx.request.GET["flow_id"]
+    except KeyError:
+        raise APIError("must specify flow_id GET parameter")
+
+    sessions = FlowSession.objects.filter(
+            course=api_ctx.course,
+            flow_id=flow_id)
+
+    result = [flow_session_to_json(sess) for sess in sessions]
+
+    return http.JsonResponse(result, safe=False)
+
+
+@with_course_api_auth("Token")
+def get_flow_session_content(
+        api_ctx: APIContext, course_identifier: str) -> http.HttpResponse:
+    if not api_ctx.has_permission(pperm.view_gradebook):
+        raise PermissionDenied("token role does not have required permissions")
+
+    try:
+        session_id_str = api_ctx.request.GET["flow_session_id"]
+    except KeyError:
+        raise APIError("must specify flow_id GET parameter")
+
+    session_id = int(session_id_str)
+
+    flow_session = get_object_or_404(FlowSession, id=session_id)
+
+    if flow_session.course != api_ctx.course:
+        raise PermissionDenied(
+                "session's course does not match auth context")
+
+    from course.content import get_course_repo
+    from course.flow import adjust_flow_session_page_data, assemble_answer_visits
+
+    with get_course_repo(api_ctx.course) as repo:
+        from course.utils import FlowContext, instantiate_flow_page_with_ctx
+        fctx = FlowContext(repo, api_ctx.course, flow_session.flow_id)
+
+        adjust_flow_session_page_data(repo, flow_session, api_ctx.course.identifier,
+                fctx.flow_desc)
+
+        from course.flow import get_all_page_data
+        all_page_data = get_all_page_data(flow_session)
+        answer_visits = assemble_answer_visits(flow_session)
+
+        pages = []
+        for i, page_data in enumerate(all_page_data):
+            page = instantiate_flow_page_with_ctx(fctx, page_data)
+
+            assert i == page_data.page_ordinal
+
+            page_data_json = {
+                    "ordinal": i,
+                    "page_type": page_data.page_type,
+                    "group_id": page_data.group_id,
+                    "page_id": page_data.page_id,
+                    "page_data": page_data.data,
+                    "title": page_data.title,
+                    "bookmarked": page_data.bookmarked,
+                    }
+            answer_json = None
+            grade_json = None
+
+            visit = answer_visits[i]
+            if visit is not None:
+                from course.page.base import PageContext
+                pctx = PageContext(api_ctx.course, repo, fctx.course_commit_sha,
+                        flow_session)
+                norm_bytes_answer_tup = page.normalized_bytes_answer(
+                        pctx, page_data.data, visit.answer)
+
+                # norm_answer needs to be JSON-encodable
+                norm_answer: Any = None
+
+                if norm_bytes_answer_tup is not None:
+                    answer_file_ext, norm_bytes_answer = norm_bytes_answer_tup
+
+                    if answer_file_ext in [".txt", ".py"]:
+                        norm_answer = norm_bytes_answer.decode("utf-8")
+                    elif answer_file_ext == ".json":
+                        import json
+                        norm_answer = json.loads(norm_bytes_answer)
+                    else:
+                        from base64 import b64encode
+                        norm_answer = [answer_file_ext,
+                                       b64encode(norm_bytes_answer).decode("utf-8")]
+
+                answer_json = {
+                        "visit_time": visit.visit_time.isoformat(),
+                        "remote_address": repr(visit.remote_address),
+                        "user": (
+                            visit.user.username if visit.user is not None else None),
+                        "impersonated_by": (
+                            visit.impersonated_by.username
+                            if visit.impersonated_by is not None else None),
+                        "is_synthetic_visit": visit.is_synthetic,
+                        "answer_data": visit.answer,
+                        "answer": norm_answer,
+                        }
+
+                grade = visit.get_most_recent_grade()
+                if grade is not None:
+                    grade_json = {
+                            "grader": (grade.grader.username
+                                if grade.grader is not None else None),
+                            "grade_time": grade.grade_time.isoformat(),
+                            "graded_at_git_commit_sha": (
+                                grade.graded_at_git_commit_sha),
+                            "max_points": grade.max_points,
+                            "correctness": grade.correctness,
+                            "feedback": grade.feedback}
+
+            pages.append({
+                "page": page_data_json,
+                "answer": answer_json,
+                "grade": grade_json,
+                })
+
+    result = {
+        "session": flow_session_to_json(flow_session),
+        "pages": pages,
+        }
+
+    return http.JsonResponse(result, safe=False)
+
+
+# vim: foldmethod=marker

course/apps.py

+from __future__ import annotations
+
 from django.apps import AppConfig
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
+
+from relate.checks import register_startup_checks, register_startup_checks_extra
 
 
 class CourseConfig(AppConfig):
-    name = 'course'
+    name = "course"
     # for translation of the name of "Course" app displayed in admin.
     verbose_name = _("Course module")
 
+    default_auto_field = "django.db.models.BigAutoField"
+
     def ready(self):
-        import course.receivers
\ No newline at end of file
+        import course.receivers  # noqa
+
+        # register all checks
+        register_startup_checks()
+        register_startup_checks_extra()

course/auth.py

-# -*- coding: utf-8 -*-
+from __future__ import annotations
 
-from __future__ import division
 
 __copyright__ = "Copyright (C) 2014 Andreas Kloeckner"
 
@@ -24,82 +23,127 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 """
 
-from typing import cast, Any, Optional, Text  # noqa
-from django.utils.translation import ugettext_lazy as _, string_concat
-from django.shortcuts import (  # noqa
-        render, get_object_or_404, redirect, resolve_url)
-from django.contrib import messages
+import re
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    cast,
+)
+
 import django.forms as forms
-from django.core.exceptions import (PermissionDenied, SuspiciousOperation,
-        ObjectDoesNotExist)
 from crispy_forms.helper import FormHelper
-from crispy_forms.layout import Submit, Layout, Div
+from crispy_forms.layout import Button, Div, Layout, Submit
+from django import http
 from django.conf import settings
-from django.contrib.auth import (get_user_model, REDIRECT_FIELD_NAME,
-        login as auth_login, logout as auth_logout)
-from django.contrib.auth.forms import \
-        AuthenticationForm as AuthenticationFormBase
-from django.contrib.sites.shortcuts import get_current_site
-from django.contrib.auth.decorators import user_passes_test
-from django.urls import reverse
-from django.core import validators
-from django.utils.http import is_safe_url
+from django.contrib import messages
+from django.contrib.auth import (
+    REDIRECT_FIELD_NAME,
+    get_user_model,
+    login as auth_login,
+    logout as auth_logout,
+)
+from django.contrib.auth.decorators import login_required, user_passes_test
+from django.contrib.auth.forms import AuthenticationForm as AuthenticationFormBase
+from django.contrib.auth.validators import ASCIIUsernameValidator
+from django.core.exceptions import (
+    MultipleObjectsReturned,
+    ObjectDoesNotExist,
+    PermissionDenied,
+    SuspiciousOperation,
+)
 from django.http import HttpResponseRedirect
+from django.shortcuts import get_object_or_404, redirect, render, resolve_url
 from django.template.response import TemplateResponse
-from django.views.decorators.debug import sensitive_post_parameters
+from django.urls import reverse
+from django.utils.http import url_has_allowed_host_and_scheme
+from django.utils.translation import gettext_lazy as _
 from django.views.decorators.cache import never_cache
 from django.views.decorators.csrf import csrf_protect
-from django import http  # noqa
-
-from djangosaml2.backends import Saml2Backend as Saml2BackendBase
+from django.views.decorators.debug import sensitive_post_parameters
+from django_select2.forms import ModelSelect2Widget
+from djangosaml2.backends import Saml2Backend
 
-from course.constants import (
-        user_status,
-        participation_status,
-        participation_permission as pperm,
-        )
-from course.models import Participation, Course  # noqa
 from accounts.models import User
+from course.constants import (
+    participation_permission as pperm,
+    participation_status,
+    user_status,
+)
+from course.models import (
+    AuthenticationToken,
+    Participation,
+    ParticipationRole,
+)
+from course.utils import CoursePageContext, course_view, render_course_page
+from relate.utils import (
+    HTML5DateTimeInput,
+    StyledForm,
+    StyledModelForm,
+    get_site_name,
+    string_concat,
+)
+
+
+if TYPE_CHECKING:
+    import datetime
+
+    from django.db.models import query
 
 
-from relate.utils import StyledForm, StyledModelForm
-from django_select2.forms import ModelSelect2Widget
+# {{{ impersonation
 
+def get_pre_impersonation_user(request):
+    is_impersonating = hasattr(
+            request, "relate_impersonate_original_user")
+    if is_impersonating:
+        return request.relate_impersonate_original_user
+    return None
 
-# {{{ impersonation
 
-def may_impersonate(impersonator, impersonee):
-    # type: (User, User) -> bool
+def get_impersonable_user_qset(impersonator: User) -> query.QuerySet:
     if impersonator.is_superuser:
-        return True
+        return User.objects.exclude(pk=impersonator.pk)
 
     my_participations = Participation.objects.filter(
-            user=impersonator,
-            status=participation_status.active)
+        user=impersonator,
+        status=participation_status.active)
 
+    impersonable_user_qset = User.objects.none()
     for part in my_participations:
-        impersonable_roles = (
-                argument
-                for perm, argument in part.permissions()
-                if perm == pperm.impersonate_role)
-
-        if Participation.objects.filter(
-                course=part.course,
-                status=participation_status.active,
-                role__in=impersonable_roles,
-                user=impersonee).count():
-            return True
+        # Notice: if a TA is not allowed to view participants'
+        # profile in one course, then he/she is not able to impersonate
+        # any user, even in courses he/she is allow to view profiles
+        # of all users.
+        if part.has_permission(pperm.view_participant_masked_profile):
+            return User.objects.none()
+        impersonable_roles = [
+            argument
+            for perm, argument in part.permissions()
+            if perm == pperm.impersonate_role]
+
+        q = (Participation.objects
+             .filter(course=part.course,
+                     status=participation_status.active,
+                     roles__identifier__in=impersonable_roles)
+             .select_related("user"))
+
+        # There can be duplicate records. Removing duplicate records is needed
+        # only when rendering ImpersonateForm
+        impersonable_user_qset = (
+            impersonable_user_qset
+            | User.objects.filter(pk__in=q.values_list("user__pk", flat=True))
+        )
 
-    return False
+    return impersonable_user_qset
 
 
-class ImpersonateMiddleware(object):
+class ImpersonateMiddleware:
     def __init__(self, get_response):
         self.get_response = get_response
 
     def __call__(self, request):
-        if 'impersonate_id' in request.session:
-            imp_id = request.session['impersonate_id']
+        if "impersonate_id" in request.session:
+            imp_id = request.session["impersonate_id"]
             impersonee = None
 
             try:
@@ -108,14 +152,18 @@ class ImpersonateMiddleware(object):
             except ObjectDoesNotExist:
                 pass
 
+            may_impersonate = False
             if impersonee is not None:
-                if may_impersonate(cast(User, request.user), impersonee):
-                    request.relate_impersonate_original_user = request.user
-                    request.user = impersonee
+                if request.user.is_superuser:
+                    may_impersonate = True
                 else:
-                    messages.add_message(request, messages.ERROR,
-                            _("Error while impersonating."))
+                    qset = get_impersonable_user_qset(cast(User, request.user))
+                    if qset.filter(pk=cast(User, impersonee).pk).count():
+                        may_impersonate = True
 
+            if may_impersonate:
+                request.relate_impersonate_original_user = request.user
+                request.user = impersonee
             else:
                 messages.add_message(request, messages.ERROR,
                         _("Error while impersonating."))
@@ -126,38 +174,36 @@ class ImpersonateMiddleware(object):
 class UserSearchWidget(ModelSelect2Widget):
     model = User
     search_fields = [
-            'username__icontains',
-            'email__icontains',
-            'first_name__icontains',
-            'last_name__icontains',
+            "username__icontains",
+            "email__icontains",
+            "first_name__icontains",
+            "last_name__icontains",
             ]
 
     def label_from_instance(self, u):
-        return (
-            (
-                # Translators: information displayed when selecting
-                # userfor impersonating. Customize how the name is
-                # shown, but leave email first to retain usability
-                # of form sorted by last name.
-                "%(full_name)s (%(username)s - %(email)s)"
-                % {
-                    "full_name": u.get_full_name(),
-                    "email": u.email,
-                    "username": u.username
-                    }))
+        if u.first_name and u.last_name:
+            return (
+                    f"{u.get_full_name()} ({u.username} - {u.email})")
+        else:
+            # for users with "None" fullname
+            return (
+                    f"{u.username} ({u.email})")
 
 
 class ImpersonateForm(StyledForm):
-    def __init__(self, *args, **kwargs):
-        # type:(*Any, **Any) -> None
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
 
-        super(ImpersonateForm, self).__init__(*args, **kwargs)
+        qset = kwargs.pop("impersonable_qset")
+        super().__init__(*args, **kwargs)
 
         self.fields["user"] = forms.ModelChoiceField(
-                queryset=User.objects.order_by("last_name"),
+                queryset=qset,
                 required=True,
                 help_text=_("Select user to impersonate."),
-                widget=UserSearchWidget(),
+                widget=UserSearchWidget(
+                    queryset=qset,
+                    attrs={"data-minimum-input-length": 0},
+                ),
                 label=_("User"))
 
         self.fields["add_impersonation_header"] = forms.BooleanField(
@@ -171,32 +217,38 @@ class ImpersonateForm(StyledForm):
         self.helper.add_input(Submit("submit", _("Impersonate")))
 
 
-def impersonate(request):
-    # type: (http.HttpRequest) -> http.HttpResponse
+def impersonate(request: http.HttpRequest) -> http.HttpResponse:
+    if not request.user.is_authenticated:
+        raise PermissionDenied()
+
+    impersonable_user_qset = get_impersonable_user_qset(cast(User, request.user))
+    if not impersonable_user_qset.count():
+        raise PermissionDenied()
 
     if hasattr(request, "relate_impersonate_original_user"):
         messages.add_message(request, messages.ERROR,
                 _("Already impersonating someone."))
-        return redirect("relate-stop_impersonating")
+        return redirect("relate-home")
 
-    if request.method == 'POST':
-        form = ImpersonateForm(request.POST)
+    # Remove duplicate and sort
+    # order_by().distinct() directly on impersonable_user_qset will not work
+    qset = (User.objects
+            .filter(pk__in=impersonable_user_qset.values_list("pk", flat=True))
+            .order_by("last_name", "first_name", "username"))
+    if request.method == "POST":
+        form = ImpersonateForm(request.POST, impersonable_qset=qset)
         if form.is_valid():
             impersonee = form.cleaned_data["user"]
 
-            if may_impersonate(cast(User, request.user), cast(User, impersonee)):
-                request.session['impersonate_id'] = impersonee.id
-                request.session['relate_impersonation_header'] = form.cleaned_data[
-                        "add_impersonation_header"]
+            request.session["impersonate_id"] = impersonee.id
+            request.session["relate_impersonation_header"] = form.cleaned_data[
+                    "add_impersonation_header"]
 
-                # Because we'll likely no longer have access to this page.
-                return redirect("relate-home")
-            else:
-                messages.add_message(request, messages.ERROR,
-                        _("Impersonating that user is not allowed."))
+            # Because we'll likely no longer have access to this page.
+            return redirect("relate-home")
 
     else:
-        form = ImpersonateForm()
+        form = ImpersonateForm(impersonable_qset=qset)
 
     return render(request, "generic-form.html", {
         "form_description": _("Impersonate user"),
@@ -204,36 +256,43 @@ def impersonate(request):
         })
 
 
-class StopImpersonatingForm(forms.Form):
-    def __init__(self, *args, **kwargs):
-        self.helper = FormHelper()
-        super(StopImpersonatingForm, self).__init__(*args, **kwargs)
+def stop_impersonating(request: http.HttpRequest) -> http.JsonResponse:
+    if request.method != "POST":
+        raise PermissionDenied(_("only AJAX POST is allowed"))
 
-        self.helper.add_input(Submit("submit", _("Stop impersonating")))
+    if not request.user.is_authenticated:
+        raise PermissionDenied()
 
+    if "stop_impersonating" not in request.POST:
+        raise SuspiciousOperation(_("odd POST parameters"))
 
-def stop_impersonating(request):
     if not hasattr(request, "relate_impersonate_original_user"):
-        messages.add_message(request, messages.ERROR,
-                _("Not currently impersonating anyone."))
-        return redirect("relate-home")
+        # prevent user without pperm to stop_impersonating
+        my_participations = Participation.objects.filter(
+            user=request.user,
+            status=participation_status.active)
 
-    if request.method == 'POST':
-        form = StopImpersonatingForm(request.POST)
-        if form.is_valid():
-            messages.add_message(request, messages.INFO,
-                    _("No longer impersonating anyone."))
-            del request.session['impersonate_id']
+        may_impersonate = False
+        for part in my_participations:
+            perms = [
+                perm
+                for perm, argument in part.permissions()
+                if perm == pperm.impersonate_role]
+            if any(perms):
+                may_impersonate = True
+                break
 
-            # Because otherwise the header will show stale data.
-            return redirect("relate-home")
-    else:
-        form = StopImpersonatingForm()
+        if not may_impersonate:
+            raise PermissionDenied(_("may not stop impersonating"))
 
-    return render(request, "generic-form.html", {
-        "form_description": _("Stop impersonating user"),
-        "form": form
-        })
+        messages.add_message(request, messages.ERROR,
+                _("Not currently impersonating anyone."))
+        return http.JsonResponse({})
+
+    del request.session["impersonate_id"]
+    messages.add_message(request, messages.INFO,
+            _("No longer impersonating anyone."))
+    return http.JsonResponse({"result": "success"})
 
 
 def impersonation_context_processor(request):
@@ -247,11 +306,10 @@ def impersonation_context_processor(request):
 # }}}
 
 
-def make_sign_in_key(user):
-    # type: (User) -> Text
+def make_sign_in_key(user: User) -> str:
     # Try to ensure these hashes aren't guessable.
-    import random
     import hashlib
+    import random
     from time import time
     m = hashlib.sha1()
     m.update(user.email.encode("utf-8"))
@@ -260,19 +318,26 @@ def make_sign_in_key(user):
     return m.hexdigest()
 
 
-def check_sign_in_key(user_id, token):
-    users = get_user_model().objects.filter(
-            id=user_id, sign_in_key=token)
-
-    assert users.count() <= 1
-    if users.count() == 0:
-        return False
-
-    return True
-
-
-class TokenBackend(object):
-    def authenticate(self, user_id=None, token=None):
+def logout_confirmation_required(
+        func=None, redirect_field_name=REDIRECT_FIELD_NAME,
+        logout_confirmation_url="relate-logout-confirmation"):
+    """
+    Decorator for views that checks that no user is logged in.
+    If a user is currently logged in, redirect him/her to the logout
+    confirmation page.
+    """
+    actual_decorator = user_passes_test(
+        lambda u: u.is_anonymous,
+        login_url=logout_confirmation_url,
+        redirect_field_name=redirect_field_name
+    )
+    if func:
+        return actual_decorator(func)
+    return actual_decorator
+
+
+class EmailedTokenBackend:
+    def authenticate(self, request, user_id=None, token=None):
         users = get_user_model().objects.filter(
                 id=user_id, sign_in_key=token)
 
@@ -297,17 +362,23 @@ class TokenBackend(object):
 
 # {{{ choice
 
-@user_passes_test(
-        lambda user: not user.username,
-        login_url='relate-logout-confirmation')
+@logout_confirmation_required
 def sign_in_choice(request, redirect_field_name=REDIRECT_FIELD_NAME):
     redirect_to = request.POST.get(redirect_field_name,
-                                   request.GET.get(redirect_field_name, ''))
+                                   request.GET.get(redirect_field_name, ""))
     next_uri = ""
     if redirect_to:
-        next_uri = "?%s=%s" % (redirect_field_name, redirect_to)
-
-    return render(request, "sign-in-choice.html", {"next_uri": next_uri})
+        next_uri = f"?{redirect_field_name}={redirect_to}"
+
+    return render(request, "sign-in-choice.html", {
+        "next_uri": next_uri,
+        "social_provider_to_logo": {
+            "google-oauth2": "google",
+            },
+        "social_provider_to_human_name": {
+            "google-oauth2": "Google",
+            },
+        })
 
 # }}}
 
@@ -323,29 +394,35 @@ class LoginForm(AuthenticationFormBase):
 
         self.helper.add_input(Submit("submit", _("Sign in")))
 
-        super(LoginForm, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
 
 @sensitive_post_parameters()
 @csrf_protect
 @never_cache
-@user_passes_test(
-        lambda user: not user.username,
-        login_url='relate-logout-confirmation')
+@logout_confirmation_required
 def sign_in_by_user_pw(request, redirect_field_name=REDIRECT_FIELD_NAME):
     """
     Displays the login form and handles the login action.
     """
+    if not settings.RELATE_SIGN_IN_BY_USERNAME_ENABLED:
+        messages.add_message(request, messages.ERROR,
+                _("Username-based sign-in is not being used"))
+        return redirect("relate-sign_in_choice")
+
     redirect_to = request.POST.get(redirect_field_name,
-                                   request.GET.get(redirect_field_name, ''))
+                                   request.GET.get(redirect_field_name, ""))
 
     if request.method == "POST":
         form = LoginForm(request, data=request.POST)
         if form.is_valid():
 
             # Ensure the user-originating redirection url is safe.
-            if not is_safe_url(url=redirect_to, host=request.get_host()):
-                redirect_to = resolve_url(settings.LOGIN_REDIRECT_URL)
+            if not url_has_allowed_host_and_scheme(
+                    url=redirect_to,
+                    allowed_hosts={request.get_host()},
+                    require_https=request.is_secure()):
+                redirect_to = resolve_url("relate-home")
 
             user = form.get_user()
 
@@ -356,18 +433,14 @@ def sign_in_by_user_pw(request, redirect_field_name=REDIRECT_FIELD_NAME):
     else:
         form = LoginForm(request)
 
-    current_site = get_current_site(request)
-
     next_uri = ""
     if redirect_to:
-        next_uri = "?%s=%s" % (redirect_field_name, redirect_to)
+        next_uri = f"?{redirect_field_name}={redirect_to}"
 
     context = {
-        'form': form,
+        "form": form,
         redirect_field_name: redirect_to,
-        'site': current_site,
-        'site_name': current_site.name,
-        'next_uri': next_uri,
+        "next_uri": next_uri,
     }
 
     return TemplateResponse(request, "course/login.html", context)
@@ -376,22 +449,14 @@ def sign_in_by_user_pw(request, redirect_field_name=REDIRECT_FIELD_NAME):
 class SignUpForm(StyledModelForm):
     username = forms.CharField(required=True, max_length=30,
             label=_("Username"),
-            validators=[
-                validators.RegexValidator('^[\\w.@+-]+$',
-                    string_concat(
-                        _('Enter a valid username.'), (' '),
-                        _('This value may contain only letters, '
-                          'numbers and @/./+/-/_ characters.')
-                        ),
-                    'invalid')
-                ])
+            validators=[ASCIIUsernameValidator()])
 
     class Meta:
         model = get_user_model()
         fields = ("email",)
 
     def __init__(self, *args, **kwargs):
-        super(SignUpForm, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         self.fields["email"].required = True
 
@@ -399,15 +464,13 @@ class SignUpForm(StyledModelForm):
                 Submit("submit", _("Send email")))
 
 
-@user_passes_test(
-        lambda user: not user.username,
-        login_url='relate-logout-confirmation')
+@logout_confirmation_required
 def sign_up(request):
     if not settings.RELATE_REGISTRATION_ENABLED:
         raise SuspiciousOperation(
                 _("self-registration is not enabled"))
 
-    if request.method == 'POST':
+    if request.method == "POST":
         form = SignUpForm(request.POST)
         if form.is_valid():
             if get_user_model().objects.filter(
@@ -415,14 +478,6 @@ def sign_up(request):
                 messages.add_message(request, messages.ERROR,
                         _("A user with that username already exists."))
 
-            elif get_user_model().objects.filter(
-                    email__iexact=form.cleaned_data["email"]).count():
-                messages.add_message(request, messages.ERROR,
-                        _("That email address is already in use. "
-                        "Would you like to "
-                        "<a href='%s'>reset your password</a> instead?")
-                        % reverse(
-                            "relate-reset_password")),
             else:
                 email = form.cleaned_data["email"]
                 user = get_user_model()(
@@ -434,8 +489,8 @@ def sign_up(request):
                 user.sign_in_key = make_sign_in_key(user)
                 user.save()
 
-                from django.template.loader import render_to_string
-                message = render_to_string("course/sign-in-email.txt", {
+                from relate.utils import render_email_template
+                message = render_email_template("course/sign-in-email.txt", {
                     "user": user,
                     "sign_in_uri": request.build_absolute_uri(
                         reverse(
@@ -448,8 +503,8 @@ def sign_up(request):
 
                 from django.core.mail import EmailMessage
                 msg = EmailMessage(
-                        string_concat("[", _("RELATE"), "] ",
-                                     _("Verify your email")),
+                        string_concat(f"[{_(get_site_name())}] ",
+                                      _("Verify your email")),
                         message,
                         getattr(settings, "NO_REPLY_EMAIL_FROM",
                                 settings.ROBOT_EMAIL_FROM),
@@ -467,6 +522,16 @@ def sign_up(request):
                         "the link."))
 
                 return redirect("relate-home")
+        else:
+            if ("email" in form.errors
+                    and "That email address is already in use."
+                    in form.errors["email"]):
+                messages.add_message(request, messages.ERROR,
+                        _("That email address is already in use. "
+                        "Would you like to "
+                        "<a href='%s'>reset your password</a> instead?")
+                        % reverse(
+                            "relate-reset_password"))
 
     else:
         form = SignUpForm()
@@ -478,10 +543,11 @@ def sign_up(request):
 
 
 class ResetPasswordFormByEmail(StyledForm):
-    email = forms.EmailField(required=True, label=_("Email"))
+    email = forms.EmailField(required=True, label=_("Email"),
+                             max_length=User._meta.get_field("email").max_length)
 
     def __init__(self, *args, **kwargs):
-        super(ResetPasswordFormByEmail, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         self.helper.add_input(
                 Submit("submit", _("Send email")))
@@ -493,7 +559,7 @@ class ResetPasswordFormByInstid(StyledForm):
                               label=_("Institutional ID"))
 
     def __init__(self, *args, **kwargs):
-        super(ResetPasswordFormByInstid, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         self.helper.add_input(
                 Submit("submit", _("Send email")))
@@ -501,96 +567,106 @@ class ResetPasswordFormByInstid(StyledForm):
 
 def masked_email(email):
     # return a masked email address
-    at = email.find('@')
+    at = email.find("@")
     return email[:2] + "*" * (len(email[3:at])-1) + email[at-1:]
 
 
-@user_passes_test(
-        lambda user: not user.username,
-        login_url='relate-logout-confirmation')
+@logout_confirmation_required
 def reset_password(request, field="email"):
     if not settings.RELATE_REGISTRATION_ENABLED:
         raise SuspiciousOperation(
                 _("self-registration is not enabled"))
 
     # return form class by string of class name
-    ResetPasswordForm = globals()["ResetPasswordFormBy" + field.title()]
-    if request.method == 'POST':
+    ResetPasswordForm = globals()["ResetPasswordFormBy" + field.title()]  # noqa
+    if request.method == "POST":
         form = ResetPasswordForm(request.POST)
+        user = None
         if form.is_valid():
+            exist_users_with_same_email = False
             if field == "instid":
                 inst_id = form.cleaned_data["instid"]
                 try:
                     user = get_user_model().objects.get(
                             institutional_id__iexact=inst_id)
                 except ObjectDoesNotExist:
-                    user = None
+                    pass
 
             if field == "email":
                 email = form.cleaned_data["email"]
                 try:
                     user = get_user_model().objects.get(email__iexact=email)
                 except ObjectDoesNotExist:
-                    user = None
+                    pass
+                except MultipleObjectsReturned:
+                    exist_users_with_same_email = True
 
-            if user is None:
-                FIELD_DICT = {
-                        "email": _("email address"),
-                        "instid": _("institutional ID")
-                        }
+            if exist_users_with_same_email:
+                # This is for backward compatibility.
                 messages.add_message(request, messages.ERROR,
-                        _("That %(field)s doesn't have an "
-                            "associated user account. Are you "
-                            "sure you've registered?")
-                        % {"field": FIELD_DICT[field]})
+                        _("Failed to send an email: multiple users were "
+                          "unexpectedly using that same "
+                          "email address. Please "
+                          "contact site staff."))
             else:
-                if not user.email:
-                    # happens when a user have an inst_id but have no email.
+                if user is None:
+                    FIELD_DICT = {  # noqa
+                            "email": _("email address"),
+                            "instid": _("institutional ID")
+                            }
                     messages.add_message(request, messages.ERROR,
-                            _("The account with that institution ID "
-                                "doesn't have an associated email."))
+                            _("That %(field)s doesn't have an "
+                                "associated user account. Are you "
+                                "sure you've registered?")
+                            % {"field": FIELD_DICT[field]})
                 else:
-                    email = user.email
-                    user.sign_in_key = make_sign_in_key(user)
-                    user.save()
-
-                    from django.template.loader import render_to_string
-                    message = render_to_string("course/sign-in-email.txt", {
-                        "user": user,
-                        "sign_in_uri": request.build_absolute_uri(
-                            reverse(
-                                "relate-reset_password_stage2",
-                                args=(user.id, user.sign_in_key,))),
-                        "home_uri": request.build_absolute_uri(
-                            reverse("relate-home"))
-                        })
-                    from django.core.mail import EmailMessage
-                    msg = EmailMessage(
-                            string_concat("[", _("RELATE"), "] ",
-                                         _("Password reset")),
-                            message,
-                            getattr(settings, "NO_REPLY_EMAIL_FROM",
-                                    settings.ROBOT_EMAIL_FROM),
-                            [email])
-
-                    from relate.utils import get_outbound_mail_connection
-                    msg.connection = (
-                            get_outbound_mail_connection("no_reply")
-                            if hasattr(settings, "NO_REPLY_EMAIL_FROM")
-                            else get_outbound_mail_connection("robot"))
-                    msg.send()
-
-                    if field == "instid":
-                        messages.add_message(request, messages.INFO,
-                            _("The email address associated with that "
-                                "account is %s.")
-                            % masked_email(email))
+                    if not user.email:
+                        messages.add_message(request, messages.ERROR,
+                                _("The account with that institution ID "
+                                    "doesn't have an associated email."))
+                    else:
+                        email = user.email
+                        user.sign_in_key = make_sign_in_key(user)
+                        user.save()
+
+                        from relate.utils import render_email_template
+                        message = render_email_template(
+                            "course/sign-in-email.txt", {
+                                "user": user,
+                                "sign_in_uri": request.build_absolute_uri(
+                                    reverse(
+                                        "relate-reset_password_stage2",
+                                        args=(user.id, user.sign_in_key,))),
+                                "home_uri": request.build_absolute_uri(
+                                    reverse("relate-home"))
+                            })
+                        from django.core.mail import EmailMessage
+                        msg = EmailMessage(
+                                string_concat(f"[{_(get_site_name())}] ",
+                                              _("Password reset")),
+                                message,
+                                getattr(settings, "NO_REPLY_EMAIL_FROM",
+                                        settings.ROBOT_EMAIL_FROM),
+                                [email])
+
+                        from relate.utils import get_outbound_mail_connection
+                        msg.connection = (
+                                get_outbound_mail_connection("no_reply")
+                                if hasattr(settings, "NO_REPLY_EMAIL_FROM")
+                                else get_outbound_mail_connection("robot"))
+                        msg.send()
+
+                        if field == "instid":
+                            messages.add_message(request, messages.INFO,
+                                _("The email address associated with that "
+                                  "account is %s.")
+                                % masked_email(email))
 
-                    messages.add_message(request, messages.INFO,
-                            _("Email sent. Please check your email and "
-                            "click the link."))
+                        messages.add_message(request, messages.INFO,
+                                _("Email sent. Please check your email and "
+                                  "click the link."))
 
-                    return redirect("relate-home")
+                        return redirect("relate-home")
     else:
         form = ResetPasswordForm()
 
@@ -598,7 +674,7 @@ def reset_password(request, field="email"):
         "field": field,
         "form_description":
             _("Password reset on %(site_name)s")
-            % {"site_name": _("RELATE")},
+            % {"site_name": _(get_site_name())},
         "form": form
         })
 
@@ -610,13 +686,13 @@ class ResetPasswordStage2Form(StyledForm):
                               label=_("Password confirmation"))
 
     def __init__(self, *args, **kwargs):
-        super(ResetPasswordStage2Form, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         self.helper.add_input(
                 Submit("submit_user", _("Update")))
 
     def clean(self):
-        cleaned_data = super(ResetPasswordStage2Form, self).clean()
+        cleaned_data = super().clean()
         password = cleaned_data.get("password")
         password_repeat = cleaned_data.get("password_repeat")
         if password and password != password_repeat:
@@ -624,26 +700,35 @@ class ResetPasswordStage2Form(StyledForm):
                     _("The two password fields didn't match."))
 
 
-@user_passes_test(
-        lambda user: not user.username,
-        login_url='relate-logout-confirmation')
+@logout_confirmation_required
 def reset_password_stage2(request, user_id, sign_in_key):
     if not settings.RELATE_REGISTRATION_ENABLED:
         raise SuspiciousOperation(
                 _("self-registration is not enabled"))
 
-    if not check_sign_in_key(user_id=int(user_id), token=sign_in_key):
-        messages.add_message(request, messages.ERROR,
-                _("Invalid sign-in token. Perhaps you've used an old token "
-                "email?"))
+    def check_sign_in_key(user_id, token):
+        user = get_user_model().objects.get(id=user_id)
+        return user.sign_in_key == token
+
+    try:
+        if not check_sign_in_key(user_id=int(user_id), token=sign_in_key):
+            messages.add_message(request, messages.ERROR,
+                    _("Invalid sign-in token. Perhaps you've used an old token "
+                    "email?"))
+            raise PermissionDenied(_("invalid sign-in token"))
+    except get_user_model().DoesNotExist:
+        messages.add_message(request, messages.ERROR, _("Account does not exist."))
         raise PermissionDenied(_("invalid sign-in token"))
 
-    if request.method == 'POST':
+    if request.method == "POST":
         form = ResetPasswordStage2Form(request.POST)
         if form.is_valid():
             from django.contrib.auth import authenticate, login
             user = authenticate(user_id=int(user_id), token=sign_in_key)
             if user is None:
+                messages.add_message(request, messages.ERROR,
+                     _("Invalid sign-in token. Perhaps you've used an old token "
+                     "email?"))
                 raise PermissionDenied(_("invalid sign-in token"))
 
             if not user.is_active:
@@ -675,7 +760,7 @@ def reset_password_stage2(request, user_id, sign_in_key):
     return render(request, "generic-form.html", {
         "form_description":
             _("Password reset on %(site_name)s")
-            % {"site_name": _("RELATE")},
+            % {"site_name": _(get_site_name())},
         "form": form
         })
 
@@ -687,41 +772,39 @@ def reset_password_stage2(request, user_id, sign_in_key):
 class SignInByEmailForm(StyledForm):
     email = forms.EmailField(required=True, label=_("Email"),
             # For now, until we upgrade to a custom user model.
-            max_length=30)
+            max_length=User._meta.get_field("email").max_length)
 
     def __init__(self, *args, **kwargs):
-        super(SignInByEmailForm, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         self.helper.add_input(
                 Submit("submit", _("Send sign-in email")))
 
 
-@user_passes_test(
-        lambda user: not user.username,
-        login_url='relate-logout-confirmation')
+@logout_confirmation_required
 def sign_in_by_email(request):
     if not settings.RELATE_SIGN_IN_BY_EMAIL_ENABLED:
         messages.add_message(request, messages.ERROR,
                 _("Email-based sign-in is not being used"))
         return redirect("relate-sign_in_choice")
 
-    if request.method == 'POST':
+    if request.method == "POST":
         form = SignInByEmailForm(request.POST)
         if form.is_valid():
             email = form.cleaned_data["email"]
             user, created = get_user_model().objects.get_or_create(
                     email__iexact=email,
-                    defaults=dict(username=email, email=email))
+                    defaults={"username": email, "email": email})
 
             if created:
                 user.set_unusable_password()
+                user.status = user_status.unconfirmed
 
-            user.status = user_status.unconfirmed,
             user.sign_in_key = make_sign_in_key(user)
             user.save()
 
-            from django.template.loader import render_to_string
-            message = render_to_string("course/sign-in-email.txt", {
+            from relate.utils import render_email_template
+            message = render_email_template("course/sign-in-email.txt", {
                 "user": user,
                 "sign_in_uri": request.build_absolute_uri(
                     reverse(
@@ -731,11 +814,12 @@ def sign_in_by_email(request):
                 })
             from django.core.mail import EmailMessage
             msg = EmailMessage(
-                    _("Your %(RELATE)s sign-in link") % {"RELATE": _("RELATE")},
+                    _("Your %(relate_site_name)s sign-in link")
+                    % {"relate_site_name": _(get_site_name())},
                     message,
                     getattr(settings, "NO_REPLY_EMAIL_FROM",
                             settings.ROBOT_EMAIL_FROM),
-                    [email])
+                    [user.email])
 
             from relate.utils import get_outbound_mail_connection
             msg.connection = (
@@ -757,9 +841,7 @@ def sign_in_by_email(request):
         })
 
 
-@user_passes_test(
-        lambda user: not user.username,
-        login_url='relate-logout-confirmation')
+@logout_confirmation_required
 def sign_in_stage2_with_token(request, user_id, sign_in_key):
     if not settings.RELATE_SIGN_IN_BY_EMAIL_ENABLED:
         messages.add_message(request, messages.ERROR,
@@ -769,9 +851,13 @@ def sign_in_stage2_with_token(request, user_id, sign_in_key):
     from django.contrib.auth import authenticate, login
     user = authenticate(user_id=int(user_id), token=sign_in_key)
     if user is None:
-        messages.add_message(request, messages.ERROR,
-                _("Invalid sign-in token. Perhaps you've used an old "
-                "token email?"))
+        if not get_user_model().objects.filter(pk=int(user_id)).count():
+            messages.add_message(request, messages.ERROR,
+                _("Account does not exist."))
+        else:
+            messages.add_message(request, messages.ERROR,
+                    _("Invalid sign-in token. Perhaps you've used an old "
+                    "token email?"))
         raise PermissionDenied(_("invalid sign-in token"))
 
     if not user.is_active:
@@ -799,8 +885,9 @@ def sign_in_stage2_with_token(request, user_id, sign_in_key):
 
 # {{{ user profile
 
-EDITABLE_INST_ID_BEFORE_VERIFICATION = \
-        settings.RELATE_EDITABLE_INST_ID_BEFORE_VERIFICATION
+def is_inst_id_editable_before_validation() -> bool:
+    return getattr(
+        settings, "RELATE_EDITABLE_INST_ID_BEFORE_VERIFICATION", True)
 
 
 class UserForm(StyledModelForm):
@@ -808,28 +895,28 @@ class UserForm(StyledModelForm):
             max_length=100,
             label=_("Institutional ID Confirmation"),
             required=False)
-    no_institutional_id = forms.BooleanField(
-            label=_("I have no Institutional ID"),
-            help_text=_("Check the checkbox if you are not a student "
-                        "or you forget your institutional id."),
-            required=False,
-            initial=False)
 
     class Meta:
         model = get_user_model()
-        fields = ("first_name", "last_name", "institutional_id",
+        fields = ("first_name", "last_name", "email", "institutional_id",
                 "editor_mode")
 
     def __init__(self, *args, **kwargs):
-        self.is_inst_id_locked = is_inst_id_locked =\
-                kwargs.pop('is_inst_id_locked')
-        super(UserForm, self).__init__(*args, **kwargs)
-
-        self.helper.layout = Layout(
-                Div("last_name", "first_name", css_class="well"),
-                Div("institutional_id", css_class="well"),
-                Div("editor_mode", css_class="well")
-                )
+        self.is_inst_id_locked = kwargs.pop("is_inst_id_locked")
+        super().__init__(*args, **kwargs)
+
+        if self.instance.name_verified:
+            self.fields["first_name"].disabled = True
+            self.fields["last_name"].disabled = True
+
+        self.fields["email"].disabled = True
+
+        if self.is_inst_id_locked:
+            self.fields["institutional_id"].disabled = True
+            self.fields["institutional_id_confirm"].disabled = True
+        else:
+            self.fields["institutional_id_confirm"].initial = (
+                self.instance.institutional_id)
 
         self.fields["institutional_id"].help_text = (
                 _("The unique ID your university or school provided, "
@@ -838,84 +925,73 @@ class UserForm(StyledModelForm):
                     "<b>Once %(submitted_or_verified)s, it cannot be "
                     "changed</b>.")
                 % {"submitted_or_verified":
-                    EDITABLE_INST_ID_BEFORE_VERIFICATION
-                    and _("verified") or _("submitted")})
-
-        def adjust_layout(is_inst_id_locked):
-            if not is_inst_id_locked:
-                self.helper.layout[1].insert(1, "institutional_id_confirm")
-                self.helper.layout[1].insert(0, "no_institutional_id")
-                self.fields["institutional_id_confirm"].initial = \
-                        self.instance.institutional_id
-            else:
-                self.fields["institutional_id"].widget.\
-                        attrs['disabled'] = True
-
-        if self.instance.name_verified:
-            self.fields["first_name"].widget.attrs['disabled'] = True
-            self.fields["last_name"].widget.attrs['disabled'] = True
+                   (is_inst_id_editable_before_validation()
+                   and _("verified")) or _("submitted")})
 
-        adjust_layout(is_inst_id_locked)
+        # {{{ build layout
 
-        self.helper.add_input(
-                Submit("submit_user", _("Update")))
+        name_fields_layout = ["last_name", "first_name", "email"]
+        fields_layout = [Div(*name_fields_layout, css_class="well")]
 
-    def clean_institutional_id(self):
-        inst_id = self.cleaned_data['institutional_id'].strip()
-        if self.is_inst_id_locked:
-            # Disabled fields are not part of form submit--so simply
-            # assume old value. At the same time, prevent smuggled-in
-            # POST parameters.
-            return self.instance.institutional_id
+        if getattr(settings, "RELATE_SHOW_INST_ID_FORM", True):
+            inst_field_group_layout = ["institutional_id"]
+            if not self.is_inst_id_locked:
+                inst_field_group_layout.append("institutional_id_confirm")
+            fields_layout.append(Div(*inst_field_group_layout, css_class="well",
+                                     css_id="institutional_id_block"))
         else:
-            return inst_id
+            # This is needed for django-crispy-form version < 1.7
+            self.fields["institutional_id"].widget = forms.HiddenInput()
 
-    def clean_first_name(self):
-        first_name = self.cleaned_data['first_name']
-        if self.instance.name_verified:
-            # Disabled fields are not part of form submit--so simply
-            # assume old value. At the same time, prevent smuggled-in
-            # POST parameters.
-            return self.instance.first_name
+        if getattr(settings, "RELATE_SHOW_EDITOR_FORM", True):
+            fields_layout.append(Div("editor_mode", css_class="well"))
         else:
-            return first_name
+            # This is needed for django-crispy-form version < 1.7
+            self.fields["editor_mode"].widget = forms.HiddenInput()
 
-    def clean_last_name(self):
-        last_name = self.cleaned_data['last_name']
-        if self.instance.name_verified:
-            # Disabled fields are not part of form submit--so simply
-            # assume old value. At the same time, prevent smuggled-in
-            # POST parameters.
-            return self.instance.last_name
-        else:
-            return last_name
+        self.helper.layout = Layout(*fields_layout)
+
+        self.helper.add_input(
+                Submit("submit_user", _("Update")))
+
+        self.helper.add_input(
+                Button("signout", _("Sign out"), css_class="btn btn-danger",
+                       onclick=(
+                           "window.location.href='{}'".format(reverse("relate-logout")))))
+
+        # }}}
 
     def clean_institutional_id_confirm(self):
-        inst_id_confirmed = self.cleaned_data.get(
-                "institutional_id_confirm")
+        inst_id_confirmed = self.cleaned_data.get("institutional_id_confirm")
 
         if not self.is_inst_id_locked:
             inst_id = self.cleaned_data.get("institutional_id")
             if inst_id and not inst_id_confirmed:
                 raise forms.ValidationError(_("This field is required."))
-            if not inst_id == inst_id_confirmed:
+            if any([inst_id, inst_id_confirmed]) and inst_id != inst_id_confirmed:
                 raise forms.ValidationError(_("Inputs do not match."))
         return inst_id_confirmed
 
+# }}}
 
-def user_profile(request):
-    if not request.user.is_authenticated:
-        raise PermissionDenied()
 
+@login_required
+def user_profile(request):
     user_form = None
 
     def is_inst_id_locked(user):
-        if EDITABLE_INST_ID_BEFORE_VERIFICATION:
+        if is_inst_id_editable_before_validation():
             return True if (user.institutional_id
                     and user.institutional_id_verified) else False
         else:
             return True if user.institutional_id else False
 
+    def is_requesting_inst_id():
+        return not is_inst_id_locked(request.user) and (
+            request.GET.get("first_login")
+            or (request.GET.get("set_inst_id")
+                and request.GET.get("referer")))
+
     if request.method == "POST":
         if "submit_user" in request.POST:
             user_form = UserForm(
@@ -924,110 +1000,133 @@ def user_profile(request):
                     is_inst_id_locked=is_inst_id_locked(request.user),
             )
             if user_form.is_valid():
-                user_form.save()
+                if user_form.has_changed():
+                    user_form.save()
+                    messages.add_message(request, messages.SUCCESS,
+                            _("Profile data updated."))
+                    request.user.refresh_from_db()
+
+                else:
+                    messages.add_message(request, messages.INFO,
+                            _("No change was made on your profile."))
 
-                messages.add_message(request, messages.INFO,
-                        _("Profile data saved."))
                 if request.GET.get("first_login"):
                     return redirect("relate-home")
+
                 if (request.GET.get("set_inst_id")
-                        and request.GET["referer"]):
+                        and request.GET.get("referer")):
                     return redirect(request.GET["referer"])
 
                 user_form = UserForm(
-                        instance=request.user,
-                        is_inst_id_locked=is_inst_id_locked(request.user))
-
-    if user_form is None:
-            user_form = UserForm(
                     instance=request.user,
                     is_inst_id_locked=is_inst_id_locked(request.user),
-            )
+                )
+
+    if user_form is None:
+        request.user.refresh_from_db()
+        user_form = UserForm(
+            instance=request.user,
+            is_inst_id_locked=is_inst_id_locked(request.user),
+        )
 
     return render(request, "user-profile-form.html", {
-        "is_inst_id_locked": is_inst_id_locked(request.user),
-        "enable_inst_id_if_not_locked": (
-            request.GET.get("first_login")
-            or (request.GET.get("set_inst_id")
-                and request.GET["referer"])
-            ),
-        "user_form": user_form,
+        "form": user_form,
+        "form_description": _("User Profile"),
+        "is_requesting_inst_id": is_requesting_inst_id(),
+        "enable_profile_form_js": (
+            not is_inst_id_locked(request.user)
+            and getattr(settings, "RELATE_SHOW_INST_ID_FORM", True))
         })
 
 # }}}
 
 
-# {{{ manage auth token
+# {{{ SAML auth backend
 
-class AuthenticationTokenForm(StyledForm):
-    def __init__(self, *args, **kwargs):
-        # type: (*Any, **Any) -> None
-        super(AuthenticationTokenForm, self).__init__(*args, **kwargs)
+# This ticks the 'verified' boxes once we've receive attribute assertions
+# through SAML2.
 
-        self.helper.add_input(Submit("reset", _("Reset")))
+class RelateSaml2Backend(Saml2Backend):
+    def get_or_create_user(self,
+            user_lookup_key, user_lookup_value, create_unknown_user,
+            idp_entityid, attributes, attribute_mapping, request):
+        user, created = super().get_or_create_user(
+                user_lookup_key, user_lookup_value, create_unknown_user,
+                idp_entityid, attributes, attribute_mapping, request)
 
+        user = self._rl_update_user(user, attributes, attribute_mapping)
 
-def manage_authentication_token(request):
-    # type: (http.HttpRequest) -> http.HttpResponse
-    if not request.user.is_authenticated:
-        raise PermissionDenied()
+        return user, created
 
-    if request.method == 'POST':
-        form = AuthenticationTokenForm(request.POST)
-        if form.is_valid():
-            token = make_sign_in_key(request.user)
-            from django.contrib.auth.hashers import make_password
-            request.user.git_auth_token_hash = make_password(token)
-            request.user.save()
+    def _rl_update_user(self, user, attributes, attribute_mapping):
+        mod = False
 
-            messages.add_message(request, messages.SUCCESS,
-                    _("A new authentication token has been set: %s.")
-                    % token)
+        mapped_attributes = {
+                mapped_key: val
+                for key, val in attributes.items()
+                for mapped_key in attribute_mapping.get(key, ())}
 
-    else:
-        if request.user.git_auth_token_hash is not None:
-            messages.add_message(request, messages.INFO,
-                    _("An authentication token has previously been set."))
-        else:
-            messages.add_message(request, messages.INFO,
-                    _("No authentication token has previously been set."))
+        if "institutional_id" in mapped_attributes:
+            if not user.institutional_id_verified:
+                user.institutional_id_verified = True
+                mod = True
 
-        form = AuthenticationTokenForm()
+        if "first_name" in mapped_attributes and "last_name" in mapped_attributes:
+            if not user.name_verified:
+                user.name_verified = True
+                mod = True
 
-    return render(request, "generic-form.html", {
-        "form_description": _("Manage Git Authentication Token"),
-        "form": form
-        })
+        if "email" in mapped_attributes:
+            from course.constants import user_status
+            if user.status != user_status.active:
+                user.status = user_status.active
+                mod = True
+
+        if mod:
+            user.save()
+
+        return user
 
 # }}}
 
 
-# {{{ SAML auth backend
+# {{{ social auth
 
-# This ticks the 'verified' boxes once we've receive attribute assertions
-# through SAML2.
+def social_set_user_email_verified(backend, details, user=None, *args, **kwargs):
+    email = details.get("email")
 
-class Saml2Backend(Saml2BackendBase):
-    def _set_attribute(self, obj, attr, value):
-        mod = super(Saml2Backend, self)._set_attribute(obj, attr, value)
+    modified = False
 
-        if attr == "institutional_id":
-            if not obj.institutional_id_verified:
-                obj.institutional_id_verified = True
-                mod = True
+    if email:
+        if email != user.email:
+            user.email = email
+            modified = True
 
-        if attr in ["first_name", "last_name"]:
-            if not obj.name_verified:
-                obj.name_verified = True
-                mod = True
+        from course.constants import user_status
+        if user.status != user_status.active:
+            user.status = user_status.active
+            modified = True
 
-        if attr == "email":
-            from course.constants import user_status
-            if obj.status != user_status.active:
-                obj.status = user_status.active
-                mod = True
+    if modified:
+        user.save()
 
-        return mod
+    # continue the social auth pipeline
+    return None
+
+
+def social_auth_check_domain_against_blacklist(backend, details, *args, **kwargs):
+    email = details.get("email")
+
+    domain_blacklist = getattr(
+            settings, "RELATE_SOCIAL_AUTH_BLACKLIST_EMAIL_DOMAINS", {})
+    if domain_blacklist and email:
+        domain = email.split("@", 1)[1]
+        if domain in domain_blacklist:
+            from social_core.exceptions import SocialAuthBaseException
+            raise SocialAuthBaseException(domain_blacklist[domain])
+
+    # continue the social auth pipeline
+    return None
 
 # }}}
 
@@ -1035,12 +1134,17 @@ class Saml2Backend(Saml2BackendBase):
 # {{{ sign-out
 
 def sign_out_confirmation(request, redirect_field_name=REDIRECT_FIELD_NAME):
+    if not request.user.is_authenticated:
+        messages.add_message(request, messages.ERROR,
+                             _("You've already signed out."))
+        return redirect("relate-home")
+
     redirect_to = request.POST.get(redirect_field_name,
-                                   request.GET.get(redirect_field_name, ''))
+                                   request.GET.get(redirect_field_name, ""))
 
     next_uri = ""
     if redirect_to:
-        next_uri = "?%s=%s" % (redirect_field_name, redirect_to)
+        next_uri = f"?{redirect_field_name}={redirect_to}"
 
     return render(request, "sign-out-confirmation.html",
                   {"next_uri": next_uri})
@@ -1048,15 +1152,20 @@ def sign_out_confirmation(request, redirect_field_name=REDIRECT_FIELD_NAME):
 
 @never_cache
 def sign_out(request, redirect_field_name=REDIRECT_FIELD_NAME):
+    if not request.user.is_authenticated:
+        messages.add_message(request, messages.ERROR,
+                             _("You've already signed out."))
+        return redirect("relate-home")
 
     redirect_to = request.POST.get(redirect_field_name,
-                                   request.GET.get(redirect_field_name, ''))
+                                   request.GET.get(redirect_field_name, ""))
     response = None
 
     if settings.RELATE_SIGN_IN_BY_SAML2_ENABLED:
-        from djangosaml2.views import _get_subject_id, logout as saml2_logout
+        from djangosaml2.views import _get_subject_id
         if _get_subject_id(request.session) is not None:
-            response = saml2_logout(request)
+            # skip auth_logout below, rely on djangosaml2 to complete logout
+            return redirect("saml2_logout")
 
     auth_logout(request)
 
@@ -1069,4 +1178,322 @@ def sign_out(request, redirect_field_name=REDIRECT_FIELD_NAME):
 
 # }}}
 
+
+# {{{ API auth
+
+class APIError(Exception):
+    pass
+
+
+def find_matching_token(
+        course_identifier: str | None = None,
+        token_id: int | None = None,
+        token_hash_str: str | None = None,
+        now_datetime: datetime.datetime | None = None
+        ) -> AuthenticationToken | None:
+    if token_id is None:
+        return None
+
+    try:
+        token = AuthenticationToken.objects.get(
+                id=token_id,
+                participation__course__identifier=course_identifier)
+    except AuthenticationToken.DoesNotExist:
+        return None
+
+    if token.token_hash is None:
+        return None
+
+    from django.contrib.auth.hashers import check_password
+    if not check_password(token_hash_str, token.token_hash):
+        return None
+
+    if token.revocation_time is not None:
+        return None
+    if token.valid_until is not None:
+        if now_datetime is None:
+            return None
+        if now_datetime > token.valid_until:
+            return None
+
+    return token
+
+
+class APIBearerTokenBackend:
+    def authenticate(self, request, course_identifier=None, token_id=None,
+            token_hash_str=None, now_datetime=None):
+        token = find_matching_token(course_identifier, token_id, token_hash_str,
+                now_datetime)
+        if token is None:
+            return None
+
+        token.last_use_time = now_datetime
+        token.save()
+
+        return token.user
+
+    def get_user(self, user_id):
+        try:
+            return get_user_model().objects.get(pk=user_id)
+        except get_user_model().DoesNotExist:
+            return None
+
+
+class APIContext:
+    def __init__(self, request, token):
+        self.request = request
+
+        self.token = token
+        self.participation = token.participation
+        self.course = self.participation.course
+
+        restrict_to_role = token.restrict_to_participation_role
+        if restrict_to_role is not None:
+            role_restriction_ok = False
+
+            if restrict_to_role in token.participation.roles.all():
+                role_restriction_ok = True
+
+            if not role_restriction_ok and self.participation.has_permission(
+                    pperm.impersonate_role, restrict_to_role.identifier):
+                role_restriction_ok = True
+
+            if not role_restriction_ok:
+                raise PermissionDenied(
+                        "API token specifies invalid role restriction")
+
+        self.restrict_to_role = restrict_to_role
+
+    def has_permission(self, perm: str, argument: str | None = None) -> bool:
+        if self.restrict_to_role is None:
+            return self.participation.has_permission(perm, argument)
+        else:
+            return self.restrict_to_role.has_permission(perm, argument)
+
+
+TOKEN_AUTH_DATA_RE = re.compile(r"^(?P<token_id>[0-9]+)_(?P<token_hash>[a-z0-9]+)$")
+BASIC_AUTH_DATA_RE = re.compile(
+    r"^(?P<username>\w+):(?P<token_id>[0-9]+)_(?P<token_hash>[a-z0-9]+)$")
+
+
+def auth_course_with_token(method, func, request,
+        course_identifier, *args, **kwargs):
+
+    from django.utils.timezone import now
+    now_datetime = now()
+
+    try:
+        auth_header = request.headers.get("authorization", None)
+        if auth_header is None:
+            raise PermissionDenied("No Authorization header provided")
+
+        auth_values = auth_header.split(" ")
+        if len(auth_values) != 2:
+            raise PermissionDenied("ill-formed Authorization header")
+
+        auth_method, auth_data = auth_values
+        if auth_method != method:
+            raise PermissionDenied("ill-formed Authorization header")
+
+        if method == "Token":
+            match = TOKEN_AUTH_DATA_RE.match(auth_data)
+
+        elif method == "Basic":
+            import binascii
+            from base64 import b64decode
+            try:
+                auth_data = b64decode(auth_data.strip()).decode(
+                        "utf-8", errors="replace")
+            except binascii.Error:
+                raise PermissionDenied("ill-formed Authorization header")
+            match = BASIC_AUTH_DATA_RE.match(auth_data)
+
+        else:
+            raise AssertionError()
+
+        if match is None:
+            raise PermissionDenied("invalid authentication token")
+
+        token_id = int(match.group("token_id"))
+        token_hash_str = match.group("token_hash")
+
+        auth_data_dict = {
+                "course_identifier": course_identifier,
+                "token_id": token_id,
+                "token_hash_str": token_hash_str,
+                "now_datetime": now_datetime}
+
+        # FIXME: Redundant db roundtrip
+        token = find_matching_token(**auth_data_dict)
+        if token is None:
+            raise PermissionDenied("invalid authentication token")
+
+        from django.contrib.auth import authenticate, login
+        user = authenticate(**auth_data_dict)
+
+        assert user is not None
+
+        if method == "Basic" and match.group("username") != user.username:
+            raise PermissionDenied("invalid authentication token")
+
+        login(request, user)
+
+        response = func(
+                APIContext(request, token),
+                course_identifier, *args, **kwargs)
+
+    except PermissionDenied as e:
+        if method == "Basic":
+            realm = _(f"Relate direct git access for {course_identifier}")
+            response = http.HttpResponse("Forbidden: " + str(e),
+                        content_type="text/plain")
+            response["WWW-Authenticate"] = f'Basic realm="{realm}"'
+            response.status_code = 401
+            return response
+
+        elif method == "Token":
+            return http.HttpResponseForbidden(
+                    "403 Forbidden: " + str(e))
+
+        else:
+            raise AssertionError()
+
+    except APIError as e:
+        return http.HttpResponseBadRequest(
+                "400 Bad Request: " + str(e))
+
+    return response
+
+
+def with_course_api_auth(method: str) -> Any:
+    def wrapper_with_method(func):
+        def wrapper(*args, **kwargs):
+            return auth_course_with_token(method, func, *args, **kwargs)
+
+        from functools import update_wrapper
+        update_wrapper(wrapper, func)
+
+        return wrapper
+    return wrapper_with_method
+
+# }}}
+
+
+# {{{ manage API auth tokens
+
+class AuthenticationTokenForm(StyledModelForm):
+    class Meta:
+        model = AuthenticationToken
+        fields = (
+                "restrict_to_participation_role",
+                "description",
+                "valid_until",
+                )
+
+        widgets = {
+                "valid_until": HTML5DateTimeInput()
+                }
+
+    def __init__(
+            self, participation: Participation, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.participation = participation
+
+        allowable_role_ids = (
+                {role.id for role in participation.roles.all()}
+                | {
+                        prole.id
+                        for prole in ParticipationRole.objects.filter(
+                            course=participation.course)
+                        if participation.has_permission(
+                            pperm.impersonate_role, prole.identifier)}
+                )
+
+        self.fields["restrict_to_participation_role"].queryset = (  # type:ignore[attr-defined]
+                ParticipationRole.objects.filter(
+                    id__in=list(allowable_role_ids)
+                    ))
+
+        self.helper.add_input(Submit("create", _("Create")))
+
+
+@course_view
+def manage_authentication_tokens(pctx: CoursePageContext) -> http.HttpResponse:
+    request = pctx.request
+
+    if not request.user.is_authenticated:
+        raise PermissionDenied()
+
+    if not pctx.has_permission(pperm.view_analytics):
+        raise PermissionDenied()
+    assert pctx.participation is not None
+
+    from course.views import get_now_or_fake_time
+    now_datetime = get_now_or_fake_time(request)
+
+    if request.method == "POST":
+        form = AuthenticationTokenForm(pctx.participation, request.POST)
+
+        revoke_prefix = "revoke_"
+        revoke_post_args = [key for key in request.POST if key.startswith("revoke_")]
+
+        if revoke_post_args:
+            token_id = int(revoke_post_args[0][len(revoke_prefix):])
+
+            auth_token = get_object_or_404(AuthenticationToken,
+                    id=token_id,
+                    user=request.user)
+
+            auth_token.revocation_time = now_datetime
+            auth_token.save()
+
+            form = AuthenticationTokenForm(pctx.participation)
+
+        elif "create" in request.POST:
+            if form.is_valid():
+                token = make_sign_in_key(request.user)
+
+                from django.contrib.auth.hashers import make_password
+                auth_token = AuthenticationToken(
+                        user=request.user,
+                        participation=pctx.participation,
+                        restrict_to_participation_role=form.cleaned_data[
+                            "restrict_to_participation_role"],
+                        description=form.cleaned_data["description"],
+                        valid_until=form.cleaned_data["valid_until"],
+                        token_hash=make_password(token))
+                auth_token.save()
+
+                user_token = "%d_%s" % (auth_token.id, token)
+
+                messages.add_message(request, messages.SUCCESS,
+                        _("A new authentication token has been created: %s. "
+                            "Please save this token, as you will not be able "
+                            "to retrieve it later.")
+                        % user_token)
+        else:
+            messages.add_message(request, messages.ERROR,
+                    _("Could not find which button was pressed."))
+
+    else:
+        form = AuthenticationTokenForm(pctx.participation)
+
+    from datetime import timedelta
+
+    from django.db.models import Q
+    tokens = AuthenticationToken.objects.filter(
+            user=request.user,
+            participation__course=pctx.course,
+            ).filter(
+                Q(revocation_time=None)
+                | Q(revocation_time__gt=now_datetime - timedelta(weeks=1)))
+
+    return render_course_page(pctx, "course/manage-auth-tokens.html", {
+        "form": form,
+        "new_token_message": "",
+        "tokens": tokens,
+        })
+
+# }}}
+
 # vim: foldmethod=marker

course/calendar.py

-# -*- coding: utf-8 -*-
+from __future__ import annotations
 
-from __future__ import division
 
 __copyright__ = "Copyright (C) 2014 Andreas Kloeckner"
 
@@ -24,28 +23,44 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 """
 
-import six
-from six.moves import range
+import datetime
 
-from django.utils.translation import (
-        ugettext_lazy as _, pgettext_lazy, string_concat)
+import django.forms as forms
+from crispy_forms.layout import Submit
+from django.contrib import messages
 from django.contrib.auth.decorators import login_required
+from django.core.exceptions import (
+    ObjectDoesNotExist,
+    PermissionDenied,
+    ValidationError,
+)
+from django.db import transaction
+from django.utils.safestring import mark_safe
+from django.utils.translation import get_language, gettext_lazy as _, pgettext_lazy
+
+from course.constants import participation_permission as pperm
+from course.models import Event
 from course.utils import course_view, render_course_page
-from django.core.exceptions import PermissionDenied, ObjectDoesNotExist
-from django.db import transaction, IntegrityError
-from django.contrib import messages  # noqa
-import django.forms as forms
+from relate.utils import HTML5DateTimeInput, StyledForm, as_local_time, string_concat
 
-from crispy_forms.layout import Submit
 
-import datetime
-from bootstrap3_datetime.widgets import DateTimePicker
+class ListTextWidget(forms.TextInput):
+    # Widget which allow free text and choices for CharField
+    def __init__(self, data_list, name, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._name = name
+        self._list = data_list
+        self.attrs.update({"list": f"list__{self._name}"})
 
-from relate.utils import StyledForm, as_local_time
-from course.constants import (
-        participation_permission as pperm,
-        )
-from course.models import Event
+    def render(self, name, value, attrs=None, renderer=None):
+        text_html = super().render(
+            name, value, attrs=attrs, renderer=renderer)
+        data_list = f'<datalist id="list__{self._name}">'
+        for item in self._list:
+            data_list += f'<option value="{item[0]}">{item[1]}</option>'
+        data_list += "</datalist>"
+
+        return mark_safe(text_html + data_list)
 
 
 # {{{ creation
@@ -56,11 +71,21 @@ class RecurringEventForm(StyledForm):
                         "allowed."),
             label=pgettext_lazy("Kind of event", "Kind of event"))
     time = forms.DateTimeField(
-            widget=DateTimePicker(
-                options={"format": "YYYY-MM-DD HH:mm", "sideBySide": True}),
+            widget=HTML5DateTimeInput(),
             label=pgettext_lazy("Starting time of event", "Starting time"))
     duration_in_minutes = forms.FloatField(required=False,
+            min_value=0,
             label=_("Duration in minutes"))
+    all_day = forms.BooleanField(
+                required=False,
+                initial=False,
+                label=_("All-day event"),
+                help_text=_("Only affects the rendering in the class calendar, "
+                "in that a start time is not shown"))
+    shown_in_calendar = forms.BooleanField(
+            required=False,
+            initial=True,
+            label=_("Shown in calendar"))
     interval = forms.ChoiceField(required=True,
             choices=(
                 ("weekly", _("Weekly")),
@@ -71,10 +96,19 @@ class RecurringEventForm(StyledForm):
             label=pgettext_lazy(
                 "Starting ordinal of recurring events", "Starting ordinal"))
     count = forms.IntegerField(required=True,
+            min_value=0,
             label=pgettext_lazy("Count of recurring events", "Count"))
 
-    def __init__(self, *args, **kwargs):
-        super(RecurringEventForm, self).__init__(*args, **kwargs)
+    def __init__(self, course_identifier, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.course_identifier = course_identifier
+
+        exist_event_choices = [(choice, choice) for choice in set(
+            Event.objects.filter(
+                course__identifier=course_identifier)
+            .values_list("kind", flat=True))]
+        self.fields["kind"].widget = ListTextWidget(data_list=exist_event_choices,
+                                                    name="event_choices")
 
         self.helper.add_input(
                 Submit("submit", _("Create")))
@@ -86,27 +120,31 @@ class EventAlreadyExists(Exception):
 
 @transaction.atomic
 def _create_recurring_events_backend(course, time, kind, starting_ordinal, interval,
-        count, duration_in_minutes):
+        count, duration_in_minutes, all_day, shown_in_calendar):
     ordinal = starting_ordinal
+    assert ordinal is not None
 
     import datetime
 
-    for i in range(count):
+    for _i in range(count):
         evt = Event()
         evt.course = course
         evt.kind = kind
         evt.ordinal = ordinal
         evt.time = time
+        evt.all_day = all_day
+        evt.shown_in_calendar = shown_in_calendar
 
         if duration_in_minutes:
             evt.end_time = evt.time + datetime.timedelta(
                     minutes=duration_in_minutes)
-        try:
-            evt.save()
-        except IntegrityError:
+
+        if Event.objects.filter(course=course, kind=kind, ordinal=ordinal).count():
             raise EventAlreadyExists(
-                _("'%(event_kind)s %(event_ordinal)d' already exists") %
-                {'event_kind': kind, 'event_ordinal': ordinal})
+                _("'%(exist_event)s' already exists")
+                % {"exist_event": evt})
+
+        evt.save()
 
         date = time.date()
         if interval == "weekly":
@@ -114,17 +152,10 @@ def _create_recurring_events_backend(course, time, kind, starting_ordinal, inter
         elif interval == "biweekly":
             date += datetime.timedelta(weeks=2)
         else:
-            raise ValueError(
-                    string_concat(
-                        pgettext_lazy(
-                            "Unkown time interval",
-                            "unknown interval"),
-                        ": %s")
-                    % interval)
-
-        time = time.tzinfo.localize(
-                datetime.datetime(date.year, date.month, date.day,
-                    time.hour, time.minute, time.second))
+            raise NotImplementedError()
+
+        time = datetime.datetime(date.year, date.month, date.day,
+                    time.hour, time.minute, time.second, tzinfo=time.tzinfo)
         del date
 
         ordinal += 1
@@ -137,9 +168,12 @@ def create_recurring_events(pctx):
         raise PermissionDenied(_("may not edit events"))
 
     request = pctx.request
+    message = None
+    message_level = None
 
     if request.method == "POST":
-        form = RecurringEventForm(request.POST, request.FILES)
+        form = RecurringEventForm(
+            pctx.course.identifier, request.POST, request.FILES)
         if form.is_valid():
             if form.cleaned_data["starting_ordinal"] is not None:
                 starting_ordinal = form.cleaned_data["starting_ordinal"]
@@ -158,36 +192,53 @@ def create_recurring_events(pctx):
                             interval=form.cleaned_data["interval"],
                             count=form.cleaned_data["count"],
                             duration_in_minutes=(
-                                form.cleaned_data["duration_in_minutes"]))
+                                form.cleaned_data["duration_in_minutes"]),
+                            all_day=form.cleaned_data["all_day"],
+                            shown_in_calendar=(
+                                form.cleaned_data["shown_in_calendar"])
+                            )
+                    message = _("Events created.")
+                    message_level = messages.SUCCESS
                 except EventAlreadyExists as e:
                     if starting_ordinal_specified:
-                        messages.add_message(request, messages.ERROR,
+                        message = (
                                 string_concat(
                                     "%(err_type)s: %(err_str)s. ",
                                     _("No events created."))
                                 % {
                                     "err_type": type(e).__name__,
                                     "err_str": str(e)})
+                        message_level = messages.ERROR
                     else:
                         starting_ordinal += 10
                         continue
 
                 except Exception as e:
-                    messages.add_message(request, messages.ERROR,
-                            string_concat(
-                                "%(err_type)s: %(err_str)s. ",
-                                _("No events created."))
-                            % {
-                                "err_type": type(e).__name__,
-                                "err_str": str(e)})
-                else:
-                    messages.add_message(request, messages.SUCCESS,
-                            _("Events created."))
-
+                    if isinstance(e, ValidationError):
+                        for field, error in e.error_dict.items():
+                            try:
+                                form.add_error(field, error)
+                            except ValueError:
+                                # This happens when ValidationError were
+                                # raised for fields which don't exist in
+                                # RecurringEventForm
+                                form.add_error(
+                                    "__all__", f"'{field}': {error}")
+                    else:
+                        message = (
+                                string_concat(
+                                    "%(err_type)s: %(err_str)s. ",
+                                    _("No events created."))
+                                % {
+                                    "err_type": type(e).__name__,
+                                    "err_str": str(e)})
+                        message_level = messages.ERROR
                 break
     else:
-        form = RecurringEventForm()
+        form = RecurringEventForm(pctx.course.identifier)
 
+    if message and message_level:
+        messages.add_message(request, message_level, message)
     return render_course_page(pctx, "course/generic-course-form.html", {
         "form": form,
         "form_description": _("Create recurring events"),
@@ -195,16 +246,30 @@ def create_recurring_events(pctx):
 
 
 class RenumberEventsForm(StyledForm):
-    kind = forms.CharField(required=True,
+    kind = forms.ChoiceField(required=True,
             help_text=_("Should be lower_case_with_underscores, no spaces "
                         "allowed."),
             label=pgettext_lazy("Kind of event", "Kind of event"))
     starting_ordinal = forms.IntegerField(required=True, initial=1,
+            help_text=_("The starting ordinal of this kind of events"),
             label=pgettext_lazy(
                 "Starting ordinal of recurring events", "Starting ordinal"))
-
-    def __init__(self, *args, **kwargs):
-        super(RenumberEventsForm, self).__init__(*args, **kwargs)
+    preserve_ordinal_order = forms.BooleanField(
+            required=False,
+            initial=False,
+            help_text=_("Tick to preserve the order of ordinals of "
+                        "existing events."),
+            label=_("Preserve ordinal order"))
+
+    def __init__(self, course_identifier, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.course_identifier = course_identifier
+
+        renumberable_event_kinds = set(Event.objects.filter(
+            course__identifier=self.course_identifier,
+            ordinal__isnull=False).values_list("kind", flat=True))
+        self.fields["kind"].choices = tuple(
+            (kind, kind) for kind in renumberable_event_kinds)
 
         self.helper.add_input(
                 Submit("submit", _("Renumber")))
@@ -219,42 +284,58 @@ def renumber_events(pctx):
 
     request = pctx.request
 
+    message = None
+    message_level = None
+
     if request.method == "POST":
-        form = RenumberEventsForm(request.POST, request.FILES)
+        form = RenumberEventsForm(
+            pctx.course.identifier, request.POST, request.FILES)
         if form.is_valid():
-            events = list(Event.objects
-                    .filter(course=pctx.course, kind=form.cleaned_data["kind"])
-                    .order_by('time'))
-
-            if events:
-                queryset = (Event.objects
-                    .filter(course=pctx.course, kind=form.cleaned_data["kind"]))
-
-                queryset.delete()
-
-                ordinal = form.cleaned_data["starting_ordinal"]
-                for event in events:
-                    new_event = Event()
-                    new_event.course = pctx.course
-                    new_event.kind = form.cleaned_data["kind"]
-                    new_event.ordinal = ordinal
-                    new_event.time = event.time
-                    new_event.end_time = event.end_time
-                    new_event.all_day = event.all_day
-                    new_event.shown_in_calendar = event.shown_in_calendar
-                    new_event.save()
-
-                    ordinal += 1
-
-                messages.add_message(request, messages.SUCCESS,
-                        _("Events renumbered."))
-            else:
-                messages.add_message(request, messages.ERROR,
-                        _("No events found."))
+            kind = form.cleaned_data["kind"]
+            order_field = "time"
+            if form.cleaned_data["preserve_ordinal_order"]:
+                order_field = "ordinal"
+            events = list(
+                Event.objects.filter(
+                    course=pctx.course, kind=kind,
+
+                    # there might be event with the same kind but no ordinal,
+                    # we don't renumber that
+                    ordinal__isnull=False)
+                .order_by(order_field))
+
+            assert events
+            queryset = (Event.objects.filter(
+                course=pctx.course, kind=kind,
+
+                # there might be event with the same kind but no ordinal,
+                # we don't renumber that
+                ordinal__isnull=False))
+
+            queryset.delete()
+
+            ordinal = form.cleaned_data["starting_ordinal"]
+            for event in events:
+                new_event = Event()
+                new_event.course = pctx.course
+                new_event.kind = kind
+                new_event.ordinal = ordinal
+                new_event.time = event.time
+                new_event.end_time = event.end_time
+                new_event.all_day = event.all_day
+                new_event.shown_in_calendar = event.shown_in_calendar
+                new_event.save()
+
+                ordinal += 1
+
+            message = _("Events renumbered.")
+            message_level = messages.SUCCESS
 
     else:
-        form = RenumberEventsForm()
+        form = RenumberEventsForm(pctx.course.identifier)
 
+    if messages and message_level:
+        messages.add_message(request, message_level, message)
     return render_course_page(pctx, "course/generic-course-form.html", {
         "form": form,
         "form_description": _("Renumber events"),
@@ -265,7 +346,7 @@ def renumber_events(pctx):
 
 # {{{ calendar
 
-class EventInfo(object):
+class EventInfo:
     def __init__(self, id, human_title, start_time, end_time, description):
         self.id = id
         self.human_title = human_title
@@ -274,19 +355,34 @@ class EventInfo(object):
         self.description = description
 
 
+def _fullcalendar_lang_code() -> str:
+    """
+    Return the fallback lang name for js files.
+    """
+
+    lang_name = get_language()
+    known_fallback_mapping = {
+        "zh-hans": "zh-cn",
+        "zh-hant": "zh-tw"}
+    return known_fallback_mapping.get(lang_name.lower(), lang_name).lower()
+
+
 @course_view
 def view_calendar(pctx):
-    from course.content import markup_to_html, parse_date_spec
+    if not pctx.has_permission(pperm.view_calendar):
+        raise PermissionDenied(_("may not view calendar"))
 
+    # must import locally for mock to work
     from course.views import get_now_or_fake_time
     now = get_now_or_fake_time(pctx.request)
 
-    if not pctx.has_permission(pperm.view_calendar):
-        raise PermissionDenied(_("may not view calendar"))
-
     events_json = []
 
-    from course.content import get_raw_yaml_from_repo
+    from course.content import (
+        get_raw_yaml_from_repo,
+        markup_to_html,
+        parse_date_spec,
+    )
     try:
         event_descr = get_raw_yaml_from_repo(pctx.repo,
                 pctx.course.events_file, pctx.course_commit_sha)
@@ -298,14 +394,19 @@ def view_calendar(pctx):
 
     event_info_list = []
 
-    for event in (Event.objects
+    events = sorted(
+            Event.objects
             .filter(
                 course=pctx.course,
-                shown_in_calendar=True)
-            .order_by("-time")):
+                shown_in_calendar=True),
+            key=lambda evt: (
+                -evt.time.year, -evt.time.month, -evt.time.day,
+                evt.time.hour, evt.time.minute, evt.time.second))
+
+    for event in events:
         kind_desc = event_kinds_desc.get(event.kind)
 
-        human_title = six.text_type(event)
+        human_title = str(event)
 
         event_json = {
                 "id": event.id,
@@ -322,11 +423,11 @@ def view_calendar(pctx):
                 if event.ordinal is not None:
                     human_title = kind_desc["title"].format(nr=event.ordinal)
                 else:
-                    human_title = kind_desc["title"]
+                    human_title = kind_desc["title"].rstrip("{nr}").strip()
 
         description = None
         show_description = True
-        event_desc = event_info_desc.get(six.text_type(event))
+        event_desc = event_info_desc.get(str(event))
         if event_desc is not None:
             if "description" in event_desc:
                 description = markup_to_html(
@@ -361,12 +462,13 @@ def view_calendar(pctx):
 
             if event.all_day:
                 start_time = start_time.date()
-                local_end_time = as_local_time(end_time)
-                end_midnight = datetime.time(tzinfo=local_end_time.tzinfo)
-                if local_end_time.time() == end_midnight:
-                    end_time = (end_time - datetime.timedelta(days=1)).date()
-                else:
-                    end_time = end_time.date()
+                if end_time is not None:
+                    local_end_time = as_local_time(end_time)
+                    end_midnight = datetime.time(tzinfo=local_end_time.tzinfo)
+                    if local_end_time.time() == end_midnight:
+                        end_time = (end_time - datetime.timedelta(days=1)).date()
+                    else:
+                        end_time = end_time.date()
 
             event_info_list.append(
                     EventInfo(
@@ -388,6 +490,7 @@ def view_calendar(pctx):
         "events_json": dumps(events_json),
         "event_info_list": event_info_list,
         "default_date": default_date.isoformat(),
+        "fullcalendar_lang_code": _fullcalendar_lang_code()
     })
 
 # }}}

course/constants.py

-# -*- coding: utf-8 -*-
+from __future__ import annotations
 
-from __future__ import division, unicode_literals
 
 __copyright__ = "Copyright (C) 2014 Andreas Kloeckner"
 
@@ -25,22 +24,28 @@ THE SOFTWARE.
 """
 
 
-from django.utils.translation import pgettext_lazy, ugettext
+from django.utils.translation import gettext, pgettext_lazy
+
+
 # Allow 10x extra credit at the very most.
 MAX_EXTRA_CREDIT_FACTOR = 10
+DEFAULT_EMAIL_APPELLATION_PRIORITY_LIST = [
+    "first_name", "email", "username", "full_name"]
 
 
 COURSE_ID_REGEX = "(?P<course_identifier>[-a-zA-Z0-9]+)"
+EVENT_KIND_REGEX = "(?P<event_kind>[_a-z0-9]+)"
 FLOW_ID_REGEX = "(?P<flow_id>[-_a-zA-Z0-9]+)"
 GRADING_OPP_ID_REGEX = "(?P<grading_opp_id>[-_a-zA-Z0-9]+)"
 # FIXME : Support page hierarchy. Add '/' here, fix validation code.
-STATICPAGE_PATH_REGEX = "(?P<page_path>[-\w]+)"
+STATICPAGE_PATH_REGEX = r"(?P<page_path>[-\w]+)"
 
 
 class user_status:  # noqa
     unconfirmed = "unconfirmed"
     active = "active"
 
+
 USER_STATUS_CHOICES = (
         (user_status.unconfirmed, pgettext_lazy("User status", "Unconfirmed")),
         (user_status.active, pgettext_lazy("User status", "Active")),
@@ -72,14 +77,13 @@ PARTICIPATION_STATUS_CHOICES = (
 
 # {{{ participation permission
 
-class participation_permission:
+class participation_permission:  # noqa
     edit_course = "edit_course"
     use_admin_interface = "use_admin_interface"
+    manage_authentication_tokens = "manage_authentication_tokens"
 
     impersonate_role = "impersonate_role"
-    # FIXME: Not yet used
     set_fake_time = "set_fake_time"
-    # FIXME: Not yet used
     set_pretend_facility = "set_pretend_facility"
 
     edit_course_permissions = "edit_course_permissions"
@@ -88,11 +92,13 @@ class participation_permission:
     send_instant_message = "send_instant_message"
     access_files_for = "access_files_for"
     included_in_grade_statistics = "included_in_grade_statistics"
+    skip_during_manual_grading = "skip_during_manual_grading"
 
     edit_exam = "edit_exam"
     issue_exam_ticket = "issue_exam_ticket"
     batch_issue_exam_ticket = "batch_issue_exam_ticket"
 
+    view_participant_masked_profile = "view_participant_masked_profile"
     view_flow_sessions_from_role = "view_flow_sessions_from_role"
     view_gradebook = "view_gradebook"
     edit_grading_opportunity = "edit_grading_opportunity"
@@ -117,6 +123,7 @@ class participation_permission:
 
     preview_content = "preview_content"
     update_content = "update_content"
+    use_git_endpoint = "use_git_endpoint"
     use_markup_sandbox = "use_markup_sandbox"
     use_page_sandbox = "use_page_sandbox"
     test_flow = "test_flow"
@@ -135,6 +142,10 @@ PARTICIPATION_PERMISSION_CHOICES = (
             pgettext_lazy("Participation permission", "Edit course")),
         (participation_permission.use_admin_interface,
             pgettext_lazy("Participation permission", "Use admin interface")),
+        (participation_permission.manage_authentication_tokens,
+            pgettext_lazy("Participation permission",
+                "Manage authentication tokens")),
+
         (participation_permission.impersonate_role,
             pgettext_lazy("Participation permission", "Impersonate role")),
         (participation_permission.set_fake_time,
@@ -154,6 +165,9 @@ PARTICIPATION_PERMISSION_CHOICES = (
         (participation_permission.included_in_grade_statistics,
             pgettext_lazy("Participation permission",
                 "Included in grade statistics")),
+        (participation_permission.skip_during_manual_grading,
+            pgettext_lazy("Participation permission",
+                "Skip during manual grading")),
 
         (participation_permission.edit_exam,
             pgettext_lazy("Participation permission", "Edit exam")),
@@ -162,9 +176,12 @@ PARTICIPATION_PERMISSION_CHOICES = (
         (participation_permission.batch_issue_exam_ticket,
             pgettext_lazy("Participation permission", "Batch issue exam ticket")),
 
+        (participation_permission.view_participant_masked_profile,
+            pgettext_lazy("Participation permission",
+                "View participants' masked profile only")),
         (participation_permission.view_flow_sessions_from_role,
             pgettext_lazy("Participation permission",
-                "View flow sessions from role ")),
+                "View flow sessions from role")),
         (participation_permission.view_gradebook,
             pgettext_lazy("Participation permission", "View gradebook")),
         (participation_permission.edit_grading_opportunity,
@@ -200,7 +217,7 @@ PARTICIPATION_PERMISSION_CHOICES = (
                 "Recalculate flow session grade")),
         (participation_permission.batch_recalculate_flow_session_grade,
             pgettext_lazy("Participation permission",
-                "Batch-recalculate flow sesssion grades")),
+                "Batch-recalculate flow session grades")),
 
         (participation_permission.reopen_flow_session,
             pgettext_lazy("Participation permission", "Reopen flow session")),
@@ -213,6 +230,8 @@ PARTICIPATION_PERMISSION_CHOICES = (
             pgettext_lazy("Participation permission", "Preview content")),
         (participation_permission.update_content,
             pgettext_lazy("Participation permission", "Update content")),
+        (participation_permission.use_git_endpoint,
+            pgettext_lazy("Participation permission", "Use direct git endpoint")),
         (participation_permission.use_markup_sandbox,
             pgettext_lazy("Participation permission", "Use markup sandbox")),
         (participation_permission.use_page_sandbox,
@@ -268,6 +287,7 @@ class flow_session_expiration_mode:  # noqa
     # allowed by special permission below
     roll_over = "roll_over"
 
+
 FLOW_SESSION_EXPIRATION_MODE_CHOICES = (
         (flow_session_expiration_mode.end,
             pgettext_lazy("Flow expiration mode", "Submit session for grading")),
@@ -277,8 +297,9 @@ FLOW_SESSION_EXPIRATION_MODE_CHOICES = (
         )
 
 
-def is_expiration_mode_allowed(expmode, permissions):
-    # type: (str, frozenset[str]) -> bool
+def is_expiration_mode_allowed(
+        expmode: str, permissions: frozenset[str]
+        ) -> bool:
     if expmode == flow_session_expiration_mode.roll_over:
         if (flow_permission.set_roll_over_expiration_mode
                 in permissions):
@@ -286,7 +307,7 @@ def is_expiration_mode_allowed(expmode, permissions):
     elif expmode == flow_session_expiration_mode.end:
         return True
     else:
-        raise ValueError(ugettext("unknown expiration mode"))
+        raise ValueError(gettext("unknown expiration mode"))
 
     return False
 
@@ -332,7 +353,7 @@ class flow_permission:  # noqa
     .. attribute:: see_answer_after_submission
 
         If present, shows the correct answer to the participant after they have
-        submitted an answer of their own.
+        submitted an answer of their own (and are no longer able to change it).
 
     .. attribute:: cannot_see_flow_result
 
@@ -354,13 +375,18 @@ class flow_permission:  # noqa
 
     .. attribute:: lock_down_as_exam_session
 
-        (Optional) Once any page of the flow has been viewed, access to all content
+        Once any page of the flow has been viewed, access to all content
         except for this session on this RELATE instance will be denied.
 
     .. attribute:: send_email_about_flow_page
 
-        (Optional) If present, the participant can send interaction emails to
-        course staffs for questions for each page with that permission.
+        If present, the participant can send interaction emails to
+        course staff for questions for each page with that permission.
+
+    .. attribute:: hide_point_count
+
+        If present, the point count for the page will not be shown to
+        the participant.
     """
     view = "view"
     end_session = "end_session"
@@ -374,6 +400,8 @@ class flow_permission:  # noqa
     see_session_time = "see_session_time"
     lock_down_as_exam_session = "lock_down_as_exam_session"
     send_email_about_flow_page = "send_email_about_flow_page"
+    hide_point_count = "hide_point_count"
+
 
 FLOW_PERMISSION_CHOICES = (
         (flow_permission.view,
@@ -406,6 +434,9 @@ FLOW_PERMISSION_CHOICES = (
         (flow_permission.send_email_about_flow_page,
          pgettext_lazy("Flow permission",
                        "Send emails about the flow page to course staff")),
+        (flow_permission.hide_point_count,
+         pgettext_lazy("Flow permission",
+                       "Hide point count")),
         )
 
 # }}}
@@ -533,5 +564,14 @@ EXAM_TICKET_STATE_CHOICES = (
 
 
 ATTRIBUTES_FILENAME = ".attributes.yml"
+DEFAULT_ACCESS_KINDS = ["public", "in_exam", "student", "ta",
+                        "unenrolled", "instructor"]
+
+
+# {{{ session attributes
+
+SESSION_LOCKED_TO_FLOW_PK = "relate_session_locked_to_exam_flow_session_pk"
+
+# }}}
 
 # vim: foldmethod=marker

course/content.py

-# -*- coding: utf-8 -*-
+from __future__ import annotations
 
-from __future__ import division
 
 __copyright__ = "Copyright (C) 2014 Andreas Kloeckner"
 
@@ -24,157 +23,617 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 """
 
-
-from django.conf import settings
-from django.utils.translation import ugettext as _
-
-import re
 import datetime
-import six
+import html.parser as html_parser
+import os
+import re
 import sys
+from typing import cast
+from xml.etree.ElementTree import Element, tostring
 
-from django.utils.timezone import now
-from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured
+import dulwich.objects
+import dulwich.repo
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist
 from django.urls import NoReverseMatch
-
+from django.utils.timezone import now
+from django.utils.translation import gettext as _
 from markdown.extensions import Extension
 from markdown.treeprocessors import Treeprocessor
+from yaml import safe_load as load_yaml
 
-from six.moves import html_parser
-
-from jinja2 import (
-        BaseLoader as BaseTemplateLoader, TemplateNotFound, FileSystemLoader)
-
-from relate.utils import dict_to_struct, Struct, SubdirRepoWrapper
 from course.constants import ATTRIBUTES_FILENAME
+from course.validation import Blob_ish, Tree_ish
+from relate.utils import Struct, SubdirRepoWrapper, dict_to_struct
 
-from yaml import load as load_yaml
 
-if sys.version_info >= (3,):
-    CACHE_KEY_ROOT = "py3"
-else:
-    CACHE_KEY_ROOT = "py2"
+CACHE_KEY_ROOT = "py4"
 
 
 # {{{ mypy
 
-from typing import (  # noqa
-        cast, Union, Any, List, Tuple, Optional, Callable, Text)
+from collections.abc import Callable, Collection, Mapping
+from typing import (
+    TYPE_CHECKING,
+    Any,
+)
 
-if False:
+
+if TYPE_CHECKING:
     # for mypy
-    from course.models import Course, Participation  # noqa
-    import dulwich  # noqa
-    from course.validation import ValidationContext  # noqa
-    from course.page.base import PageBase  # noqa
-    from relate.utils import Repo_ish  # noqa
+    import dulwich
+
+    from course.models import Course, Participation
+    from course.page.base import PageBase
+    from course.validation import FileSystemFakeRepoTree, ValidationContext
+    from relate.utils import Repo_ish
 
-    Date_ish = Union[datetime.datetime, datetime.date]
-    Datespec = Union[datetime.datetime, datetime.date, Text]
+Date_ish = datetime.datetime | datetime.date
+Datespec = datetime.datetime | datetime.date | str
 
 
 class ChunkRulesDesc(Struct):
-    if_has_role = None  # type: List[Text]
-    if_before = None  # type: Datespec
-    if_after = None  # type: Datespec
-    if_in_facility = None  # type: Text
-    roles = None  # type: List[Text]
-    start = None  # type: Datespec
-    end = None  # type: Datespec
-    shown = None  # type: bool
-    weight = None  # type: float
+    if_has_role: list[str]
+    if_before: Datespec
+    if_after: Datespec
+    if_in_facility: str
+    if_has_participation_tags_any: list[str]
+    if_has_participation_tags_all: list[str]
+    roles: list[str]
+    start: Datespec
+    end: Datespec
+    shown: bool
+    weight: float
 
 
 class ChunkDesc(Struct):
-    weight = None  # type: float
-    shown = None  # type: bool
-    title = None  # type: Optional[Text]
-    content = None  # type: Text
-    rules = None  # type: List[ChunkRulesDesc]
+    weight: float
+    shown: bool
+    title: str | None
+    content: str
+    rules: list[ChunkRulesDesc]
 
-    html_content = None  # type: Text
+    html_content: str
 
 
 class StaticPageDesc(Struct):
-    chunks = None  # type: List[ChunkDesc]
-    content = None  # type: Text
+    chunks: list[ChunkDesc]
+    content: str
 
 
 class CourseDesc(StaticPageDesc):
     pass
 
+# }}}
+
+
+# {{{ mypy: flow start rule
 
 class FlowSessionStartRuleDesc(Struct):
-    if_after = None  # type: Date_ish
-    if_before = None  # type: Date_ish
-    if_has_role = None  # type: list
-    if_in_facility = None  # type: Text
-    if_has_in_progress_session = None  # type: bool
-    if_has_session_tagged = None  # type: Optional[Text]
-    if_has_fewer_sessions_than = None  # type: int
-    if_has_fewer_tagged_sessions_than = None  # type: int
-    if_signed_in_with_matching_exam_ticket = None  # type: bool
-    tag_session = None  # type: Optional[Text]
-    may_start_new_session = None  # type: bool
-    may_list_existing_sessions = None  # type: bool
-    lock_down_as_exam_session = None  # type: bool
-    default_expiration_mode = None  # type: Text
+    """Rules that govern when a new session may be started and whether
+    existing sessions may be listed.
+
+    Found in the ``start`` attribute of :class:`FlowRulesDesc`.
+
+    .. rubric:: Conditions
+
+    .. attribute:: if_after
+
+        (Optional) A :ref:`datespec <datespec>` that determines a date/time
+        after which this rule applies.
+
+    .. attribute:: if_before
+
+        (Optional) A :ref:`datespec <datespec>` that determines a date/time
+        before which this rule applies.
+
+    .. attribute:: if_has_role
+
+        (Optional) A list of a subset of the roles defined in the course, by
+        default ``unenrolled``, ``ta``, ``student``, ``instructor``.
+
+    .. attribute:: if_has_participation_tags_any
+
+        (Optional) A list of participation tags. Rule applies when the
+        participation has at least one tag in this list.
+
+    .. attribute:: if_has_participation_tags_all
+
+        (Optional) A list of participation tags. Rule applies if only the
+        participation's tags include all items in this list.
+
+    .. attribute:: if_in_facility
+
+        (Optional) Name of a facility known to the RELATE web page. This rule allows
+        (for example) restricting flow starting based on whether a user is physically
+        located in a computer-based testing center (which RELATE can
+        recognize based on IP ranges).
+
+    .. attribute:: if_has_in_progress_session
+
+        (Optional) A Boolean (True/False) value, indicating that the rule only
+        applies if the participant has an in-progress session.
+
+    .. attribute:: if_has_session_tagged
+
+        (Optional) An identifier (or ``null``) indicating that the rule only applies
+        if the participant has a session with the corresponding tag.
+
+    .. attribute:: if_has_fewer_sessions_than
+
+        (Optional) An integer. The rule applies if the participant has fewer
+        than this number of sessions.
+
+    .. attribute:: if_has_fewer_tagged_sessions_than
+
+        (Optional) An integer. The rule applies if the participant has fewer
+        than this number of sessions with access rule tags.
+
+    .. attribute:: if_signed_in_with_matching_exam_ticket
+
+        (Optional) The rule applies if the participant signed in with an exam
+        ticket matching this flow.
+
+    .. rubric:: Rules specified
+
+    .. attribute:: may_start_new_session
+
+        (Mandatory) A Boolean (True/False) value indicating whether, if the
+        rule applies, the participant may start a new session.
 
+    .. attribute:: may_list_existing_sessions
+
+        (Mandatory) A Boolean (True/False) value indicating whether, if the
+        rule applies, the participant may view a list of existing sessions.
+
+    .. attribute:: tag_session
+
+        (Optional) An identifier that will be applied to a newly-created
+        session as a "tag".  This can be used by
+        :attr:`FlowSessionAccessRuleDesc.if_has_tag` and
+        :attr:`FlowSessionGradingRuleDesc.if_has_tag`.
+
+    .. attribute:: default_expiration_mode
+
+        (Optional) One of :class:`~course.constants.flow_session_expiration_mode`.
+        The expiration mode applied when a session is first created or rolled
+        over.
+    """
+
+    # conditions
+    if_after: Date_ish
+    if_before: Date_ish
+    if_has_role: list[str]
+    if_has_participation_tags_any: list[str]
+    if_has_participation_tags_all: list[str]
+    if_in_facility: str
+    if_has_in_progress_session: bool
+    if_has_session_tagged: str | None
+    if_has_fewer_sessions_than: int
+    if_has_fewer_tagged_sessions_than: int
+    if_signed_in_with_matching_exam_ticket: bool
+
+    # rules specified
+    tag_session: str | None
+    may_start_new_session: bool
+    may_list_existing_sessions: bool
+    lock_down_as_exam_session: bool
+    default_expiration_mode: str
+
+# }}}
+
+
+# {{{ mypy: flow access rule
 
 class FlowSessionAccessRuleDesc(Struct):
-    permissions = None  # type: list
-    if_after = None  # type: Date_ish
-    if_before = None  # type: Date_ish
-    if_started_before = None  # type: Date_ish
-    if_has_role = None  # type: List[Text]
-    if_in_facility = None  # type: Text
-    if_has_tag = None  # type: Optional[Text]
-    if_in_progress = None  # type: bool
-    if_completed_before = None  # type: Date_ish
-    if_expiration_mode = None  # type: Text
-    if_session_duration_shorter_than_minutes = None  # type: float
-    if_signed_in_with_matching_exam_ticket = None  # type: bool
-    message = None  # type: Text
+    """Rules that govern what a user may do with an existing session.
+
+    Found in the ``access`` attribute of :class:`FlowRulesDesc`.
+
+    .. rubric:: Conditions
+
+    .. attribute:: if_after
+
+        (Optional) A :ref:`datespec <datespec>` that determines a date/time
+        after which this rule applies.
+
+    .. attribute:: if_before
+
+        (Optional) A :ref:`datespec <datespec>` that determines a date/time
+        before which this rule applies.
+
+    .. attribute:: if_started_before
+
+        (Optional) A :ref:`datespec <datespec>`. Rule applies if the session
+        was started before this time.
+
+    .. attribute:: if_has_role
+
+        (Optional) A list of a subset of ``[unenrolled, ta, student, instructor]``.
+
+    .. attribute:: if_has_participation_tags_any
+
+        (Optional) A list of participation tags. Rule applies when the
+        participation has at least one tag in this list.
+
+    .. attribute:: if_has_participation_tags_all
+
+        (Optional) A list of participation tags. Rule applies if only the
+        participation's tags include all items in this list.
+
+    .. attribute:: if_in_facility
+
+        (Optional) Name of a facility known to the RELATE web page. This rule allows
+        (for example) restricting flow access based on whether a user is physically
+        located in a computer-based testing center (which RELATE can
+        recognize based on IP ranges).
+
+    .. attribute:: if_has_tag
+
+        (Optional) Rule applies if session has this tag (see
+        :attr:`FlowSessionStartRuleDesc.tag_session`), an identifier.
+
+    .. attribute:: if_in_progress
+
+        (Optional) A Boolean (True/False) value. Rule applies if the session's
+        in-progress status matches this Boolean value.
+
+    .. attribute:: if_completed_before
+
+        (Optional) A :ref:`datespec <datespec>`. Rule applies if the session
+        was completed before this time.
+
+    .. attribute:: if_expiration_mode
+
+        (Optional) One of :class:`~course.constants.flow_session_expiration_mode`.
+        Rule applies if the expiration mode (see :ref:`flow-life-cycle`)
+        matches.
+
+    .. attribute:: if_session_duration_shorter_than_minutes
+
+        (Optional) The rule applies if the current session has been going on for
+        less than the specified number of minutes. Fractional values (e.g. "0.5")
+        are accepted here.
+
+    .. attribute:: if_signed_in_with_matching_exam_ticket
+
+        (Optional) The rule applies if the participant signed in with an exam
+        ticket matching this flow.
+
+    .. rubric:: Rules specified
+
+    .. attribute:: permissions
+
+        A list of :class:`~course.constants.flow_permission`.
+
+        :attr:`~course.constants.flow_permission.submit_answer`
+        and :attr:`~course.constants.flow_permission.end_session`
+        are automatically removed from a finished (i.e. not 'in-progress')
+        session.
 
+    .. attribute:: message
+
+        (Optional) Some text in :ref:`markup` that is shown to the student in
+        an 'alert' box at the top of the page if this rule applies.
+    """
+
+    # conditions
+    if_after: Date_ish
+    if_before: Date_ish
+    if_started_before: Date_ish
+    if_has_role: list[str]
+    if_has_participation_tags_any: list[str]
+    if_has_participation_tags_all: list[str]
+    if_in_facility: str
+    if_has_tag: str | None
+    if_in_progress: bool
+    if_completed_before: Date_ish
+    if_expiration_mode: str
+    if_session_duration_shorter_than_minutes: float
+    if_signed_in_with_matching_exam_ticket: bool
+
+    # rules specified
+    permissions: list
+    message: str
+
+# }}}
+
+
+# {{{ mypy: flow grading rule
 
 class FlowSessionGradingRuleDesc(Struct):
-    grade_identifier = None  # type: Optional[Text]
-    grade_aggregation_strategy = None  # type: Optional[Text]
+    """ Rules that govern how (permanent) grades are generated from the
+    results of a flow.
+
+    Found in the ``grading`` attribute of :class:`FlowRulesDesc`.
+
+    .. rubric:: Conditions
+
+    .. attribute:: if_has_role
+
+        (Optional) A list of a subset of ``[unenrolled, ta, student, instructor]``.
+
+    .. attribute:: if_has_participation_tags_any
+
+        (Optional) A list of participation tags. Rule applies when the
+        participation has at least one tag in this list.
+
+    .. attribute:: if_has_participation_tags_all
+
+        (Optional) A list of participation tags. Rule applies if only the
+        participation's tags include all items in this list.
+
+    .. attribute:: if_started_before
+
+        (Optional) A :ref:`datespec <datespec>`. Rule applies if the session
+        was started before this time.
+
+    .. attribute:: if_has_tag
+
+        (Optional) Rule applies if session has this tag (see
+        :attr:`FlowSessionStartRuleDesc.tag_session`), an identifier.
+
+    .. attribute:: if_completed_before
+
+        (Optional) A :ref:`datespec <datespec>`. Rule applies if the session
+        was completed before this time.
+
+        When evaluating this condition for in-progress sessions, the current time,
+        or, if :attr:`use_last_activity_as_completion_time` is set, the time of the
+        last activity is used.
+
+        Since September 2017, this respects
+        :attr:`use_last_activity_as_completion_time`.
+
+    .. rubric:: Rules specified
+
+    .. attribute:: credit_percent
+
+        (Optional) A number indicating the percentage of credit assigned for
+        this flow.  Defaults to 100 if not present. This is applied *after*
+        point modifiers such as :attr:`bonus_points` and
+        :attr:`max_points_enforced_cap`.
+
+    .. attribute:: due
+
+        A :ref:`datespec <datespec>` indicating the due date of the flow. This
+        is shown to the participant and also used to batch-expire 'past-due'
+        flows.
+
+    .. attribute:: generates_grade
+
+        (Optional) A Boolean indicating whether a grade will be recorded when this
+        flow is ended. Note that the value of this rule must never change over
+        the lifetime of a flow. I.e. a flow that, at some point during its lifetime,
+        *may* have been set to generate a grade must *always* be set to generate
+        a grade. Defaults to ``true``.
+
+    .. attribute:: use_last_activity_as_completion_time
+
+        (Optional) A Boolean indicating whether the last time a participant made
+        a change to their flow should be used as the completion time.
+
+        Defaults to ``false`` to match past behavior. ``true`` is probably the more
+        sensible value for this.
+
+    .. attribute:: description
+
+        (Optional) A description of this set of grading rules being applied to
+        the flow.  Shown to the participant on the flow start page.
+
+    .. attribute:: max_points
+
+        (Optional, an integer or floating point number if given)
+        The number of points on the flow which constitute
+        "100% of the achievable points". If not given, this is automatically
+        computed by summing point values from all constituent pages.
+
+        This may be used to 'grade out of N points', where N is a number that
+        is lower than the actually achievable count.
+
+    .. attribute:: max_points_enforced_cap
+
+        (Optional, an integer or floating point number if given)
+        No participant will have a grade higher than this recorded for this flow.
+        This may be used to limit the amount of 'extra credit' achieved beyond
+        :attr:`max_points`.
+
+    .. attribute:: bonus_points
+
+        (Optional, an integer or floating point number if given)
+        This number of points will be added to every participant's score.
+
+    """
+    # conditions
+    if_has_role: list[str]
+    if_has_participation_tags_any: list[str]
+    if_has_participation_tags_all: list[str]
+    if_started_after: Date_ish
+    if_has_tag: str | None
+    if_completed_before: Date_ish
+
+    # rules specified
+    credit_percent: int | float | None
+    due: Date_ish
+    generates_grade: bool | None
+    use_last_activity_as_completion_time: bool
+    description: str
+    max_points: int | float | None
+    max_points_enforced_cap: int | float | None
+    bonus_points: int | float | None
+
+# }}}
 
 
+# {{{ mypy: flow rules
+
 class FlowRulesDesc(Struct):
-    start = None  # type: List[FlowSessionStartRuleDesc]
-    access = None  # type: List[FlowSessionAccessRuleDesc]
-    grading = None  # type: List[FlowSessionGradingRuleDesc]
-    grade_identifier = None  # type: Optional[Text]
-    grade_aggregation_strategy = None  # type: Optional[Text]
+    """
+    Found in the ``rules`` attribute of a :class:`FlowDesc`.
+
+    .. attribute:: start
+
+        Rules that govern when a new session may be started and whether
+        existing sessions may be listed.
+
+        A list of :class:`FlowSessionStartRuleDesc`
+
+        Rules are tested from top to bottom. The first rule
+        whose conditions apply determines the access.
+
+    .. attribute:: access
+
+        Rules that govern what a user may do while they are interacting with an
+        existing session.
+
+        A list of :class:`FlowSessionAccessRuleDesc`.
+
+        Rules are tested from top to bottom. The first rule
+        whose conditions apply determines the access.
+
+    .. rubric:: Grading-Related
+
+    .. attribute:: grade_identifier
+
+        (Required) The identifier of the grade to be generated once the
+        participant completes the flow.  If ``null``, no grade is generated.
+
+    .. attribute:: grade_aggregation_strategy
+
+        (Required if :attr:`grade_identifier` is not ``null``)
+
+        One of :class:`grade_aggregation_strategy`.
+
+    .. attribute:: grading
+
+        Rules that govern how (permanent) overall grades are generated from the
+        results of a flow. These rules apply once a flow session ends/is submitted
+        for grading. See :ref:`flow-life-cycle`.
+
+        (Required if grade_identifier is not ``null``)
+        A list of :class:`FlowSessionGradingRuleDesc`
+
+        Rules are tested from top to bottom. The first rule
+        whose conditions apply determines the access.
+    """
+    start: list[FlowSessionStartRuleDesc]
+    access: list[FlowSessionAccessRuleDesc]
+    grading: list[FlowSessionGradingRuleDesc]
+    grade_identifier: str | None
+    grade_aggregation_strategy: str | None
+
+# }}}
+
+
+# {{{ mypy: flow
+
+class TabDesc(Struct):
+    """
+    .. attribute:: title
+
+        (Required) Title to be displayed on the tab.
+
+    .. attribute:: url
+
+        (Required) The URL of the external web page.
+    """
+
+    def __init__(self, title: str, url: str) -> None:
+        self.title = title
+        self.url = url
+
+    title: str
+    url: str
 
 
 class FlowPageDesc(Struct):
-    id = None  # type: Text
-    type = None  # type: Text
+    id: str
+    type: str
 
 
 class FlowPageGroupDesc(Struct):
-    id = None  # type: Text
-    pages = None  # type: List[FlowPageDesc]
+    """
+    .. attribute:: id
+
+        (Required) A symbolic name for the page group.
+
+    .. attribute:: pages
+
+        (Required) A list of :ref:`flow-page`
+
+    .. attribute:: shuffle
+
+        (Optional) A boolean (True/False) indicating whether the order
+        of pages should be as in the list :attr:`pages` or
+        determined by random shuffling
+
+    .. attribute:: max_page_count
+
+        (Optional) An integer limiting the page count of this group
+        to a certain value. Allows selection of a random subset by combining
+        with :attr:`shuffle`.
+    """
+
+    id: str
+    pages: list[FlowPageDesc]
 
 
 class FlowDesc(Struct):
-    title = None  # type: Text
-    rules = None  # type: FlowRulesDesc
-    pages = None  # type: List[FlowPageDesc]
-    groups = None  # type: List[FlowPageGroupDesc]
-    notify_on_submit = None  # type: Optional[List[Text]]
+    """
+    .. attribute:: title
+
+        A plain-text title of the flow
+
+    .. attribute:: description
+
+        A description in :ref:`markup` shown on the start page of the flow.
+
+    .. attribute:: completion_text
+
+        (Optional) Some text in :ref:`markup` shown once a student has
+        completed the flow.
+
+    .. attribute:: notify_on_submit
+
+        (Optional) A list of email addresses which to notify about a flow
+        submission by a participant.
 
+    .. attribute:: rules
+
+        (Optional) Some rules governing students' use and grading of the flow.
+        See :ref:`flow-rules`.
+
+    .. attribute:: groups
+
+        A list of :class:`FlowPageGroupDesc`.  Exactly one of
+        :attr:`groups` or :class:`pages` must be given.
+
+    .. attribute:: pages
+
+        A list of :ref:`pages <flow-page>`. If you specify this, a single
+        :class:`FlowPageGroupDesc` will be implicitly created. Exactly one of
+        :attr:`groups` or :class:`pages` must be given.
+
+    .. attribute:: external_resources
+
+        A list of :class:`TabDesc`. These are links to external
+        resources that are displayed as tabs on the flow tabbed page.
+    """
+
+    title: str
+    description: str
+    rules: FlowRulesDesc
+    pages: list[FlowPageDesc]
+    groups: list[FlowPageGroupDesc]
+    external_resources: list[TabDesc]
+    notify_on_submit: list[str] | None
 
 # }}}
 
 
 # {{{ repo blob getting
 
-def get_true_repo_and_path(repo, path):
-    # type: (Repo_ish, Text) -> Tuple[dulwich.Repo, Text]
+def get_true_repo_and_path(repo: Repo_ish, path: str) -> tuple[dulwich.repo.Repo, str]:
 
     if isinstance(repo, SubdirRepoWrapper):
         if path:
@@ -188,15 +647,12 @@ def get_true_repo_and_path(repo, path):
         return repo, path
 
 
-def get_course_repo_path(course):
-    # type: (Course) -> Text
+def get_course_repo_path(course: Course) -> str:
 
-    from os.path import join
-    return join(settings.GIT_ROOT, course.identifier)
+    return os.path.join(settings.GIT_ROOT, course.identifier)
 
 
-def get_course_repo(course):
-    # type: (Course) -> Repo_ish
+def get_course_repo(course: Course) -> Repo_ish:
 
     from dulwich.repo import Repo
     repo = Repo(get_course_repo_path(course))
@@ -207,9 +663,65 @@ def get_course_repo(course):
         return repo
 
 
-def get_repo_blob(repo, full_name, commit_sha, allow_tree=True):
-    # type: (Repo_ish, Text, bytes, bool) -> dulwich.Blob
+def look_up_git_object(repo: dulwich.repo.Repo,
+        root_tree: dulwich.objects.Tree | FileSystemFakeRepoTree,
+        full_name: str, _max_symlink_depth: int | None = None):
+    """Traverse git directory tree from *root_tree*, respecting symlinks."""
+
+    if _max_symlink_depth is None:
+        _max_symlink_depth = 20
+    if _max_symlink_depth == 0:
+        raise ObjectDoesNotExist(_("symlink nesting depth exceeded "
+            "while locating '%s'") % full_name)
+
+    # https://github.com/inducer/relate/pull/556
+    # FIXME: https://github.com/inducer/relate/issues/767
+    name_parts = os.path.normpath(full_name).split(os.sep)
 
+    processed_name_parts: list[str] = []
+
+    from dulwich.objects import Tree
+
+    from course.validation import FileSystemFakeRepoTree
+
+    cur_lookup = root_tree
+
+    from stat import S_ISLNK
+    while name_parts:
+        if not isinstance(cur_lookup, Tree | FileSystemFakeRepoTree):
+            raise ObjectDoesNotExist(
+                    _("'%s' is not a directory, cannot lookup nested names")
+                    % os.sep.join(processed_name_parts))
+
+        name_part = name_parts.pop(0)
+
+        if not name_part:
+            # tolerate empty path components (begrudgingly)
+            continue
+        elif name_part == ".":
+            return cur_lookup
+
+        encoded_name_part = name_part.encode()
+        try:
+            mode_sha = cur_lookup[encoded_name_part]
+        except KeyError:
+            raise ObjectDoesNotExist(_("resource '%s' not found") % full_name)
+
+        mode, cur_lookup_sha = mode_sha
+
+        if S_ISLNK(mode):
+            link_target = os.sep.join(
+                        [*processed_name_parts, repo[cur_lookup_sha].data.decode()])
+            cur_lookup = look_up_git_object(repo, root_tree, link_target,
+                    _max_symlink_depth=_max_symlink_depth-1)
+        else:
+            processed_name_parts.append(name_part)
+            cur_lookup = repo[cur_lookup_sha]
+
+    return cur_lookup
+
+
+def get_repo_tree(repo: Repo_ish, full_name: str, commit_sha: bytes) -> Tree_ish:
     """
     :arg full_name: A Unicode string indicating the file name.
     :arg commit_sha: A byte string containing the commit hash
@@ -218,74 +730,72 @@ def get_repo_blob(repo, full_name, commit_sha, allow_tree=True):
 
     dul_repo, full_name = get_true_repo_and_path(repo, full_name)
 
-    names = full_name.split("/")
-
-    # Allow non-ASCII file name
-    full_name_bytes = full_name.encode('utf-8')
-
     try:
         tree_sha = dul_repo[commit_sha].tree
     except KeyError:
         raise ObjectDoesNotExist(
                 _("commit sha '%s' not found") % commit_sha.decode())
 
-    tree = dul_repo[tree_sha]
+    git_obj = look_up_git_object(
+            dul_repo, root_tree=dul_repo[tree_sha], full_name=full_name)
 
-    def access_directory_content(maybe_tree, name):
-        # type: (Any, Text) -> Any
-        try:
-            mode_and_blob_sha = tree[name.encode()]
-        except TypeError:
-            raise ObjectDoesNotExist(_("resource '%s' is a file, "
-                "not a directory") % full_name)
+    from dulwich.objects import Tree
 
-        mode, blob_sha = mode_and_blob_sha
-        return mode_and_blob_sha
+    from course.validation import FileSystemFakeRepoTree
+
+    msg_full_name = full_name if full_name else _("(repo root)")
+
+    if isinstance(git_obj, Tree | FileSystemFakeRepoTree):
+        return git_obj
+    else:
+        raise ObjectDoesNotExist(_("resource '%s' is not a tree") % msg_full_name)
 
-    if not full_name_bytes:
-        if allow_tree:
-            return tree
-        else:
-            raise ObjectDoesNotExist(
-                    _("repo root is a directory, not a file"))
+
+def get_repo_blob(repo: Repo_ish, full_name: str, commit_sha: bytes) -> Blob_ish:
+    """
+    :arg full_name: A Unicode string indicating the file name.
+    :arg commit_sha: A byte string containing the commit hash
+    :arg allow_tree: Allow the resulting object to be a directory
+    """
+
+    dul_repo, full_name = get_true_repo_and_path(repo, full_name)
 
     try:
-        for name in names[:-1]:
-            if not name:
-                # tolerate empty path components (begrudgingly)
-                continue
+        tree_sha = dul_repo[commit_sha].tree
+    except KeyError:
+        raise ObjectDoesNotExist(
+                _("commit sha '%s' not found") % commit_sha.decode())
 
-            mode, blob_sha = access_directory_content(tree, name)
-            tree = dul_repo[blob_sha]
+    git_obj = look_up_git_object(
+            dul_repo, root_tree=dul_repo[tree_sha], full_name=full_name)
 
-        mode, blob_sha = access_directory_content(tree, names[-1])
+    from dulwich.objects import Blob
 
-        result = dul_repo[blob_sha]
-        if not allow_tree and not hasattr(result, "data"):
-            raise ObjectDoesNotExist(
-                    _("resource '%s' is a directory, not a file") % full_name)
+    from course.validation import FileSystemFakeRepoFile
 
-        return result
+    msg_full_name = full_name if full_name else _("(repo root)")
 
-    except KeyError:
-        raise ObjectDoesNotExist(_("resource '%s' not found") % full_name)
+    if isinstance(git_obj, Blob | FileSystemFakeRepoFile):
+        return git_obj
+    else:
+        raise ObjectDoesNotExist(_("resource '%s' is not a file") % msg_full_name)
 
 
-def get_repo_blob_data_cached(repo, full_name, commit_sha):
-    # type: (Repo_ish, Text, bytes) -> bytes
+def get_repo_blob_data_cached(
+        repo: Repo_ish, full_name: str, commit_sha: bytes) -> bytes:
     """
     :arg commit_sha: A byte string containing the commit hash
     """
 
-    if isinstance(commit_sha, six.binary_type):
-        from six.moves.urllib.parse import quote_plus
-        cache_key = "%s%R%1".join((
+    if isinstance(commit_sha, bytes):
+        from urllib.parse import quote_plus
+        cache_key: str | None = "%s%R%1".join((
             CACHE_KEY_ROOT,
             quote_plus(repo.controldir()),
             quote_plus(full_name),
             commit_sha.decode(),
             ".".join(str(s) for s in sys.version_info[:2]),
-            ))  # type: Optional[Text]
+            ))
     else:
         cache_key = None
 
@@ -294,10 +804,10 @@ def get_repo_blob_data_cached(repo, full_name, commit_sha):
     except ImproperlyConfigured:
         cache_key = None
 
+    result: bytes | None = None
     if cache_key is None:
-        result = get_repo_blob(repo, full_name, commit_sha,
-                allow_tree=False).data
-        assert isinstance(result, six.binary_type)
+        result = get_repo_blob(repo, full_name, commit_sha).data
+        assert isinstance(result, bytes)
         return result
 
     # Byte string is wrapped in a tuple to force pickling because memcache's
@@ -306,28 +816,29 @@ def get_repo_blob_data_cached(repo, full_name, commit_sha):
 
     def_cache = cache.caches["default"]
 
-    result = None
     # Memcache is apparently limited to 250 characters.
     if len(cache_key) < 240:
-        result = def_cache.get(cache_key)
-    if result is not None:
-        (result,) = result
-        assert isinstance(result, six.binary_type), cache_key
-        return result
+        cached_result = def_cache.get(cache_key)
 
-    result = get_repo_blob(repo, full_name, commit_sha,
-            allow_tree=False).data
+        if cached_result is not None:
+            (result,) = cached_result
+            assert isinstance(result, bytes), cache_key
+            return result
+
+    result = get_repo_blob(repo, full_name, commit_sha).data
+    assert result is not None
 
     if len(result) <= getattr(settings, "RELATE_CACHE_MAX_BYTES", 0):
         def_cache.add(cache_key, (result,), None)
 
-    assert isinstance(result, six.binary_type)
+    assert isinstance(result, bytes)
 
     return result
 
 
-def is_repo_file_accessible_as(access_kinds, repo, commit_sha, path):
-    # type: (List[Text], Repo_ish, bytes, Text) -> bool
+def is_repo_file_accessible_as(
+        access_kinds: list[str], repo: Repo_ish, commit_sha: bytes, path: str
+        ) -> bool:
     """
     Check of a file in a repo directory is accessible.  For example,
     'instructor' can access anything listed in the attributes.
@@ -338,8 +849,7 @@ def is_repo_file_accessible_as(access_kinds, repo, commit_sha, path):
     """
 
     # set the path to .attributes.yml
-    from os.path import dirname, basename, join
-    attributes_path = join(dirname(path), ATTRIBUTES_FILENAME)
+    attributes_path = os.path.join(os.path.dirname(path), ATTRIBUTES_FILENAME)
 
     # retrieve the .attributes.yml structure
     try:
@@ -349,18 +859,18 @@ def is_repo_file_accessible_as(access_kinds, repo, commit_sha, path):
         # no attributes file: not accessible
         return False
 
-    path_basename = basename(path)
+    path_basename = os.path.basename(path)
 
     # "public" is a deprecated alias for "unenrolled".
 
-    access_patterns = []  # type: List[Text]
+    access_patterns: list[str] = []
     for kind in access_kinds:
         access_patterns += attributes.get(kind, [])
 
     from fnmatch import fnmatch
     if isinstance(access_patterns, list):
         for pattern in access_patterns:
-            if isinstance(pattern, six.string_types):
+            if isinstance(pattern, str):
                 if fnmatch(path_basename, pattern):
                     return True
 
@@ -376,17 +886,17 @@ JINJA_YAML_RE = re.compile(
     re.MULTILINE | re.DOTALL)
 YAML_BLOCK_START_SCALAR_RE = re.compile(
     r"(:\s*[|>])"
-    "(J?)"
-    "((?:[0-9][-+]?|[-+][0-9]?)?)"
-    "(?:\s*\#.*)?"
-    "$")
+    r"(J?)"
+    r"((?:[0-9][-+]?|[-+][0-9]?)?)"
+    r"(?:\s*\#.*)?"
+    r"$")
 
+IN_BLOCK_END_RAW_RE = re.compile(r"(.*)({%-?\s*endraw\s*-?%})(.*)")
 GROUP_COMMENT_START = re.compile(r"^\s*#\s*\{\{\{")
 LEADING_SPACES_RE = re.compile(r"^( *)")
 
 
-def process_yaml_for_expansion(yaml_str):
-    # type: (Text) -> Text
+def process_yaml_for_expansion(yaml_str: str) -> str:
 
     lines = yaml_str.split("\n")
     jinja_lines = []
@@ -395,34 +905,40 @@ def process_yaml_for_expansion(yaml_str):
     line_count = len(lines)
 
     while i < line_count:
-        l = lines[i]
-        yaml_block_scalar_match = YAML_BLOCK_START_SCALAR_RE.search(l)
+        ln = lines[i].rstrip()
+        yaml_block_scalar_match = YAML_BLOCK_START_SCALAR_RE.search(ln)
 
         if yaml_block_scalar_match is not None:
             unprocessed_block_lines = []
             allow_jinja = bool(yaml_block_scalar_match.group(2))
-            l = YAML_BLOCK_START_SCALAR_RE.sub(
-                    r"\1\3", l)
+            ln = YAML_BLOCK_START_SCALAR_RE.sub(
+                    r"\1\3", ln)
 
-            unprocessed_block_lines.append(l)
+            unprocessed_block_lines.append(ln)
 
-            block_start_indent = len(LEADING_SPACES_RE.match(l).group(1))
+            leading_spaces_match = LEADING_SPACES_RE.match(ln)
+            assert leading_spaces_match
+            block_start_indent = len(leading_spaces_match.group(1))
 
             i += 1
 
             while i < line_count:
-                l = lines[i]
+                ln = lines[i]
 
-                if not l.rstrip():
-                    unprocessed_block_lines.append(l)
+                if not ln.rstrip():
+                    unprocessed_block_lines.append(ln)
                     i += 1
                     continue
 
-                line_indent = len(LEADING_SPACES_RE.match(l).group(1))
+                leading_spaces_match = LEADING_SPACES_RE.match(ln)
+                assert leading_spaces_match
+                line_indent = len(leading_spaces_match.group(1))
                 if line_indent <= block_start_indent:
                     break
                 else:
-                    unprocessed_block_lines.append(l)
+                    ln = IN_BLOCK_END_RAW_RE.sub(
+                        r"\1{% endraw %}{{ '\2' }}{% raw %}\3", ln)
+                    unprocessed_block_lines.append(ln.rstrip())
                     i += 1
 
             if not allow_jinja:
@@ -431,97 +947,82 @@ def process_yaml_for_expansion(yaml_str):
             if not allow_jinja:
                 jinja_lines.append("{% endraw %}")
 
-        elif GROUP_COMMENT_START.match(l):
+        elif GROUP_COMMENT_START.match(ln):
             jinja_lines.append("{% raw %}")
-            jinja_lines.append(l)
+            jinja_lines.append(ln)
             jinja_lines.append("{% endraw %}")
             i += 1
 
         else:
-            jinja_lines.append(l)
+            jinja_lines.append(ln)
             i += 1
-
     return "\n".join(jinja_lines)
 
 
-class GitTemplateLoader(BaseTemplateLoader):
-    def __init__(self, repo, commit_sha):
-        # type: (Repo_ish, bytes) -> None
+class GitTemplateLoader:
+    def __init__(self, repo: Repo_ish, commit_sha: bytes) -> None:
         self.repo = repo
         self.commit_sha = commit_sha
 
-    def get_source(self, environment, template):
-        try:
-            data = get_repo_blob_data_cached(self.repo, template, self.commit_sha)
-        except ObjectDoesNotExist:
-            raise TemplateNotFound(template)
-
-        source = data.decode('utf-8')
+    def __call__(self, template):
+        data = get_repo_blob_data_cached(self.repo, template, self.commit_sha)
 
-        def is_up_to_date():
-            # There's not much point to caching here, because we create
-            # a new loader for every request anyhow...
-            return False
-
-        return source, None, is_up_to_date
+        return data.decode("utf-8")
 
 
 class YamlBlockEscapingGitTemplateLoader(GitTemplateLoader):
     # https://github.com/inducer/relate/issues/130
 
-    def get_source(self, environment, template):
-        source, path, is_up_to_date = \
-                super(YamlBlockEscapingGitTemplateLoader, self).get_source(
-                        environment, template)
+    def __call__(self, template):
+        source = super().__call__(template)
 
-        from os.path import splitext
-        _, ext = splitext(template)
+        _, ext = os.path.splitext(template)
         ext = ext.lower()
 
         if ext in [".yml", ".yaml"]:
             source = process_yaml_for_expansion(source)
 
-        return source, path, is_up_to_date
+        return source
 
 
-class YamlBlockEscapingFileSystemLoader(FileSystemLoader):
+class YamlBlockEscapingFileSystemLoader:
     # https://github.com/inducer/relate/issues/130
 
-    def get_source(self, environment, template):
-        source, path, is_up_to_date = \
-                super(YamlBlockEscapingFileSystemLoader, self).get_source(
-                        environment, template)
+    def __init__(self, root):
+        self.root = root
+
+    def __call__(self, template):
+        with open(os.path.join(self.root, template)) as inf:
+            source = inf.read()
 
-        from os.path import splitext
-        _, ext = splitext(template)
+        _, ext = os.path.splitext(template)
         ext = ext.lower()
 
         if ext in [".yml", ".yaml"]:
             source = process_yaml_for_expansion(source)
 
-        return source, path, is_up_to_date
+        return source
 
 
-def expand_yaml_macros(repo, commit_sha, yaml_str):
-    # type: (Repo_ish, bytes, Text) -> Text
+def expand_yaml_macros(repo: Repo_ish, commit_sha: bytes, yaml_str: str) -> str:
 
-    if isinstance(yaml_str, six.binary_type):
+    if isinstance(yaml_str, bytes):
         yaml_str = yaml_str.decode("utf-8")
 
-    from jinja2 import Environment, StrictUndefined
+    from minijinja import Environment
     jinja_env = Environment(
             loader=YamlBlockEscapingGitTemplateLoader(repo, commit_sha),
-            undefined=StrictUndefined)
+            undefined_behavior="strict",
+            auto_escape_callback=lambda fn: False)
 
     # {{{ process explicit [JINJA] tags (deprecated)
 
-    def compute_replacement(match):
-        template = jinja_env.from_string(match.group(1))
-        return template.render()
+    def compute_replacement(match):  # pragma: no cover  # deprecated
+        return jinja_env.render_str(match.group(1))
 
     yaml_str, count = JINJA_YAML_RE.subn(compute_replacement, yaml_str)
 
-    if count:
+    if count:  # pragma: no cover  # deprecated
         # The file uses explicit [JINJA] tags. Assume that it doesn't
         # want anything else processed through YAML.
         return yaml_str
@@ -529,8 +1030,7 @@ def expand_yaml_macros(repo, commit_sha, yaml_str):
     # }}}
 
     jinja_str = process_yaml_for_expansion(yaml_str)
-    template = jinja_env.from_string(jinja_str)
-    yaml_str = template.render()
+    yaml_str = jinja_env.render_str(jinja_str)
 
     return yaml_str
 
@@ -539,15 +1039,15 @@ def expand_yaml_macros(repo, commit_sha, yaml_str):
 
 # {{{ repo yaml getting
 
-def get_raw_yaml_from_repo(repo, full_name, commit_sha):
-    # type: (Repo_ish, Text, bytes) -> Any
+def get_raw_yaml_from_repo(
+        repo: Repo_ish, full_name: str, commit_sha: bytes) -> Any:
     """Return decoded YAML data structure from
     the given file in *repo* at *commit_sha*.
 
     :arg commit_sha: A byte string containing the commit hash
     """
 
-    from six.moves.urllib.parse import quote_plus
+    from urllib.parse import quote_plus
     cache_key = "%RAW%%2".join((
         CACHE_KEY_ROOT,
         quote_plus(repo.controldir()), quote_plus(full_name), commit_sha.decode(),
@@ -555,56 +1055,74 @@ def get_raw_yaml_from_repo(repo, full_name, commit_sha):
 
     import django.core.cache as cache
     def_cache = cache.caches["default"]
-    result = None
+
+    result: Any | None = None
     # Memcache is apparently limited to 250 characters.
     if len(cache_key) < 240:
         result = def_cache.get(cache_key)
     if result is not None:
         return result
 
-    result = load_yaml(
-            expand_yaml_macros(
+    yaml_str = expand_yaml_macros(
                 repo, commit_sha,
-                get_repo_blob(repo, full_name, commit_sha,
-                    allow_tree=False).data))
+                get_repo_blob(repo, full_name, commit_sha).data)
+
+    result = load_yaml(yaml_str)  # type: ignore
 
     def_cache.add(cache_key, result, None)
 
     return result
 
 
-def get_yaml_from_repo(repo, full_name, commit_sha, cached=True):
-    # type: (Repo_ish, Text, bytes, bool) -> Any
+LINE_HAS_INDENTING_TABS_RE = re.compile(r"^\s*\t\s*", re.MULTILINE)
+
 
+def get_yaml_from_repo(
+        repo: Repo_ish, full_name: str, commit_sha: bytes, cached: bool = True,
+        tolerate_tabs: bool = False) -> Any:
     """Return decoded, struct-ified YAML data structure from
     the given file in *repo* at *commit_sha*.
 
     See :class:`relate.utils.Struct` for more on
     struct-ification.
+
+    :arg tolerate_tabs: At one point, Relate accepted tabs
+        in indentation, but it no longer does. In places where legacy compatibility
+        matters, you may set *tolerate_tabs* to *True*.
     """
 
     if cached:
-        from six.moves.urllib.parse import quote_plus
-        cache_key = "%%%2".join(
-                (CACHE_KEY_ROOT,
-                    quote_plus(repo.controldir()), quote_plus(full_name),
-                    commit_sha.decode()))
+        try:
+            import django.core.cache as cache
+        except ImproperlyConfigured:
+            cached = False
+        else:
+            from urllib.parse import quote_plus
+            cache_key = "%%%2".join(
+                    (CACHE_KEY_ROOT,
+                        quote_plus(repo.controldir()), quote_plus(full_name),
+                        commit_sha.decode()))
 
-        import django.core.cache as cache
-        def_cache = cache.caches["default"]
-        result = None
-        # Memcache is apparently limited to 250 characters.
-        if len(cache_key) < 240:
-            result = def_cache.get(cache_key)
-        if result is not None:
-            return result
+            def_cache = cache.caches["default"]
+            result = None
+            # Memcache is apparently limited to 250 characters.
+            if len(cache_key) < 240:
+                result = def_cache.get(cache_key)
+            if result is not None:
+                return result
+
+    yaml_bytestream = get_repo_blob(
+            repo, full_name, commit_sha).data
+    yaml_text = yaml_bytestream.decode("utf-8")
+
+    if not tolerate_tabs and LINE_HAS_INDENTING_TABS_RE.search(yaml_text):
+        raise ValueError("File uses tabs in indentation. "
+                "This is not allowed.")
 
-    expanded = expand_yaml_macros(
-            repo, commit_sha,
-            get_repo_blob(repo, full_name, commit_sha,
-                allow_tree=False).data)
+    expanded = expand_yaml_macros(repo, commit_sha, yaml_bytestream)
 
-    result = dict_to_struct(load_yaml(expanded))
+    yaml_data = load_yaml(expanded)  # type:ignore
+    result = dict_to_struct(yaml_data)
 
     if cached:
         def_cache.add(cache_key, result, None)
@@ -620,14 +1138,18 @@ def get_yaml_from_repo(repo, full_name, commit_sha, cached=True):
 def _attr_to_string(key, val):
     if val is None:
         return key
-    elif "\"" in val:
-        return "%s='%s'" % (key, val)
+    elif '"' in val:
+        return f"{key}='{val}'"
     else:
-        return "%s=\"%s\"" % (key, val)
+        return f'{key}="{val}"'
 
 
 class TagProcessingHTMLParser(html_parser.HTMLParser):
-    def __init__(self, out_file, process_tag_func):
+    def __init__(
+                self,
+                out_file,
+                process_tag_func: Callable[[str, Mapping[str, str]], Mapping[str, str]]
+            ) -> None:
         html_parser.HTMLParser.__init__(self)
 
         self.out_file = out_file
@@ -637,44 +1159,44 @@ class TagProcessingHTMLParser(html_parser.HTMLParser):
         attrs = dict(attrs)
         attrs.update(self.process_tag_func(tag, attrs))
 
-        self.out_file.write("<%s %s>" % (tag, " ".join(
-            _attr_to_string(k, v) for k, v in six.iteritems(attrs))))
+        self.out_file.write("<{} {}>".format(tag, " ".join(
+            _attr_to_string(k, v) for k, v in attrs.items())))
 
     def handle_endtag(self, tag):
-        self.out_file.write("</%s>" % tag)
+        self.out_file.write(f"</{tag}>")
 
     def handle_startendtag(self, tag, attrs):
         attrs = dict(attrs)
         attrs.update(self.process_tag_func(tag, attrs))
 
-        self.out_file.write("<%s %s/>" % (tag, " ".join(
-            _attr_to_string(k, v) for k, v in six.iteritems(attrs))))
+        self.out_file.write("<{} {}/>".format(tag, " ".join(
+            _attr_to_string(k, v) for k, v in attrs.items())))
 
     def handle_data(self, data):
         self.out_file.write(data)
 
     def handle_entityref(self, name):
-        self.out_file.write("&%s;" % name)
+        self.out_file.write(f"&{name};")
 
     def handle_charref(self, name):
-        self.out_file.write("&#%s;" % name)
+        self.out_file.write(f"&#{name};")
 
     def handle_comment(self, data):
-        self.out_file.write("<!--%s-->" % data)
+        self.out_file.write(f"<!--{data}-->")
 
     def handle_decl(self, decl):
-        self.out_file.write("<!%s>" % decl)
+        self.out_file.write(f"<!{decl}>")
 
     def handle_pi(self, data):
         raise NotImplementedError(
                 _("I have no idea what a processing instruction is."))
 
     def unknown_decl(self, data):
-        self.out_file.write("<![%s]>" % data)
+        self.out_file.write(f"<![{data}]>")
 
 
-class PreserveFragment(object):
-    def __init__(self, s):
+class PreserveFragment:
+    def __init__(self, s: str) -> None:
         self.s = s
 
 
@@ -686,7 +1208,7 @@ class LinkFixerTreeprocessor(Treeprocessor):
         self.commit_sha = commit_sha
         self.reverse_func = reverse_func
 
-    def reverse(self, viewname, args):
+    def reverse(self, viewname: str, args: tuple[Any, ...]) -> str:
         frag = None
 
         new_args = []
@@ -709,13 +1231,13 @@ class LinkFixerTreeprocessor(Treeprocessor):
 
         return result
 
-    def get_course_identifier(self):
+    def get_course_identifier(self) -> str:
         if self.course is None:
             return "bogus-course-identifier"
         else:
             return self.course.identifier
 
-    def process_url(self, url):
+    def process_url(self, url: str) -> str | None:
         try:
             if url.startswith("course:"):
                 course_id = url[7:]
@@ -743,7 +1265,7 @@ class LinkFixerTreeprocessor(Treeprocessor):
                 return self.reverse("relate-get_media",
                             args=(
                                 self.get_course_identifier(),
-                                self.commit_sha,
+                                self.commit_sha.decode(),
                                 PreserveFragment(media_path)))
 
             elif url.startswith("repo:"):
@@ -751,7 +1273,7 @@ class LinkFixerTreeprocessor(Treeprocessor):
                 return self.reverse("relate-get_repo_file",
                             args=(
                                 self.get_course_identifier(),
-                                self.commit_sha,
+                                self.commit_sha.decode(),
                                 PreserveFragment(path)))
 
             elif url.startswith("repocur:"):
@@ -765,17 +1287,18 @@ class LinkFixerTreeprocessor(Treeprocessor):
                 return self.reverse("relate-view_calendar",
                             args=(self.get_course_identifier(),))
 
+            else:
+                return None
+
         except NoReverseMatch:
             from base64 import b64encode
             message = ("Invalid character in RELATE URL: " + url).encode("utf-8")
             return "data:text/plain;base64,"+b64encode(message).decode()
 
-        return None
-
-    def process_tag(self, tag_name, attrs):
+    def process_tag(self, tag_name: str, attrs: Mapping[str, str]) -> Mapping[str, str]:
         changed_attrs = {}
 
-        if tag_name == "table":
+        if tag_name == "table" and attrs.get("bootstrap") != "no":
             changed_attrs["class"] = "table table-condensed"
 
         if tag_name in ["a", "link"] and "href" in attrs:
@@ -798,49 +1321,55 @@ class LinkFixerTreeprocessor(Treeprocessor):
 
         return changed_attrs
 
-    def process_etree_element(self, element):
+    def process_etree_element(self, element: Element) -> None:
         changed_attrs = self.process_tag(element.tag, element.attrib)
 
-        for key, val in six.iteritems(changed_attrs):
+        for key, val in changed_attrs.items():
             element.set(key, val)
 
-    def walk_and_process_tree(self, root):
+    def walk_and_process_tree(self, root: Element) -> None:
         self.process_etree_element(root)
 
         for child in root:
             self.walk_and_process_tree(child)
 
-    def run(self, root):
+    def run(self, root: Element) -> None:
         self.walk_and_process_tree(root)
 
         # root through and process Markdown's HTML stash (gross!)
-        from six.moves import cStringIO
+        from io import StringIO
 
-        for i, (html, safe) in enumerate(self.md.htmlStash.rawHtmlBlocks):
-            outf = cStringIO()
+        for i, html in enumerate(self.md.htmlStash.rawHtmlBlocks):
+            outf = StringIO()
             parser = TagProcessingHTMLParser(outf, self.process_tag)
+
+            # According to
+            # https://github.com/python/typeshed/blob/61ba4de28f1469d6a642c983d5a7674479c12444/stubs/Markdown/markdown/util.pyi#L44
+            # this should not happen, but... *shrug*
+            if isinstance(html, Element):
+                html = tostring(html).decode("utf-8")
             parser.feed(html)
 
-            self.md.htmlStash.rawHtmlBlocks[i] = (outf.getvalue(), safe)
+            self.md.htmlStash.rawHtmlBlocks[i] = outf.getvalue()
 
 
 class LinkFixerExtension(Extension):
-    def __init__(self, course, commit_sha, reverse_func):
-        # type: (Optional[Course], bytes, Optional[Callable]) -> None
-
+    def __init__(
+            self, course: Course | None,
+            commit_sha: bytes, reverse_func: Callable | None) -> None:
         Extension.__init__(self)
         self.course = course
         self.commit_sha = commit_sha
         self.reverse_func = reverse_func
 
-    def extendMarkdown(self, md, md_globals):  # noqa
-        md.treeprocessors["relate_link_fixer"] = \
-                LinkFixerTreeprocessor(md, self.course, self.commit_sha,
-                        reverse_func=self.reverse_func)
+    def extendMarkdown(self, md):  # noqa
+        md.treeprocessors.register(
+            LinkFixerTreeprocessor(md, self.course, self.commit_sha,
+                                    reverse_func=self.reverse_func),
+            "relate_link_fixer", 0)
 
 
-def remove_prefix(prefix, s):
-    # type: (Text, Text) -> Text
+def remove_prefix(prefix: str, s: str) -> str:
     if s.startswith(prefix):
         return s[len(prefix):]
     else:
@@ -850,21 +1379,80 @@ def remove_prefix(prefix, s):
 JINJA_PREFIX = "[JINJA]"
 
 
-def markup_to_html(
-        course,  # type: Optional[Course]
-        repo,  # type: Repo_ish
-        commit_sha,  # type: bytes
-        text,  # type: Text
-        reverse_func=None,  # type: Callable
-        validate_only=False,  # type: bool
-        use_jinja=True,  # type: bool
-        jinja_env={},  # type: Dict
-        ):
-    # type: (...) -> Text
+def expand_markup(
+        course: Course | None,
+        repo: Repo_ish,
+        commit_sha: bytes,
+        text: str,
+        use_jinja: bool = True,
+        jinja_env: dict | None = None,
+        ) -> str:
 
-    if reverse_func is None:
-        from django.urls import reverse
-        reverse_func = reverse
+    if jinja_env is None:
+        jinja_env = {}
+
+    if not isinstance(text, str):
+        text = str(text)
+
+    # {{{ process through Jinja
+
+    if use_jinja:
+        from minijinja import Environment
+        env = Environment(
+                loader=GitTemplateLoader(repo, commit_sha),
+                undefined_behavior="strict")
+
+        def render_notebook_cells(*args, **kwargs):
+            return "[The ability to render notebooks was removed.]"
+
+        env.add_function("render_notebook_cells", render_notebook_cells)
+
+        text = env.render_str(text, **jinja_env)
+
+    # }}}
+
+    return text
+
+
+def filter_html_attributes(tag, name, value):
+    from bleach.sanitizer import ALLOWED_ATTRIBUTES
+
+    allowed_attrs = ALLOWED_ATTRIBUTES.get(tag, [])
+    result = name in allowed_attrs
+
+    if tag == "a":
+        result = (result
+                or (name == "role" and value == "button")
+                or (name == "class" and value.startswith("btn btn-")))
+    elif tag == "img":
+        result = result or name == "src"
+    elif tag == "div":
+        result = result or (name == "class" and value == "well")
+    elif tag == "i":
+        result = result or (name == "class" and value.startswith("bi bi-"))
+    elif tag == "table":
+        result = (result or (name == "class") or (name == "bootstrap"))
+
+    return result
+
+
+def markup_to_html(
+        course: Course | None,
+        repo: Repo_ish,
+        commit_sha: bytes,
+        text: str,
+        reverse_func: Callable | None = None,
+        validate_only: bool = False,
+        use_jinja: bool = True,
+        jinja_env: dict | None = None,
+        ) -> str:
+
+    if jinja_env is None:
+        jinja_env = {}
+
+    disable_codehilite = bool(
+        getattr(settings,
+                "RELATE_DISABLE_CODEHILITE_MARKDOWN_EXTENSION", True))
 
     if course is not None and not jinja_env:
         try:
@@ -873,14 +1461,17 @@ def markup_to_html(
             cache_key = None
         else:
             import hashlib
-            cache_key = ("markup:v6:%s:%d:%s:%s"
-                    % (CACHE_KEY_ROOT, course.id, str(commit_sha),
-                        hashlib.md5(text.encode("utf-8")).hexdigest()))
+            cache_key = ("markup:v9:%s:%d:%s:%s:%s%s"
+                    % (CACHE_KEY_ROOT,
+                       course.id, course.trusted_for_markup, str(commit_sha),
+                       hashlib.md5(text.encode("utf-8")).hexdigest(),
+                       ":NOCODEHILITE" if disable_codehilite else ""
+                       ))
 
             def_cache = cache.caches["default"]
             result = def_cache.get(cache_key)
             if result is not None:
-                assert isinstance(result, six.text_type)
+                assert isinstance(result, str)
                 return result
 
         if text.lstrip().startswith(JINJA_PREFIX):
@@ -888,51 +1479,56 @@ def markup_to_html(
     else:
         cache_key = None
 
-    if not isinstance(text, six.text_type):
-        text = six.text_type(text)
-
-    # {{{ process through Jinja
-
-    if use_jinja:
-        from jinja2 import Environment, StrictUndefined
-        env = Environment(
-                loader=GitTemplateLoader(repo, commit_sha),
-                undefined=StrictUndefined)
-        template = env.from_string(text)
-        text = template.render(**jinja_env)
+    text = expand_markup(
+            course, repo, commit_sha, text, use_jinja=use_jinja, jinja_env=jinja_env)
 
-    # }}}
+    if reverse_func is None:
+        from django.urls import reverse
+        reverse_func = reverse
 
     if validate_only:
         return ""
 
-    from course.mdx_mathjax import MathJaxExtension
     import markdown
+
+    from course.mdx_mathjax import MathJaxExtension
+
+    extensions: list[markdown.Extension | str] = [
+        LinkFixerExtension(course, commit_sha, reverse_func=reverse_func),
+        MathJaxExtension(),
+        "markdown.extensions.extra",
+    ]
+
     result = markdown.markdown(text,
-        extensions=[
-            LinkFixerExtension(course, commit_sha, reverse_func=reverse_func),
-            MathJaxExtension(),
-            "markdown.extensions.extra",
-            "markdown.extensions.codehilite",
-            ],
-        output_format="html5")
-
-    assert isinstance(result, six.text_type)
+        extensions=extensions,
+        output_format="html")
+
+    if course is None or not course.trusted_for_markup:
+        import bleach
+        result = bleach.clean(result,
+                tags=[*bleach.ALLOWED_TAGS, "div", "span", "p", "img",
+                    "h1", "h2", "h3", "h4", "h5", "h6",
+                    "table", "td", "tr", "th",
+                    "pre", "details", "summary", "thead", "tbody"],
+                attributes=filter_html_attributes)
+
+    result = f"<div class='relate-markup'>{result}</div>"
+
+    assert isinstance(result, str)
     if cache_key is not None:
         def_cache.add(cache_key, result, None)
 
     return result
 
 
-TITLE_RE = re.compile(r"^\#+\s*(\w.*)", re.UNICODE)
+TITLE_RE = re.compile(r"^\#+\s*(.+)", re.UNICODE)
 
 
-def extract_title_from_markup(markup_text):
-    # type: (Text) -> Optional[Text]
+def extract_title_from_markup(markup_text: str) -> str | None:
     lines = markup_text.split("\n")
 
-    for l in lines[:10]:
-        match = TITLE_RE.match(l)
+    for ln in lines[:10]:
+        match = TITLE_RE.match(ln)
         if match is not None:
             return match.group(1)
 
@@ -955,14 +1551,12 @@ class InvalidDatespec(ValueError):
         self.datespec = datespec
 
 
-class DatespecPostprocessor(object):
+class DatespecPostprocessor:
     @classmethod
-    def parse(cls, s):
-        # type: (Text) -> Tuple[Text, Optional[DatespecPostprocessor]]
+    def parse(cls, s: str) -> tuple[str, DatespecPostprocessor | None]:
         raise NotImplementedError()
 
-    def apply(self, dtm):
-        # type: (datetime.datetime) -> datetime.datetime
+    def apply(self, dtm: datetime.datetime) -> datetime.datetime:
         raise NotImplementedError()
 
 
@@ -970,8 +1564,7 @@ AT_TIME_RE = re.compile(r"^(.*)\s*@\s*([0-2]?[0-9])\:([0-9][0-9])\s*$")
 
 
 class AtTimePostprocessor(DatespecPostprocessor):
-    def __init__(self, hour, minute, second=0):
-        # type: (int, int, int) -> None
+    def __init__(self, hour: int, minute: int, second: int = 0) -> None:
         self.hour = hour
         self.minute = minute
         self.second = second
@@ -993,9 +1586,9 @@ class AtTimePostprocessor(DatespecPostprocessor):
         else:
             return s, None
 
-    def apply(self, dtm):
-        from pytz import timezone
-        server_tz = timezone(settings.TIME_ZONE)
+    def apply(self, dtm: datetime.datetime) -> datetime.datetime:
+        from zoneinfo import ZoneInfo
+        server_tz = ZoneInfo(settings.TIME_ZONE)
 
         return dtm.astimezone(server_tz).replace(
                     hour=self.hour,
@@ -1004,12 +1597,11 @@ class AtTimePostprocessor(DatespecPostprocessor):
 
 
 PLUS_DELTA_RE = re.compile(r"^(.*)\s*([+-])\s*([0-9]+)\s+"
-    "(weeks?|days?|hours?|minutes?)$")
+    r"(weeks?|days?|hours?|minutes?)$")
 
 
 class PlusDeltaPostprocessor(DatespecPostprocessor):
-    def __init__(self, count, period):
-        # type: (int, Text) -> None
+    def __init__(self, count: int, period: str) -> None:
 
         self.count = count
         self.period = period
@@ -1034,35 +1626,31 @@ class PlusDeltaPostprocessor(DatespecPostprocessor):
             d = datetime.timedelta(days=self.count)
         elif self.period.startswith("hour"):
             d = datetime.timedelta(hours=self.count)
-        elif self.period.startswith("minute"):
-            d = datetime.timedelta(minutes=self.count)
         else:
-            raise InvalidDatespec(_("invalid period: %s" % self.period))
-
+            assert self.period.startswith("minute")
+            d = datetime.timedelta(minutes=self.count)
         return dtm + d
 
 
-DATESPEC_POSTPROCESSORS = [
+DATESPEC_POSTPROCESSORS: list[Any] = [
         AtTimePostprocessor,
         PlusDeltaPostprocessor,
-        ]  # type: List[Any]
+        ]
 
 
 def parse_date_spec(
-        course,  # type: Optional[Course]
-        datespec,  # type: Union[Text, datetime.date, datetime.datetime]
-        vctx=None,  # type: Optional[ValidationContext]
-        location=None,  # type: Optional[Text]
-        ):
-    # type: (...)  -> datetime.datetime
+        course: Course | None,
+        datespec: str | datetime.date | datetime.datetime,
+        vctx: ValidationContext | None = None,
+        location: str | None = None,
+        ) -> datetime.datetime:
 
     if datespec is None:
         return None
 
     orig_datespec = datespec
 
-    def localize_if_needed(d):
-        # type: (datetime.datetime) -> datetime.datetime
+    def localize_if_needed(d: datetime.datetime) -> datetime.datetime:
         if d.tzinfo is None:
             from relate.utils import localize_datetime
             return localize_datetime(d)
@@ -1075,11 +1663,11 @@ def parse_date_spec(
         return localize_if_needed(
                 datetime.datetime.combine(datespec, datetime.time.min))
 
-    datespec_str = cast(Text, datespec).strip()
+    datespec_str = cast(str, datespec).strip()
 
     # {{{ parse postprocessors
 
-    postprocs = []  # type: List[DatespecPostprocessor]
+    postprocs: list[DatespecPostprocessor] = []
     while True:
         parsed_one = False
         for pp_class in DATESPEC_POSTPROCESSORS:
@@ -1096,8 +1684,7 @@ def parse_date_spec(
 
     # }}}
 
-    def apply_postprocs(dtime):
-        # type: (datetime.datetime) -> datetime.datetime
+    def apply_postprocs(dtime: datetime.datetime) -> datetime.datetime:
         for postproc in postprocs:
             dtime = postproc.apply(dtime)
 
@@ -1122,7 +1709,7 @@ def parse_date_spec(
         # event with numeral
 
         event_kind = match.group(1)
-        ordinal = int(match.group(2))  # type: Optional[int]
+        ordinal: int | None = int(match.group(2))
 
     else:
         # event without numeral
@@ -1132,7 +1719,7 @@ def parse_date_spec(
 
     if vctx is not None:
         from course.validation import validate_identifier
-        validate_identifier(vctx, "%s: event kind" % location, event_kind)
+        validate_identifier(vctx, f"{location}: event kind", event_kind)
 
     if course is None:
         return now()
@@ -1149,8 +1736,9 @@ def parse_date_spec(
         if vctx is not None:
             vctx.add_warning(
                     location,
-                    _("unrecognized date/time specification: '%s' "
-                    "(interpreted as 'now')")
+                    _("Unrecognized date/time specification: '%s' "
+                    "(interpreted as 'now'). "
+                    "You should add an event with this name.")
                     % orig_datespec)
         return now()
 
@@ -1177,13 +1765,12 @@ def parse_date_spec(
 # {{{ page chunks
 
 def compute_chunk_weight_and_shown(
-        course,  # type:  Course
-        chunk,  # type: ChunkDesc
-        roles,  # type: List[Text]
-        now_datetime,  # type: datetime.datetime
-        facilities,  # type: frozenset[Text]
-        ):
-    # type: (...) -> Tuple[float, bool]
+        course: Course,
+        chunk: ChunkDesc,
+        roles: list[str],
+        now_datetime: datetime.datetime,
+        facilities: Collection[str],
+        ) -> tuple[float, bool]:
     if not hasattr(chunk, "rules"):
         return 0, True
 
@@ -1208,16 +1795,16 @@ def compute_chunk_weight_and_shown(
 
         # {{{ deprecated
 
-        if hasattr(rule, "roles"):
+        if hasattr(rule, "roles"):  # pragma: no cover  # deprecated
             if all(role not in rule.roles for role in roles):
                 continue
 
-        if hasattr(rule, "start"):
+        if hasattr(rule, "start"):  # pragma: no cover  # deprecated
             start_date = parse_date_spec(course, rule.start)
             if now_datetime < start_date:
                 continue
 
-        if hasattr(rule, "end"):
+        if hasattr(rule, "end"):  # pragma: no cover  # deprecated
             end_date = parse_date_spec(course, rule.end)
             if end_date < now_datetime:
                 continue
@@ -1234,15 +1821,14 @@ def compute_chunk_weight_and_shown(
 
 
 def get_processed_page_chunks(
-        course,  # type: Course
-        repo,  # type: Repo_ish
-        commit_sha,  # type: bytes
-        page_desc,  # type: StaticPageDesc
-        roles,  # type: List[Text]
-        now_datetime,  # type: datetime.datetime
-        facilities,  # type: frozenset[Text]
-        ):
-    # type: (...) -> List[ChunkDesc]
+        course: Course,
+        repo: Repo_ish,
+        commit_sha: bytes,
+        page_desc: StaticPageDesc,
+        roles: list[str],
+        now_datetime: datetime.datetime,
+        facilities: Collection[str],
+        ) -> list[ChunkDesc]:
     for chunk in page_desc.chunks:
         chunk.weight, chunk.shown = \
                 compute_chunk_weight_and_shown(
@@ -1263,11 +1849,10 @@ def get_processed_page_chunks(
 
 # {{{ repo desc getting
 
-def normalize_page_desc(page_desc):
-    # type: (StaticPageDesc) -> StaticPageDesc
+def normalize_page_desc(page_desc: StaticPageDesc) -> StaticPageDesc:
     if hasattr(page_desc, "content"):
         content = page_desc.content
-        from relate.utils import struct_to_dict, Struct
+        from relate.utils import Struct, struct_to_dict
         d = struct_to_dict(page_desc)
         del d["content"]
         d["chunks"] = [Struct({"id": "main", "content": content})]
@@ -1276,28 +1861,27 @@ def normalize_page_desc(page_desc):
     return page_desc
 
 
-def get_staticpage_desc(repo, course, commit_sha, filename):
-    # type: (Repo_ish, Course, bytes, Text) -> StaticPageDesc
+def get_staticpage_desc(
+        repo: Repo_ish, course: Course, commit_sha: bytes, filename: str
+        ) -> StaticPageDesc:
 
     page_desc = get_yaml_from_repo(repo, filename, commit_sha)
     page_desc = normalize_page_desc(page_desc)
     return page_desc
 
 
-def get_course_desc(repo, course, commit_sha):
-    # type: (Repo_ish, Course, bytes) -> CourseDesc
+def get_course_desc(repo: Repo_ish, course: Course, commit_sha: bytes) -> CourseDesc:
 
     return cast(
             CourseDesc,
             get_staticpage_desc(repo, course, commit_sha, course.course_file))
 
 
-def normalize_flow_desc(flow_desc):
-    # type: (FlowDesc) -> FlowDesc
+def normalize_flow_desc(flow_desc: FlowDesc) -> FlowDesc:
 
     if hasattr(flow_desc, "pages"):
         pages = flow_desc.pages
-        from relate.utils import struct_to_dict, Struct
+        from relate.utils import Struct, struct_to_dict
         d = struct_to_dict(flow_desc)
         del d["pages"]
         d["groups"] = [Struct({"id": "main", "pages": pages})]
@@ -1305,7 +1889,7 @@ def normalize_flow_desc(flow_desc):
 
     if hasattr(flow_desc, "rules"):
         rules = flow_desc.rules
-        if not hasattr(rules, "grade_identifier"):
+        if not hasattr(rules, "grade_identifier"):  # pragma: no cover  # deprecated
             # Legacy content with grade_identifier in grading rule,
             # move first found grade_identifier up to rules.
 
@@ -1313,30 +1897,35 @@ def normalize_flow_desc(flow_desc):
             rules.grade_aggregation_strategy = None
 
             for grule in rules.grading:
-                if grule.grade_identifier is not None:
-                    rules.grade_identifier = grule.grade_identifier
-                    rules.grade_aggregation_strategy = \
-                            grule.grade_aggregation_strategy
+                if grule.grade_identifier is not None:  # type: ignore
+                    rules.grade_identifier = grule.grade_identifier  # type: ignore
+                    rules.grade_aggregation_strategy = (  # type: ignore
+                            grule.grade_aggregation_strategy)  # type: ignore
                     break
 
     return flow_desc
 
 
-def get_flow_desc(repo, course, flow_id, commit_sha):
-    # type: (Repo_ish, Course, Text, bytes) -> FlowDesc
+def get_flow_desc(
+        repo: Repo_ish, course: Course, flow_id: str,
+        commit_sha: bytes, tolerate_tabs: bool = False) -> FlowDesc:
+    """
+    :arg tolerate_tabs: At one point, Relate accepted tabs
+        in indentation, but it no longer does. In places where legacy
+        compatibility matters, you may set *tolerate_tabs* to *True*.
+    """
 
-    flow_desc = get_yaml_from_repo(repo, "flows/%s.yml" % flow_id, commit_sha)
+    # FIXME: extension should be case-insensitive
+    flow_desc = get_yaml_from_repo(repo, f"flows/{flow_id}.yml", commit_sha,
+            tolerate_tabs=tolerate_tabs)
 
     flow_desc = normalize_flow_desc(flow_desc)
 
-    flow_desc.description_html = markup_to_html(
-            course, repo, commit_sha, getattr(flow_desc, "description", None))
     return flow_desc
 
 
-def get_flow_page_desc(flow_id, flow_desc, group_id, page_id):
-    # type: (Text, FlowDesc, Text, Text) -> FlowPageDesc
-
+def get_flow_page_desc(flow_id: str, flow_desc: FlowDesc,
+        group_id: str, page_id: str) -> FlowPageDesc:
     for grp in flow_desc.groups:
         if grp.id == group_id:
             for page in grp.pages:
@@ -1345,9 +1934,9 @@ def get_flow_page_desc(flow_id, flow_desc, group_id, page_id):
 
     raise ObjectDoesNotExist(
             _("page '%(group_id)s/%(page_id)s' in flow '%(flow_id)s'") % {
-                'group_id': group_id,
-                'page_id': page_id,
-                'flow_id': flow_id
+                "group_id": group_id,
+                "page_id": page_id,
+                "flow_id": flow_id
                 })
 
 # }}}
@@ -1359,31 +1948,39 @@ class ClassNotFoundError(RuntimeError):
     pass
 
 
-def import_class(name):
-    # type: (Text) -> type
-    components = name.split('.')
+def import_class(name: str) -> type:
+    components = name.split(".")
 
     if len(components) < 2:
         # need at least one module plus class name
         raise ClassNotFoundError(name)
 
-    module_name = ".".join(components[:-1])
-    try:
-        mod = __import__(module_name)
-    except ImportError:
-        raise ClassNotFoundError(name)
-
-    for comp in components[1:]:
+    from importlib import import_module
+    mod_components = len(components) - 1
+    while mod_components:
+        module_name = ".".join(components[:mod_components])
         try:
-            mod = getattr(mod, comp)
-        except AttributeError:
-            raise ClassNotFoundError(name)
+            mod = import_module(module_name)
+        except ImportError:
+            mod_components -= 1
+            continue
+
+        sym = mod
+        for cls_comp in components[mod_components:]:
+            try:
+                sym = getattr(sym, cls_comp)
+            except AttributeError:
+                raise ClassNotFoundError(name)
 
-    return mod
+        if isinstance(sym, type):
+            return sym
+        else:
+            raise ClassNotFoundError(f"'{name}' does not name a type")
+
+    raise ClassNotFoundError(name)
 
 
-def get_flow_page_class(repo, typename, commit_sha):
-    # type: (Repo_ish, Text, bytes) -> type
+def get_flow_page_class(repo: Repo_ish, typename: str, commit_sha: bytes) -> type:
 
     # look among default page types
     import course.page
@@ -1398,80 +1995,76 @@ def get_flow_page_class(repo, typename, commit_sha):
     except ClassNotFoundError:
         pass
 
-    if typename.startswith("repo:"):
-        stripped_typename = typename[5:]
-
-        components = stripped_typename.split(".")
-        if len(components) != 2:
-            raise ClassNotFoundError(
-                    _("repo page class must conist of two "
-                    "dotted components (invalid: '%s')")
-                    % typename)
+    raise ClassNotFoundError(typename)
 
-        module, classname = components
-        module_name = "code/"+module+".py"
-        module_code = get_repo_blob(repo, module_name, commit_sha,
-                allow_tree=False).data
-
-        module_dict = {}  # type: Dict
-
-        exec(compile(module_code, module_name, 'exec'), module_dict)
-
-        try:
-            return module_dict[classname]
-        except AttributeError:
-            raise ClassNotFoundError(typename)
-    else:
-        raise ClassNotFoundError(typename)
 
-
-def instantiate_flow_page(location, repo, page_desc, commit_sha):
-    # type: (Text, Repo_ish, FlowPageDesc, bytes) -> PageBase
+def instantiate_flow_page(
+        location: str, repo: Repo_ish, page_desc: FlowPageDesc, commit_sha: bytes
+        ) -> PageBase:
     class_ = get_flow_page_class(repo, page_desc.type, commit_sha)
 
+    from course.page.base import PageBase
+    if not issubclass(class_, PageBase):
+        raise ClassNotFoundError(f"'{page_desc.type}' is not a PageBase subclass")
+
     return class_(None, location, page_desc)
 
 # }}}
 
 
-def get_course_commit_sha(course, participation):
-    # type: (Course, Optional[Participation]) -> bytes
+class CourseCommitSHADoesNotExist(Exception):
+    pass
 
-    # logic duplicated in course.utils.CoursePageContext
 
+def get_course_commit_sha(
+        course: Course,
+        participation: Participation | None,
+        repo: Repo_ish | None = None,
+        raise_on_nonexistent_preview_commit: bool | None = False
+        ) -> bytes:
     sha = course.active_git_commit_sha
 
+    def is_commit_sha_valid(repo: Repo_ish, commit_sha: str) -> bool:
+        if isinstance(repo, SubdirRepoWrapper):
+            repo = repo.repo
+        try:
+            repo[commit_sha.encode()]
+        except KeyError:
+            if raise_on_nonexistent_preview_commit:
+                raise CourseCommitSHADoesNotExist(
+                    _("Preview revision '{}' does not exist--"
+                      "showing active course content instead.").format(commit_sha))
+            return False
+
+        return True
+
     if participation is not None:
         if participation.preview_git_commit_sha:
             preview_sha = participation.preview_git_commit_sha
 
-            repo = get_course_repo(course)
-            if isinstance(repo, SubdirRepoWrapper):
-                repo = repo.repo
-
-            try:
-                repo[preview_sha.encode()]
-            except KeyError:
-                preview_sha = None
+            if repo is not None:
+                preview_sha_valid = is_commit_sha_valid(repo, preview_sha)
+            else:
+                with get_course_repo(course) as repo:
+                    preview_sha_valid = is_commit_sha_valid(repo, preview_sha)
 
-            if preview_sha is not None:
+            if preview_sha_valid:
                 sha = preview_sha
 
     return sha.encode()
 
 
-def list_flow_ids(repo, commit_sha):
-    # type: (Repo_ish, bytes) -> List[Text]
+def list_flow_ids(repo: Repo_ish, commit_sha: bytes) -> list[str]:
     flow_ids = []
     try:
-        flows_tree = get_repo_blob(repo, "flows", commit_sha)
+        flows_tree = get_repo_tree(repo, "flows", commit_sha)
     except ObjectDoesNotExist:
         # That's OK--no flows yet.
         pass
     else:
         for entry in flows_tree.items():
             if entry.path.endswith(b".yml"):
-                flow_ids.append(entry.path[:-4])
+                flow_ids.append(entry.path[:-4].decode("utf-8"))
 
     return sorted(flow_ids)