content.py

# -*- coding: utf-8 -*-

from __future__ import division

__copyright__ = "Copyright (C) 2014 Andreas Kloeckner"

__license__ = """
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

from typing import cast, Union, Text

from django.conf import settings
from django.utils.translation import gettext as _

import os
import re
import datetime
import sys

from django.utils.timezone import now
from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured
from django.urls import NoReverseMatch

from markdown.extensions import Extension
from markdown.treeprocessors import Treeprocessor

import html.parser as html_parser

from jinja2 import (
        BaseLoader as BaseTemplateLoader, TemplateNotFound, FileSystemLoader)

from relate.utils import dict_to_struct, Struct, SubdirRepoWrapper
from course.constants import ATTRIBUTES_FILENAME

from yaml import safe_load as load_yaml

if sys.version_info >= (3,):
    CACHE_KEY_ROOT = "py3"
else:
    CACHE_KEY_ROOT = "py2"


# {{{ mypy

from typing import (  # noqa
    Any, List, Tuple, Optional, Callable, Text, Dict, FrozenSet, TYPE_CHECKING)
if TYPE_CHECKING:
    # for mypy
    from course.models import Course, Participation  # noqa
    import dulwich  # noqa
    from course.validation import ValidationContext  # noqa
    from course.page.base import PageBase  # noqa
    from relate.utils import Repo_ish  # noqa

    Date_ish = Union[datetime.datetime, datetime.date]
    Datespec = Union[datetime.datetime, datetime.date, Text]


class ChunkRulesDesc(Struct):
    if_has_role = None  # type: List[Text]
    if_before = None  # type: Datespec
    if_after = None  # type: Datespec
    if_in_facility = None  # type: Text
    if_has_participation_tags_any = None  # type: List[Text]
    if_has_participation_tags_all = None  # type: List[Text]
    roles = None  # type: List[Text]
    start = None  # type: Datespec
    end = None  # type: Datespec
    shown = None  # type: bool
    weight = None  # type: float


class ChunkDesc(Struct):
    weight = None  # type: float
    shown = None  # type: bool
    title = None  # type: Optional[Text]
    content = None  # type: Text
    rules = None  # type: List[ChunkRulesDesc]

    html_content = None  # type: Text


class StaticPageDesc(Struct):
    chunks = None  # type: List[ChunkDesc]
    content = None  # type: Text


class CourseDesc(StaticPageDesc):
    pass


class FlowSessionStartRuleDesc(Struct):
    if_after = None  # type: Date_ish
    if_before = None  # type: Date_ish
    if_has_role = None  # type: list
    if_has_participation_tags_any = None  # type: List[Text]
    if_has_participation_tags_all = None  # type: List[Text]
    if_in_facility = None  # type: Text
    if_has_in_progress_session = None  # type: bool
    if_has_session_tagged = None  # type: Optional[Text]
    if_has_fewer_sessions_than = None  # type: int
    if_has_fewer_tagged_sessions_than = None  # type: int
    if_signed_in_with_matching_exam_ticket = None  # type: bool
    tag_session = None  # type: Optional[Text]
    may_start_new_session = None  # type: bool
    may_list_existing_sessions = None  # type: bool
    lock_down_as_exam_session = None  # type: bool
    default_expiration_mode = None  # type: Text


class FlowSessionAccessRuleDesc(Struct):
    permissions = None  # type: list
    if_after = None  # type: Date_ish
    if_before = None  # type: Date_ish
    if_started_before = None  # type: Date_ish
    if_has_role = None  # type: List[Text]
    if_has_participation_tags_any = None  # type: List[Text]
    if_has_participation_tags_all = None  # type: List[Text]
    if_in_facility = None  # type: Text
    if_has_tag = None  # type: Optional[Text]
    if_in_progress = None  # type: bool
    if_completed_before = None  # type: Date_ish
    if_expiration_mode = None  # type: Text
    if_session_duration_shorter_than_minutes = None  # type: float
    if_signed_in_with_matching_exam_ticket = None  # type: bool
    message = None  # type: Text


class FlowSessionGradingRuleDesc(Struct):
    grade_identifier = None  # type: Optional[Text]
    grade_aggregation_strategy = None  # type: Optional[Text]


class FlowRulesDesc(Struct):
    start = None  # type: List[FlowSessionStartRuleDesc]
    access = None  # type: List[FlowSessionAccessRuleDesc]
    grading = None  # type: List[FlowSessionGradingRuleDesc]
    grade_identifier = None  # type: Optional[Text]
    grade_aggregation_strategy = None  # type: Optional[Text]


class FlowPageDesc(Struct):
    id = None  # type: Text
    type = None  # type: Text


class FlowPageGroupDesc(Struct):
    id = None  # type: Text
    pages = None  # type: List[FlowPageDesc]


class FlowDesc(Struct):
    title = None  # type: Text
    rules = None  # type: FlowRulesDesc
    pages = None  # type: List[FlowPageDesc]
    groups = None  # type: List[FlowPageGroupDesc]
    notify_on_submit = None  # type: Optional[List[Text]]


# }}}


# {{{ repo blob getting

def get_true_repo_and_path(repo, path):
    # type: (Repo_ish, Text) -> Tuple[dulwich.Repo, Text]

    if isinstance(repo, SubdirRepoWrapper):
        if path:
            path = repo.subdir + "/" + path
        else:
            path = repo.subdir

        return repo.repo, path

    else:
        return repo, path


def get_course_repo_path(course):
    # type: (Course) -> Text

    return os.path.join(settings.GIT_ROOT, course.identifier)


def get_course_repo(course):
    # type: (Course) -> Repo_ish

    from dulwich.repo import Repo
    repo = Repo(get_course_repo_path(course))

    if course.course_root_path:
        return SubdirRepoWrapper(repo, course.course_root_path)
    else:
        return repo


def get_repo_blob(repo, full_name, commit_sha, allow_tree=True):
    # type: (Repo_ish, Text, bytes, bool) -> dulwich.Blob

    """
    :arg full_name: A Unicode string indicating the file name.
    :arg commit_sha: A byte string containing the commit hash
    :arg allow_tree: Allow the resulting object to be a directory
    """

    dul_repo, full_name = get_true_repo_and_path(repo, full_name)

    # https://github.com/inducer/relate/pull/556
    names = os.path.normpath(full_name).split(os.sep)

    # Allow non-ASCII file name
    full_name_bytes = full_name.encode('utf-8')

    try:
        tree_sha = dul_repo[commit_sha].tree
    except KeyError:
        raise ObjectDoesNotExist(
                _("commit sha '%s' not found") % commit_sha.decode())

    tree = dul_repo[tree_sha]

    def access_directory_content(maybe_tree, name):
        # type: (Any, Text) -> Any
        try:
            mode_and_blob_sha = maybe_tree[name.encode()]
        except TypeError:
            raise ObjectDoesNotExist(_("resource '%s' is a file, "
                "not a directory") % full_name)

        mode, blob_sha = mode_and_blob_sha
        return mode_and_blob_sha

    if not full_name_bytes:
        if allow_tree:
            return tree
        else:
            raise ObjectDoesNotExist(
                    _("repo root is a directory, not a file"))

    try:
        for name in names[:-1]:
            if not name:
                # tolerate empty path components (begrudgingly)
                continue

            mode, blob_sha = access_directory_content(tree, name)
            tree = dul_repo[blob_sha]

        mode, blob_sha = access_directory_content(tree, names[-1])

        result = dul_repo[blob_sha]
        if not allow_tree and not hasattr(result, "data"):
            raise ObjectDoesNotExist(
                    _("resource '%s' is a directory, not a file") % full_name)

        return result

    except KeyError:
        raise ObjectDoesNotExist(_("resource '%s' not found") % full_name)


def get_repo_blob_data_cached(repo, full_name, commit_sha):
    # type: (Repo_ish, Text, bytes) -> bytes
    """
    :arg commit_sha: A byte string containing the commit hash
    """

    if isinstance(commit_sha, bytes):
        from urllib.parse import quote_plus
        cache_key = "%s%R%1".join((
            CACHE_KEY_ROOT,
            quote_plus(repo.controldir()),
            quote_plus(full_name),
            commit_sha.decode(),
            ".".join(str(s) for s in sys.version_info[:2]),
            ))  # type: Optional[Text]
    else:
        cache_key = None

    try:
        import django.core.cache as cache
    except ImproperlyConfigured:
        cache_key = None

    result = None  # type: Optional[bytes]
    if cache_key is None:
        result = get_repo_blob(repo, full_name, commit_sha,
                allow_tree=False).data
        assert isinstance(result, bytes)
        return result

    # Byte string is wrapped in a tuple to force pickling because memcache's
    # python wrapper appears to auto-decode/encode string values, thus trying
    # to decode our byte strings. Grr.

    def_cache = cache.caches["default"]

    # Memcache is apparently limited to 250 characters.
    if len(cache_key) < 240:
        cached_result = def_cache.get(cache_key)

        if cached_result is not None:
            (result,) = cached_result
            assert isinstance(result, bytes), cache_key
            return result

    result = get_repo_blob(repo, full_name, commit_sha,
            allow_tree=False).data
    assert result is not None

    if len(result) <= getattr(settings, "RELATE_CACHE_MAX_BYTES", 0):
        def_cache.add(cache_key, (result,), None)

    assert isinstance(result, bytes)

    return result


def is_repo_file_accessible_as(access_kinds, repo, commit_sha, path):
    # type: (List[Text], Repo_ish, bytes, Text) -> bool
    """
    Check of a file in a repo directory is accessible.  For example,
    'instructor' can access anything listed in the attributes.
    'student' can access 'student' and 'unenrolled'.  The 'unenrolled' role
    can only access 'unenrolled'.

    :arg commit_sha: A byte string containing the commit hash
    """

    # set the path to .attributes.yml
    attributes_path = os.path.join(os.path.dirname(path), ATTRIBUTES_FILENAME)

    # retrieve the .attributes.yml structure
    try:
        attributes = get_raw_yaml_from_repo(repo, attributes_path,
                                            commit_sha)
    except ObjectDoesNotExist:
        # no attributes file: not accessible
        return False

    path_basename = os.path.basename(path)

    # "public" is a deprecated alias for "unenrolled".

    access_patterns = []  # type: List[Text]
    for kind in access_kinds:
        access_patterns += attributes.get(kind, [])

    from fnmatch import fnmatch
    if isinstance(access_patterns, list):
        for pattern in access_patterns:
            if isinstance(pattern, str):
                if fnmatch(path_basename, pattern):
                    return True

    return False

# }}}


# {{{ jinja interaction

JINJA_YAML_RE = re.compile(
    r"^\[JINJA\]\s*$(.*?)^\[\/JINJA\]\s*$",
    re.MULTILINE | re.DOTALL)
YAML_BLOCK_START_SCALAR_RE = re.compile(
    r"(:\s*[|>])"
    "(J?)"
    "((?:[0-9][-+]?|[-+][0-9]?)?)"
    r"(?:\s*\#.*)?"
    "$")

IN_BLOCK_END_RAW_RE = re.compile(r"(.*)({%-?\s*endraw\s*-?%})(.*)")
GROUP_COMMENT_START = re.compile(r"^\s*#\s*\{\{\{")
LEADING_SPACES_RE = re.compile(r"^( *)")


def process_yaml_for_expansion(yaml_str):
    # type: (Text) -> Text

    lines = yaml_str.split("\n")
    jinja_lines = []

    i = 0
    line_count = len(lines)

    while i < line_count:
        ln = lines[i].rstrip()
        yaml_block_scalar_match = YAML_BLOCK_START_SCALAR_RE.search(ln)

        if yaml_block_scalar_match is not None:
            unprocessed_block_lines = []
            allow_jinja = bool(yaml_block_scalar_match.group(2))
            ln = YAML_BLOCK_START_SCALAR_RE.sub(
                    r"\1\3", ln)

            unprocessed_block_lines.append(ln)

            leading_spaces_match = LEADING_SPACES_RE.match(ln)
            assert leading_spaces_match
            block_start_indent = len(leading_spaces_match.group(1))

            i += 1

            while i < line_count:
                ln = lines[i]

                if not ln.rstrip():
                    unprocessed_block_lines.append(ln)
                    i += 1
                    continue

                leading_spaces_match = LEADING_SPACES_RE.match(ln)
                assert leading_spaces_match
                line_indent = len(leading_spaces_match.group(1))
                if line_indent <= block_start_indent:
                    break
                else:
                    ln = IN_BLOCK_END_RAW_RE.sub(
                        r"\1{% endraw %}{{ '\2' }}{% raw %}\3", ln)
                    unprocessed_block_lines.append(ln.rstrip())
                    i += 1

            if not allow_jinja:
                jinja_lines.append("{% raw %}")
            jinja_lines.extend(unprocessed_block_lines)
            if not allow_jinja:
                jinja_lines.append("{% endraw %}")

        elif GROUP_COMMENT_START.match(ln):
            jinja_lines.append("{% raw %}")
            jinja_lines.append(ln)
            jinja_lines.append("{% endraw %}")
            i += 1

        else:
            jinja_lines.append(ln)
            i += 1
    return "\n".join(jinja_lines)


class GitTemplateLoader(BaseTemplateLoader):
    def __init__(self, repo, commit_sha):
        # type: (Repo_ish, bytes) -> None
        self.repo = repo
        self.commit_sha = commit_sha

    def get_source(self, environment, template):
        try:
            data = get_repo_blob_data_cached(self.repo, template, self.commit_sha)
        except ObjectDoesNotExist:
            raise TemplateNotFound(template)

        source = data.decode('utf-8')

        def is_up_to_date():
            # There's not much point to caching here, because we create
            # a new loader for every request anyhow...
            return False

        return source, None, is_up_to_date


class YamlBlockEscapingGitTemplateLoader(GitTemplateLoader):
    # https://github.com/inducer/relate/issues/130

    def get_source(self, environment, template):
        source, path, is_up_to_date = \
                super(YamlBlockEscapingGitTemplateLoader, self).get_source(
                        environment, template)

        _, ext = os.path.splitext(template)
        ext = ext.lower()

        if ext in [".yml", ".yaml"]:
            source = process_yaml_for_expansion(source)

        return source, path, is_up_to_date


class YamlBlockEscapingFileSystemLoader(FileSystemLoader):
    # https://github.com/inducer/relate/issues/130

    def get_source(self, environment, template):
        source, path, is_up_to_date = \
                super(YamlBlockEscapingFileSystemLoader, self).get_source(
                        environment, template)

        _, ext = os.path.splitext(template)
        ext = ext.lower()

        if ext in [".yml", ".yaml"]:
            source = process_yaml_for_expansion(source)

        return source, path, is_up_to_date


def expand_yaml_macros(repo, commit_sha, yaml_str):
    # type: (Repo_ish, bytes, Text) -> Text

    if isinstance(yaml_str, bytes):
        yaml_str = yaml_str.decode("utf-8")

    from jinja2 import Environment, StrictUndefined
    jinja_env = Environment(
            loader=YamlBlockEscapingGitTemplateLoader(repo, commit_sha),
            undefined=StrictUndefined)

    # {{{ process explicit [JINJA] tags (deprecated)

    def compute_replacement(match):  # pragma: no cover  # deprecated
        template = jinja_env.from_string(match.group(1))
        return template.render()

    yaml_str, count = JINJA_YAML_RE.subn(compute_replacement, yaml_str)

    if count:  # pragma: no cover  # deprecated
        # The file uses explicit [JINJA] tags. Assume that it doesn't
        # want anything else processed through YAML.
        return yaml_str

    # }}}

    jinja_str = process_yaml_for_expansion(yaml_str)
    template = jinja_env.from_string(jinja_str)
    yaml_str = template.render()

    return yaml_str

# }}}


# {{{ repo yaml getting

def get_raw_yaml_from_repo(repo, full_name, commit_sha):
    # type: (Repo_ish, Text, bytes) -> Any
    """Return decoded YAML data structure from
    the given file in *repo* at *commit_sha*.

    :arg commit_sha: A byte string containing the commit hash
    """

    from urllib.parse import quote_plus
    cache_key = "%RAW%%2".join((
        CACHE_KEY_ROOT,
        quote_plus(repo.controldir()), quote_plus(full_name), commit_sha.decode(),
        ))

    import django.core.cache as cache
    def_cache = cache.caches["default"]

    result = None  # type: Optional[Any]
    # Memcache is apparently limited to 250 characters.
    if len(cache_key) < 240:
        result = def_cache.get(cache_key)
    if result is not None:
        return result

    yaml_str = expand_yaml_macros(
                repo, commit_sha,
                get_repo_blob(repo, full_name, commit_sha,
                    allow_tree=False).data)

    result = load_yaml(yaml_str)  # type: ignore

    def_cache.add(cache_key, result, None)

    return result


LINE_HAS_INDENTING_TABS_RE = re.compile(r"^\s*\t\s*", re.MULTILINE)


def get_yaml_from_repo(repo, full_name, commit_sha, cached=True):
    # type: (Repo_ish, Text, bytes, bool) -> Any

    """Return decoded, struct-ified YAML data structure from
    the given file in *repo* at *commit_sha*.

    See :class:`relate.utils.Struct` for more on
    struct-ification.
    """

    if cached:
        try:
            import django.core.cache as cache
        except ImproperlyConfigured:
            cached = False
        else:
            from urllib.parse import quote_plus
            cache_key = "%%%2".join(
                    (CACHE_KEY_ROOT,
                        quote_plus(repo.controldir()), quote_plus(full_name),
                        commit_sha.decode()))

            def_cache = cache.caches["default"]
            result = None
            # Memcache is apparently limited to 250 characters.
            if len(cache_key) < 240:
                result = def_cache.get(cache_key)
            if result is not None:
                return result

    yaml_bytestream = get_repo_blob(
            repo, full_name, commit_sha, allow_tree=False).data
    yaml_text = yaml_bytestream.decode("utf-8")

    if LINE_HAS_INDENTING_TABS_RE.search(yaml_text):
        raise ValueError("File uses tabs in indentation. "
                "This is not allowed.")

    expanded = expand_yaml_macros(
            repo, commit_sha, yaml_bytestream)

    yaml_data = load_yaml(expanded)  # type:ignore
    result = dict_to_struct(yaml_data)

    if cached:
        def_cache.add(cache_key, result, None)

    return result


# }}}


# {{{ markup

def _attr_to_string(key, val):
    if val is None:
        return key
    elif "\"" in val:
        return "%s='%s'" % (key, val)
    else:
        return "%s=\"%s\"" % (key, val)


class TagProcessingHTMLParser(html_parser.HTMLParser):
    def __init__(self, out_file, process_tag_func):
        html_parser.HTMLParser.__init__(self)

        self.out_file = out_file
        self.process_tag_func = process_tag_func

    def handle_starttag(self, tag, attrs):
        attrs = dict(attrs)
        attrs.update(self.process_tag_func(tag, attrs))

        self.out_file.write("<%s %s>" % (tag, " ".join(
            _attr_to_string(k, v) for k, v in attrs.items())))

    def handle_endtag(self, tag):
        self.out_file.write("</%s>" % tag)

    def handle_startendtag(self, tag, attrs):
        attrs = dict(attrs)
        attrs.update(self.process_tag_func(tag, attrs))

        self.out_file.write("<%s %s/>" % (tag, " ".join(
            _attr_to_string(k, v) for k, v in attrs.items())))

    def handle_data(self, data):
        self.out_file.write(data)

    def handle_entityref(self, name):
        self.out_file.write("&%s;" % name)

    def handle_charref(self, name):
        self.out_file.write("&#%s;" % name)

    def handle_comment(self, data):
        self.out_file.write("<!--%s-->" % data)

    def handle_decl(self, decl):
        self.out_file.write("<!%s>" % decl)

    def handle_pi(self, data):
        raise NotImplementedError(
                _("I have no idea what a processing instruction is."))

    def unknown_decl(self, data):
        self.out_file.write("<![%s]>" % data)


class PreserveFragment(object):
    def __init__(self, s):
        self.s = s


class LinkFixerTreeprocessor(Treeprocessor):
    def __init__(self, md, course, commit_sha, reverse_func):
        Treeprocessor.__init__(self)
        self.md = md
        self.course = course
        self.commit_sha = commit_sha
        self.reverse_func = reverse_func

    def reverse(self, viewname, args):
        frag = None

        new_args = []
        for arg in args:
            if isinstance(arg, PreserveFragment):
                s = arg.s
                frag_index = s.find("#")
                if frag_index != -1:
                    frag = s[frag_index:]
                    s = s[:frag_index]

                new_args.append(s)
            else:
                new_args.append(arg)

        result = self.reverse_func(viewname, args=new_args)

        if frag is not None:
            result += frag

        return result

    def get_course_identifier(self):
        if self.course is None:
            return "bogus-course-identifier"
        else:
            return self.course.identifier

    def process_url(self, url):
        try:
            if url.startswith("course:"):
                course_id = url[7:]
                if course_id:
                    return self.reverse("relate-course_page",
                                args=(course_id,))
                else:
                    return self.reverse("relate-course_page",
                                args=(self.get_course_identifier(),))

            elif url.startswith("flow:"):
                flow_id = url[5:]
                return self.reverse("relate-view_start_flow",
                            args=(self.get_course_identifier(), flow_id))

            elif url.startswith("staticpage:"):
                page_path = url[11:]
                return self.reverse("relate-content_page",
                            args=(
                                self.get_course_identifier(),
                                PreserveFragment(page_path)))

            elif url.startswith("media:"):
                media_path = url[6:]
                return self.reverse("relate-get_media",
                            args=(
                                self.get_course_identifier(),
                                self.commit_sha.decode(),
                                PreserveFragment(media_path)))

            elif url.startswith("repo:"):
                path = url[5:]
                return self.reverse("relate-get_repo_file",
                            args=(
                                self.get_course_identifier(),
                                self.commit_sha.decode(),
                                PreserveFragment(path)))

            elif url.startswith("repocur:"):
                path = url[8:]
                return self.reverse("relate-get_current_repo_file",
                            args=(
                                self.get_course_identifier(),
                                PreserveFragment(path)))

            elif url.strip() == "calendar:":
                return self.reverse("relate-view_calendar",
                            args=(self.get_course_identifier(),))

            else:
                return None

        except NoReverseMatch:
            from base64 import b64encode
            message = ("Invalid character in RELATE URL: " + url).encode("utf-8")
            return "data:text/plain;base64,"+b64encode(message).decode()

    def process_tag(self, tag_name, attrs):
        changed_attrs = {}

        if tag_name == "table" and attrs.get("bootstrap") != "no":
            changed_attrs["class"] = "table table-condensed"

        if tag_name in ["a", "link"] and "href" in attrs:
            new_href = self.process_url(attrs["href"])

            if new_href is not None:
                changed_attrs["href"] = new_href

        elif tag_name == "img" and "src" in attrs:
            new_src = self.process_url(attrs["src"])

            if new_src is not None:
                changed_attrs["src"] = new_src

        elif tag_name == "object" and "data" in attrs:
            new_data = self.process_url(attrs["data"])

            if new_data is not None:
                changed_attrs["data"] = new_data

        return changed_attrs

    def process_etree_element(self, element):
        changed_attrs = self.process_tag(element.tag, element.attrib)

        for key, val in changed_attrs.items():
            element.set(key, val)

    def walk_and_process_tree(self, root):
        self.process_etree_element(root)

        for child in root:
            self.walk_and_process_tree(child)

    def run(self, root):
        self.walk_and_process_tree(root)

        # root through and process Markdown's HTML stash (gross!)
        from io import StringIO

        for i, (html, safe) in enumerate(self.md.htmlStash.rawHtmlBlocks):
            outf = StringIO()
            parser = TagProcessingHTMLParser(outf, self.process_tag)
            parser.feed(html)

            self.md.htmlStash.rawHtmlBlocks[i] = (outf.getvalue(), safe)


class LinkFixerExtension(Extension):
    def __init__(self, course, commit_sha, reverse_func):
        # type: (Optional[Course], bytes, Optional[Callable]) -> None

        Extension.__init__(self)
        self.course = course
        self.commit_sha = commit_sha
        self.reverse_func = reverse_func

    def extendMarkdown(self, md, md_globals):  # noqa
        md.treeprocessors["relate_link_fixer"] = \
                LinkFixerTreeprocessor(md, self.course, self.commit_sha,
                        reverse_func=self.reverse_func)


def remove_prefix(prefix, s):
    # type: (Text, Text) -> Text
    if s.startswith(prefix):
        return s[len(prefix):]
    else:
        return s


JINJA_PREFIX = "[JINJA]"


def expand_markup(
        course,  # type: Optional[Course]
        repo,  # type: Repo_ish
        commit_sha,  # type: bytes
        text,  # type: Text
        use_jinja=True,  # type: bool
        jinja_env={},  # type: Dict
        ):
    # type: (...) -> Text

    if not isinstance(text, str):
        text = str(text)

    # {{{ process through Jinja

    if use_jinja:
        from jinja2 import Environment, StrictUndefined
        env = Environment(
                loader=GitTemplateLoader(repo, commit_sha),
                undefined=StrictUndefined)

        template = env.from_string(text)
        kwargs = {}
        if jinja_env:
            kwargs.update(jinja_env)

        from course.utils import IpynbJinjaMacro
        kwargs[IpynbJinjaMacro.name] = IpynbJinjaMacro(course, repo, commit_sha)

        text = template.render(**kwargs)

    # }}}

    return text


def markup_to_html(
        course,  # type: Optional[Course]
        repo,  # type: Repo_ish
        commit_sha,  # type: bytes
        text,  # type: Text
        reverse_func=None,  # type: Callable
        validate_only=False,  # type: bool
        use_jinja=True,  # type: bool
        jinja_env={},  # type: Dict
        ):
    # type: (...) -> Text

    disable_codehilite = bool(
        getattr(settings,
                "RELATE_DISABLE_CODEHILITE_MARKDOWN_EXTENSION", True))

    if course is not None and not jinja_env:
        try:
            import django.core.cache as cache
        except ImproperlyConfigured:
            cache_key = None
        else:
            import hashlib
            cache_key = ("markup:v7:%s:%d:%s:%s%s"
                    % (CACHE_KEY_ROOT,
                       course.id, str(commit_sha),
                       hashlib.md5(text.encode("utf-8")).hexdigest(),
                       ":NOCODEHILITE" if disable_codehilite else ""
                       ))

            def_cache = cache.caches["default"]
            result = def_cache.get(cache_key)
            if result is not None:
                assert isinstance(result, str)
                return result

        if text.lstrip().startswith(JINJA_PREFIX):
            text = remove_prefix(JINJA_PREFIX, text.lstrip())
    else:
        cache_key = None

    text = expand_markup(
            course, repo, commit_sha, text, use_jinja=use_jinja, jinja_env=jinja_env)

    if reverse_func is None:
        from django.urls import reverse
        reverse_func = reverse

    if validate_only:
        return ""

    from course.mdx_mathjax import MathJaxExtension
    from course.utils import NBConvertExtension
    import markdown

    extensions = [
        LinkFixerExtension(course, commit_sha, reverse_func=reverse_func),
        MathJaxExtension(),
        NBConvertExtension(),
        "markdown.extensions.extra",
    ]

    if not disable_codehilite:
        # Note: no matter whether disable_codehilite, the code in
        # the rendered ipython notebook will be highlighted.
        # "css_class=highlight" is to ensure that, when codehilite extension
        # is enabled, code out side of notebook uses the same html class
        # attribute as the default highlight class (i.e., `highlight`)
        # used by rendered ipynb notebook cells, Thus we don't need to
        # make 2 copies of css for the highlight.
        extensions += ["markdown.extensions.codehilite(css_class=highlight)"]

    result = markdown.markdown(text,
        extensions=extensions,
        output_format="html5")

    assert isinstance(result, str)
    if cache_key is not None:
        def_cache.add(cache_key, result, None)

    return result


TITLE_RE = re.compile(r"^\#+\s*(.+)", re.UNICODE)