content.py

from __future__ import annotations


__copyright__ = "Copyright (C) 2014 Andreas Kloeckner"

__license__ = """
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import datetime
import html.parser as html_parser
import os
import re
import sys
from typing import cast
from xml.etree.ElementTree import Element, tostring

import dulwich.objects
import dulwich.repo
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist
from django.urls import NoReverseMatch
from django.utils.timezone import now
from django.utils.translation import gettext as _
from markdown.extensions import Extension
from markdown.treeprocessors import Treeprocessor
from yaml import safe_load as load_yaml

from course.constants import ATTRIBUTES_FILENAME
from course.validation import Blob_ish, Tree_ish
from relate.utils import Struct, SubdirRepoWrapper, dict_to_struct


CACHE_KEY_ROOT = "py4"


# {{{ mypy

from collections.abc import Callable, Collection, Mapping
from typing import (
    TYPE_CHECKING,
    Any,
)


if TYPE_CHECKING:
    # for mypy
    import dulwich

    from course.models import Course, Participation
    from course.page.base import PageBase
    from course.validation import FileSystemFakeRepoTree, ValidationContext
    from relate.utils import Repo_ish

Date_ish = datetime.datetime | datetime.date
Datespec = datetime.datetime | datetime.date | str


class ChunkRulesDesc(Struct):
    if_has_role: list[str]
    if_before: Datespec
    if_after: Datespec
    if_in_facility: str
    if_has_participation_tags_any: list[str]
    if_has_participation_tags_all: list[str]
    roles: list[str]
    start: Datespec
    end: Datespec
    shown: bool
    weight: float


class ChunkDesc(Struct):
    weight: float
    shown: bool
    title: str | None
    content: str
    rules: list[ChunkRulesDesc]

    html_content: str


class StaticPageDesc(Struct):
    chunks: list[ChunkDesc]
    content: str


class CourseDesc(StaticPageDesc):
    pass

# }}}


# {{{ mypy: flow start rule

class FlowSessionStartRuleDesc(Struct):
    """Rules that govern when a new session may be started and whether
    existing sessions may be listed.

    Found in the ``start`` attribute of :class:`FlowRulesDesc`.

    .. rubric:: Conditions

    .. attribute:: if_after

        (Optional) A :ref:`datespec <datespec>` that determines a date/time
        after which this rule applies.

    .. attribute:: if_before

        (Optional) A :ref:`datespec <datespec>` that determines a date/time
        before which this rule applies.

    .. attribute:: if_has_role

        (Optional) A list of a subset of the roles defined in the course, by
        default ``unenrolled``, ``ta``, ``student``, ``instructor``.

    .. attribute:: if_has_participation_tags_any

        (Optional) A list of participation tags. Rule applies when the
        participation has at least one tag in this list.

    .. attribute:: if_has_participation_tags_all

        (Optional) A list of participation tags. Rule applies if only the
        participation's tags include all items in this list.

    .. attribute:: if_in_facility

        (Optional) Name of a facility known to the RELATE web page. This rule allows
        (for example) restricting flow starting based on whether a user is physically
        located in a computer-based testing center (which RELATE can
        recognize based on IP ranges).

    .. attribute:: if_has_in_progress_session

        (Optional) A Boolean (True/False) value, indicating that the rule only
        applies if the participant has an in-progress session.

    .. attribute:: if_has_session_tagged

        (Optional) An identifier (or ``null``) indicating that the rule only applies
        if the participant has a session with the corresponding tag.

    .. attribute:: if_has_fewer_sessions_than

        (Optional) An integer. The rule applies if the participant has fewer
        than this number of sessions.

    .. attribute:: if_has_fewer_tagged_sessions_than

        (Optional) An integer. The rule applies if the participant has fewer
        than this number of sessions with access rule tags.

    .. attribute:: if_signed_in_with_matching_exam_ticket

        (Optional) The rule applies if the participant signed in with an exam
        ticket matching this flow.

    .. rubric:: Rules specified

    .. attribute:: may_start_new_session

        (Mandatory) A Boolean (True/False) value indicating whether, if the
        rule applies, the participant may start a new session.

    .. attribute:: may_list_existing_sessions

        (Mandatory) A Boolean (True/False) value indicating whether, if the
        rule applies, the participant may view a list of existing sessions.

    .. attribute:: tag_session

        (Optional) An identifier that will be applied to a newly-created
        session as a "tag".  This can be used by
        :attr:`FlowSessionAccessRuleDesc.if_has_tag` and
        :attr:`FlowSessionGradingRuleDesc.if_has_tag`.

    .. attribute:: default_expiration_mode

        (Optional) One of :class:`~course.constants.flow_session_expiration_mode`.
        The expiration mode applied when a session is first created or rolled
        over.
    """

    # conditions
    if_after: Date_ish
    if_before: Date_ish
    if_has_role: list[str]
    if_has_participation_tags_any: list[str]
    if_has_participation_tags_all: list[str]
    if_in_facility: str
    if_has_in_progress_session: bool
    if_has_session_tagged: str | None
    if_has_fewer_sessions_than: int
    if_has_fewer_tagged_sessions_than: int
    if_signed_in_with_matching_exam_ticket: bool

    # rules specified
    tag_session: str | None
    may_start_new_session: bool
    may_list_existing_sessions: bool
    lock_down_as_exam_session: bool
    default_expiration_mode: str

# }}}


# {{{ mypy: flow access rule

class FlowSessionAccessRuleDesc(Struct):
    """Rules that govern what a user may do with an existing session.

    Found in the ``access`` attribute of :class:`FlowRulesDesc`.

    .. rubric:: Conditions

    .. attribute:: if_after

        (Optional) A :ref:`datespec <datespec>` that determines a date/time
        after which this rule applies.

    .. attribute:: if_before

        (Optional) A :ref:`datespec <datespec>` that determines a date/time
        before which this rule applies.

    .. attribute:: if_started_before

        (Optional) A :ref:`datespec <datespec>`. Rule applies if the session
        was started before this time.

    .. attribute:: if_has_role

        (Optional) A list of a subset of ``[unenrolled, ta, student, instructor]``.

    .. attribute:: if_has_participation_tags_any

        (Optional) A list of participation tags. Rule applies when the
        participation has at least one tag in this list.

    .. attribute:: if_has_participation_tags_all

        (Optional) A list of participation tags. Rule applies if only the
        participation's tags include all items in this list.

    .. attribute:: if_in_facility

        (Optional) Name of a facility known to the RELATE web page. This rule allows
        (for example) restricting flow access based on whether a user is physically
        located in a computer-based testing center (which RELATE can
        recognize based on IP ranges).

    .. attribute:: if_has_tag

        (Optional) Rule applies if session has this tag (see
        :attr:`FlowSessionStartRuleDesc.tag_session`), an identifier.

    .. attribute:: if_in_progress

        (Optional) A Boolean (True/False) value. Rule applies if the session's
        in-progress status matches this Boolean value.

    .. attribute:: if_completed_before

        (Optional) A :ref:`datespec <datespec>`. Rule applies if the session
        was completed before this time.

    .. attribute:: if_expiration_mode

        (Optional) One of :class:`~course.constants.flow_session_expiration_mode`.
        Rule applies if the expiration mode (see :ref:`flow-life-cycle`)
        matches.

    .. attribute:: if_session_duration_shorter_than_minutes

        (Optional) The rule applies if the current session has been going on for
        less than the specified number of minutes. Fractional values (e.g. "0.5")
        are accepted here.

    .. attribute:: if_signed_in_with_matching_exam_ticket

        (Optional) The rule applies if the participant signed in with an exam
        ticket matching this flow.

    .. rubric:: Rules specified

    .. attribute:: permissions

        A list of :class:`~course.constants.flow_permission`.

        :attr:`~course.constants.flow_permission.submit_answer`
        and :attr:`~course.constants.flow_permission.end_session`
        are automatically removed from a finished (i.e. not 'in-progress')
        session.

    .. attribute:: message

        (Optional) Some text in :ref:`markup` that is shown to the student in
        an 'alert' box at the top of the page if this rule applies.
    """

    # conditions
    if_after: Date_ish
    if_before: Date_ish
    if_started_before: Date_ish
    if_has_role: list[str]
    if_has_participation_tags_any: list[str]
    if_has_participation_tags_all: list[str]
    if_in_facility: str
    if_has_tag: str | None
    if_in_progress: bool
    if_completed_before: Date_ish
    if_expiration_mode: str
    if_session_duration_shorter_than_minutes: float
    if_signed_in_with_matching_exam_ticket: bool

    # rules specified
    permissions: list
    message: str

# }}}


# {{{ mypy: flow grading rule

class FlowSessionGradingRuleDesc(Struct):
    """ Rules that govern how (permanent) grades are generated from the
    results of a flow.

    Found in the ``grading`` attribute of :class:`FlowRulesDesc`.

    .. rubric:: Conditions

    .. attribute:: if_has_role

        (Optional) A list of a subset of ``[unenrolled, ta, student, instructor]``.

    .. attribute:: if_has_participation_tags_any

        (Optional) A list of participation tags. Rule applies when the
        participation has at least one tag in this list.

    .. attribute:: if_has_participation_tags_all

        (Optional) A list of participation tags. Rule applies if only the
        participation's tags include all items in this list.

    .. attribute:: if_started_before

        (Optional) A :ref:`datespec <datespec>`. Rule applies if the session
        was started before this time.

    .. attribute:: if_has_tag

        (Optional) Rule applies if session has this tag (see
        :attr:`FlowSessionStartRuleDesc.tag_session`), an identifier.

    .. attribute:: if_completed_before

        (Optional) A :ref:`datespec <datespec>`. Rule applies if the session
        was completed before this time.

        When evaluating this condition for in-progress sessions, the current time,
        or, if :attr:`use_last_activity_as_completion_time` is set, the time of the
        last activity is used.

        Since September 2017, this respects
        :attr:`use_last_activity_as_completion_time`.

    .. rubric:: Rules specified

    .. attribute:: credit_percent

        (Optional) A number indicating the percentage of credit assigned for
        this flow.  Defaults to 100 if not present. This is applied *after*
        point modifiers such as :attr:`bonus_points` and
        :attr:`max_points_enforced_cap`.

    .. attribute:: due

        A :ref:`datespec <datespec>` indicating the due date of the flow. This
        is shown to the participant and also used to batch-expire 'past-due'
        flows.

    .. attribute:: generates_grade

        (Optional) A Boolean indicating whether a grade will be recorded when this
        flow is ended. Note that the value of this rule must never change over
        the lifetime of a flow. I.e. a flow that, at some point during its lifetime,
        *may* have been set to generate a grade must *always* be set to generate
        a grade. Defaults to ``true``.

    .. attribute:: use_last_activity_as_completion_time

        (Optional) A Boolean indicating whether the last time a participant made
        a change to their flow should be used as the completion time.

        Defaults to ``false`` to match past behavior. ``true`` is probably the more
        sensible value for this.

    .. attribute:: description

        (Optional) A description of this set of grading rules being applied to
        the flow.  Shown to the participant on the flow start page.

    .. attribute:: max_points

        (Optional, an integer or floating point number if given)
        The number of points on the flow which constitute
        "100% of the achievable points". If not given, this is automatically
        computed by summing point values from all constituent pages.

        This may be used to 'grade out of N points', where N is a number that
        is lower than the actually achievable count.

    .. attribute:: max_points_enforced_cap

        (Optional, an integer or floating point number if given)
        No participant will have a grade higher than this recorded for this flow.
        This may be used to limit the amount of 'extra credit' achieved beyond
        :attr:`max_points`.

    .. attribute:: bonus_points

        (Optional, an integer or floating point number if given)
        This number of points will be added to every participant's score.

    """
    # conditions
    if_has_role: list[str]
    if_has_participation_tags_any: list[str]
    if_has_participation_tags_all: list[str]
    if_started_after: Date_ish
    if_has_tag: str | None
    if_completed_before: Date_ish

    # rules specified
    credit_percent: int | float | None
    due: Date_ish
    generates_grade: bool | None
    use_last_activity_as_completion_time: bool
    description: str
    max_points: int | float | None
    max_points_enforced_cap: int | float | None
    bonus_points: int | float | None

# }}}


# {{{ mypy: flow rules

class FlowRulesDesc(Struct):
    """
    Found in the ``rules`` attribute of a :class:`FlowDesc`.

    .. attribute:: start

        Rules that govern when a new session may be started and whether
        existing sessions may be listed.

        A list of :class:`FlowSessionStartRuleDesc`

        Rules are tested from top to bottom. The first rule
        whose conditions apply determines the access.

    .. attribute:: access

        Rules that govern what a user may do while they are interacting with an
        existing session.

        A list of :class:`FlowSessionAccessRuleDesc`.

        Rules are tested from top to bottom. The first rule
        whose conditions apply determines the access.

    .. rubric:: Grading-Related

    .. attribute:: grade_identifier

        (Required) The identifier of the grade to be generated once the
        participant completes the flow.  If ``null``, no grade is generated.

    .. attribute:: grade_aggregation_strategy

        (Required if :attr:`grade_identifier` is not ``null``)

        One of :class:`grade_aggregation_strategy`.

    .. attribute:: grading

        Rules that govern how (permanent) overall grades are generated from the
        results of a flow. These rules apply once a flow session ends/is submitted
        for grading. See :ref:`flow-life-cycle`.

        (Required if grade_identifier is not ``null``)
        A list of :class:`FlowSessionGradingRuleDesc`

        Rules are tested from top to bottom. The first rule
        whose conditions apply determines the access.
    """
    start: list[FlowSessionStartRuleDesc]
    access: list[FlowSessionAccessRuleDesc]
    grading: list[FlowSessionGradingRuleDesc]
    grade_identifier: str | None
    grade_aggregation_strategy: str | None

# }}}


# {{{ mypy: flow

class TabDesc(Struct):
    """
    .. attribute:: title

        (Required) Title to be displayed on the tab.

    .. attribute:: url

        (Required) The URL of the external web page.
    """

    def __init__(self, title: str, url: str) -> None:
        self.title = title
        self.url = url

    title: str
    url: str


class FlowPageDesc(Struct):
    id: str
    type: str


class FlowPageGroupDesc(Struct):
    """
    .. attribute:: id

        (Required) A symbolic name for the page group.

    .. attribute:: pages

        (Required) A list of :ref:`flow-page`

    .. attribute:: shuffle

        (Optional) A boolean (True/False) indicating whether the order
        of pages should be as in the list :attr:`pages` or
        determined by random shuffling

    .. attribute:: max_page_count

        (Optional) An integer limiting the page count of this group
        to a certain value. Allows selection of a random subset by combining
        with :attr:`shuffle`.
    """

    id: str
    pages: list[FlowPageDesc]


class FlowDesc(Struct):
    """
    .. attribute:: title

        A plain-text title of the flow

    .. attribute:: description

        A description in :ref:`markup` shown on the start page of the flow.

    .. attribute:: completion_text

        (Optional) Some text in :ref:`markup` shown once a student has
        completed the flow.

    .. attribute:: notify_on_submit

        (Optional) A list of email addresses which to notify about a flow
        submission by a participant.

    .. attribute:: rules

        (Optional) Some rules governing students' use and grading of the flow.
        See :ref:`flow-rules`.

    .. attribute:: groups

        A list of :class:`FlowPageGroupDesc`.  Exactly one of
        :attr:`groups` or :class:`pages` must be given.

    .. attribute:: pages

        A list of :ref:`pages <flow-page>`. If you specify this, a single
        :class:`FlowPageGroupDesc` will be implicitly created. Exactly one of
        :attr:`groups` or :class:`pages` must be given.

    .. attribute:: external_resources

        A list of :class:`TabDesc`. These are links to external
        resources that are displayed as tabs on the flow tabbed page.
    """

    title: str
    description: str
    rules: FlowRulesDesc
    pages: list[FlowPageDesc]
    groups: list[FlowPageGroupDesc]
    external_resources: list[TabDesc]
    notify_on_submit: list[str] | None

# }}}


# {{{ repo blob getting

def get_true_repo_and_path(repo: Repo_ish, path: str) -> tuple[dulwich.repo.Repo, str]:

    if isinstance(repo, SubdirRepoWrapper):
        if path:
            path = repo.subdir + "/" + path
        else:
            path = repo.subdir

        return repo.repo, path

    else:
        return repo, path


def get_course_repo_path(course: Course) -> str:

    return os.path.join(settings.GIT_ROOT, course.identifier)


def get_course_repo(course: Course) -> Repo_ish:

    from dulwich.repo import Repo
    repo = Repo(get_course_repo_path(course))

    if course.course_root_path:
        return SubdirRepoWrapper(repo, course.course_root_path)
    else:
        return repo


def look_up_git_object(repo: dulwich.repo.Repo,
        root_tree: dulwich.objects.Tree | FileSystemFakeRepoTree,
        full_name: str, _max_symlink_depth: int | None = None):
    """Traverse git directory tree from *root_tree*, respecting symlinks."""

    if _max_symlink_depth is None:
        _max_symlink_depth = 20
    if _max_symlink_depth == 0:
        raise ObjectDoesNotExist(_("symlink nesting depth exceeded "
            "while locating '%s'") % full_name)

    # https://github.com/inducer/relate/pull/556
    # FIXME: https://github.com/inducer/relate/issues/767
    name_parts = os.path.normpath(full_name).split(os.sep)

    processed_name_parts: list[str] = []

    from dulwich.objects import Tree

    from course.validation import FileSystemFakeRepoTree

    cur_lookup = root_tree

    from stat import S_ISLNK
    while name_parts:
        if not isinstance(cur_lookup, Tree | FileSystemFakeRepoTree):
            raise ObjectDoesNotExist(
                    _("'%s' is not a directory, cannot lookup nested names")
                    % os.sep.join(processed_name_parts))

        name_part = name_parts.pop(0)

        if not name_part:
            # tolerate empty path components (begrudgingly)
            continue
        elif name_part == ".":
            return cur_lookup

        encoded_name_part = name_part.encode()
        try:
            mode_sha = cur_lookup[encoded_name_part]
        except KeyError:
            raise ObjectDoesNotExist(_("resource '%s' not found") % full_name)

        mode, cur_lookup_sha = mode_sha

        if S_ISLNK(mode):
            link_target = os.sep.join(
                        [*processed_name_parts, repo[cur_lookup_sha].data.decode()])
            cur_lookup = look_up_git_object(repo, root_tree, link_target,
                    _max_symlink_depth=_max_symlink_depth-1)
        else:
            processed_name_parts.append(name_part)
            cur_lookup = repo[cur_lookup_sha]

    return cur_lookup


def get_repo_tree(repo: Repo_ish, full_name: str, commit_sha: bytes) -> Tree_ish:
    """
    :arg full_name: A Unicode string indicating the file name.
    :arg commit_sha: A byte string containing the commit hash
    :arg allow_tree: Allow the resulting object to be a directory
    """

    dul_repo, full_name = get_true_repo_and_path(repo, full_name)

    try:
        tree_sha = dul_repo[commit_sha].tree
    except KeyError:
        raise ObjectDoesNotExist(
                _("commit sha '%s' not found") % commit_sha.decode())

    git_obj = look_up_git_object(
            dul_repo, root_tree=dul_repo[tree_sha], full_name=full_name)

    from dulwich.objects import Tree

    from course.validation import FileSystemFakeRepoTree

    msg_full_name = full_name if full_name else _("(repo root)")

    if isinstance(git_obj, Tree | FileSystemFakeRepoTree):
        return git_obj
    else:
        raise ObjectDoesNotExist(_("resource '%s' is not a tree") % msg_full_name)


def get_repo_blob(repo: Repo_ish, full_name: str, commit_sha: bytes) -> Blob_ish:
    """
    :arg full_name: A Unicode string indicating the file name.
    :arg commit_sha: A byte string containing the commit hash
    :arg allow_tree: Allow the resulting object to be a directory
    """

    dul_repo, full_name = get_true_repo_and_path(repo, full_name)

    try:
        tree_sha = dul_repo[commit_sha].tree
    except KeyError:
        raise ObjectDoesNotExist(
                _("commit sha '%s' not found") % commit_sha.decode())

    git_obj = look_up_git_object(
            dul_repo, root_tree=dul_repo[tree_sha], full_name=full_name)

    from dulwich.objects import Blob

    from course.validation import FileSystemFakeRepoFile

    msg_full_name = full_name if full_name else _("(repo root)")

    if isinstance(git_obj, Blob | FileSystemFakeRepoFile):
        return git_obj
    else:
        raise ObjectDoesNotExist(_("resource '%s' is not a file") % msg_full_name)


def get_repo_blob_data_cached(
        repo: Repo_ish, full_name: str, commit_sha: bytes) -> bytes:
    """
    :arg commit_sha: A byte string containing the commit hash
    """

    if isinstance(commit_sha, bytes):
        from urllib.parse import quote_plus
        cache_key: str | None = "%s%R%1".join((
            CACHE_KEY_ROOT,
            quote_plus(repo.controldir()),
            quote_plus(full_name),
            commit_sha.decode(),
            ".".join(str(s) for s in sys.version_info[:2]),
            ))
    else:
        cache_key = None

    try:
        import django.core.cache as cache
    except ImproperlyConfigured:
        cache_key = None

    result: bytes | None = None
    if cache_key is None:
        result = get_repo_blob(repo, full_name, commit_sha).data
        assert isinstance(result, bytes)
        return result

    # Byte string is wrapped in a tuple to force pickling because memcache's
    # python wrapper appears to auto-decode/encode string values, thus trying
    # to decode our byte strings. Grr.

    def_cache = cache.caches["default"]

    # Memcache is apparently limited to 250 characters.
    if len(cache_key) < 240:
        cached_result = def_cache.get(cache_key)

        if cached_result is not None:
            (result,) = cached_result
            assert isinstance(result, bytes), cache_key
            return result

    result = get_repo_blob(repo, full_name, commit_sha).data
    assert result is not None

    if len(result) <= getattr(settings, "RELATE_CACHE_MAX_BYTES", 0):
        def_cache.add(cache_key, (result,), None)

    assert isinstance(result, bytes)

    return result


def is_repo_file_accessible_as(
        access_kinds: list[str], repo: Repo_ish, commit_sha: bytes, path: str
        ) -> bool:
    """
    Check of a file in a repo directory is accessible.  For example,
    'instructor' can access anything listed in the attributes.
    'student' can access 'student' and 'unenrolled'.  The 'unenrolled' role
    can only access 'unenrolled'.

    :arg commit_sha: A byte string containing the commit hash
    """

    # set the path to .attributes.yml
    attributes_path = os.path.join(os.path.dirname(path), ATTRIBUTES_FILENAME)

    # retrieve the .attributes.yml structure
    try:
        attributes = get_raw_yaml_from_repo(repo, attributes_path,
                                            commit_sha)
    except ObjectDoesNotExist:
        # no attributes file: not accessible
        return False

    path_basename = os.path.basename(path)

    # "public" is a deprecated alias for "unenrolled".

    access_patterns: list[str] = []
    for kind in access_kinds:
        access_patterns += attributes.get(kind, [])

    from fnmatch import fnmatch
    if isinstance(access_patterns, list):
        for pattern in access_patterns:
            if isinstance(pattern, str):
                if fnmatch(path_basename, pattern):
                    return True

    return False

# }}}


# {{{ jinja interaction

JINJA_YAML_RE = re.compile(
    r"^\[JINJA\]\s*$(.*?)^\[\/JINJA\]\s*$",
    re.MULTILINE | re.DOTALL)
YAML_BLOCK_START_SCALAR_RE = re.compile(
    r"(:\s*[|>])"
    r"(J?)"
    r"((?:[0-9][-+]?|[-+][0-9]?)?)"
    r"(?:\s*\#.*)?"
    r"$")

IN_BLOCK_END_RAW_RE = re.compile(r"(.*)({%-?\s*endraw\s*-?%})(.*)")
GROUP_COMMENT_START = re.compile(r"^\s*#\s*\{\{\{")
LEADING_SPACES_RE = re.compile(r"^( *)")


def process_yaml_for_expansion(yaml_str: str) -> str:

    lines = yaml_str.split("\n")
    jinja_lines = []

    i = 0
    line_count = len(lines)

    while i < line_count:
        ln = lines[i].rstrip()
        yaml_block_scalar_match = YAML_BLOCK_START_SCALAR_RE.search(ln)

        if yaml_block_scalar_match is not None:
            unprocessed_block_lines = []
            allow_jinja = bool(yaml_block_scalar_match.group(2))
            ln = YAML_BLOCK_START_SCALAR_RE.sub(
                    r"\1\3", ln)

            unprocessed_block_lines.append(ln)

            leading_spaces_match = LEADING_SPACES_RE.match(ln)
            assert leading_spaces_match
            block_start_indent = len(leading_spaces_match.group(1))

            i += 1

            while i < line_count:
                ln = lines[i]

                if not ln.rstrip():
                    unprocessed_block_lines.append(ln)
                    i += 1
                    continue

                leading_spaces_match = LEADING_SPACES_RE.match(ln)
                assert leading_spaces_match
                line_indent = len(leading_spaces_match.group(1))
                if line_indent <= block_start_indent:
                    break
                else:
                    ln = IN_BLOCK_END_RAW_RE.sub(
                        r"\1{% endraw %}{{ '\2' }}{% raw %}\3", ln)
                    unprocessed_block_lines.append(ln.rstrip())
                    i += 1

            if not allow_jinja:
                jinja_lines.append("{% raw %}")
            jinja_lines.extend(unprocessed_block_lines)
            if not allow_jinja:
                jinja_lines.append("{% endraw %}")

        elif GROUP_COMMENT_START.match(ln):
            jinja_lines.append("{% raw %}")
            jinja_lines.append(ln)
            jinja_lines.append("{% endraw %}")
            i += 1

        else:
            jinja_lines.append(ln)
            i += 1
    return "\n".join(jinja_lines)


class GitTemplateLoader:
    def __init__(self, repo: Repo_ish, commit_sha: bytes) -> None:
        self.repo = repo
        self.commit_sha = commit_sha

    def __call__(self, template):
        data = get_repo_blob_data_cached(self.repo, template, self.commit_sha)

        return data.decode("utf-8")


class YamlBlockEscapingGitTemplateLoader(GitTemplateLoader):
    # https://github.com/inducer/relate/issues/130

    def __call__(self, template):
        source = super().__call__(template)

        _, ext = os.path.splitext(template)
        ext = ext.lower()

        if ext in [".yml", ".yaml"]:
            source = process_yaml_for_expansion(source)

        return source


class YamlBlockEscapingFileSystemLoader:
    # https://github.com/inducer/relate/issues/130

    def __init__(self, root):
        self.root = root

    def __call__(self, template):
        with open(os.path.join(self.root, template)) as inf:
            source = inf.read()

        _, ext = os.path.splitext(template)
        ext = ext.lower()