flow.py

from __future__ import annotations


__copyright__ = "Copyright (C) 2014 Andreas Kloeckner"

__license__ = """
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, cast

from crispy_forms.helper import FormHelper
from crispy_forms.layout import Submit
from django import forms, http
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core.exceptions import (
    ObjectDoesNotExist,
    PermissionDenied,
    SuspiciousOperation,
)
from django.db import transaction
from django.db.models import query
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.utils.safestring import mark_safe
from django.utils.translation import gettext, gettext_lazy as _
from django_select2.forms import Select2Widget

from course.constants import (
    FLOW_SESSION_EXPIRATION_MODE_CHOICES,
    GRADE_AGGREGATION_STRATEGY_CHOICES,
    SESSION_LOCKED_TO_FLOW_PK,
    flow_permission,
    flow_session_expiration_mode,
    flow_session_interaction_kind,
    grade_aggregation_strategy,
    is_expiration_mode_allowed,
    participation_permission as pperm,
)
from course.content import FlowPageDesc, TabDesc
from course.exam import get_login_exam_ticket
from course.models import (
    Course,
    FlowPageData,
    FlowPageVisit,
    FlowPageVisitGrade,
    FlowSession,
    GradeChange,
    Participation,
    get_feedback_for_grade,
    update_bulk_feedback,
)
from course.page import InvalidPageData
from course.utils import (
    FlowContext,
    FlowPageContext,
    FlowSessionGradingRule,
    LanguageOverride,
    PageOrdinalOutOfRange,
    course_view,
    get_session_access_rule,
    get_session_grading_rule,
    get_session_start_rule,
    instantiate_flow_page_with_ctx,
    render_course_page,
)
from course.views import get_now_or_fake_time
from relate.utils import (
    StyledForm,
    as_local_time,
    format_datetime_local,
    local_now,
    not_none,
    remote_address_from_request,
    retry_transaction_decorator,
    string_concat,
)


# {{{ mypy

if TYPE_CHECKING:
    import datetime

    from accounts.models import User
    from course.content import FlowDesc
    from course.models import Course
    from course.page.base import AnswerFeedback, PageBase, PageBehavior
    from course.utils import CoursePageContext, FlowSessionStartRule
    from relate.utils import Repo_ish

# }}}


# {{{ page data wrangling

@retry_transaction_decorator(serializable=True)
def _adjust_flow_session_page_data_inner(
        repo: Repo_ish, flow_session: FlowSession,
        course_identifier: str, flow_desc, commit_sha: bytes) -> int:
    """
    :returns: new page count
    """
    from course.page.base import PageContext
    pctx = PageContext(
            course=flow_session.course,
            repo=repo,
            commit_sha=commit_sha,
            flow_session=flow_session,
            in_sandbox=False,
            page_uri=None)

    desc_group_ids = []

    # {{{ helper functions

    def remove_page(fpd: FlowPageData) -> None:
        if fpd.page_ordinal is not None:
            fpd.page_ordinal = None
            fpd.save()

    def find_page_desc(page_id: str) -> FlowPageDesc:
        new_page_desc = None

        for page_desc in grp.pages:  # pragma: no branch
            if page_desc.id == page_id:
                new_page_desc = page_desc
                break

        assert new_page_desc is not None

        return new_page_desc

    def instantiate_page(page_desc: FlowPageDesc) -> PageBase:
        from course.content import instantiate_flow_page
        return instantiate_flow_page(
                f"course '{course_identifier}', flow '{flow_session.flow_id}', "
                f"page '{grp.id}/{page_desc.id}'",
                repo, page_desc, commit_sha)

    def create_fpd(new_page_desc: FlowPageDesc) -> FlowPageData:
        page = instantiate_page(new_page_desc)

        data = page.initialize_page_data(pctx)
        return FlowPageData(
                flow_session=flow_session,
                page_ordinal=None,
                page_type=new_page_desc.type,
                group_id=grp.id,
                page_id=new_page_desc.id,
                data=data,
                title=page.title(pctx, data))

    def add_page(fpd: FlowPageData) -> None:
        if fpd.page_ordinal != ordinal[0]:
            fpd.page_ordinal = ordinal[0]
            fpd.save()

        page_desc = find_page_desc(fpd.page_id)
        page = instantiate_page(page_desc)
        title = page.title(pctx, fpd.data)

        if fpd.title != title:
            fpd.title = title
            fpd.save()

        ordinal[0] += 1
        available_page_ids.remove(fpd.page_id)

        group_pages.append(fpd)

    # }}}

    ordinal = [0]
    for grp in flow_desc.groups:
        desc_group_ids.append(grp.id)

        shuffle = getattr(grp, "shuffle", False)
        max_page_count = getattr(grp, "max_page_count", None)

        available_page_ids = [page_desc.id for page_desc in grp.pages]

        if max_page_count is None:
            max_page_count = len(available_page_ids)

        group_pages: list[FlowPageData] = []

        if shuffle:
            # {{{ maintain order of existing pages as much as possible

            for fpd in (FlowPageData.objects
                    .filter(
                        flow_session=flow_session,
                        group_id=grp.id,
                        page_ordinal__isnull=False)
                    .order_by("page_ordinal")):

                if (fpd.page_id in available_page_ids
                        and len(group_pages) < max_page_count):
                    add_page(fpd)
                else:
                    remove_page(fpd)

            assert len(group_pages) <= max_page_count

            # }}}

            # {{{ then add randomly chosen new pages

            from random import choice

            while len(group_pages) < max_page_count and available_page_ids:
                new_page_id = choice(available_page_ids)

                new_page_fpds = (FlowPageData.objects
                        .filter(
                            flow_session=flow_session,
                            group_id=grp.id,
                            page_id=new_page_id))

                if new_page_fpds.count():
                    # We already have FlowPageData for this page, revive it
                    new_page_fpd, = new_page_fpds
                    assert new_page_fpd.page_id == new_page_id
                else:
                    # Make a new FlowPageData instance
                    page_desc = find_page_desc(new_page_id)
                    assert page_desc.id == new_page_id
                    new_page_fpd = create_fpd(page_desc)
                    assert new_page_fpd.page_id == new_page_id

                add_page(new_page_fpd)

            # }}}

        else:
            # {{{ reorder pages to order in flow

            id_to_fpd = {
                    (fpd.group_id, fpd.page_id): fpd
                    for fpd in FlowPageData.objects.filter(
                        flow_session=flow_session,
                        group_id=grp.id)}

            for page_desc in grp.pages:
                key = (grp.id, page_desc.id)

                if key in id_to_fpd:
                    fpd = id_to_fpd.pop(key)
                else:
                    fpd = create_fpd(page_desc)

                if len(group_pages) < max_page_count:
                    add_page(fpd)

            for fpd in id_to_fpd.values():
                remove_page(fpd)

            # }}}

    # {{{ remove pages orphaned because of group renames

    for fpd in (
            FlowPageData.objects
            .filter(
                flow_session=flow_session,
                page_ordinal__isnull=False)
            .exclude(group_id__in=desc_group_ids)
            ):
        remove_page(fpd)

    # }}}

    return ordinal[0]  # new page count


def adjust_flow_session_page_data(repo: Repo_ish, flow_session: FlowSession,
        course_identifier: str, flow_desc: FlowDesc | None = None,
        respect_preview: bool = True) -> None:

    """
    The caller may *not* be in a transaction that has a weaker isolation
    level than *serializable*.
    """

    from course.content import get_course_commit_sha, get_flow_desc
    commit_sha = get_course_commit_sha(
            flow_session.course,
            flow_session.participation if respect_preview else None)
    revision_key = "2:"+commit_sha.decode()

    if flow_desc is None:
        flow_desc = get_flow_desc(repo, flow_session.course,
                flow_session.flow_id, commit_sha)

    if flow_session.page_data_at_revision_key == revision_key:
        return

    new_page_count = _adjust_flow_session_page_data_inner(
            repo, flow_session, course_identifier, flow_desc,
            commit_sha)

    # These are idempotent, so they don't need to be guarded by a seqcst
    # transaction.
    flow_session.page_count = new_page_count
    flow_session.page_data_at_revision_key = revision_key
    flow_session.save()

# }}}


# {{{ grade page visit

def grade_page_visit(visit: FlowPageVisit,
        visit_grade_model: type = FlowPageVisitGrade,
        grade_data: Any = None, respect_preview: bool = True) -> None:
    if not visit.is_submitted_answer:
        raise RuntimeError(_("cannot grade ungraded answer"))

    flow_session = visit.flow_session
    course = flow_session.course
    page_data = visit.page_data

    most_recent_grade: FlowPageVisitGrade | None = visit.get_most_recent_grade()
    if most_recent_grade is not None and grade_data is None:
        grade_data = most_recent_grade.grade_data

    from course.content import (
        get_course_commit_sha,
        get_course_repo,
        get_flow_desc,
        get_flow_page_desc,
        instantiate_flow_page,
    )

    with get_course_repo(course) as repo:
        course_commit_sha = get_course_commit_sha(
                course, flow_session.participation if respect_preview else None)

        flow_desc = get_flow_desc(repo, course,
                flow_session.flow_id, course_commit_sha)

        page_desc = get_flow_page_desc(
                flow_session.flow_id,
                flow_desc,
                page_data.group_id, page_data.page_id)

        page = instantiate_flow_page(
                location=(f"flow '{flow_session.flow_id}', "
                    f"group, '{page_data.group_id}', page '{page_data.page_id}'"),
                repo=repo, page_desc=page_desc,
                commit_sha=course_commit_sha)

        assert page.expects_answer()
        if not page.is_answer_gradable():
            return

        from course.page import PageContext
        grading_page_context = PageContext(
                course=course,
                repo=repo,
                commit_sha=course_commit_sha,
                flow_session=flow_session)

        with LanguageOverride(course=course):
            answer_feedback = page.grade(
                    grading_page_context, visit.page_data.data,
                    visit.answer, grade_data=grade_data)

        grade = visit_grade_model()
        grade.visit = visit
        grade.grade_data = grade_data
        grade.max_points = page.max_points(visit.page_data)
        grade.graded_at_git_commit_sha = course_commit_sha.decode()

        bulk_feedback_json = None
        if answer_feedback is not None:
            grade.correctness = answer_feedback.correctness
            grade.feedback, bulk_feedback_json = answer_feedback.as_json()

        grade.save()

        update_bulk_feedback(page_data, grade, bulk_feedback_json)

# }}}


# {{{ start flow

def start_flow(
        repo: Repo_ish,
        course: Course,
        participation: Participation | None,
        user: Any,
        flow_id: str,
        flow_desc: FlowDesc,
        session_start_rule: FlowSessionStartRule,
        now_datetime: datetime.datetime,
        ) -> FlowSession:

    # This function does not need to be transactionally atomic.
    # The only essential part is the creation of the session.
    # The remainder of the function (opportunity creation and
    # page setup) is atomic and gets retried.

    from course.content import get_course_commit_sha
    course_commit_sha = get_course_commit_sha(course, participation)

    if participation is not None:
        assert participation.user == user

    exp_mode = flow_session_expiration_mode.end
    if session_start_rule.default_expiration_mode is not None:
        exp_mode = session_start_rule.default_expiration_mode

    assert exp_mode in dict(FLOW_SESSION_EXPIRATION_MODE_CHOICES)

    session = FlowSession(
        course=course,
        participation=participation,
        user=user,
        active_git_commit_sha=course_commit_sha.decode(),
        flow_id=flow_id,
        start_time=now_datetime,
        in_progress=True,
        expiration_mode=exp_mode,
        access_rules_tag=session_start_rule.tag_session)

    session.save()

    # Create flow grading opportunity. This makes the flow
    # show up in the grade book.

    rules = getattr(flow_desc, "rules", None)
    if rules is not None:
        identifier = rules.grade_identifier

        if identifier is not None:
            from course.models import get_flow_grading_opportunity
            get_flow_grading_opportunity(
                    course, flow_id, flow_desc,
                    identifier,
                    rules.grade_aggregation_strategy)

    # will implicitly modify and save the session if there are changes
    adjust_flow_session_page_data(repo, session,
            course.identifier, flow_desc, respect_preview=True)

    return session

# }}}


# {{{ finish flow

def get_multiple_flow_session_graded_answers_qset(
        flow_sessions: list[FlowSession]) -> query.QuerySet:

    from django.db.models import Q
    qset = (FlowPageVisit.objects
            .filter(flow_session__in=flow_sessions)
            .filter(Q(answer__isnull=False) | Q(is_synthetic=True))
            .order_by("flow_session__id"))

    # Ungraded answers *can* show up in non-in-progress flows as a result
    # of a race between a 'save' and the 'end session'. If this happens,
    # we'll go ahead and ignore those.
    qset = qset.filter(
        (Q(flow_session__in_progress=False) & Q(is_submitted_answer=True))
        | Q(flow_session__in_progress=True))

    return qset


def get_flow_session_graded_answers_qset(
        flow_session: FlowSession) -> query.QuerySet:

    return get_multiple_flow_session_graded_answers_qset([flow_session])


def get_prev_answer_visits_qset(page_data: FlowPageData) -> query.QuerySet:
    return (
            get_flow_session_graded_answers_qset(page_data.flow_session)
            .filter(page_data=page_data)
            .order_by("-visit_time"))


def get_first_from_qset(qset: query.QuerySet) -> Any | None:
    for item in qset[:1]:
        return item

    return None


def get_prev_answer_visit(page_data):
    return get_first_from_qset(get_prev_answer_visits_qset(page_data))


def assemble_page_grades(
        flow_sessions: list[FlowSession]
        ) -> list[list[FlowPageVisitGrade | None]]:
    """
    Given a list of flow sessions, return a list of lists of FlowPageVisitGrade
    objects corresponding to the most recent page grades for each page of the
    flow session.  If a page is not graded, the corresponding entry is None.

    Note that, even if the flow sessions belong to the same flow, the length
    of the lists may vary since the flow page count may vary per session.
    """
    id_to_fsess_idx = {fsess.id: i for i, fsess in enumerate(flow_sessions)}
    answer_visit_ids: list[list[int | None]] = [
            [None] * not_none(fsess.page_count) for fsess in flow_sessions
            ]

    # Get all answer visits corresponding to the sessions. The query result is
    # typically very large.
    all_answer_visits = (
        get_multiple_flow_session_graded_answers_qset(flow_sessions)
        .order_by("visit_time")
        .values("id", "flow_session_id", "page_data__page_ordinal",
                "is_submitted_answer"))

    for answer_visit in all_answer_visits:
        fsess_idx = id_to_fsess_idx[answer_visit["flow_session_id"]]
        page_ordinal = answer_visit["page_data__page_ordinal"]
        if page_ordinal is not None:
            answer_visit_ids[fsess_idx][page_ordinal] = answer_visit["id"]

        if not flow_sessions[fsess_idx].in_progress:
            assert answer_visit["is_submitted_answer"] is True

    flat_answer_visit_ids = []
    for visit_id_list in answer_visit_ids:
        for visit_id in visit_id_list:
            if visit_id is not None:
                flat_answer_visit_ids.append(visit_id)

    # Get all grade visits associated with the answer visits.
    grades = (FlowPageVisitGrade.objects
              .filter(visit__in=flat_answer_visit_ids)
              .order_by("visit__id")
              .order_by("grade_time"))

    grades_by_answer_visit: dict[int | None, FlowPageVisitGrade] = {}
    for grade in grades:
        grades_by_answer_visit[grade.visit_id] = grade

    def get_grades_for_visit_group(
            visit_group: list[int | None]
            ) -> list[FlowPageVisitGrade | None]:

        return [grades_by_answer_visit.get(visit_id)
            for visit_id in visit_group]

    return [get_grades_for_visit_group(group) for group in answer_visit_ids]


def assemble_answer_visits(
        flow_session: FlowSession) -> list[FlowPageVisit | None]:

    answer_visits: list[FlowPageVisit | None] = [
        cast(FlowPageVisit | None, None)] * not_none(flow_session.page_count)

    answer_page_visits = (
            get_flow_session_graded_answers_qset(flow_session)
            .order_by("visit_time"))

    for page_visit in answer_page_visits:
        if page_visit.page_data.page_ordinal is not None:
            answer_visits[page_visit.page_data.page_ordinal] = page_visit

        if not flow_session.in_progress:
            assert page_visit.is_submitted_answer is True

    return answer_visits


def get_all_page_data(flow_session: FlowSession) -> Iterable[FlowPageData]:

    return (FlowPageData.objects
            .filter(
                flow_session=flow_session,
                page_ordinal__isnull=False)
            .order_by("page_ordinal"))


def get_interaction_kind(
        fctx: FlowContext,
        flow_session: FlowSession,
        flow_generates_grade: bool,
        all_page_data: Iterable[FlowPageData],
        ) -> str:

    has_interactive = False
    has_gradable = False

    for i, page_data in enumerate(all_page_data):
        assert i == page_data.page_ordinal

        page = instantiate_flow_page_with_ctx(fctx, page_data)
        if page.expects_answer():
            has_interactive = True
            if page.is_answer_gradable():
                has_gradable = True

    if has_interactive:
        if has_gradable:
            if flow_generates_grade:
                return flow_session_interaction_kind.permanent_grade
            else:
                return flow_session_interaction_kind.practice_grade
        else:
            return flow_session_interaction_kind.ungraded
    else:
        return flow_session_interaction_kind.noninteractive


def get_session_answered_page_data(
        fctx: FlowContext,
        flow_session: FlowSession,
        answer_visits: list[FlowPageVisit | None]
        ) -> tuple[list[FlowPageData], list[FlowPageData], bool]:
    all_page_data = get_all_page_data(flow_session)

    answered_page_data_list: list[FlowPageData] = []
    unanswered_page_data_list: list[FlowPageData] = []
    is_interactive_flow: bool = False

    for i, page_data in enumerate(all_page_data):
        assert i == page_data.page_ordinal

        avisit = answer_visits[i]
        if avisit is not None:
            answer_data = avisit.answer
        else:
            answer_data = None

        page = instantiate_flow_page_with_ctx(fctx, page_data)
        if page.expects_answer():
            is_interactive_flow = True
            if not page.is_optional_page:
                if answer_data is None:
                    unanswered_page_data_list.append(page_data)
                else:
                    answered_page_data_list.append(page_data)

    return (answered_page_data_list, unanswered_page_data_list, is_interactive_flow)


class GradeInfo:
    """An object to hold a tally of points and page counts of various types in
    a flow.

    .. attribute:: points

        The final grade, in points. May be *None* if the grade is not yet
        final.

    .. attribute:: provisional_points

        The number of points still attainable for this flow if all points
        were awarded for as yet ungraded (e.g. human-graded) pages.

    .. attribute:: max_reachable_points

        The maximum number of actually attainable points on the flow, subject
        to :attr:`FlowSessionGradingRule.max_points_enforced_cap`.
    """

    def __init__(
            self,
            points: float | None,
            provisional_points: float | None,
            max_points: float | None,
            max_reachable_points: float | None,
            fully_correct_count: int,
            partially_correct_count: int,
            incorrect_count: int,
            unknown_count: int,
            optional_fully_correct_count: int = 0,
            optional_partially_correct_count: int = 0,
            optional_incorrect_count: int = 0,
            optional_unknown_count: int = 0,
            ) -> None:
        self.points = points
        self.provisional_points = provisional_points
        self.max_points = max_points
        self.max_reachable_points = max_reachable_points
        self.fully_correct_count = fully_correct_count
        self.partially_correct_count = partially_correct_count
        self.incorrect_count = incorrect_count
        self.unknown_count = unknown_count
        self.optional_fully_correct_count = optional_fully_correct_count
        self.optional_partially_correct_count = optional_partially_correct_count
        self.optional_incorrect_count = optional_incorrect_count
        self.optional_unknown_count = optional_unknown_count

    # Rounding to larger than 100% will break the percent bars on the
    # flow results page.
    FULL_PERCENT = 99.99

    # {{{ point percentages

    def points_percent(self):
        """Only to be used for visualization purposes."""

        if self.max_points is None or self.max_points == 0:
            if self.points == 0:
                return 100
            else:
                return 0
        else:
            return self.FULL_PERCENT*self.provisional_points/self.max_points

    def missed_points_percent(self):
        """Only to be used for visualization purposes."""

        return (self.FULL_PERCENT
                - self.points_percent()
                - self.unreachable_points_percent())

    def unreachable_points_percent(self):
        """Only to be used for visualization purposes."""

        if (self.max_points is None
                or self.max_reachable_points is None
                or self.max_points == 0):
            return 0
        else:
            return self.FULL_PERCENT*(
                    self.max_points - self.max_reachable_points)/self.max_points

    def total_points_percent(self):
        return (
                self.points_percent()
                + self.missed_points_percent()
                + self.unreachable_points_percent())

    # }}}

    # {{{ page counts / percentages

    def total_count(self) -> int:
        return (self.fully_correct_count
                + self.partially_correct_count
                + self.incorrect_count
                + self.unknown_count)

    def fully_correct_percent(self) -> float:
        """Only to be used for visualization purposes."""
        return self.FULL_PERCENT*self.fully_correct_count/self.total_count()

    def partially_correct_percent(self) -> float:
        """Only to be used for visualization purposes."""
        return self.FULL_PERCENT*self.partially_correct_count/self.total_count()

    def incorrect_percent(self) -> float:
        """Only to be used for visualization purposes."""
        return self.FULL_PERCENT*self.incorrect_count/self.total_count()

    def unknown_percent(self) -> float:
        """Only to be used for visualization purposes."""
        return self.FULL_PERCENT*self.unknown_count/self.total_count()

    def optional_total_count(self) -> int:
        return (self.optional_fully_correct_count
                + self.optional_partially_correct_count
                + self.optional_incorrect_count
                + self.optional_unknown_count)

    def optional_fully_correct_percent(self) -> float:
        """Only to be used for visualization purposes."""
        return self.FULL_PERCENT * self.optional_fully_correct_count\
               / self.optional_total_count()

    def optional_partially_correct_percent(self) -> float:
        """Only to be used for visualization purposes."""
        return self.FULL_PERCENT * self.optional_partially_correct_count\
               / self.optional_total_count()

    def optional_incorrect_percent(self) -> float:
        """Only to be used for visualization purposes."""
        return self.FULL_PERCENT * self.optional_incorrect_count\
               / self.optional_total_count()

    def optional_unknown_percent(self) -> float:
        """Only to be used for visualization purposes."""
        return self.FULL_PERCENT * self.optional_unknown_count\
               / self.optional_total_count()

    # }}}


def gather_grade_info(
        fctx: FlowContext,
        flow_session: FlowSession,
        grading_rule: FlowSessionGradingRule,
        answer_visits: list[FlowPageVisit | None],
        ) -> GradeInfo:
    all_page_data = get_all_page_data(flow_session)

    bonus_points = grading_rule.bonus_points
    points: float | None = bonus_points
    provisional_points = bonus_points
    max_points = bonus_points
    max_reachable_points = bonus_points

    fully_correct_count = 0
    partially_correct_count = 0
    incorrect_count = 0
    unknown_count = 0

    optional_fully_correct_count = 0
    optional_partially_correct_count = 0
    optional_incorrect_count = 0
    optional_unknown_count = 0

    for i, page_data in enumerate(all_page_data):
        page = instantiate_flow_page_with_ctx(fctx, page_data)

        assert i == page_data.page_ordinal

        av = answer_visits[i]

        if av is None:
            # This is true in principle, but early code to deal with survey questions
            # didn't generate synthetic answer visits for survey questions, so this
            # can't actually be enforced.

            # assert not page.expects_answer()
            continue

        if not page.is_answer_gradable():
            continue

        grade = av.get_most_recent_grade()
        assert grade is not None
        assert grade.max_points is not None

        feedback = get_feedback_for_grade(grade)

        if page.is_optional_page:
            if feedback is None or feedback.correctness is None:
                optional_unknown_count += 1
                continue

            if feedback.correctness == 1:
                optional_fully_correct_count += 1
            elif feedback.correctness == 0:
                optional_incorrect_count += 1
            else:
                optional_partially_correct_count += 1

        else:
            max_points += grade.max_points

            if feedback is None or feedback.correctness is None:
                unknown_count += 1
                points = None
                continue

            max_reachable_points += grade.max_points

            page_points = grade.max_points*feedback.correctness

            if points is not None:
                points += page_points

            provisional_points += page_points

            if grade.max_points > 0:
                if feedback.correctness == 1:
                    fully_correct_count += 1
                elif feedback.correctness == 0:
                    incorrect_count += 1
                else:
                    partially_correct_count += 1

    # {{{ adjust max_points if requested

    if grading_rule.max_points is not None:
        max_points = grading_rule.max_points

    # }}}

    # {{{ enforce points cap

    if grading_rule.max_points_enforced_cap is not None:
        max_reachable_points = min(
                max_reachable_points, grading_rule.max_points_enforced_cap)
        if points is not None:
            points = min(
                    points, grading_rule.max_points_enforced_cap)
        assert provisional_points is not None
        provisional_points = min(
                provisional_points, grading_rule.max_points_enforced_cap)

    # }}}

    return GradeInfo(
            points=points,
            provisional_points=provisional_points,
            max_points=max_points,
            max_reachable_points=max_reachable_points,

            fully_correct_count=fully_correct_count,
            partially_correct_count=partially_correct_count,
            incorrect_count=incorrect_count,
            unknown_count=unknown_count,

            optional_fully_correct_count=optional_fully_correct_count,
            optional_partially_correct_count=optional_partially_correct_count,
            optional_incorrect_count=optional_incorrect_count,
            optional_unknown_count=optional_unknown_count)


@transaction.atomic
def grade_page_visits(
        fctx: FlowContext,
        flow_session: FlowSession,
        answer_visits: list[FlowPageVisit | None],
        force_regrade: bool = False,
        respect_preview: bool = True,
        ) -> None:
    for i in range(len(answer_visits)):
        answer_visit = answer_visits[i]

        if answer_visit is not None:
            answer_visit.is_submitted_answer = True
            answer_visit.save()

        else:
            page_data = flow_session.page_data.get(page_ordinal=i)
            page = instantiate_flow_page_with_ctx(fctx, page_data)

            if not page.expects_answer():
                continue

            # Create a synthetic visit to attach a grade
            new_answer_visit = FlowPageVisit()

            new_answer_visit.flow_session = flow_session
            new_answer_visit.page_data = page_data
            new_answer_visit.is_synthetic = True
            new_answer_visit.answer = None
            new_answer_visit.is_submitted_answer = True
            new_answer_visit.save()

            answer_visits[i] = answer_visit = new_answer_visit

            if not page.is_answer_gradable():
                continue

        assert answer_visit is not None
        if not answer_visit.grades.count() or force_regrade:  # type: ignore
            grade_page_visit(answer_visit, respect_preview=respect_preview)


@retry_transaction_decorator()
def finish_flow_session(fctx, flow_session, grading_rule,
        force_regrade=False, now_datetime=None, respect_preview=True):
    """
    :returns: :class:`GradeInfo`
    """
    # Do not be tempted to call adjust_flow_session_page_data in here.
    # This function may be called from within a transaction.

    if not flow_session.in_progress:
        raise RuntimeError(_("Can't end a session that's already ended"))

    assert isinstance(grading_rule, FlowSessionGradingRule)

    if now_datetime is None:
        from django.utils.timezone import now
        now_datetime = now()

    answer_visits = assemble_answer_visits(flow_session)

    grade_page_visits(fctx, flow_session, answer_visits,
            force_regrade=force_regrade,
            respect_preview=respect_preview)