base_test_mixins.py

            course.refresh_from_db()

    @classmethod
    def get_view_start_flow_url(cls, flow_id, course_identifier=None):
        course_identifier = course_identifier or cls.get_default_course_identifier()
        kwargs = {"course_identifier": course_identifier,
                  "flow_id": flow_id}
        return reverse("relate-view_start_flow", kwargs=kwargs)

    @classmethod
    def start_flow(cls, flow_id, course_identifier=None,
                   ignore_cool_down=True, assume_success=True):
        existing_session_count = FlowSession.objects.all().count()
        if ignore_cool_down:
            cool_down_seconds = 0
        else:
            cool_down_seconds = settings.RELATE_SESSION_RESTART_COOLDOWN_SECONDS
        with override_settings(
                RELATE_SESSION_RESTART_COOLDOWN_SECONDS=cool_down_seconds):
            resp = cls.c.post(
                cls.get_view_start_flow_url(flow_id, course_identifier))

        if assume_success:
            assert resp.status_code == 302, resp.content
            new_session_count = FlowSession.objects.all().count()
            assert new_session_count == existing_session_count + 1
            _, _, params = resolve(resp.url)
            del params["page_ordinal"]
            cls.default_flow_params = params
            cls.update_default_flow_session_id(course_identifier)

        return resp

    @classmethod
    def end_flow(cls, course_identifier=None, flow_session_id=None):
        params = {}
        if not course_identifier or not flow_session_id:
            if cls.default_flow_params is None:
                raise RuntimeError("There's no started flow_sessions.")
            params = deepcopy(cls.default_flow_params)
        if course_identifier:
            params["course_identifier"] = course_identifier
        if flow_session_id:
            params["flow_session_id"] = flow_session_id
        resp = cls.c.post(reverse("relate-finish_flow_session_view",
                                  kwargs=params), {'submit': ['']})
        return resp

    @classmethod
    def get_flow_params(cls, course_identifier=None, flow_session_id=None):
        course_identifier = (
                course_identifier or cls.get_default_course_identifier())
        if flow_session_id is None:
            flow_session_id = cls.get_default_flow_session_id(course_identifier)
        return {
            "course_identifier": course_identifier,
            "flow_session_id": flow_session_id
        }

    @classmethod
    def get_page_params(cls, course_identifier=None, flow_session_id=None,
                        page_ordinal=None):
        page_params = cls.get_flow_params(course_identifier, flow_session_id)
        if page_ordinal is None:
            page_ordinal = 0
        page_params.update({"page_ordinal": page_ordinal})
        return page_params

    @classmethod
    def get_page_ordinal_via_page_id(
            cls, page_id, course_identifier=None, flow_session_id=None,
            with_group_id=False):
        flow_params = cls.get_flow_params(course_identifier, flow_session_id)
        return (
            get_flow_page_ordinal_from_page_id(
                flow_params["flow_session_id"], page_id,
                with_group_id=with_group_id))

    @classmethod
    def get_page_id_via_page_oridnal(
            cls, page_ordinal, course_identifier=None, flow_session_id=None,
            with_group_id=False):
        flow_params = cls.get_flow_params(course_identifier, flow_session_id)
        return (
            get_flow_page_id_from_page_ordinal(
                flow_params["flow_session_id"], page_ordinal,
                with_group_id=with_group_id))

    @classmethod
    def get_page_view_url_by_ordinal(
            cls, viewname, page_ordinal, course_identifier=None,
            flow_session_id=None):
        page_params = cls.get_page_params(
            course_identifier, flow_session_id, page_ordinal)
        return reverse(viewname, kwargs=page_params)

    @classmethod
    def get_page_view_url_by_page_id(
            cls, viewname, page_id, course_identifier=None, flow_session_id=None):
        page_ordinal = cls.get_page_ordinal_via_page_id(
            page_id, course_identifier, flow_session_id)
        return cls.get_page_view_url_by_ordinal(
            viewname, page_ordinal, course_identifier, flow_session_id)

    @classmethod
    def get_page_url_by_ordinal(
            cls, page_ordinal, course_identifier=None, flow_session_id=None):
        return cls.get_page_view_url_by_ordinal(
            "relate-view_flow_page",
            page_ordinal, course_identifier, flow_session_id)

    @classmethod
    def get_page_url_by_page_id(
            cls, page_id, course_identifier=None, flow_session_id=None):
        page_ordinal = cls.get_page_ordinal_via_page_id(
            page_id, course_identifier, flow_session_id)
        return cls.get_page_url_by_ordinal(
            page_ordinal, course_identifier, flow_session_id)

    @classmethod
    def get_page_grading_url_by_ordinal(
            cls, page_ordinal, course_identifier=None, flow_session_id=None):
        return cls.get_page_view_url_by_ordinal(
            "relate-grade_flow_page",
            page_ordinal, course_identifier, flow_session_id)

    @classmethod
    def get_page_grading_url_by_page_id(
            cls, page_id, course_identifier=None, flow_session_id=None):
        page_ordinal = cls.get_page_ordinal_via_page_id(
            page_id, course_identifier, flow_session_id)
        return cls.get_page_grading_url_by_ordinal(
            page_ordinal, course_identifier, flow_session_id)

    @classmethod
    def post_answer_by_ordinal(
            cls, page_ordinal, answer_data,
            course_identifier=None, flow_session_id=None):
        submit_data = answer_data
        submit_data.update({"submit": ["Submit final answer"]})
        resp = cls.c.post(
            cls.get_page_url_by_ordinal(
                page_ordinal, course_identifier, flow_session_id),
            submit_data)
        return resp

    @classmethod
    def post_answer_by_page_id(cls, page_id, answer_data,
                               course_identifier=None, flow_session_id=None):
        page_ordinal = cls.get_page_ordinal_via_page_id(
            page_id, course_identifier, flow_session_id)
        return cls.post_answer_by_ordinal(
            page_ordinal, answer_data, course_identifier, flow_session_id)

    @classmethod
    def post_answer_by_ordinal_class(cls, page_ordinal, answer_data,
                                     course_identifier, flow_session_id):
        submit_data = answer_data
        submit_data.update({"submit": ["Submit final answer"]})
        page_params = {
            "course_identifier": course_identifier,
            "flow_session_id": flow_session_id,
            "page_ordinal": page_ordinal
        }
        page_url = reverse("relate-view_flow_page", kwargs=page_params)
        resp = cls.c.post(page_url, submit_data)
        return resp

    @classmethod
    def post_answer_by_page_id_class(cls, page_id, answer_data,
                                     course_identifier, flow_session_id):
        page_ordinal = get_flow_page_ordinal_from_page_id(flow_session_id, page_id)
        return cls.post_answer_by_ordinal_class(page_ordinal, answer_data,
                                                course_identifier, flow_session_id)

    @classmethod
    def post_grade_by_ordinal(cls, page_ordinal, grade_data,
                              course_identifier=None, flow_session_id=None,
                              force_login_instructor=True):
        post_data = {"submit": [""]}
        post_data.update(grade_data)

        page_params = cls.get_page_params(
            course_identifier, flow_session_id, page_ordinal)

        force_login_user = cls.get_logged_in_user()
        if force_login_instructor:
            force_login_user = cls.get_default_instructor_user(
                page_params["course_identifier"])

        with cls.temporarily_switch_to_user(force_login_user):
            response = cls.c.post(
                cls.get_page_grading_url_by_ordinal(**page_params),
                data=post_data,
                follow=True)
        return response

    @classmethod
    def post_grade_by_page_id(cls, page_id, grade_data,
                              course_identifier=None, flow_session_id=None,
                              force_login_instructor=True):
        page_ordinal = cls.get_page_ordinal_via_page_id(
            page_id, course_identifier, flow_session_id)

        return cls.post_grade_by_ordinal(
            page_ordinal, grade_data, course_identifier,
            flow_session_id, force_login_instructor)

    @classmethod
    def assertSessionScoreEqual(  # noqa
            cls, expected_score, course_identifier=None, flow_session_id=None):
        if flow_session_id is None:
            flow_params = cls.get_flow_params(course_identifier, flow_session_id)
            flow_session_id = flow_params["flow_session_id"]
        flow_session = FlowSession.objects.get(id=flow_session_id)
        if expected_score is not None:
            from decimal import Decimal
            assert flow_session.points == Decimal(str(expected_score)), (
                "The flow session got '%s' in stead of '%s'"
                % (str(flow_session.points), str(Decimal(str(expected_score))))
            )
        else:
            assert flow_session.points is None, (
                    "This flow session unexpectedly got %s instead of None"
                    % flow_session.points)

    @classmethod
    def get_page_submit_history_url_by_ordinal(
            cls, page_ordinal, course_identifier=None, flow_session_id=None):
        return cls.get_page_view_url_by_ordinal(
            "relate-get_prev_answer_visits_dropdown_content",
            page_ordinal, course_identifier, flow_session_id)

    @classmethod
    def get_page_grade_history_url_by_ordinal(
            cls, page_ordinal, course_identifier=None, flow_session_id=None):
        return cls.get_page_view_url_by_ordinal(
            "relate-get_prev_grades_dropdown_content",
            page_ordinal, course_identifier, flow_session_id)

    @classmethod
    def get_page_submit_history_by_ordinal(
            cls, page_ordinal, course_identifier=None, flow_session_id=None):
        resp = cls.c.get(
            cls.get_page_submit_history_url_by_ordinal(
                page_ordinal, course_identifier, flow_session_id),
            HTTP_X_REQUESTED_WITH='XMLHttpRequest')
        return resp

    @classmethod
    def get_page_grade_history_by_ordinal(
            cls, page_ordinal, course_identifier=None, flow_session_id=None):
        resp = cls.c.get(
            cls.get_page_grade_history_url_by_ordinal(
                page_ordinal, course_identifier, flow_session_id),
            HTTP_X_REQUESTED_WITH='XMLHttpRequest')
        return resp

    def assertSubmitHistoryItemsCount(  # noqa
            self, page_ordinal, expected_count, course_identifier=None,
            flow_session_id=None):
        resp = self.get_page_submit_history_by_ordinal(
            page_ordinal, course_identifier, flow_session_id)
        import json
        result = json.loads(resp.content.decode())["result"]
        self.assertEqual(len(result), expected_count)

    def assertGradeHistoryItemsCount(  # noqa
            self, page_ordinal, expected_count,
            course_identifier=None,
            flow_session_id=None,
            force_login_instructor=True):

        if course_identifier is None:
            course_identifier = self.get_default_course_identifier()

        if force_login_instructor:
            switch_to = self.get_default_instructor_user(course_identifier)
        else:
            switch_to = self.get_logged_in_user()

        with self.temporarily_switch_to_user(switch_to):
            resp = self.get_page_grade_history_by_ordinal(
                page_ordinal, course_identifier, flow_session_id)

        import json
        result = json.loads(resp.content.decode())["result"]
        self.assertEqual(len(result), expected_count)

    @classmethod
    def get_update_course_url(cls, course_identifier=None):
        if course_identifier is None:
            course_identifier = cls.get_default_course_identifier()
        return reverse("relate-update_course", args=[course_identifier])

    @classmethod
    def get_course_commit_sha(cls, participation, course=None):
        course = course or cls.get_default_course()
        from course.content import get_course_commit_sha
        return get_course_commit_sha(course, participation)

    @classmethod
    def post_update_course_content(cls, commit_sha,
                                   prevent_discarding_revisions=True,
                                   force_login_instructor=True,
                                   course=None,
                                   command="update",
                                   ):
        # course instead of course_identifier because we need to do
        # refresh_from_db
        course = course or cls.get_default_course()

        try:
            commit_sha = commit_sha.decode()
        except Exception:
            pass

        data = {"new_sha": [commit_sha]}

        if not prevent_discarding_revisions:
            data["prevent_discarding_revisions"] = ["on"]

        # normally, command should be in
        # ["fetch", "fetch_update", "update", "fetch_preview", "preview",
        #  "end_preview"]
        data[command] = 'on'

        force_login_user = cls.get_logged_in_user()
        if force_login_instructor:
            force_login_user = cls.get_default_instructor_user(course.identifier)

        with cls.temporarily_switch_to_user(force_login_user):
            response = cls.c.post(
                cls.get_update_course_url(course.identifier), data)
            course.refresh_from_db()

        return response

    @classmethod
    def get_page_data_by_page_id(
            cls, page_id, course_identifier=None, flow_session_id=None):
        flow_params = cls.get_flow_params(course_identifier, flow_session_id)
        return FlowPageData.objects.get(
            flow_session_id=flow_params["flow_session_id"], page_id=page_id)

    @classmethod
    def get_page_visits(cls, course_identifier=None,
                        flow_session_id=None, page_ordinal=None, page_id=None,
                        **kwargs):
        query_kwargs = {}
        if kwargs.get("answer_visit", False):
            query_kwargs.update({"answer__isnull": False})
        flow_params = cls.get_flow_params(course_identifier, flow_session_id)
        query_kwargs.update({"flow_session_id": flow_params["flow_session_id"]})
        if page_ordinal is not None:
            query_kwargs.update({"page_data__page_ordinal": page_ordinal})
        elif page_id is not None:
            query_kwargs.update({"page_data__page_id": page_id})
        return FlowPageVisit.objects.filter(**query_kwargs)

    @classmethod
    def get_last_answer_visit(cls, course_identifier=None,
                              flow_session_id=None, page_ordinal=None,
                              page_id=None, assert_not_none=True):
        result_qset = cls.get_page_visits(course_identifier,
                                          flow_session_id, page_ordinal, page_id,
                                          answer_visit=True).order_by('-pk')[:1]
        if result_qset:
            result = result_qset[0]
        else:
            result = None
        if assert_not_none:
            assert result is not None, "The query returns None"
        return result

    @classmethod
    def download_all_submissions_url(cls, flow_id, course_identifier):
        params = {"course_identifier": course_identifier,
                  "flow_id": flow_id}
        return reverse("relate-download_all_submissions", kwargs=params)

    @classmethod
    def get_download_all_submissions(cls, flow_id, course_identifier=None):
        if course_identifier is None:
            course_identifier = cls.get_default_course_identifier()

        return cls.c.get(
            cls.download_all_submissions_url(flow_id, course_identifier))

    @classmethod
    def post_download_all_submissions_by_group_page_id(
            cls, group_page_id, flow_id, course_identifier=None, **kwargs):
        """
        :param group_page_id: format: group_id/page_id
        :param flow_id:
        :param course_identifier:
        :param kwargs: for updating the default post_data
        :return: response
        """
        if course_identifier is None:
            course_identifier = cls.get_default_course_identifier()

        data = {'restrict_to_rules_tag': '<<<ALL>>>',
                'which_attempt': 'last',
                'extra_file': '', 'download': 'Download',
                'page_id': group_page_id,
                'non_in_progress_only': 'on'}

        data.update(kwargs)

        return cls.c.post(
            cls.download_all_submissions_url(flow_id, course_identifier),
            data=data
        )

    @classmethod
    def get_flow_page_analytics(cls, flow_id, group_id, page_id,
                                course_identifier=None):
        if course_identifier is None:
            course_identifier = cls.get_default_course_identifier()

        params = {"course_identifier": course_identifier,
                  "flow_id": flow_id,
                  "group_id": group_id,
                  "page_id": page_id}

        return cls.c.get(reverse("relate-page_analytics", kwargs=params))


class SingleCourseTestMixin(CoursesTestMixinBase):
    courses_setup_list = SINGLE_COURSE_SETUP_LIST

    @classmethod
    def setUpTestData(cls):  # noqa
        super(SingleCourseTestMixin, cls).setUpTestData()
        assert len(cls.course_qset) == 1
        cls.course = cls.course_qset.first()
        cls.instructor_participation = Participation.objects.filter(
            course=cls.course,
            roles__identifier="instructor",
            status=participation_status.active
        ).first()
        assert cls.instructor_participation

        cls.student_participation = Participation.objects.filter(
            course=cls.course,
            roles__identifier="student",
            status=participation_status.active
        ).first()
        assert cls.student_participation

        cls.ta_participation = Participation.objects.filter(
            course=cls.course,
            roles__identifier="ta",
            status=participation_status.active
        ).first()
        assert cls.ta_participation
        cls.c.logout()
        cls.course_page_url = cls.get_course_page_url()

    def setUp(self):  # noqa
        super(SingleCourseTestMixin, self).setUp()

        # reload objects created during setUpTestData in case they were modified in
        # tests. Ref: https://goo.gl/AuzJRC#django.test.TestCase.setUpTestData
        self.course.refresh_from_db()
        self.instructor_participation.refresh_from_db()
        self.student_participation.refresh_from_db()
        self.ta_participation.refresh_from_db()

    @classmethod
    def get_default_course(cls):
        return cls.course

    @classmethod
    def get_default_course_identifier(cls):
        return cls.get_default_course().identifier

    def copy_course_dict_and_set_attrs_for_post(self, attrs_dict={}):
        from course.models import Course
        kwargs = Course.objects.first().__dict__
        kwargs.update(attrs_dict)

        import six
        for k, v in six.iteritems(kwargs):
            if v is None:
                kwargs[k] = ""
        return kwargs


class TwoCourseTestMixin(CoursesTestMixinBase):
    courses_setup_list = TWO_COURSE_SETUP_LIST

    @classmethod
    def setUpTestData(cls):  # noqa
        super(TwoCourseTestMixin, cls).setUpTestData()
        assert len(cls.course_qset) == 2, (
            "'courses_setup_list' should contain two courses")
        cls.course1 = cls.course_qset.first()
        cls.course1_instructor_participation = Participation.objects.filter(
            course=cls.course1,
            roles__identifier="instructor",
            status=participation_status.active
        ).first()
        assert cls.course1_instructor_participation

        cls.course1_student_participation = Participation.objects.filter(
            course=cls.course1,
            roles__identifier="student",
            status=participation_status.active
        ).first()
        assert cls.course1_student_participation

        cls.course1_ta_participation = Participation.objects.filter(
            course=cls.course1,
            roles__identifier="ta",
            status=participation_status.active
        ).first()
        assert cls.course1_ta_participation
        cls.course1_page_url = cls.get_course_page_url(cls.course1.identifier)

        cls.course2 = cls.course_qset.last()
        cls.course2_instructor_participation = Participation.objects.filter(
            course=cls.course2,
            roles__identifier="instructor",
            status=participation_status.active
        ).first()
        assert cls.course2_instructor_participation

        cls.course2_student_participation = Participation.objects.filter(
            course=cls.course2,
            roles__identifier="student",
            status=participation_status.active
        ).first()
        assert cls.course2_student_participation

        cls.course2_ta_participation = Participation.objects.filter(
            course=cls.course2,
            roles__identifier="ta",
            status=participation_status.active
        ).first()
        assert cls.course2_ta_participation
        cls.course2_page_url = cls.get_course_page_url(cls.course2.identifier)

        cls.c.logout()

    def setUp(self):  # noqa
        super(TwoCourseTestMixin, self).setUp()
        # reload objects created during setUpTestData in case they were modified in
        # tests. Ref: https://goo.gl/AuzJRC#django.test.TestCase.setUpTestData
        self.course1.refresh_from_db()
        self.course1_instructor_participation.refresh_from_db()
        self.course1_student_participation.refresh_from_db()
        self.course1_ta_participation.refresh_from_db()

        self.course2.refresh_from_db()
        self.course2_instructor_participation.refresh_from_db()
        self.course2_student_participation.refresh_from_db()
        self.course2_ta_participation.refresh_from_db()


class SingleCoursePageTestMixin(SingleCourseTestMixin):
    # This serves as cache
    _default_session_id = None

    flow_id = QUIZ_FLOW_ID

    @classmethod
    def update_default_flow_session_id(cls, course_identifier):
        cls._default_session_id = cls.default_flow_params["flow_session_id"]

    @classmethod
    def get_default_flow_session_id(cls, course_identifier):
        if cls._default_session_id is not None:
            return cls._default_session_id
        cls._default_session_id = cls.get_latest_session_id(course_identifier)
        return cls._default_session_id


class TwoCoursePageTestMixin(TwoCourseTestMixin):
    _course1_default_session_id = None
    _course2_default_session_id = None

    @property
    def flow_id(self):
        raise NotImplementedError

    @classmethod
    def get_default_flow_session_id(cls, course_identifier):
        if course_identifier == cls.course1.identifier:
            if cls._course1_default_session_id is not None:
                return cls._course1_default_session_id
            cls._course1_default_session_id = (
                cls.get_last_session_id(course_identifier))
            return cls._course1_default_session_id
        if course_identifier == cls.course2.identifier:
            if cls._course2_default_session_id is not None:
                return cls._course2_default_session_id
            cls._course2_default_session_id = (
                cls.get_last_session_id(course_identifier))
            return cls._course2_default_session_id

    @classmethod
    def update_default_flow_session_id(cls, course_identifier):
        new_session_id = cls.default_flow_params["flow_session_id"]
        if course_identifier == cls.course1.identifier:
            cls._course1_default_session_id = new_session_id
        elif course_identifier == cls.course2.identifier:
            cls._course2_default_session_id = new_session_id


class SingleCourseQuizPageTestMixin(SingleCoursePageTestMixin):

    skip_code_question = True

    @classmethod
    def ensure_grading_ui_get(cls, page_id):
        with cls.temporarily_switch_to_user(cls.instructor_participation.user):
            url = cls.get_page_grading_url_by_page_id(page_id)
            resp = cls.c.get(url)
            assert resp.status_code == 200

    @classmethod
    def ensure_analytic_page_get(cls, group_id, page_id):
        with cls.temporarily_switch_to_user(cls.instructor_participation.user):
            resp = cls.get_flow_page_analytics(
                flow_id=cls.flow_id, group_id=group_id,
                page_id=page_id)
            assert resp.status_code == 200

    @classmethod
    def ensure_download_submission(
            cls, group_id, page_id, dl_file_extension=None,
            file_with_ext_count=None):
        with cls.temporarily_switch_to_user(cls.instructor_participation.user):
            group_page_id = "%s/%s" % (group_id, page_id)
            resp = cls.post_download_all_submissions_by_group_page_id(
                group_page_id=group_page_id, flow_id=cls.flow_id)
            assert resp.status_code == 200
            prefix, zip_file = resp["Content-Disposition"].split('=')
            assert prefix == "attachment; filename"
            assert resp.get('Content-Type') == "application/zip"
            if dl_file_extension:
                buf = six.BytesIO(resp.content)
                import zipfile
                with zipfile.ZipFile(buf, 'r') as zf:
                    assert zf.testzip() is None
                    # todo: make more assertions in terms of file content

                    for f in zf.filelist:
                        assert f.file_size > 0

                    if file_with_ext_count is None:
                        assert len([f for f in zf.filelist if
                                 f.filename.endswith(dl_file_extension)]) > 0
                    else:
                        assert (
                                len([f for f in zf.filelist if
                                     f.filename.endswith(dl_file_extension)])
                                == file_with_ext_count)

    @classmethod
    def submit_page_answer_by_ordinal_and_test(
            cls, page_ordinal, use_correct_answer=True, answer_data=None,
            skip_code_question=True,
            expected_grades=None, expected_post_answer_status_code=200,
            do_grading=False, do_human_grade=False, grade_data=None,
            grade_data_extra_kwargs=None,
            dl_file_extension=None,
            ensure_grading_ui_get_before_grading=False,
            ensure_grading_ui_get_after_grading=False,
            ensure_analytic_page_get_before_submission=False,
            ensure_analytic_page_get_after_submission=False,
            ensure_analytic_page_get_before_grading=False,
            ensure_analytic_page_get_after_grading=False,
            ensure_download_before_submission=False,
            ensure_download_after_submission=False,
            ensure_download_before_grading=False,
            ensure_download_after_grading=False,
            dl_file_with_ext_count=None):
        page_id = cls.get_page_id_via_page_oridnal(page_ordinal)

        return cls.submit_page_answer_by_page_id_and_test(
            page_id, use_correct_answer,
            answer_data, skip_code_question, expected_grades,
            expected_post_answer_status_code,
            do_grading, do_human_grade,
            grade_data, grade_data_extra_kwargs, dl_file_extension,
            ensure_grading_ui_get_before_grading,
            ensure_grading_ui_get_after_grading,
            ensure_analytic_page_get_before_submission,
            ensure_analytic_page_get_after_submission,
            ensure_analytic_page_get_before_grading,
            ensure_analytic_page_get_after_grading,
            ensure_download_before_submission,
            ensure_download_after_submission,
            ensure_download_before_grading,
            ensure_download_after_grading,
            dl_file_with_ext_count)

    @classmethod
    def submit_page_answer_by_page_id_and_test(
            cls, page_id, use_correct_answer=True, answer_data=None,
            skip_code_question=True,
            expected_grades=None, expected_post_answer_status_code=200,
            do_grading=False, do_human_grade=False, grade_data=None,
            grade_data_extra_kwargs=None,
            dl_file_extension=None,
            ensure_grading_ui_get_before_grading=False,
            ensure_grading_ui_get_after_grading=False,
            ensure_analytic_page_get_before_submission=False,
            ensure_analytic_page_get_after_submission=False,
            ensure_analytic_page_get_before_grading=False,
            ensure_analytic_page_get_after_grading=False,
            ensure_download_before_submission=False,
            ensure_download_after_submission=False,
            ensure_download_before_grading=False,
            ensure_download_after_grading=False,
            dl_file_with_ext_count=None):

        if answer_data is not None:
            assert isinstance(answer_data, dict)
            use_correct_answer = False

        submit_answer_response = None
        post_grade_response = None

        for page_tuple in TEST_PAGE_TUPLE:
            if skip_code_question and page_tuple.need_runpy:
                continue
            if page_id == page_tuple.page_id:
                group_id = page_tuple.group_id
                if ensure_grading_ui_get_before_grading:
                    cls.ensure_grading_ui_get(page_id)

                if ensure_analytic_page_get_before_submission:
                    cls.ensure_analytic_page_get(group_id, page_id)

                if ensure_download_before_submission:
                    cls.ensure_download_submission(group_id, page_id)

                if page_tuple.correct_answer is not None:

                    if answer_data is None:
                        answer_data = page_tuple.correct_answer

                    if page_id in ["anyup", "proof"]:
                        file_path = answer_data["uploaded_file"]
                        if not file_path:
                            # submitting an empty answer
                            submit_answer_response = (
                                cls.post_answer_by_page_id(page_id, answer_data))
                        else:
                            if isinstance(file_path, list):
                                file_path, = file_path

                            file_path = file_path.strip()
                            with open(file_path, 'rb') as fp:
                                answer_data = {"uploaded_file": fp}
                                submit_answer_response = (
                                    cls.post_answer_by_page_id(
                                        page_id, answer_data))
                    else:
                        submit_answer_response = (
                            cls.post_answer_by_page_id(page_id, answer_data))

                    # Fixed #514
                    # https://github.com/inducer/relate/issues/514
                    submit_answer_response.context["form"].as_p()

                    assert (submit_answer_response.status_code
                            == expected_post_answer_status_code), (
                            "%s != %s" % (submit_answer_response.status_code,
                                          expected_post_answer_status_code))

                    if ensure_analytic_page_get_after_submission:
                        cls.ensure_analytic_page_get(group_id, page_id)

                    if ensure_download_after_submission:
                        cls.ensure_download_submission(group_id, page_id)

                if not do_grading:
                    break

                assert cls.end_flow().status_code == 200

                if ensure_analytic_page_get_before_grading:
                    cls.ensure_analytic_page_get(group_id, page_id)

                if ensure_download_before_grading:
                    cls.ensure_download_submission(group_id, page_id)

                if page_tuple.correct_answer is not None:
                    if use_correct_answer:
                        expected_grades = page_tuple.full_points

                    if page_tuple.need_human_grade:
                        if not do_human_grade:
                            cls.assertSessionScoreEqual(None)
                            break
                        if grade_data is not None:
                            assert isinstance(grade_data, dict)
                        else:
                            grade_data = page_tuple.grade_data.copy()

                        if grade_data_extra_kwargs:
                            assert isinstance(grade_data_extra_kwargs, dict)
                            grade_data.update(grade_data_extra_kwargs)

                        post_grade_response = cls.post_grade_by_page_id(
                            page_id, grade_data)
                    cls.assertSessionScoreEqual(expected_grades)

                    if not dl_file_extension:
                        dl_file_extension = page_tuple.dl_file_extension

                    if ensure_download_after_grading:
                        cls.ensure_download_submission(
                            group_id, page_id,
                            dl_file_extension=dl_file_extension,
                            file_with_ext_count=dl_file_with_ext_count)

                if ensure_analytic_page_get_after_grading:
                    cls.ensure_analytic_page_get(group_id, page_id)

                if ensure_grading_ui_get_after_grading:
                    cls.ensure_grading_ui_get(page_id)

        return submit_answer_response, post_grade_response

    def default_submit_page_answer_by_page_id_and_test(self, page_id,
                                                       answer_data=None,
                                                       expected_grade=None,
                                                       do_grading=True,
                                                       grade_data=None,
                                                       grade_data_extra_kwargs=None,
                                                       ):
        return self.submit_page_answer_by_page_id_and_test(
            page_id, answer_data=answer_data,
            skip_code_question=self.skip_code_question,
            expected_grades=expected_grade, expected_post_answer_status_code=200,
            do_grading=do_grading, do_human_grade=True, grade_data=grade_data,
            grade_data_extra_kwargs=grade_data_extra_kwargs,
            ensure_grading_ui_get_before_grading=True,
            ensure_grading_ui_get_after_grading=True,
            ensure_analytic_page_get_before_submission=True,
            ensure_analytic_page_get_after_submission=True,
            ensure_analytic_page_get_before_grading=True,
            ensure_analytic_page_get_after_grading=True,
            ensure_download_before_submission=True,
            ensure_download_after_submission=True,
            ensure_download_before_grading=True,
            ensure_download_after_grading=True)

    @classmethod
    def submit_page_human_grading_by_page_id_and_test(
            cls, page_id,
            expected_post_grading_status_code=200,
            grade_data=None,
            expected_grades=None,
            do_session_score_equal_assersion=True,
            grade_data_extra_kwargs=None,
            force_login_instructor=True,
            ensure_grading_ui_get_before_grading=False,
            ensure_grading_ui_get_after_grading=False,
            ensure_analytic_page_get_before_grading=False,
            ensure_analytic_page_get_after_grading=False,
            ensure_download_before_grading=False,
            ensure_download_after_grading=False):

        # this helper is expected to be used when the session is finished

        post_grade_response = None

        for page_tuple in TEST_PAGE_TUPLE:
            if page_id == page_tuple.page_id:
                group_id = page_tuple.group_id
                if ensure_grading_ui_get_before_grading:
                    cls.ensure_grading_ui_get(page_id)

                if ensure_analytic_page_get_before_grading:
                    cls.ensure_analytic_page_get(group_id, page_id)

                if ensure_download_before_grading:
                    cls.ensure_download_submission(group_id, page_id)

                if not page_tuple.need_human_grade:
                    break

                assign_full_grades = True

                if grade_data is not None:
                    assert isinstance(grade_data, dict)
                    assign_full_grades = False
                else:
                    grade_data = page_tuple.grade_data.copy()

                if assign_full_grades:
                    expected_grades = page_tuple.full_points

                if grade_data_extra_kwargs:
                    assert isinstance(grade_data_extra_kwargs, dict)
                    grade_data.update(grade_data_extra_kwargs)

                post_grade_response = cls.post_grade_by_page_id(
                    page_id, grade_data,
                    force_login_instructor=force_login_instructor)

                assert (post_grade_response.status_code
                        == expected_post_grading_status_code)

                if post_grade_response.status_code == 200:
                    if do_session_score_equal_assersion:
                        cls.assertSessionScoreEqual(expected_grades)

                if ensure_download_after_grading:
                    cls.ensure_download_submission(group_id, page_id)

                if ensure_analytic_page_get_after_grading:
                    cls.ensure_analytic_page_get(group_id, page_id)

                if ensure_grading_ui_get_after_grading:
                    cls.ensure_grading_ui_get(page_id)

        return post_grade_response


class FallBackStorageMessageTestMixin(object):
    # In case other message storage are used, the following is the default
    # storage used by django and RELATE. Tests which concerns the message
    # should not include this mixin.
    storage = 'django.contrib.messages.storage.fallback.FallbackStorage'

    def setUp(self):  # noqa
        super(FallBackStorageMessageTestMixin, self).setUp()
        self.msg_settings_override = override_settings(MESSAGE_STORAGE=self.storage)
        self.msg_settings_override.enable()
        self.addCleanup(self.msg_settings_override.disable)

    def get_listed_storage_from_response(self, response):
        return list(self.get_response_context_value_by_name(response, 'messages'))

    def clear_message_response_storage(self, response):
        # this should only be used for debug, because we are using private method
        # which might change
        try:
            storage = self.get_response_context_value_by_name(response, 'messages')
        except AssertionError:
            # message doesn't exist in response context
            return
        if hasattr(storage, '_loaded_data'):
            storage._loaded_data = []
        elif hasattr(storage, '_loaded_message'):
            storage._loaded_messages = []

        if hasattr(storage, '_queued_messages'):
            storage._queued_messages = []

        self.assertEqual(len(storage), 0)

    def assertResponseMessagesCount(self, response, expected_count):  # noqa
        storage = self.get_listed_storage_from_response(response)
        self.assertEqual(len(storage), expected_count)

    def assertResponseMessagesEqual(self, response, expected_messages):  # noqa
        storage = self.get_listed_storage_from_response(response)
        if not isinstance(expected_messages, list):
            expected_messages = [expected_messages]
        self.assertEqual(len([m for m in storage]), len(expected_messages))
        self.assertEqual([m.message for m in storage], expected_messages)

    def assertResponseMessagesEqualRegex(self, response, expected_message_regexs):  # noqa
        storage = self.get_listed_storage_from_response(response)
        if not isinstance(expected_message_regexs, list):
            expected_message_regexs = [expected_message_regexs]
        self.assertEqual(len([m for m in storage]), len(expected_message_regexs))
        messages = [m.message for m in storage]
        for idx, m in enumerate(messages):
            six.assertRegex(self, m, expected_message_regexs[idx])

    def assertResponseMessagesContains(self, response, expected_messages,  # noqa
                                       loose=False):
        storage = self.get_listed_storage_from_response(response)
        if isinstance(expected_messages, str):
            expected_messages = [expected_messages]
        messages = [m.message for m in storage]
        if loose:
            from django.utils.encoding import force_text
            messages = " ".join([force_text(m) for m in messages])
        for em in expected_messages:
            self.assertIn(em, messages)

    def assertResponseMessageLevelsEqual(self, response, expected_levels):  # noqa
        storage = self.get_listed_storage_from_response(response)
        self.assertEqual([m.level for m in storage], expected_levels)

    def debug_print_response_messages(self, response):
        """
        For debugging :class:`django.contrib.messages` objects in post response
        :param response: response