diff --git a/.gitignore b/.gitignore
index 72cffcc6bc57b982c9ad6d4493543bcdd519a48f..cdd667e65391cb80ced01c93e79f32850648ab31 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,5 @@ components
build
*.pyz
+
+*.pem
diff --git a/course/admin.py b/course/admin.py
index b2e66784febb10899333bded6b963565e765c3d2..5e89a2debf85e230b1eabdaf5de0c10ccf5d7ce6 100644
--- a/course/admin.py
+++ b/course/admin.py
@@ -295,15 +295,16 @@ class ParticipationAdmin(admin.ModelAdmin):
from django.conf import settings
return string_concat(
- "<a href='%(link)s'>", _("%(last_name)s, %(first_name)s"),
- "</a>"
- ) % {
- "link": reverse(
- "admin:%s_change" % settings.AUTH_USER_MODEL.replace(".", "_")
- .lower(),
- args=(obj.user.id,)),
- "last_name": verbose_blank(obj.user.last_name),
- "first_name": verbose_blank(obj.user.first_name)}
+ "<a href='%(link)s'>", _("%(last_name)s, %(first_name)s"),
+ "</a>"
+ ) % {
+ "link": reverse(
+ "admin:%s_change"
+ % settings.AUTH_USER_MODEL.replace(".", "_")
+ .lower(),
+ args=(obj.user.id,)),
+ "last_name": verbose_blank(obj.user.last_name),
+ "first_name": verbose_blank(obj.user.first_name)}
get_user.short_description = pgettext("real name of a user", "Name")
get_user.admin_order_field = "user__last_name"
@@ -389,7 +390,14 @@ admin.site.register(ParticipationPreapproval, ParticipationPreapprovalAdmin)
class InstantFlowRequestAdmin(admin.ModelAdmin):
- pass
+ list_display = ("course", "flow_id", "start_time", "end_time", "cancelled")
+ list_filter = ("course",)
+
+ date_hierarchy = "start_time"
+
+ search_fields = (
+ "email",
+ )
admin.site.register(InstantFlowRequest, InstantFlowRequestAdmin)
diff --git a/course/content.py b/course/content.py
index b198c21c1855c3527a1df77dc6c91d85af1f557f..f60a6c9fefffbdc14618e5b753d51f0f2760d5a9 100644
--- a/course/content.py
+++ b/course/content.py
@@ -30,6 +30,7 @@ from django.utils.translation import ugettext as _
import re
import datetime
import six
+import sys
from django.utils.timezone import now
from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured
@@ -123,10 +124,12 @@ def get_repo_blob_data_cached(repo, full_name, commit_sha):
if isinstance(commit_sha, six.binary_type):
from six.moves.urllib.parse import quote_plus
- cache_key = "%%%1".join((
+ cache_key = "%R%1".join((
quote_plus(repo.controldir()),
quote_plus(full_name),
- commit_sha.decode()))
+ commit_sha.decode(),
+ ".".join(str(s) for s in sys.version_info[:2]),
+ ))
else:
cache_key = None
@@ -136,7 +139,13 @@ def get_repo_blob_data_cached(repo, full_name, commit_sha):
cache_key = None
if cache_key is None:
- return get_repo_blob(repo, full_name, commit_sha).data
+ result = get_repo_blob(repo, full_name, commit_sha).data
+ assert isinstance(result, six.binary_type)
+ return result
+
+ # Byte string is wrapped in a tuple to force pickling because memcache's
+ # python wrapper appears to auto-decode/encode string values, thus trying
+ # to decode our byte strings. Grr.
def_cache = cache.caches["default"]
@@ -145,12 +154,16 @@ def get_repo_blob_data_cached(repo, full_name, commit_sha):
if len(cache_key) < 240:
result = def_cache.get(cache_key)
if result is not None:
+ (result,) = result
+ assert isinstance(result, six.binary_type), cache_key
return result
result = get_repo_blob(repo, full_name, commit_sha).data
if len(result) <= getattr(settings, "RELATE_CACHE_MAX_BYTES", 0):
- def_cache.add(cache_key, result, None)
+ def_cache.add(cache_key, (result,), None)
+
+ assert isinstance(result, six.binary_type)
return result
@@ -324,7 +337,8 @@ def get_raw_yaml_from_repo(repo, full_name, commit_sha):
from six.moves.urllib.parse import quote_plus
cache_key = "%RAW%%2".join((
- quote_plus(repo.controldir()), quote_plus(full_name), commit_sha.decode()))
+ quote_plus(repo.controldir()), quote_plus(full_name), commit_sha.decode(),
+ ))
import django.core.cache as cache
def_cache = cache.caches["default"]
diff --git a/course/models.py b/course/models.py
index 011f62b94e88f7fe395d0697dde357d631291a5e..97314e199ab0efaeed1fb1029e4d1aa7b9fc97dc 100644
--- a/course/models.py
+++ b/course/models.py
@@ -448,6 +448,18 @@ class InstantFlowRequest(models.Model):
verbose_name = _("Instant flow request")
verbose_name_plural = _("Instant flow requests")
+ def __unicode__(self):
+ return _("Instant flow request for "
+ "%(flow_id)s in %(course)s at %(start_time)s") \
+ % {
+ "flow_id": self.flow_id,
+ "course": self.course,
+ "start_time": self.start_time,
+ }
+
+ if six.PY3:
+ __str__ = __unicode__
+
# {{{ flow session
diff --git a/course/page/code.py b/course/page/code.py
index fc0ead65192159da8d9b055373e32631ec4b7fc4..6488eb773eacd06f0067db968d50244fdf9856f7 100644
--- a/course/page/code.py
+++ b/course/page/code.py
@@ -93,10 +93,15 @@ def request_python_run(run_req, run_timeout, image=None):
# DEBUGGING SWITCH: 1 for 'spawn container', 0 for 'static container'
if 1:
+ docker_url = getattr(settings, "RELATE_DOCKER_URL",
+ "unix://var/run/docker.sock")
+ docker_tls = getattr(settings, "RELATE_DOCKER_TLS_CONFIG",
+ None)
docker_cnx = docker.Client(
- base_url=getattr(settings, "RELATE_DOCKER_URL",
- "unix://var/run/docker.sock"),
- version='1.12', timeout=docker_timeout)
+ base_url=docker_url,
+ tls=docker_tls,
+ timeout=docker_timeout,
+ version="1.19")
if image is None:
image = settings.RELATE_DOCKER_RUNPY_IMAGE
@@ -106,22 +111,34 @@ def request_python_run(run_req, run_timeout, image=None):
command=[
"/opt/runpy/runpy",
"-1"],
- mem_limit=256*10**6,
+ host_config={
+ "Memory": 256*10**6,
+ "MemorySwap": -1,
+ "PublishAllPorts": True,
+ "ReadonlyRootfs": True,
+ },
user="runpy")
container_id = dresult["Id"]
else:
container_id = None
+ connect_host_ip = 'localhost'
+
try:
# FIXME: Prohibit networking
if container_id is not None:
- docker_cnx.start(
- container_id,
- port_bindings={RUNPY_PORT: ('127.0.0.1',)})
+ docker_cnx.start(container_id)
+
+ container_props = docker_cnx.inspect_container(container_id)
+ (port_info,) = (container_props
+ ["NetworkSettings"]["Ports"]["%d/tcp" % RUNPY_PORT])
+ port_host_ip = port_info.get("HostIp")
+
+ if port_host_ip != "0.0.0.0":
+ connect_host_ip = port_host_ip
- port_info, = docker_cnx.port(container_id, RUNPY_PORT)
port = int(port_info["HostPort"])
else:
port = RUNPY_PORT
@@ -133,44 +150,44 @@ def request_python_run(run_req, run_timeout, image=None):
from traceback import format_exc
+ def check_timeout():
+ if time() - start_time < docker_timeout:
+ sleep(0.1)
+ # and retry
+ else:
+ return {
+ "result": "uncaught_error",
+ "message": "Timeout waiting for container.",
+ "traceback": "".join(format_exc()),
+ }
+
while True:
try:
- connection = http_client.HTTPConnection('localhost', port)
+ connection = http_client.HTTPConnection(connect_host_ip, port)
connection.request('GET', '/ping')
response = connection.getresponse()
- response_data = response.read().decode("utf-8")
+ response_data = response.read().decode()
if response_data != "OK":
raise InvalidPingResponse()
break
+ except (http_client.BadStatusLine, InvalidPingResponse):
+ ct_res = check_timeout()
+ if ct_res is not None:
+ return ct_res
+
except socket.error as e:
if e.errno in [errno.ECONNRESET, errno.ECONNREFUSED]:
- if time() - start_time < docker_timeout:
- sleep(0.1)
- # and retry
- else:
- return {
- "result": "uncaught_error",
- "message": "Timeout waiting for container.",
- "traceback": "".join(format_exc()),
- }
- else:
- raise
+ ct_res = check_timeout()
+ if ct_res is not None:
+ return ct_res
- except (http_client.BadStatusLine, InvalidPingResponse):
- if time() - start_time < docker_timeout:
- sleep(0.1)
- # and retry
else:
- return {
- "result": "uncaught_error",
- "message": "Timeout waiting for container.",
- "traceback": "".join(format_exc()),
- }
+ raise
# }}}
@@ -178,7 +195,7 @@ def request_python_run(run_req, run_timeout, image=None):
try:
# Add a second to accommodate 'wire' delays
- connection = http_client.HTTPConnection('localhost', port,
+ connection = http_client.HTTPConnection(connect_host_ip, port,
timeout=1 + run_timeout)
headers = {'Content-type': 'application/json'}
diff --git a/course/versioning.py b/course/versioning.py
index 4ca2706d4fb4ca71a2e43fc63b0d379e5eaf9510..ff4888f0a6e2c1b6e2f5b030524b43973a045197 100644
--- a/course/versioning.py
+++ b/course/versioning.py
@@ -81,46 +81,15 @@ def transfer_remote_refs(repo, remote_refs):
del repo[ref]
-# {{{ shell quoting
-
-# Adapted from
-# https://github.com/python/cpython/blob/8cd133c63f156451eb3388b9308734f699f4f1af/Lib/shlex.py#L278
-
-def is_shell_safe(s):
- import re
- import sys
-
- flags = 0
- if sys.version_info >= (3,):
- flags = re.ASCII
-
- unsafe_re = re.compile(br'[^\w@%+=:,./-]', flags)
-
- return unsafe_re.search(s) is None
-
-
-def shell_quote(s):
- """Return a shell-escaped version of the byte string *s*."""
-
- # Unconditionally quotes because that's apparently git's behavior, too,
- # and some code hosting sites (notably Bitbucket) appear to rely on that.
-
- if not s:
- return b"''"
-
- # use single quotes, and put single quotes into double quotes
- # the string $'b is then quoted as '$'"'"'b'
- return b"'" + s.replace(b"'", b"'\"'\"'") + b"'"
-
-# }}}
-
-
class DulwichParamikoSSHVendor(object):
def __init__(self, ssh_kwargs):
self.ssh_kwargs = ssh_kwargs
def run_command(self, host, command, username=None, port=None,
progress_stderr=None):
+ if not isinstance(command, bytes):
+ raise TypeError(command)
+
if port is None:
port = 22
@@ -132,15 +101,6 @@ class DulwichParamikoSSHVendor(object):
channel = client.get_transport().open_session()
- assert command
- assert is_shell_safe(command[0])
-
- command = (
- command[0]
- + b' '
- + b' '.join(
- shell_quote(c) for c in command[1:]))
-
channel.exec_command(command)
def progress_stderr(s):
diff --git a/local_settings.py.example b/local_settings.py.example
index f7c5d35d5326303598e1be3c6c2429c5b3a4e436..32d3db611a1ce02e4ed63402f491bab4e1a37e83 100644
--- a/local_settings.py.example
+++ b/local_settings.py.example
@@ -83,6 +83,25 @@ RELATE_DOCKER_RUNPY_IMAGE = "inducer/relate-runpy-i386"
# to spawn containers for student code.
RELATE_DOCKER_URL = "unix://var/run/docker.sock"
+RELATE_DOCKER_TLS_CONFIG = None
+
+# Example setup for targeting remote Docker instances
+# with TLS authentication:
+
+# RELATE_DOCKER_URL = "https://relate.cs.illinois.edu:2375"
+#
+# import os.path
+# pki_base_dir = os.path.dirname(__file__)
+#
+# import docker.tls
+# RELATE_DOCKER_TLS_CONFIG = docker.tls.TLSConfig(
+# client_cert=(
+# os.path.join(pki_base_dir, "client-cert.pem"),
+# os.path.join(pki_base_dir, "client-key.pem"),
+# ),
+# ca_cert=os.path.join(pki_base_dir, "ca.pem"),
+# verify=True)
+
RELATE_MAINTENANCE_MODE = False
# May be set to a string to set a sitewide announcement visible on every page.
diff --git a/relate/settings.py b/relate/settings.py
index a6253a82b46098c41e471076d035de01e4824ebd..82086092ef215dbea85a51b832a9c9e20d6703db 100644
--- a/relate/settings.py
+++ b/relate/settings.py
@@ -15,9 +15,12 @@ import os
from os.path import join
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
-local_settings = {}
+_local_settings_file = join(BASE_DIR, "local_settings.py")
+local_settings = {
+ "__file__": _local_settings_file,
+ }
try:
- with open(join(BASE_DIR, "local_settings.py")) as inf:
+ with open(_local_settings_file) as inf:
local_settings_contents = inf.read()
except IOError:
pass
diff --git a/requirements.txt b/requirements.txt
index 7bdab3f318cf628f57bf3fd4c88fa52ad4ec17ab..6a312cf0ae5b415cce6fccd9668c9dcff2c2ea7b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -27,7 +27,7 @@ pytz
pyyaml
# dulwich (git for Py3 support)
-git+https://github.com/jelmer/dulwich
+dulwich>=0.12
ecdsa
paramiko