wrap_cl.hpp

              "invalid length of pitch tuple");
        pitch = py::cast<size_t>(pitches[0]);
      }

      // check buffer size
      cl_int itemsize = get_image_format_item_size(fmt);
      if (buf && std::max(pitch, width*itemsize)*height > cl_uint(len))
          throw pyopencl::error("Image", CL_INVALID_VALUE,
              "buffer too small");

      PYOPENCL_PRINT_CALL_TRACE("clCreateImage2D");
      PYOPENCL_RETRY_IF_MEM_ERROR(
          {
            mem = clCreateImage2D(ctx.data(), flags, &fmt,
                width, height, pitch, buf, &status_code);
            if (status_code != CL_SUCCESS)
              throw pyopencl::error("clCreateImage2D", status_code);
          } );

    }
    else if (dims == 3)
    {
      size_t width = py::cast<size_t>(shape[0]);
      size_t height = py::cast<size_t>(shape[1]);
      size_t depth = py::cast<size_t>(shape[2]);

      size_t pitch_x = 0;
      size_t pitch_y = 0;

      if (pitches.ptr() != Py_None)
      {
        if (py::len(pitches) != 2)
          throw pyopencl::error("Image", CL_INVALID_VALUE,
              "invalid length of pitch tuple");

        pitch_x = py::cast<size_t>(pitches[0]);
        pitch_y = py::cast<size_t>(pitches[1]);
      }

      // check buffer size
      cl_int itemsize = get_image_format_item_size(fmt);
      if (buf &&
          std::max(std::max(pitch_x, width*itemsize)*height, pitch_y)
          * depth > cl_uint(len))
        throw pyopencl::error("Image", CL_INVALID_VALUE,
            "buffer too small");

      PYOPENCL_PRINT_CALL_TRACE("clCreateImage3D");
      PYOPENCL_RETRY_IF_MEM_ERROR(
          {
            mem = clCreateImage3D(ctx.data(), flags, &fmt,
              width, height, depth, pitch_x, pitch_y, buf, &status_code);
            if (status_code != CL_SUCCESS)
              throw pyopencl::error("clCreateImage3D", status_code);
          } );
    }
    else
      throw pyopencl::error("Image", CL_INVALID_VALUE,
          "invalid dimension");

    if (!(flags & CL_MEM_USE_HOST_PTR))
      retained_buf_obj.reset();

    try
    {
      return new image(mem, false, std::move(retained_buf_obj));
    }
    catch (...)
    {
      PYOPENCL_CALL_GUARDED(clReleaseMemObject, (mem));
      throw;
    }
  }

#if PYOPENCL_CL_VERSION >= 0x1020

  inline
  image *create_image_from_desc(
      context const &ctx,
      cl_mem_flags flags,
      cl_image_format const &fmt,
      cl_image_desc &desc,
      py::object buffer)
  {
    if (buffer.ptr() != Py_None &&
        !(flags & (CL_MEM_USE_HOST_PTR | CL_MEM_COPY_HOST_PTR)))
      PyErr_Warn(PyExc_UserWarning, "'hostbuf' was passed, "
          "but no memory flags to make use of it.");

    void *buf = 0;

    std::unique_ptr<py_buffer_wrapper> retained_buf_obj;
    if (buffer.ptr() != Py_None)
    {
      retained_buf_obj = std::unique_ptr<py_buffer_wrapper>(new py_buffer_wrapper);

      int py_buf_flags = PyBUF_ANY_CONTIGUOUS;
      if ((flags & CL_MEM_USE_HOST_PTR)
          && ((flags & CL_MEM_READ_WRITE)
            || (flags & CL_MEM_WRITE_ONLY)))
        py_buf_flags |= PyBUF_WRITABLE;

      retained_buf_obj->get(buffer.ptr(), py_buf_flags);

      buf = retained_buf_obj->m_buf.buf;
    }

    PYOPENCL_PRINT_CALL_TRACE("clCreateImage");
    cl_int status_code;
    cl_mem mem = clCreateImage(ctx.data(), flags, &fmt, &desc, buf, &status_code);
    if (status_code != CL_SUCCESS)
      throw pyopencl::error("clCreateImage", status_code);

    if (!(flags & CL_MEM_USE_HOST_PTR))
      retained_buf_obj.reset();

    try
    {
      return new image(mem, false, std::move(retained_buf_obj));
    }
    catch (...)
    {
      PYOPENCL_CALL_GUARDED(clReleaseMemObject, (mem));
      throw;
    }
  }

#endif

  // }}}

  // {{{ image transfers

  inline
  event *enqueue_read_image(
      command_queue &cq,
      image &img,
      py::object py_origin, py::object py_region,
      py::object buffer,
      size_t row_pitch, size_t slice_pitch,
      py::object py_wait_for,
      bool is_blocking)
  {
    PYOPENCL_PARSE_WAIT_FOR;
    COPY_PY_COORD_TRIPLE(origin);
    COPY_PY_REGION_TRIPLE(region);

    void *buf;

    std::unique_ptr<py_buffer_wrapper> ward(new py_buffer_wrapper);

    ward->get(buffer.ptr(), PyBUF_ANY_CONTIGUOUS | PyBUF_WRITABLE);

    buf = ward->m_buf.buf;

    cl_event evt;

    PYOPENCL_RETRY_IF_MEM_ERROR(
      PYOPENCL_CALL_GUARDED(clEnqueueReadImage, (
            cq.data(),
            img.data(),
            PYOPENCL_CAST_BOOL(is_blocking),
            origin, region, row_pitch, slice_pitch, buf,
            PYOPENCL_WAITLIST_ARGS, &evt
            ));
      );
    PYOPENCL_RETURN_NEW_NANNY_EVENT(evt, ward);
  }




  inline
  event *enqueue_write_image(
      command_queue &cq,
      image &img,
      py::object py_origin, py::object py_region,
      py::object buffer,
      size_t row_pitch, size_t slice_pitch,
      py::object py_wait_for,
      bool is_blocking)
  {
    PYOPENCL_PARSE_WAIT_FOR;
    COPY_PY_COORD_TRIPLE(origin);
    COPY_PY_REGION_TRIPLE(region);

    const void *buf;

    std::unique_ptr<py_buffer_wrapper> ward(new py_buffer_wrapper);

    ward->get(buffer.ptr(), PyBUF_ANY_CONTIGUOUS);

    buf = ward->m_buf.buf;

    cl_event evt;
    PYOPENCL_RETRY_IF_MEM_ERROR(
      PYOPENCL_CALL_GUARDED(clEnqueueWriteImage, (
            cq.data(),
            img.data(),
            PYOPENCL_CAST_BOOL(is_blocking),
            origin, region, row_pitch, slice_pitch, buf,
            PYOPENCL_WAITLIST_ARGS, &evt
            ));
      );
    PYOPENCL_RETURN_NEW_NANNY_EVENT(evt, ward);
  }




  inline
  event *enqueue_copy_image(
      command_queue &cq,
      memory_object_holder &src,
      memory_object_holder &dest,
      py::object py_src_origin,
      py::object py_dest_origin,
      py::object py_region,
      py::object py_wait_for
      )
  {
    PYOPENCL_PARSE_WAIT_FOR;
    COPY_PY_COORD_TRIPLE(src_origin);
    COPY_PY_COORD_TRIPLE(dest_origin);
    COPY_PY_REGION_TRIPLE(region);

    cl_event evt;
    PYOPENCL_RETRY_IF_MEM_ERROR(
      PYOPENCL_CALL_GUARDED(clEnqueueCopyImage, (
            cq.data(), src.data(), dest.data(),
            src_origin, dest_origin, region,
            PYOPENCL_WAITLIST_ARGS, &evt
            ));
      );
    PYOPENCL_RETURN_NEW_EVENT(evt);
  }




  inline
  event *enqueue_copy_image_to_buffer(
      command_queue &cq,
      memory_object_holder &src,
      memory_object_holder &dest,
      py::object py_origin,
      py::object py_region,
      size_t offset,
      py::object py_wait_for
      )
  {
    PYOPENCL_PARSE_WAIT_FOR;
    COPY_PY_COORD_TRIPLE(origin);
    COPY_PY_REGION_TRIPLE(region);

    cl_event evt;
    PYOPENCL_RETRY_IF_MEM_ERROR(
      PYOPENCL_CALL_GUARDED(clEnqueueCopyImageToBuffer, (
            cq.data(), src.data(), dest.data(),
            origin, region, offset,
            PYOPENCL_WAITLIST_ARGS, &evt
            ));
      );
    PYOPENCL_RETURN_NEW_EVENT(evt);
  }




  inline
  event *enqueue_copy_buffer_to_image(
      command_queue &cq,
      memory_object_holder &src,
      memory_object_holder &dest,
      size_t offset,
      py::object py_origin,
      py::object py_region,
      py::object py_wait_for
      )
  {
    PYOPENCL_PARSE_WAIT_FOR;
    COPY_PY_COORD_TRIPLE(origin);
    COPY_PY_REGION_TRIPLE(region);

    cl_event evt;
    PYOPENCL_RETRY_IF_MEM_ERROR(
      PYOPENCL_CALL_GUARDED(clEnqueueCopyBufferToImage, (
            cq.data(), src.data(), dest.data(),
            offset, origin, region,
            PYOPENCL_WAITLIST_ARGS, &evt
            ));
      );
    PYOPENCL_RETURN_NEW_EVENT(evt);
  }

  // }}}

#if PYOPENCL_CL_VERSION >= 0x1020
  inline
  event *enqueue_fill_image(
      command_queue &cq,
      memory_object_holder &mem,
      py::object color,
      py::object py_origin, py::object py_region,
      py::object py_wait_for
      )
  {
    PYOPENCL_PARSE_WAIT_FOR;

    COPY_PY_COORD_TRIPLE(origin);
    COPY_PY_REGION_TRIPLE(region);

    const void *color_buf;

    std::unique_ptr<py_buffer_wrapper> ward(new py_buffer_wrapper);

    ward->get(color.ptr(), PyBUF_ANY_CONTIGUOUS);

    color_buf = ward->m_buf.buf;

    cl_event evt;
    PYOPENCL_RETRY_IF_MEM_ERROR(
      PYOPENCL_CALL_GUARDED(clEnqueueFillImage, (
            cq.data(),
            mem.data(),
            color_buf, origin, region,
            PYOPENCL_WAITLIST_ARGS, &evt
            ));
      );
    PYOPENCL_RETURN_NEW_EVENT(evt);
  }
#endif

  // }}}


  // {{{ pipe

  class pipe : public memory_object
  {
    public:
      pipe(cl_mem mem, bool retain)
        : memory_object(mem, retain)
      { }

#if PYOPENCL_CL_VERSION < 0x2000
      typedef void* cl_pipe_info;
#endif

      py::object get_pipe_info(cl_pipe_info param_name) const
      {
#if PYOPENCL_CL_VERSION >= 0x2000
        switch (param_name)
        {
          case CL_PIPE_PACKET_SIZE:
          case CL_PIPE_MAX_PACKETS:
            PYOPENCL_GET_TYPED_INFO(Pipe, data(), param_name, cl_uint);

          default:
            throw error("Pipe.get_pipe_info", CL_INVALID_VALUE);
        }
#else
        throw error("Pipes not available. PyOpenCL was not compiled against a CL2+ header.",
            CL_INVALID_VALUE);
#endif
      }
  };

#if PYOPENCL_CL_VERSION >= 0x2000
  inline
  pipe *create_pipe(
      context const &ctx,
      cl_mem_flags flags,
      cl_uint pipe_packet_size,
      cl_uint pipe_max_packets,
      py::sequence py_props)
  {
#if 0
    PYOPENCL_STACK_CONTAINER(cl_pipe_properties, props, py::len(py_props) + 1);
    {
      size_t i = 0;
      for (auto prop: py_props)
        props[i++] = py::cast<cl_pipe_properties>(prop);
      props[i++] = 0;
    }
#endif
    if (py::len(py_props) != 0)
      throw pyopencl::error("Pipe", CL_INVALID_VALUE, "non-empty properties "
          "argument to Pipe not allowed");

    cl_int status_code;
    PYOPENCL_PRINT_CALL_TRACE("clCreatePipe");

    cl_mem mem = clCreatePipe(
        ctx.data(),
        flags,
        pipe_packet_size,
        pipe_max_packets,
        nullptr,
        &status_code);

    if (status_code != CL_SUCCESS)
      throw pyopencl::error("Pipe", status_code);

    try
    {
      return new pipe(mem, false);
    }
    catch (...)
    {
      PYOPENCL_CALL_GUARDED(clReleaseMemObject, (mem));
      throw;
    }
}
#endif

  // }}}


  // {{{ maps
  class memory_map
  {
    private:
      bool m_valid;
      std::shared_ptr<command_queue> m_queue;
      memory_object m_mem;
      void *m_ptr;

    public:
      memory_map(std::shared_ptr<command_queue> cq, memory_object const &mem, void *ptr)
        : m_valid(true), m_queue(cq), m_mem(mem), m_ptr(ptr)
      {
      }

      ~memory_map()
      {
        if (m_valid)
          delete release(0, py::none());
      }

      event *release(command_queue *cq, py::object py_wait_for)
      {
        PYOPENCL_PARSE_WAIT_FOR;

        if (cq == 0)
          cq = m_queue.get();

        cl_event evt;
        PYOPENCL_CALL_GUARDED(clEnqueueUnmapMemObject, (
              cq->data(), m_mem.data(), m_ptr,
              PYOPENCL_WAITLIST_ARGS, &evt
              ));

        m_valid = false;

        PYOPENCL_RETURN_NEW_EVENT(evt);
      }
  };




  // FIXME: Reenable in pypy
#ifndef PYPY_VERSION
  inline
  py::object enqueue_map_buffer(
      std::shared_ptr<command_queue> cq,
      memory_object_holder &buf,
      cl_map_flags flags,
      size_t offset,
      py::object py_shape, py::object dtype,
      py::object py_order, py::object py_strides,
      py::object py_wait_for,
      bool is_blocking
      )
  {
    PYOPENCL_PARSE_WAIT_FOR;
    PYOPENCL_PARSE_NUMPY_ARRAY_SPEC;

    npy_uintp size_in_bytes = tp_descr->elsize;
    for (npy_intp sdim: shape)
      size_in_bytes *= sdim;

    py::object result;

    cl_event evt;
    cl_int status_code;
    PYOPENCL_PRINT_CALL_TRACE("clEnqueueMapBuffer");
    void *mapped;

    PYOPENCL_RETRY_IF_MEM_ERROR(
        {
          {
            py::gil_scoped_release release;
            mapped = clEnqueueMapBuffer(
                  cq->data(), buf.data(),
                  PYOPENCL_CAST_BOOL(is_blocking), flags,
                  offset, size_in_bytes,
                  PYOPENCL_WAITLIST_ARGS, &evt,
                  &status_code);
          }
          if (status_code != CL_SUCCESS)
            throw pyopencl::error("clEnqueueMapBuffer", status_code);
        } );

    event evt_handle(evt, false);

    std::unique_ptr<memory_map> map;
    try
    {
      result = py::object(py::reinterpret_steal<py::object>(PyArray_NewFromDescr(
          &PyArray_Type, tp_descr,
          shape.size(),
          shape.empty() ? nullptr : &shape.front(),
          strides.empty() ? nullptr : &strides.front(),
          mapped, ary_flags, /*obj*/nullptr)));

      if (size_in_bytes != (npy_uintp) PyArray_NBYTES(result.ptr()))
        throw pyopencl::error("enqueue_map_buffer", CL_INVALID_VALUE,
            "miscalculated numpy array size (not contiguous?)");

       map = std::unique_ptr<memory_map>(new memory_map(cq, buf, mapped));
    }
    catch (...)
    {
      PYOPENCL_CALL_GUARDED_CLEANUP(clEnqueueUnmapMemObject, (
            cq->data(), buf.data(), mapped, 0, 0, 0));
      throw;
    }

    py::object map_py(handle_from_new_ptr(map.release()));
    PyArray_BASE(result.ptr()) = map_py.ptr();
    Py_INCREF(map_py.ptr());

    return py::make_tuple(
        result,
        handle_from_new_ptr(new event(evt_handle)));
  }
#endif




  // FIXME: Reenable in pypy
#ifndef PYPY_VERSION
  inline
  py::object enqueue_map_image(
      std::shared_ptr<command_queue> cq,
      memory_object_holder &img,
      cl_map_flags flags,
      py::object py_origin,
      py::object py_region,
      py::object py_shape, py::object dtype,
      py::object py_order, py::object py_strides,
      py::object py_wait_for,
      bool is_blocking
      )
  {
    PYOPENCL_PARSE_WAIT_FOR;
    PYOPENCL_PARSE_NUMPY_ARRAY_SPEC;
    COPY_PY_COORD_TRIPLE(origin);
    COPY_PY_REGION_TRIPLE(region);

    cl_event evt;
    cl_int status_code;
    PYOPENCL_PRINT_CALL_TRACE("clEnqueueMapImage");
    size_t row_pitch, slice_pitch;
    void *mapped;
    PYOPENCL_RETRY_IF_MEM_ERROR(
      {
        {
          py::gil_scoped_release release;
          mapped = clEnqueueMapImage(
                cq->data(), img.data(),
                PYOPENCL_CAST_BOOL(is_blocking), flags,
                origin, region, &row_pitch, &slice_pitch,
                PYOPENCL_WAITLIST_ARGS, &evt,
                &status_code);
        }
        if (status_code != CL_SUCCESS)
          throw pyopencl::error("clEnqueueMapImage", status_code);
      } );

    event evt_handle(evt, false);

    std::unique_ptr<memory_map> map;
    try
    {
       map = std::unique_ptr<memory_map>(new memory_map(cq, img, mapped));
    }
    catch (...)
    {
      PYOPENCL_CALL_GUARDED_CLEANUP(clEnqueueUnmapMemObject, (
            cq->data(), img.data(), mapped, 0, 0, 0));
      throw;
    }

    py::object result = py::reinterpret_steal<py::object>(PyArray_NewFromDescr(
        &PyArray_Type, tp_descr,
        shape.size(),
        shape.empty() ? nullptr : &shape.front(),
        strides.empty() ? nullptr : &strides.front(),
        mapped, ary_flags, /*obj*/nullptr));

    py::object map_py(handle_from_new_ptr(map.release()));
    PyArray_BASE(result.ptr()) = map_py.ptr();
    Py_INCREF(map_py.ptr());

    return py::make_tuple(
        result,
        handle_from_new_ptr(new event(evt_handle)),
        row_pitch, slice_pitch);
  }
#endif

  // }}}


#if PYOPENCL_CL_VERSION >= 0x2000

  // {{{ svm pointer

  class size_not_available { };

  class svm_pointer
  {
    public:
      virtual void *svm_ptr() const = 0;
      // may throw size_not_available
      virtual size_t size() const = 0;
      virtual ~svm_pointer()
      { }
  };

  // }}}


  // {{{ svm_arg_wrapper

  class svm_arg_wrapper : public svm_pointer
  {
    private:
      void *m_ptr;
      PYOPENCL_BUFFER_SIZE_T m_size;
      std::unique_ptr<py_buffer_wrapper> ward;

    public:
      svm_arg_wrapper(py::object holder)
      {
        ward = std::unique_ptr<py_buffer_wrapper>(new py_buffer_wrapper);
#ifdef PYPY_VERSION
        // FIXME: get a read-only buffer
        // Not quite honest, but Pypy doesn't consider numpy arrays
        // created from objects with the __aray_interface__ writeable.
        ward->get(holder.ptr(), PyBUF_ANY_CONTIGUOUS);
#else
        ward->get(holder.ptr(), PyBUF_ANY_CONTIGUOUS | PyBUF_WRITABLE);
#endif
        m_ptr = ward->m_buf.buf;
        m_size = ward->m_buf.len;
      }

      void *svm_ptr() const
      {
        return m_ptr;
      }
      size_t size() const
      {
        return m_size;
      }
  };

  // }}}


  // {{{ svm_allocation

  class svm_allocation : public svm_pointer
  {
    private:
      std::shared_ptr<context> m_context;
      void *m_allocation;
      size_t m_size;
      command_queue_ref m_queue;
      // FIXME Should maybe also allow keeping a list of events so that we can
      // wait for users to finish in the case of out-of-order queues.

    public:
      svm_allocation(std::shared_ptr<context> const &ctx, size_t size, cl_uint alignment,
          cl_svm_mem_flags flags, const command_queue *queue = nullptr)
        : m_context(ctx), m_size(size)
      {
        if (queue)
        {
          m_queue.set(queue->data());
          if (is_queue_out_of_order(m_queue.data()))
            throw error("SVMAllocation.__init__", CL_INVALID_VALUE,
                "supplying an out-of-order queue to SVMAllocation is invalid");
        }

        int try_count = 0;
        while (try_count < 2)
        {
          PYOPENCL_PRINT_CALL_TRACE("clSVMalloc");
          m_allocation = clSVMAlloc(
              ctx->data(),
              flags, size, alignment);
          if (m_allocation)
            return;

          ++try_count;
          run_python_gc();
        }

        if (!m_allocation)
          throw pyopencl::error("clSVMAlloc", CL_OUT_OF_RESOURCES);
      }

      svm_allocation(std::shared_ptr<context> const &ctx, void *allocation, size_t size,
           const cl_command_queue queue)
        : m_context(ctx), m_allocation(allocation), m_size(size)
      {
        if (queue)
        {
          if (is_queue_out_of_order(queue))
          {
            release();
            throw error("SVMAllocation.__init__", CL_INVALID_VALUE,
                "supplying an out-of-order queue to SVMAllocation is invalid");
          }
          m_queue.set(queue);
        }
      }

      svm_allocation(const svm_allocation &) = delete;
      svm_allocation &operator=(const svm_allocation &) = delete;

      ~svm_allocation()
      {
        if (m_allocation)
          release();
      }

      void release()
      {
        if (!m_allocation)
          throw error("SVMAllocation.release", CL_INVALID_VALUE,
              "trying to double-unref svm allocation");

        if (m_queue.is_valid())
        {
          PYOPENCL_CALL_GUARDED_CLEANUP(clEnqueueSVMFree, (
                m_queue.data(), 1, &m_allocation,
                nullptr, nullptr,
                0, nullptr, nullptr));
          m_queue.reset();
        }
        else
        {
          PYOPENCL_PRINT_CALL_TRACE("clSVMFree");
          clSVMFree(m_context->data(), m_allocation);
        }
        m_allocation = nullptr;
      }

      event *enqueue_release(command_queue *queue, py::object py_wait_for)
      {
        PYOPENCL_PARSE_WAIT_FOR;

        if (!m_allocation)
          throw error("SVMAllocation.enqueue_release", CL_INVALID_VALUE,
              "trying to enqueue_release on an already-freed allocation");

        cl_command_queue use_queue;
        if (queue)
          use_queue = queue->data();
        else
        {
          if (m_queue.is_valid())
            use_queue = m_queue.data();
          else
            throw error("SVMAllocation.enqueue_release", CL_INVALID_VALUE,
                "no implicit queue available, must be provided explicitly");
        }

        cl_event evt;

        PYOPENCL_CALL_GUARDED_CLEANUP(clEnqueueSVMFree, (
              use_queue, 1, &m_allocation,
              nullptr, nullptr,
              PYOPENCL_WAITLIST_ARGS, &evt));

        m_allocation = nullptr;

        PYOPENCL_RETURN_NEW_EVENT(evt);
      }

      void *svm_ptr() const
      {
        return m_allocation;
      }

      size_t size() const
      {
        return m_size;
      }

      bool operator==(svm_allocation const &other) const
      {
        return m_allocation == other.m_allocation;
      }

      bool operator!=(svm_allocation const &other) const
      {
        return m_allocation != other.m_allocation;
      }

      void bind_to_queue(command_queue const &queue)
      {
        if (is_queue_out_of_order(queue.data()))
          throw error("SVMAllocation.bind_to_queue", CL_INVALID_VALUE,
              "supplying an out-of-order queue to SVMAllocation is invalid");

        if (m_queue.is_valid())
        {
          if (m_queue.data() != queue.data())
          {
            // make sure synchronization promises stay valid in new queue
            cl_event evt;

            PYOPENCL_CALL_GUARDED(clEnqueueMarker, (m_queue.data(), &evt));
            PYOPENCL_CALL_GUARDED(clEnqueueMarkerWithWaitList,
                (queue.data(), 1, &evt, nullptr));
          }
        }

        m_queue.set(queue.data());
      }

      void unbind_from_queue()
      {
        if (m_queue.is_valid())
          PYOPENCL_CALL_GUARDED_THREADED(clFinish, (m_queue.data()));

        m_queue.reset();
      }

      // only use for testing/diagnostic/debugging purposes!
      cl_command_queue queue() const
      {
        if (m_queue.is_valid())
          return m_queue.data();
        else
          return nullptr;
      }
  };

  // }}}


  // {{{ svm operations

  inline
  event *enqueue_svm_memcpy(
      command_queue &cq,
      cl_bool is_blocking,
      svm_pointer &dst, svm_pointer &src,
      py::object py_wait_for,
      py::object byte_count_py
      )
  {
    PYOPENCL_PARSE_WAIT_FOR;

    // {{{ process size

    PYOPENCL_GET_SVM_SIZE(src);
    PYOPENCL_GET_SVM_SIZE(dst);

    size_t size;
    bool have_size = false;

    if (src_has_size)
    {
      size = src_size;
      have_size = true;
    }
    if (dst_has_size)
    {
      if (have_size)
      {
        if (!byte_count_py.is_none())
          size = std::min(size, dst_size);
        else if (size != dst_size)
          throw error("_enqueue_svm_memcpy", CL_INVALID_VALUE,
              "sizes of source and destination buffer do not match");
      }
      else
      {
        size = dst_size;
        have_size = true;
      }
    }

    if (!byte_count_py.is_none())
    {
      size_t byte_count = py::cast<size_t>(byte_count_py);
      if (have_size && byte_count > size)
        throw error("_enqueue_svm_memcpy", CL_INVALID_VALUE,
            "specified byte_count larger than size of source or destination buffers");
      size = byte_count;
      have_size = true;
    }

    if (!have_size)
      throw error("_enqueue_svm_memcpy", CL_INVALID_VALUE,
          "size not passed and could not be determined");

    // }}}

    cl_event evt;
    PYOPENCL_CALL_GUARDED(
        clEnqueueSVMMemcpy,
        (
          cq.data(),
          is_blocking,
          dst.svm_ptr(), src.svm_ptr(),
          size,
          PYOPENCL_WAITLIST_ARGS,
          &evt
        ));

    PYOPENCL_RETURN_NEW_EVENT(evt);
  }


  inline
  event *enqueue_svm_memfill(
      command_queue &cq,
      svm_pointer &dst, py::object py_pattern,
      py::object byte_count,
      py::object py_wait_for
      )
  {
    PYOPENCL_PARSE_WAIT_FOR;

    const void *pattern_ptr;
    PYOPENCL_BUFFER_SIZE_T pattern_len;

    std::unique_ptr<py_buffer_wrapper> pattern_ward(new py_buffer_wrapper);

    pattern_ward->get(py_pattern.ptr(), PyBUF_ANY_CONTIGUOUS);

    pattern_ptr = pattern_ward->m_buf.buf;
    pattern_len = pattern_ward->m_buf.len;

    // {{{ process size

    PYOPENCL_GET_SVM_SIZE(dst);

    size_t size;
    bool have_size = false;
    if (dst_has_size)
    {
      size = dst_size;
      have_size = true;
    }
    if (!byte_count.is_none())
    {
      size_t user_size = py::cast<size_t>(byte_count);
      if (have_size && user_size > size)
        throw error("enqueue_svm_memfill", CL_INVALID_VALUE,
            "byte_count too large for specified SVM buffer");
    }

    if (!have_size)
    {
      throw error("enqueue_svm_memfill", CL_INVALID_VALUE,
          "byte_count not passed and could not be determined");
    }

    // }}}

    cl_event evt;
    PYOPENCL_CALL_GUARDED(
        clEnqueueSVMMemFill,
        (
          cq.data(),
          dst.svm_ptr(), pattern_ptr,
          pattern_len,
          size,
          PYOPENCL_WAITLIST_ARGS,
          &evt
        ));

    PYOPENCL_RETURN_NEW_EVENT(evt);
  }


  inline
  event *enqueue_svm_map(