src/compute/skc/platforms/cl_12/extent_cl_12.c

/*
 * Copyright 2017 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can
 * be found in the LICENSE file.
 *
 */

//
//
//

#include <stdlib.h>

#include "common/cl/assert_cl.h"
#include "extent_cl_12.h"
#include "runtime_cl_12.h"

//
// DURABLE R/W HOST EXTENT -- STANDARD CACHED MEMORY
//

void
skc_extent_phrw_alloc(struct skc_runtime     * const runtime,
                      struct skc_extent_phrw * const extent,
                      size_t                   const size)
{
  extent->hrw = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE,size);
}

void
skc_extent_phrw_free(struct skc_runtime     * const runtime,
                     struct skc_extent_phrw * const extent)
{
  skc_runtime_host_perm_free(runtime,extent->hrw);
}

//
// DURABLE R/W DEVICE EXTENT -- ALLOCATED FROM DEVICE HEAP
//

void
skc_extent_pdrw_alloc(struct skc_runtime     * const runtime,
                      struct skc_extent_pdrw * const extent,
                      size_t                   const size)
{
  extent->drw = skc_runtime_device_perm_alloc(runtime,
                                              CL_MEM_READ_WRITE | CL_MEM_HOST_NO_ACCESS,
                                              size);
}

void
skc_extent_pdrw_free(struct skc_runtime     * const runtime,
                     struct skc_extent_pdrw * const extent)
{
  skc_runtime_device_perm_free(runtime,extent->drw);
}

//
// EPHEMERAL DEVICE R/W EXTENT -- ALLOCATED QUICKLY FROM A MANAGED RING
//

void
skc_extent_tdrw_alloc(struct skc_runtime     * const runtime,
                      struct skc_extent_tdrw * const extent,
                      size_t                   const size)
{
  extent->size = size;
  extent->drw  = skc_runtime_device_temp_alloc(runtime,
                                               CL_MEM_READ_WRITE | CL_MEM_HOST_NO_ACCESS,
                                               size,&extent->id,NULL);
}

void
skc_extent_tdrw_free(struct skc_runtime     * const runtime,
                     struct skc_extent_tdrw * const extent)
{
  skc_runtime_device_temp_free(runtime,extent->drw,extent->id);
}

void
skc_extent_tdrw_zero(struct skc_extent_tdrw * const extent,
                     cl_command_queue         const cq,
                     cl_event               * const event)
{
  if (extent->size == 0)
    return;

  skc_uint const zero = 0;

  cl(EnqueueFillBuffer(cq,
                       extent->drw,
                       &zero,
                       sizeof(zero),
                       0,
                       extent->size,
                       0,NULL,event));
}

//
// DURABLE SMALL EXTENTS BACKING ATOMICS
//

void
skc_extent_phr_pdrw_alloc(struct skc_runtime         * const runtime,
                          struct skc_extent_phr_pdrw * const extent,
                          size_t                       const size)
{
  extent->size = size;
  extent->hr   = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_ONLY,size);
  extent->drw  = skc_runtime_device_perm_alloc(runtime,CL_MEM_READ_WRITE,size);
}

void
skc_extent_phr_pdrw_free(struct skc_runtime         * const runtime,
                         struct skc_extent_phr_pdrw * const extent)
{
  skc_runtime_host_perm_free(runtime,extent->hr);
  skc_runtime_device_perm_free(runtime,extent->drw);
}

void
skc_extent_phr_pdrw_read(struct skc_extent_phr_pdrw * const extent,
                         cl_command_queue             const cq,
                         cl_event                   * const event)
{
  if (extent->size == 0)
    return;

  cl(EnqueueReadBuffer(cq,
                       extent->drw,
                       CL_FALSE,
                       0,
                       extent->size,
                       extent->hr,
                       0,NULL,event));
}

void
skc_extent_phr_pdrw_zero(struct skc_extent_phr_pdrw * const extent,
                         cl_command_queue             const cq,
                         cl_event                   * const event)
{
  if (extent->size == 0)
    return;

  skc_uint const zero = 0;

  cl(EnqueueFillBuffer(cq,
                       extent->drw,
                       &zero,
                       sizeof(zero),
                       0,
                       extent->size,
                       0,NULL,event));
}

//
// EPHEMERAL SMALL EXTENTS BACKING ATOMICS
//

void
skc_extent_thr_tdrw_alloc(struct skc_runtime         * const runtime,
                          struct skc_extent_thr_tdrw * const extent,
                          size_t                       const size)
{
  extent->size = size;
  extent->hr   = skc_runtime_host_temp_alloc(runtime,
                                             SKC_MEM_FLAGS_READ_ONLY,
                                             size,&extent->id.hr,NULL);
  extent->drw  = skc_runtime_device_temp_alloc(runtime,
                                               CL_MEM_READ_WRITE,
                                               size,
                                               &extent->id.drw,
                                               NULL);
}

void
skc_extent_thr_tdrw_free(struct skc_runtime         * const runtime,
                         struct skc_extent_thr_tdrw * const extent)
{
  skc_runtime_host_temp_free(runtime,extent->hr,extent->id.hr);
  skc_runtime_device_temp_free(runtime,extent->drw,extent->id.drw);
}

void
skc_extent_thr_tdrw_read(struct skc_extent_thr_tdrw * const extent,
                         cl_command_queue             const cq,
                         cl_event                   * const event)
{
  if (extent->size == 0)
    return;

  cl(EnqueueReadBuffer(cq,
                       extent->drw,
                       CL_FALSE,
                       0,
                       extent->size,
                       extent->hr,
                       0,NULL,event));
}

void
skc_extent_thr_tdrw_zero(struct skc_extent_thr_tdrw * const extent,
                         cl_command_queue             const cq,
                         cl_event                   * const event)
{
  if (extent->size == 0)
    return;

  skc_uint const zero = 0;

  cl(EnqueueFillBuffer(cq,
                       extent->drw,
                       &zero,
                       sizeof(zero),
                       0,
                       extent->size,
                       0,NULL,event));
}

//
// DURABLE W/1 HOST RING WITH AN EPHEMERAL R/N DEVICE SNAPSHOT
//

void
skc_extent_phw1g_tdrNs_alloc(struct skc_runtime            * const runtime,
                             struct skc_extent_phw1g_tdrNs * const extent,
                             size_t                          const size)
{
  extent->hw1 = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_WRITE_ONLY,size);
}

void
skc_extent_phw1g_tdrNs_free(struct skc_runtime            * const runtime,
                            struct skc_extent_phw1g_tdrNs * const extent)
{
  skc_runtime_host_perm_free(runtime,extent->hw1);
}

void
skc_extent_phw1g_tdrNs_snap_init(struct skc_runtime                 * const runtime,
                                 struct skc_extent_ring             * const ring,
                                 struct skc_extent_phw1g_tdrNs_snap * const snap)
{
  snap->snap = skc_extent_ring_snap_alloc(runtime,ring);
}

void
skc_extent_phw1g_tdrNs_snap_alloc(struct skc_runtime                 * const runtime,
                                  struct skc_extent_phw1g_tdrNs      * const extent,
                                  struct skc_extent_phw1g_tdrNs_snap * const snap,
                                  cl_command_queue                     const cq,
                                  cl_event                           * const event)
{
  struct skc_extent_ring const * const ring = snap->snap->ring;

  skc_uint const count = skc_extent_ring_snap_count(snap->snap);
  size_t   const size  = count * ring->size.elem;

  snap->drN = skc_runtime_device_temp_alloc(runtime,
                                            CL_MEM_READ_ONLY | CL_MEM_HOST_WRITE_ONLY,
                                            size,&snap->id,NULL);

  if (count == 0)
    return;

  // possibly two copies
  skc_uint const index_lo  = snap->snap->reads & ring->size.mask;
  skc_uint const count_max = ring->size.pow2 - index_lo;
  skc_uint const count_lo  = min(count_max,count);
  size_t   const bytes_lo  = count_lo * ring->size.elem;

  if (count > count_max)
    {
      skc_uint const bytes_hi = (count - count_max) * ring->size.elem;

      cl(EnqueueWriteBuffer(cq,
                            snap->drN,
                            CL_FALSE,
                            bytes_lo,
                            bytes_hi,
                            extent->hw1, // offset_hi = 0
                            0,NULL,NULL));
    }

  size_t const offset_lo = index_lo * ring->size.elem;

  cl(EnqueueWriteBuffer(cq,
                        snap->drN,
                        CL_FALSE,
                        0,
                        bytes_lo,
                        (skc_uchar*)extent->hw1 + offset_lo,
                        0,NULL,event));

}

void
skc_extent_phw1g_tdrNs_snap_free(struct skc_runtime                 * const runtime,
                                 struct skc_extent_phw1g_tdrNs_snap * const snap)
{
  skc_runtime_device_temp_free(runtime,snap->drN,snap->id);
  skc_extent_ring_snap_free(runtime,snap->snap);
}

//
// DURABLE R/W HOST RING WITH AN EPHEMERAL R/N DEVICE SNAPSHOT
//

void
skc_extent_phrwg_tdrNs_alloc(struct skc_runtime            * const runtime,
                             struct skc_extent_phrwg_tdrNs * const extent,
                             size_t                          const size)
{
  extent->hrw = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE,size); // WRITE-ONCE
}

void
skc_extent_phrwg_tdrNs_free(struct skc_runtime            * const runtime,
                            struct skc_extent_phrwg_tdrNs * const extent)
{
  skc_runtime_host_perm_free(runtime,extent->hrw);
}

void
skc_extent_phrwg_tdrNs_snap_init(struct skc_runtime                 * const runtime,
                                 struct skc_extent_ring             * const ring,
                                 struct skc_extent_phrwg_tdrNs_snap * const snap)
{
  snap->snap = skc_extent_ring_snap_alloc(runtime,ring);
}

void
skc_extent_phrwg_tdrNs_snap_alloc(struct skc_runtime                 * const runtime,
                                  struct skc_extent_phrwg_tdrNs      * const extent,
                                  struct skc_extent_phrwg_tdrNs_snap * const snap,
                                  cl_command_queue                     const cq,
                                  cl_event                           * const event)
{
  struct skc_extent_ring const * const ring = snap->snap->ring;

  skc_uint const count = skc_extent_ring_snap_count(snap->snap);
  size_t   const size  = count * ring->size.elem;

  snap->drN = skc_runtime_device_temp_alloc(runtime,
                                            CL_MEM_READ_ONLY | CL_MEM_HOST_WRITE_ONLY,
                                            size,&snap->id,NULL);

  if (count == 0)
    return;

  // possibly two copies
  skc_uint const index_lo  = snap->snap->reads & ring->size.mask;
  skc_uint const count_max = ring->size.pow2 - index_lo;
  skc_uint const count_lo  = min(count_max,count);
  size_t   const bytes_lo  = count_lo * ring->size.elem;

  if (count > count_max)
    {
      skc_uint const count_hi = count - count_max;
      skc_uint const bytes_hi = count_hi * ring->size.elem;

      cl(EnqueueWriteBuffer(cq,
                            snap->drN,
                            CL_FALSE,
                            bytes_lo,
                            bytes_hi,
                            extent->hrw, // offset_hi = 0
                            0,NULL,NULL));
    }

  size_t offset_lo = index_lo * ring->size.elem;

  cl(EnqueueWriteBuffer(cq,
                        snap->drN,
                        CL_FALSE,
                        0,
                        bytes_lo,
                        (skc_uchar*)extent->hrw + offset_lo,
                        0,NULL,event));

}

void
skc_extent_phrwg_tdrNs_snap_free(struct skc_runtime                 * const runtime,
                                 struct skc_extent_phrwg_tdrNs_snap * const snap)
{
  skc_runtime_device_temp_free(runtime,snap->drN,snap->id);
  skc_extent_ring_snap_free(runtime,snap->snap);
}

//
// DURABLE HOST R/W RING WITH AN EPHEMERAL HOST R/1 SNAPSHOT
//
// Note that because the ring and snapshot are both in host memory and
// the snapshot blocks progress until freed we can simply point the
// fake ephemeral snapshot at the ring's durable extent.
//

void
skc_extent_phrwg_thr1s_alloc(struct skc_runtime            * const runtime,
                             struct skc_extent_phrwg_thr1s * const extent,
                             size_t                          const size)
{
  extent->hrw = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE,size); // WRITE-ONCE
}

void
skc_extent_phrwg_thr1s_free(struct skc_runtime            * const runtime,
                            struct skc_extent_phrwg_thr1s * const extent)
{
  skc_runtime_host_perm_free(runtime,extent->hrw);
}

void
skc_extent_phrwg_thr1s_snap_init(struct skc_runtime                 * const runtime,
                                 struct skc_extent_ring             * const ring,
                                 struct skc_extent_phrwg_thr1s_snap * const snap)
{
  snap->snap = skc_extent_ring_snap_alloc(runtime,ring);
}

void
skc_extent_phrwg_thr1s_snap_alloc(struct skc_runtime                 * const runtime,
                                  struct skc_extent_phrwg_thr1s      * const extent,
                                  struct skc_extent_phrwg_thr1s_snap * const snap)
{
  struct skc_extent_ring const * const ring = snap->snap->ring;

  skc_uint const count     = skc_extent_ring_snap_count(snap->snap);
  skc_uint const index_lo  = snap->snap->reads & ring->size.mask;
  skc_uint const count_max = ring->size.pow2 - index_lo;

  snap->count.lo = min(count_max,count);
  snap->hr1.lo   = (skc_uchar*)extent->hrw + (index_lo * ring->size.elem);

  if (count > count_max)
    {
      snap->count.hi = count - count_max;
      snap->hr1.hi   = extent->hrw;
    }
  else
    {
      snap->count.hi = 0;
      snap->hr1.hi   = NULL;
    }
}

void
skc_extent_phrwg_thr1s_snap_free(struct skc_runtime                 * const runtime,
                                 struct skc_extent_phrwg_thr1s_snap * const snap)
{
  skc_extent_ring_snap_free(runtime,snap->snap);
}

//
//
//