| /* |
| * Copyright 2017 Google Inc. |
| * |
| * Use of this source code is governed by a BSD-style license that can |
| * be found in the LICENSE file. |
| * |
| */ |
| |
| // |
| // |
| // |
| |
| #include <stddef.h> |
| #include <stdlib.h> |
| #include <string.h> |
| #include <float.h> |
| #include <stdio.h> |
| |
| #include "common/cl/assert_cl.h" |
| |
| #include "context.h" |
| #include "handle.h" |
| #include "grid.h" |
| #include "path.h" |
| #include "path_builder.h" |
| |
| #include "config_cl.h" |
| #include "export_cl_12.h" |
| #include "runtime_cl_12.h" |
| #include "path_builder_cl_12.h" |
| |
| // |
| // OpenCL 1.2 devices support mapping of buffers into the host address |
| // space. |
| // |
| // Mapped buffers must be aligned on MIN_DATA_TYPE_ALIGN_SIZE bit |
| // boundary (e.g. 128 bytes). This complicates coordinating sharing |
| // of data between the host and the device. |
| // |
| // Some OpenCL 2.0 devices support fine-grained shared virtual memory |
| // pointers with byte-addressing and allow simpler coordination |
| // strategies at the cost of maintaining cache coherency. |
| // |
| // The path builder is focused on moving bulk path data from the host |
| // into the device-managed "block" memory pool and arranging it into a |
| // SIMT/SIMD-friendly data structure that can be efficiently read by |
| // the rasterizer. |
| // |
| // Note that one simplifying assumption is that the maximum length of |
| // a *single* path can't be larger than what fits in the single extent |
| // (which is split into M subbuffers). This would be a very long path |
| // and a legitimate size limitation. |
| // |
| // For some systems, it may be appropriate to never pull path data |
| // into the device-managed block pool and instead present the path |
| // data to the device in a temporarily available allocated memory |
| // "zone" of paths that can be discarded all at once. |
| // |
| // For other systems, it may be appropriate to simply copy the path |
| // data from host to device. |
| // |
| // But the majority of OpenCL (and VK, MTL, DX12) devices we'll be |
| // targeting support basic map/unmap functionality similar to OpenCL |
| // 1.2. Furthermore, not all OpenCL 2.0 devices support fine-grained |
| // sharing of memory and still require a map/unmap step... but note |
| // that they all support byte-aligned mapping and subbuffers. |
| // |
| // The general strategy that this particular CL_12 implementation uses |
| // is to allocate a large mappable bulk-data path buffer and an |
| // auxilary mappable command buffer. |
| // |
| // The buffers are split into a reasonable number of properly aligned |
| // subbuffers to enable simultaneous host and device access. |
| // |
| |
| // |
| // Blocks: |
| // 1 extent |
| // M mapped subbuffers (configurable) to allow for concurrency |
| // |
| // Commands: |
| // 1 extent |
| // M mapped subbuffers (configurable) to allow for concurrency |
| // |
| // Spans: |
| // M hi/lo structures |
| // |
| // { cl_sub, void*, event, base } |
| // |
| // - size of sub buffer |
| // - remaining |
| // |
| // - counts |
| // |
| |
| // |
| // For any kernel launch, at most one path will be discontiguous and |
| // defined across two sub-buffers. |
| // |
| // Nodes are updated locally until full and then stored so they will |
| // never be incomplete. Headers are stored locally until the path is |
| // ended so they will never be incomplete. |
| // |
| // A line, quad or cubic acquires 4/6/8 segments which may be spread |
| // across one or more congtiguous blocks. |
| // |
| // If a flush() occurs then the remaining columns of multi-segment |
| // paths are initialized with zero-length line, quad, cubic elements. |
| // |
| // Every block's command word has a type and a count acquired from a |
| // rolling counter. |
| // |
| // The kernel is passed two spans of blocks { base, count } to |
| // process. The grid is must process (lo.count + hi.count) blocks. |
| // |
| |
| struct skc_subbuffer_blocks |
| { |
| cl_mem device; |
| void * host; |
| }; |
| |
| struct skc_subbuffer_cmds |
| { |
| cl_mem device; |
| void * host; |
| cl_event map; |
| }; |
| |
| // |
| // ringdex is an index with range [0, blocks-per-subbuf * subbufs-per-buffer ) |
| // |
| |
| typedef skc_uint skc_ringdex_t; |
| |
| union skc_ringdex_expand |
| { |
| div_t qr; |
| |
| struct { |
| #ifndef SKC_DIV_REM_BEFORE_QUOT // offsetof(div_t,quot) != 0 |
| skc_uint subbuf; |
| skc_uint block; |
| #else |
| skc_uint block; |
| skc_uint subbuf; |
| #endif |
| }; |
| }; |
| |
| // |
| // this record is executed by the grid |
| // |
| |
| struct skc_release_record |
| { |
| struct skc_path_builder_impl * impl; // back pointer to impl |
| |
| skc_grid_t grid; // pointer to scheduled grid |
| |
| skc_uint from; // inclusive starting index : [from,to) |
| skc_uint to; // non-inclusive ending index : [from,to) |
| }; |
| |
| // |
| // |
| // |
| |
| struct skc_path_builder_impl |
| { |
| struct skc_path_builder * path_builder; |
| |
| struct skc_runtime * runtime; |
| |
| cl_command_queue cq; |
| |
| struct { |
| cl_kernel alloc; |
| cl_kernel copy; |
| } kernels; |
| |
| // |
| // FIXME -- make this pointer to constant config |
| // |
| // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv |
| struct { |
| skc_uint subbufs; // how many subbufs in the buffer? |
| |
| struct { |
| skc_uint buffer; // how many blocks in the buffer? |
| skc_uint subbuf; // how many blocks in a subbuf? |
| } blocks_per; |
| } ring; |
| // |
| // ^^^^^^^^^^^ don't duplicate these constants ^^^^^^^^^^^^^^^^^^ |
| // |
| |
| struct { |
| cl_mem buffer; // backing buffer for blocks |
| struct skc_subbuffer_blocks * subbufs; // array of structures |
| } blocks; |
| |
| struct { |
| cl_mem buffer; // backing buffer for commands |
| struct skc_subbuffer_cmds * subbufs; // array of structures |
| } cmds; |
| |
| struct { |
| struct skc_release_record * records; // max release records is equal to max subbufs |
| skc_path_t * paths; // max paths is less than or equal to max commands |
| } release; |
| |
| cl_mem reads; // each kernel only requires one word to store the block pool "base" |
| |
| struct { |
| skc_uint rolling; // rolling counter used by cmds to map to block pool alloc |
| skc_ringdex_t from; |
| skc_ringdex_t to; |
| } prev; |
| |
| struct { |
| skc_ringdex_t from; |
| skc_ringdex_t to; |
| } curr; |
| |
| struct { |
| struct skc_path_head * head; // pointer to local path header -- not written until path end |
| struct skc_path_node * node; // pointer to local node -- may alias head until head is full |
| |
| struct { |
| skc_uint rolling; // rolling counter of wip node -- valid after one node is allocated |
| union skc_tagged_block_id * next; // next slot in node -- may initially point to head.ids |
| skc_uint rem; // how many id slots left in node block |
| } ids; |
| |
| struct { |
| skc_uint rem; // how many subblocks left in block? |
| skc_uint rolling; // rolling counter of block of subblocks |
| float * next; // next subblock in current subblock block |
| skc_uint idx; // index of next subblock |
| } subblocks; |
| |
| struct { |
| skc_uint one; // .block = 1 |
| skc_uint next; // rolling counter used by cmds to map to block pool alloc |
| } rolling; |
| |
| skc_ringdex_t to; // ringdex of _next_available_ command/block in ring -- FIXME -- should be current |
| } wip; |
| }; |
| |
| // |
| // FIXME -- move to a pow2 subbuffer size and dispense with division |
| // and modulo operations |
| // |
| |
| static |
| union skc_ringdex_expand |
| skc_ringdex_expand(struct skc_path_builder_impl * const impl, |
| skc_ringdex_t const ringdex) |
| { |
| return (union skc_ringdex_expand){ |
| .qr = div(ringdex,impl->ring.blocks_per.subbuf) |
| }; |
| } |
| |
| static |
| void |
| skc_ringdex_wip_to_block_inc(struct skc_path_builder_impl * const impl) |
| { |
| // |
| // FIXME - which is faster? |
| // |
| #if 1 |
| impl->wip.to = (impl->wip.to + 1) % impl->ring.blocks_per.buffer; |
| #else |
| impl->wip.to -= (impl->wip.to < impl->ring.blocks_per.buffer) ? -1 : impl->wip.to; |
| #endif |
| |
| // this path is too long -- for now assert() and die |
| assert(impl->wip.to != impl->curr.from); |
| } |
| |
| static |
| skc_ringdex_t |
| skc_ringdex_span(struct skc_path_builder_impl * const impl, |
| skc_ringdex_t const from, |
| skc_ringdex_t const to) |
| { |
| return (to - from) % impl->ring.blocks_per.buffer; |
| } |
| |
| static |
| void |
| skc_ringdex_wip_to_subbuf_inc(struct skc_path_builder_impl * const impl) |
| { |
| union skc_ringdex_expand const to = skc_ringdex_expand(impl,impl->wip.to); |
| |
| // nothing to do if this is the first block in the subbuf |
| if (to.block == 0) |
| return; |
| |
| skc_uint const new_subbuf = (to.subbuf + 1) % impl->ring.subbufs; |
| |
| // otherwise increment and mod |
| impl->wip.to = new_subbuf * impl->ring.blocks_per.subbuf; |
| } |
| |
| static |
| skc_bool |
| skc_ringdex_curr_is_equal(struct skc_path_builder_impl * const impl) |
| { |
| return impl->curr.from == impl->curr.to; |
| } |
| |
| static |
| skc_bool |
| skc_ringdex_prev_is_equal(struct skc_path_builder_impl * const impl) |
| { |
| return impl->prev.from == impl->prev.to; |
| } |
| |
| static |
| skc_uint |
| skc_ringdex_dont_map_last(struct skc_path_builder_impl * const impl, |
| skc_uint const to_block) |
| { |
| // no blocks acquired OR this is last block in subbuf |
| return !((impl->wip.to == impl->curr.to) || (to_block == 0)); |
| } |
| |
| // |
| // |
| // |
| |
| static |
| struct skc_release_record * |
| skc_release_curr(struct skc_path_builder_impl * const impl) |
| { |
| union skc_ringdex_expand curr_from = skc_ringdex_expand(impl,impl->curr.from); |
| |
| return impl->release.records + curr_from.subbuf; |
| } |
| |
| // |
| // FIXME -- get rid of all distant config references -- grab them at all at creation time |
| // |
| |
| static |
| void |
| skc_path_builder_pfn_begin(struct skc_path_builder_impl * const impl) |
| { |
| // init header counters // { handle, blocks, nodes, prims } |
| impl->wip.head->header = (union skc_path_header){ |
| .handle = 0, |
| .blocks = 0, |
| .nodes = 0, |
| .prims = 0 |
| }; |
| |
| // FIXME -- BOUNDS SHOULD USE SIMD4 TRICK AND NEGATE ONE OF THE CORNERS |
| impl->wip.head->bounds = (union skc_path_bounds){ +FLT_MIN, +FLT_MIN, -FLT_MIN, -FLT_MIN }; |
| |
| // point wip ids at local head node |
| impl->wip.ids.next = impl->wip.head->tag_ids; // point to local head node |
| impl->wip.ids.rem = impl->runtime->config->block.words - SKC_PATH_HEAD_WORDS; // FIXME -- save this constant somewhere |
| |
| // start with no subblocks |
| impl->wip.subblocks.rem = 0; |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void |
| skc_path_builder_impl_finalize_node(struct skc_path_builder_impl * const impl) |
| { |
| #if 1 |
| // |
| // FIXME -- a Duff's device might be optimal here but would have to |
| // be customized per device since node's could be 16-128+ words |
| // |
| while (impl->wip.ids.rem > 0) |
| { |
| impl->wip.ids.rem -= 1; |
| impl->wip.ids.next->u32 = SKC_TAGGED_BLOCK_ID_INVALID; |
| impl->wip.ids.next += 1; |
| } |
| #else |
| memset(&impl->wip.ids.next->u32, |
| SKC_TAGGED_BLOCK_ID_INVALID, // 0xFF |
| sizeof(impl->wip.ids.next->u32) * impl->wip.ids.rem); |
| |
| impl->wip.ids.next += impl->wip.ids.rem; |
| impl->wip.ids.rem = 0; |
| #endif |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void |
| skc_zero_float(skc_float * p, skc_uint rem) |
| { |
| memset(p,0,sizeof(*p)*rem); |
| } |
| |
| static |
| void |
| skc_path_builder_finalize_subblocks(struct skc_path_builder * const path_builder) |
| { |
| // |
| // FIXME -- it might be more performant to zero the remaining |
| // columns in a subblock -- a subblock at a time -- instead of the |
| // same column across all the subblocks |
| // |
| #if 0 |
| while (path_builder->line.rem > 0) |
| { |
| --path_builder->line.rem; |
| |
| *path_builder->line.coords[0]++ = 0.0f; |
| *path_builder->line.coords[1]++ = 0.0f; |
| *path_builder->line.coords[2]++ = 0.0f; |
| *path_builder->line.coords[3]++ = 0.0f; |
| } |
| |
| while (path_builder->quad.rem > 0) |
| { |
| --path_builder->quad.rem; |
| |
| *path_builder->line.coords[0]++ = 0.0f; |
| *path_builder->line.coords[1]++ = 0.0f; |
| *path_builder->line.coords[2]++ = 0.0f; |
| *path_builder->line.coords[3]++ = 0.0f; |
| *path_builder->line.coords[4]++ = 0.0f; |
| *path_builder->line.coords[5]++ = 0.0f; |
| } |
| |
| while (path_builder->cubic.rem > 0) |
| { |
| --path_builder->cubic.rem; |
| |
| *path_builder->line.coords[0]++ = 0.0f; |
| *path_builder->line.coords[1]++ = 0.0f; |
| *path_builder->line.coords[2]++ = 0.0f; |
| *path_builder->line.coords[3]++ = 0.0f; |
| *path_builder->line.coords[4]++ = 0.0f; |
| *path_builder->line.coords[5]++ = 0.0f; |
| *path_builder->line.coords[6]++ = 0.0f; |
| *path_builder->line.coords[7]++ = 0.0f; |
| } |
| #else |
| if (path_builder->line.rem > 0) |
| { |
| skc_zero_float(path_builder->line.coords[0],path_builder->line.rem); |
| skc_zero_float(path_builder->line.coords[1],path_builder->line.rem); |
| skc_zero_float(path_builder->line.coords[2],path_builder->line.rem); |
| skc_zero_float(path_builder->line.coords[3],path_builder->line.rem); |
| |
| path_builder->line.rem = 0; |
| } |
| |
| if (path_builder->quad.rem > 0) |
| { |
| skc_zero_float(path_builder->quad.coords[0],path_builder->quad.rem); |
| skc_zero_float(path_builder->quad.coords[1],path_builder->quad.rem); |
| skc_zero_float(path_builder->quad.coords[2],path_builder->quad.rem); |
| skc_zero_float(path_builder->quad.coords[3],path_builder->quad.rem); |
| skc_zero_float(path_builder->quad.coords[4],path_builder->quad.rem); |
| skc_zero_float(path_builder->quad.coords[5],path_builder->quad.rem); |
| |
| path_builder->quad.rem = 0; |
| } |
| |
| if (path_builder->cubic.rem > 0) |
| { |
| skc_zero_float(path_builder->cubic.coords[0],path_builder->cubic.rem); |
| skc_zero_float(path_builder->cubic.coords[1],path_builder->cubic.rem); |
| skc_zero_float(path_builder->cubic.coords[2],path_builder->cubic.rem); |
| skc_zero_float(path_builder->cubic.coords[3],path_builder->cubic.rem); |
| skc_zero_float(path_builder->cubic.coords[4],path_builder->cubic.rem); |
| skc_zero_float(path_builder->cubic.coords[5],path_builder->cubic.rem); |
| skc_zero_float(path_builder->cubic.coords[6],path_builder->cubic.rem); |
| skc_zero_float(path_builder->cubic.coords[7],path_builder->cubic.rem); |
| |
| path_builder->cubic.rem = 0; |
| } |
| #endif |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void |
| skc_path_builder_impl_unmap(struct skc_path_builder_impl * const impl, |
| skc_uint from, |
| skc_uint to) |
| { |
| // to might be out of range |
| to = to % impl->ring.subbufs; |
| |
| #if 0 |
| fprintf(stderr,"unmap: [%2u,%2u)\n",from,to); |
| #endif |
| |
| while (from != to) // 'to' might be out of range |
| { |
| // bring 'from' back in range |
| from = from % impl->ring.subbufs; |
| |
| struct skc_subbuffer_blocks * const blocks = impl->blocks.subbufs + from; |
| struct skc_subbuffer_cmds * const cmds = impl->cmds .subbufs + from; |
| |
| cl(EnqueueUnmapMemObject(impl->cq, |
| blocks->device, |
| blocks->host, |
| 0,NULL,NULL)); |
| |
| cl(EnqueueUnmapMemObject(impl->cq, |
| cmds->device, |
| cmds->host, |
| 0,NULL,NULL)); |
| |
| // bring from back in range |
| from = (from + 1) % impl->ring.subbufs; |
| } |
| } |
| |
| // |
| // FIXME -- reuse this in create() |
| // |
| |
| static |
| void |
| skc_path_builder_impl_map(struct skc_path_builder_impl * const impl, |
| skc_uint from, |
| skc_uint to) |
| { |
| // to might be out of range |
| to = to % impl->ring.subbufs; |
| |
| #if 0 |
| fprintf(stderr," map: [%2u,%2u)\n",from,to); |
| #endif |
| |
| while (from != to) |
| { |
| cl_int cl_err; |
| |
| struct skc_subbuffer_blocks * const blocks = impl->blocks.subbufs + from; |
| struct skc_subbuffer_cmds * const cmds = impl->cmds .subbufs + from; |
| |
| blocks->host = clEnqueueMapBuffer(impl->cq, |
| blocks->device, |
| CL_FALSE, |
| CL_MAP_WRITE_INVALIDATE_REGION, |
| 0,impl->runtime->config->paths_copy.block.subbuf, |
| 0,NULL,NULL, |
| &cl_err); cl_ok(cl_err); |
| |
| cl(ReleaseEvent(cmds->map)); |
| |
| cmds->host = clEnqueueMapBuffer(impl->cq, |
| cmds->device, |
| CL_FALSE, |
| CL_MAP_WRITE_INVALIDATE_REGION, |
| 0,impl->runtime->config->paths_copy.command.subbuf, |
| 0,NULL,&cmds->map, |
| &cl_err); cl_ok(cl_err); |
| |
| // bring from back in range |
| from = (from + 1) % impl->ring.subbufs; |
| } |
| // |
| // FIXME -- when we switch to out of order queues we'll need a barrier here |
| // |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void |
| skc_path_builder_release_dispose(struct skc_release_record * const release, |
| struct skc_path_builder_impl * const impl) |
| { |
| struct skc_runtime * runtime = impl->runtime; |
| |
| if (release->from <= release->to) // no wrap |
| { |
| skc_path_t const * paths = impl->release.paths + release->from; |
| skc_uint count = release->to - release->from; |
| |
| skc_grid_deps_unmap(runtime->deps,paths,count); |
| skc_runtime_path_device_release(runtime,paths,count); |
| } |
| else // from > to implies wrap |
| { |
| skc_path_t const * paths_lo = impl->release.paths + release->from; |
| skc_uint count_lo = impl->ring.blocks_per.buffer - release->from; |
| |
| skc_grid_deps_unmap(runtime->deps,paths_lo,count_lo); |
| skc_runtime_path_device_release(runtime,paths_lo,count_lo); |
| |
| skc_grid_deps_unmap(runtime->deps,impl->release.paths,release->to); |
| skc_runtime_path_device_release(runtime,impl->release.paths,release->to); |
| } |
| |
| release->to = release->from; |
| } |
| |
| static |
| void |
| skc_path_builder_grid_pfn_dispose(skc_grid_t const grid) |
| { |
| struct skc_release_record * const release = skc_grid_get_data(grid); |
| struct skc_path_builder_impl * const impl = release->impl; |
| |
| skc_path_builder_release_dispose(release,impl); |
| } |
| |
| static |
| void |
| // skc_path_builder_complete(struct skc_release_record * const release) |
| skc_path_builder_complete(skc_grid_t grid) |
| { |
| // |
| // notify deps that this grid is complete enough for other grids to |
| // proceed |
| // |
| // the path builder still has some cleanup to do before all its |
| // resources can be reused |
| // |
| skc_grid_complete(grid); |
| } |
| |
| static |
| void |
| skc_path_builder_paths_copy_cb(cl_event event, cl_int status, skc_grid_t grid) |
| { |
| SKC_CL_CB(status); |
| |
| struct skc_release_record * const release = skc_grid_get_data(grid); |
| |
| SKC_SCHEDULER_SCHEDULE(release->impl->runtime->scheduler,skc_path_builder_complete,grid); |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void |
| skc_path_builder_grid_pfn_waiting(skc_grid_t const grid) |
| { |
| struct skc_release_record * const release = skc_grid_get_data(grid); |
| struct skc_path_builder_impl * const impl = release->impl; |
| |
| // 1. flush incomplete subblocks of path elements |
| // 2. unmap subbuffer on cq.unmap |
| // 3. flush cq.unmap |
| // 4. launch kernel on cq.kernel but wait for unmap completion |
| // 5. flush cq.kernel |
| // 6. remap relevant subbuffers on cq.map but wait for kernel completion |
| // 7. flush cq.map |
| |
| // |
| // FIXME -- can be smarter about flushing if the wip paths are not |
| // in the same subbuf as curr.to |
| // |
| // THIS IS IMPORTANT TO FIX |
| // |
| |
| // flush incomplete subblocks |
| skc_path_builder_finalize_subblocks(impl->path_builder); |
| |
| // |
| // get range of subbufs that need to be unmapped |
| // |
| // note that impl->prev subbufs have already been unmapped |
| // |
| union skc_ringdex_expand curr_from = skc_ringdex_expand(impl,impl->curr.from); |
| union skc_ringdex_expand curr_to = skc_ringdex_expand(impl,impl->curr.to); |
| skc_uint const is_partial = curr_to.block > 0; |
| skc_uint const unmap_to = curr_to.subbuf + is_partial; |
| |
| // |
| // unmap all subbufs in range [from,to) |
| // |
| skc_path_builder_impl_unmap(impl,curr_from.subbuf,unmap_to); |
| |
| // |
| // launch kernels |
| // |
| skc_uint const pb_prev_span = skc_ringdex_span(impl,impl->prev.from,impl->prev.to); |
| skc_uint const pb_curr_span = skc_ringdex_span(impl,impl->curr.from,impl->curr.to); |
| skc_uint const pb_cmds = pb_prev_span + pb_curr_span; |
| |
| // |
| // 1) allocate blocks from pool |
| // |
| |
| // |
| // FIXME -- pack integers into struct/vector |
| // |
| cl(SetKernelArg(impl->kernels.alloc,0,SKC_CL_ARG(impl->runtime->block_pool.atomics.drw))); |
| cl(SetKernelArg(impl->kernels.alloc,1,SKC_CL_ARG(impl->reads))); |
| cl(SetKernelArg(impl->kernels.alloc,2,SKC_CL_ARG(curr_from.subbuf))); |
| cl(SetKernelArg(impl->kernels.alloc,3,SKC_CL_ARG(pb_cmds))); |
| |
| skc_device_enqueue_kernel(impl->runtime->device, |
| SKC_DEVICE_KERNEL_ID_PATHS_ALLOC, |
| impl->cq, |
| impl->kernels.alloc, |
| 1, |
| 0,NULL,NULL); |
| |
| // |
| // 2) copy blocks from unmapped device-accessible memory |
| // |
| |
| // |
| // FIXME -- pack integers into struct/vector and reduce 13 arguments down to 7 |
| // |
| cl(SetKernelArg(impl->kernels.copy, 0,SKC_CL_ARG(impl->runtime->handle_pool.map.drw))); |
| |
| cl(SetKernelArg(impl->kernels.copy, 1,SKC_CL_ARG(impl->runtime->block_pool.ids.drw))); |
| cl(SetKernelArg(impl->kernels.copy, 2,SKC_CL_ARG(impl->runtime->block_pool.blocks.drw))); |
| cl(SetKernelArg(impl->kernels.copy, 3,SKC_CL_ARG(impl->runtime->block_pool.size->ring_mask))); |
| |
| cl(SetKernelArg(impl->kernels.copy, 4,SKC_CL_ARG(impl->reads))); |
| cl(SetKernelArg(impl->kernels.copy, 5,SKC_CL_ARG(curr_from.subbuf))); |
| |
| cl(SetKernelArg(impl->kernels.copy, 6,SKC_CL_ARG(impl->cmds.buffer))); |
| cl(SetKernelArg(impl->kernels.copy, 7,SKC_CL_ARG(impl->blocks.buffer))); |
| |
| cl(SetKernelArg(impl->kernels.copy, 8,SKC_CL_ARG(impl->ring.blocks_per.buffer))); |
| cl(SetKernelArg(impl->kernels.copy, 9,SKC_CL_ARG(impl->prev.rolling))); |
| |
| cl(SetKernelArg(impl->kernels.copy,10,SKC_CL_ARG(impl->prev.from))); |
| cl(SetKernelArg(impl->kernels.copy,11,SKC_CL_ARG(pb_prev_span))); |
| cl(SetKernelArg(impl->kernels.copy,12,SKC_CL_ARG(impl->curr.from))); |
| |
| cl_event complete; |
| |
| skc_device_enqueue_kernel(impl->runtime->device, |
| SKC_DEVICE_KERNEL_ID_PATHS_COPY, |
| impl->cq, |
| impl->kernels.copy, |
| pb_cmds, |
| 0,NULL,&complete); |
| |
| // set a callback on completion |
| cl(SetEventCallback(complete,CL_COMPLETE, |
| skc_path_builder_paths_copy_cb, |
| grid)); |
| |
| // immediately release |
| cl(ReleaseEvent(complete)); |
| |
| // |
| // remap as many subbuffers as possible after the kernel completes |
| // |
| // note that remaps are async and enqueued on the same command queue |
| // as the kernel launch |
| // |
| // we can't remap subbuffers that are in the possibly empty range |
| // |
| // cases: |
| // |
| // - curr.to == wip.to which means no blocks have been acquired |
| // - curr.to points to first block in (next) subbuf |
| // - otherwise, wip acquired blocks in the curr.to subbuf |
| // |
| // check for these first 2 cases! |
| // |
| union skc_ringdex_expand const prev_from = skc_ringdex_expand(impl,impl->prev.from); |
| skc_uint const no_wip = impl->curr.to == impl->wip.to; |
| skc_uint map_to = curr_to.subbuf + (is_partial && no_wip); |
| |
| // remap all subbufs in range [from,to) |
| skc_path_builder_impl_map(impl,prev_from.subbuf,map_to); |
| |
| // flush command queue |
| cl(Flush(impl->cq)); |
| |
| // save rolling |
| impl->prev.rolling = impl->wip.rolling.next; |
| |
| // update prev and curr |
| if (no_wip) |
| { |
| // |
| // if there was no wip then round up to the next subbuf |
| // |
| skc_ringdex_wip_to_subbuf_inc(impl); |
| |
| // |
| // update prev/curr with with incremented wip |
| // |
| impl->prev.from = impl->prev.to = impl->wip.to; |
| impl->curr.from = impl->curr.to = impl->wip.to; |
| } |
| else |
| { |
| // |
| // update prev with wip partials |
| // |
| impl->prev.from = impl->curr.to; |
| impl->prev.to = impl->wip .to; |
| |
| // |
| // start curr on a new subbuf boundary |
| // |
| skc_ringdex_wip_to_subbuf_inc(impl); |
| |
| impl->curr.from = impl->wip.to; |
| impl->curr.to = impl->wip.to; |
| } |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void |
| skc_path_builder_impl_acquire_subbuffer(struct skc_path_builder_impl * const impl, |
| skc_uint const subbuf) |
| { |
| // |
| // FIXME -- move to a power-of-two subbuf size and kickstart path |
| // copies as early as possible |
| // |
| // FIXME -- the subbufs "self-clock" (flow control) the kernel |
| // launches and accounting. Combine all the subbuffers and release |
| // records into a single indexable struct instead of 3. |
| // |
| struct skc_subbuffer_cmds * const sc = impl->cmds.subbufs + subbuf; |
| struct skc_release_record * const release = impl->release.records + subbuf; |
| struct skc_scheduler * const scheduler = impl->runtime->scheduler; |
| |
| // can't proceed until the paths have been released |
| SKC_SCHEDULER_WAIT_WHILE(scheduler,release->from != release->to); |
| |
| // throw in a scheduler yield ... FIXME -- get rid of |
| skc_scheduler_yield(scheduler); |
| |
| // can't proceed until the subbuffer is mapped |
| cl(WaitForEvents(1,&sc->map)); |
| } |
| |
| // |
| // |
| // |
| |
| static |
| union skc_ringdex_expand |
| skc_path_builder_impl_acquire_block(struct skc_path_builder_impl * const impl) |
| { |
| // break ringdex into components |
| union skc_ringdex_expand const to = skc_ringdex_expand(impl,impl->wip.to); |
| |
| // does wip ringdex point to a new subbuffer? |
| if (to.block == 0) |
| { |
| // potentially spin/block waiting for subbuffer |
| skc_path_builder_impl_acquire_subbuffer(impl,to.subbuf); |
| } |
| |
| // post increment wip.to |
| skc_ringdex_wip_to_block_inc(impl); |
| |
| return to; |
| } |
| |
| // |
| // |
| // |
| |
| static |
| skc_uint |
| skc_rolling_block(skc_uint const rolling, skc_uint const tag) |
| { |
| return rolling | tag; |
| } |
| |
| static |
| skc_uint |
| skc_rolling_subblock(skc_uint const rolling, skc_uint const subblock, skc_uint const tag) |
| { |
| return rolling | (subblock << SKC_TAGGED_BLOCK_ID_BITS_TAG) | tag; |
| } |
| |
| static |
| void |
| skc_rolling_inc(struct skc_path_builder_impl * const impl) |
| { |
| impl->wip.rolling.next += impl->wip.rolling.one; |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void * |
| skc_path_builder_impl_new_command(struct skc_path_builder_impl * const impl, |
| skc_uint const rolling, |
| skc_cmd_paths_copy_tag const tag) |
| { |
| // bump blocks count |
| impl->wip.head->header.blocks += 1; |
| |
| // acquire a block |
| union skc_ringdex_expand const to = skc_path_builder_impl_acquire_block(impl); |
| |
| // make a pointer |
| union skc_tagged_block_id * const cmds_subbuf = impl->cmds.subbufs[to.subbuf].host; |
| |
| // store command for block |
| cmds_subbuf[to.block].u32 = skc_rolling_block(rolling,tag); |
| |
| #if 0 |
| // store command for block |
| cmds_subbuf[to.block].u32 = skc_rolling_block(impl->wip.rolling.next,tag); |
| |
| // increment rolling |
| skc_rolling_inc(impl); |
| #endif |
| |
| // return pointer to block |
| float * const blocks_subbuf = impl->blocks.subbufs[to.subbuf].host; |
| |
| // FIXME -- make it easier to get config constant |
| return blocks_subbuf + (to.block * impl->runtime->config->block.words); |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void |
| skc_path_builder_impl_flush_node(struct skc_path_builder_impl * const impl) |
| { |
| // store command to subbuf and get pointer to blocks subbuf |
| void * const block = skc_path_builder_impl_new_command(impl,impl->wip.ids.rolling, |
| SKC_CMD_PATHS_COPY_TAG_NODE); |
| |
| // copy head to blocks subbuf -- write-only |
| memcpy(block,impl->wip.node,impl->runtime->config->block.bytes); |
| } |
| |
| static |
| void |
| skc_path_builder_impl_flush_head(struct skc_path_builder_impl * const impl) |
| { |
| // store command to subbuf and get pointer to blocks subbuf |
| void * const block = skc_path_builder_impl_new_command(impl,impl->wip.rolling.next, |
| SKC_CMD_PATHS_COPY_TAG_HEAD); |
| |
| // copy head to blocks subbuf -- write-only |
| memcpy(block,impl->wip.head,impl->runtime->config->block.bytes); |
| |
| // increment rolling |
| skc_rolling_inc(impl); |
| |
| // the 'to' index is non-inclusive so assign wip.to after flush_head |
| impl->curr.to = impl->wip.to; |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void |
| skc_path_builder_impl_new_node_block(struct skc_path_builder_impl * const impl) |
| { |
| // update final block id in node |
| impl->wip.ids.next->u32 = skc_rolling_block(impl->wip.rolling.next,SKC_BLOCK_ID_TAG_PATH_NEXT); |
| |
| // if wip.ids is not the header then flush now full wip node |
| if (impl->wip.head->header.nodes > 0) |
| skc_path_builder_impl_flush_node(impl); |
| |
| // bump node count |
| impl->wip.head->header.nodes += 1; |
| |
| // save current rolling |
| impl->wip.ids.rolling = impl->wip.rolling.next; |
| |
| // increment rolling |
| skc_rolling_inc(impl); |
| |
| // update wip.ids.* |
| impl->wip.ids.next = impl->wip.node->tag_ids; |
| impl->wip.ids.rem = impl->runtime->config->block.words; |
| } |
| |
| static |
| void |
| skc_path_builder_impl_new_segs_block(struct skc_path_builder_impl * const impl) |
| { |
| impl->wip.subblocks.rem = impl->runtime->config->block.subblocks; // FIXME -- move constants closer to structure |
| impl->wip.subblocks.rolling = impl->wip.rolling.next; |
| impl->wip.subblocks.next = skc_path_builder_impl_new_command(impl,impl->wip.rolling.next, |
| SKC_CMD_PATHS_COPY_TAG_SEGS); |
| impl->wip.subblocks.idx = 0; |
| |
| // increment rolling |
| skc_rolling_inc(impl); |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void |
| skc_path_builder_impl_acquire_subblocks(struct skc_path_builder_impl * const impl, |
| skc_block_id_tag tag, |
| skc_uint vertices, |
| float * * subblocks) |
| { |
| // |
| // FIRST TAG RECORDS THE ELEMENT TYPE |
| // |
| while (true) |
| { |
| // if only one block id left in node then acquire new node block |
| // and append its block id as with a next tag |
| if (impl->wip.ids.rem == 1) |
| skc_path_builder_impl_new_node_block(impl); |
| |
| // if zero subblocks left then acquire a new subblock block and |
| // append its block id |
| if (impl->wip.subblocks.rem == 0) |
| skc_path_builder_impl_new_segs_block(impl); |
| |
| // save first command -- tag and subblocks may have been updated |
| impl->wip.ids.next->u32 = skc_rolling_subblock(impl->wip.subblocks.rolling,impl->wip.subblocks.idx,tag); |
| |
| // increment node block subblock pointer |
| impl->wip.ids.next += 1; |
| impl->wip.ids.rem -= 1; |
| |
| // how many vertices can we store |
| skc_uint rem = min(vertices,impl->wip.subblocks.rem); |
| |
| // decrement vertices |
| vertices -= rem; |
| impl->wip.subblocks.rem -= rem; |
| impl->wip.subblocks.idx += rem; |
| |
| // assign subblocks |
| do { |
| *subblocks++ = impl->wip.subblocks.next; |
| impl->wip.subblocks.next += impl->runtime->config->subblock.words; |
| // FIXME -- move constants closer to structure |
| } while (--rem > 0); |
| |
| // anything left to do? |
| if (vertices == 0) |
| break; |
| |
| // any tag after this will be a caboose command |
| tag = SKC_BLOCK_ID_TAG_PATH_NEXT; |
| } |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void |
| skc_path_builder_pfn_end(struct skc_path_builder_impl * const impl, skc_path_t * const path) |
| { |
| // finalize incomplete active subblocks -- we don't care about any |
| // remaining unused subblocks in block |
| skc_path_builder_finalize_subblocks(impl->path_builder); |
| |
| // mark remaining wips.ids in the head or node as invalid |
| skc_path_builder_impl_finalize_node(impl); |
| |
| // flush node if rem > 0 and node is not actually head |
| if (impl->wip.head->header.nodes >= 1) |
| skc_path_builder_impl_flush_node(impl); |
| |
| // acquire path host id |
| *path = skc_runtime_handle_device_acquire(impl->runtime); // FIXME -- MAY WANT TO GRAB AN ID ON BEGIN |
| |
| // save path host handle |
| impl->wip.head->header.handle = *path; |
| |
| // flush head -- acquires a block and bumps head->header.blocks |
| skc_path_builder_impl_flush_head(impl); |
| |
| // get current release |
| struct skc_release_record * const release = skc_release_curr(impl); |
| |
| // acquire grid if null |
| if (release->grid == NULL) |
| { |
| release->grid = |
| SKC_GRID_DEPS_ATTACH(impl->runtime->deps, |
| &release->grid, // NULL on start/force |
| release, // data payload |
| skc_path_builder_grid_pfn_waiting, |
| NULL, // no execute pfn |
| skc_path_builder_grid_pfn_dispose); |
| } |
| |
| // update grid map |
| skc_grid_map(release->grid,*path); |
| |
| // update path release |
| impl->release.paths[release->to] = *path; |
| |
| // increment release.to |
| release->to = (release->to + 1) % impl->ring.blocks_per.buffer; |
| |
| // add guard bit |
| *path |= SKC_TYPED_HANDLE_TYPE_IS_PATH; |
| |
| #if 1 |
| // |
| // eager kernel launch? |
| // |
| { |
| union skc_ringdex_expand const curr_from = skc_ringdex_expand(impl,impl->curr.from); |
| union skc_ringdex_expand const curr_to = skc_ringdex_expand(impl,impl->curr.to); |
| |
| if (curr_from.subbuf != curr_to.subbuf) |
| { |
| skc_grid_start(release->grid); |
| // skc_scheduler_yield(impl->runtime->scheduler); |
| } |
| } |
| #endif |
| } |
| |
| // |
| // FIXME -- clean up accessing of CONFIG constants in these 3 routines |
| // |
| |
| static |
| void |
| skc_path_builder_pfn_new_line(struct skc_path_builder_impl * const impl) |
| { |
| // acquire subblock pointers |
| skc_path_builder_impl_acquire_subblocks(impl,SKC_BLOCK_ID_TAG_PATH_LINE,4, |
| impl->path_builder->line.coords); |
| |
| // increment line count |
| impl->wip.head->header.prims += 1; |
| |
| // update rem_count_xxx count |
| impl->path_builder->line.rem = impl->runtime->config->subblock.words; |
| } |
| |
| static |
| void |
| skc_path_builder_pfn_new_quad(struct skc_path_builder_impl * const impl) |
| { |
| // acquire subblock pointers |
| skc_path_builder_impl_acquire_subblocks(impl,SKC_BLOCK_ID_TAG_PATH_QUAD,6, |
| impl->path_builder->quad.coords); |
| |
| // increment line count |
| impl->wip.head->header.prims += 1; |
| |
| // update rem_count_xxx count |
| impl->path_builder->quad.rem = impl->runtime->config->subblock.words; |
| } |
| |
| static |
| void |
| skc_path_builder_pfn_new_cubic(struct skc_path_builder_impl * const impl) |
| { |
| // acquire subblock pointers |
| skc_path_builder_impl_acquire_subblocks(impl,SKC_BLOCK_ID_TAG_PATH_CUBIC,8, |
| impl->path_builder->cubic.coords); |
| |
| // increment line count |
| impl->wip.head->header.prims += 1; |
| |
| // update rem_count_xxx count |
| impl->path_builder->cubic.rem = impl->runtime->config->subblock.words; |
| } |
| |
| // |
| // |
| // |
| |
| static |
| void |
| skc_path_builder_pfn_release(struct skc_path_builder_impl * const impl) |
| { |
| // decrement reference count |
| if (--impl->path_builder->refcount != 0) |
| return; |
| |
| // |
| // otherwise, dispose of everything |
| // |
| struct skc_runtime * const runtime = impl->runtime; |
| |
| // free path builder |
| skc_runtime_host_perm_free(impl->runtime,impl->path_builder); |
| |
| // release cq |
| skc_runtime_release_cq_in_order(runtime,impl->cq); |
| |
| // release kernels |
| cl(ReleaseKernel(impl->kernels.alloc)); |
| cl(ReleaseKernel(impl->kernels.copy)); |
| |
| // free blocks extents |
| cl(ReleaseMemObject(impl->blocks.buffer)); |
| skc_runtime_host_perm_free(runtime,impl->blocks.subbufs); |
| |
| cl(ReleaseMemObject(impl->cmds.buffer)); |
| skc_runtime_host_perm_free(runtime,impl->cmds.subbufs); |
| |
| // free records |
| skc_runtime_host_perm_free(runtime,impl->release.records); |
| skc_runtime_host_perm_free(runtime,impl->release.paths); |
| |
| // release staging head and node |
| skc_runtime_host_perm_free(runtime,impl->wip.head); |
| skc_runtime_host_perm_free(runtime,impl->wip.node); |
| |
| // release reads scratch array |
| cl(ReleaseMemObject(impl->reads)); |
| |
| // for all subbuffers |
| // unmap subbuffer |
| // release subbuffer |
| // printf("%s not releasing subbuffers\n",__func__); |
| |
| skc_runtime_host_perm_free(impl->runtime,impl); |
| } |
| |
| // |
| // |
| // |
| |
| skc_err |
| skc_path_builder_cl_12_create(struct skc_context * const context, |
| struct skc_path_builder * * const path_builder) |
| { |
| // |
| // retain the context |
| // skc_context_retain(context); |
| // |
| struct skc_runtime * const runtime = context->runtime; |
| |
| // allocate path builder |
| (*path_builder) = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE,sizeof(**path_builder)); |
| |
| // init state |
| SKC_ASSERT_STATE_INIT((*path_builder),SKC_PATH_BUILDER_STATE_READY); |
| |
| (*path_builder)->context = context; |
| |
| // save opaque impl-specific pointers |
| (*path_builder)->begin = skc_path_builder_pfn_begin; |
| (*path_builder)->end = skc_path_builder_pfn_end; |
| (*path_builder)->new_line = skc_path_builder_pfn_new_line; |
| (*path_builder)->new_quad = skc_path_builder_pfn_new_quad; |
| (*path_builder)->new_cubic = skc_path_builder_pfn_new_cubic; |
| (*path_builder)->release = skc_path_builder_pfn_release; |
| |
| // initialize path builder counts |
| (*path_builder)->line.rem = 0; |
| (*path_builder)->quad.rem = 0; |
| (*path_builder)->cubic.rem = 0; |
| |
| (*path_builder)->refcount = 1; |
| |
| struct skc_path_builder_impl * const impl = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE,sizeof(*impl)); |
| |
| (*path_builder)->impl = impl; |
| |
| // |
| // init impl |
| // |
| impl->path_builder = *path_builder; |
| impl->runtime = runtime; |
| |
| impl->cq = skc_runtime_acquire_cq_in_order(runtime); |
| |
| impl->kernels.alloc = skc_device_acquire_kernel(runtime->device,SKC_DEVICE_KERNEL_ID_PATHS_ALLOC); |
| impl->kernels.copy = skc_device_acquire_kernel(runtime->device,SKC_DEVICE_KERNEL_ID_PATHS_COPY); |
| |
| // |
| // FIXME -- let these config constants remain constant and in place |
| // |
| struct skc_config const * const config = runtime->config; |
| |
| impl->ring.subbufs = config->paths_copy.buffer.count; |
| impl->ring.blocks_per.buffer = config->paths_copy.subbuf.count * config->paths_copy.buffer.count; |
| impl->ring.blocks_per.subbuf = config->paths_copy.subbuf.count; |
| // |
| // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| // |
| |
| cl_int cl_err; |
| |
| // allocate large device-side extent for path data |
| impl->blocks.buffer = clCreateBuffer(runtime->cl.context, |
| CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR, |
| config->paths_copy.block.buffer, // FIXME -- either use config or local constants everywhere |
| NULL,&cl_err); cl_ok(cl_err); |
| |
| // allocate small host-side array of pointers to mapped subbufs |
| impl->blocks.subbufs = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE, |
| impl->ring.subbufs * |
| sizeof(*impl->blocks.subbufs)); |
| |
| // allocate large device-side extent for path copy commands |
| impl->cmds.buffer = clCreateBuffer(runtime->cl.context, |
| CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR, |
| config->paths_copy.command.buffer, |
| NULL,&cl_err); cl_ok(cl_err); |
| |
| // allocate small host-side array of pointers to mapped subbufs |
| impl->cmds.subbufs = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE, |
| impl->ring.subbufs * |
| sizeof(*impl->cmds.subbufs)); |
| |
| // allocate small host-side array of intervals of path handles |
| impl->release.records = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE, |
| impl->ring.subbufs * |
| sizeof(*impl->release.records)); |
| |
| // allocate large host-side array that is max # of path handles in flight |
| impl->release.paths = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE, |
| impl->ring.blocks_per.buffer * |
| sizeof(*impl->release.paths)); |
| |
| // small scratch used by kernels |
| impl->reads = clCreateBuffer(runtime->cl.context, |
| CL_MEM_READ_WRITE | CL_MEM_HOST_NO_ACCESS, |
| sizeof(skc_uint) * impl->ring.subbufs, |
| NULL,&cl_err); cl_ok(cl_err); |
| |
| // initialize release record with impl backpointer |
| for (skc_uint ii=0; ii<impl->ring.subbufs; ii++) |
| { |
| struct skc_release_record * record = impl->release.records + ii; |
| |
| record->impl = impl; |
| record->grid = NULL; |
| record->from = record->to = ii * impl->ring.blocks_per.subbuf; |
| } |
| |
| // |
| // allocate and map subbuffers -- we always check the command |
| // subbuffer's map/unmap events before touching it or its associated |
| // block subbuffer. |
| // |
| struct skc_subbuffer_blocks * sb = impl->blocks.subbufs; |
| struct skc_subbuffer_cmds * sc = impl->cmds .subbufs; |
| |
| cl_buffer_region rb = { 0, config->paths_copy.block.subbuf }; |
| cl_buffer_region rc = { 0, config->paths_copy.command.subbuf }; |
| |
| // for each subbuffer |
| for (skc_uint ii=0; ii<config->paths_copy.buffer.count; ii++) |
| { |
| sb->device = clCreateSubBuffer(impl->blocks.buffer, |
| CL_MEM_HOST_WRITE_ONLY, |
| CL_BUFFER_CREATE_TYPE_REGION, |
| &rb, |
| &cl_err); cl_ok(cl_err); |
| |
| sb->host = clEnqueueMapBuffer(impl->cq, |
| sb->device, |
| CL_FALSE, |
| CL_MAP_WRITE_INVALIDATE_REGION, |
| 0,rb.size, |
| 0,NULL,NULL, |
| &cl_err); cl_ok(cl_err); |
| |
| sc->device = clCreateSubBuffer(impl->cmds.buffer, |
| CL_MEM_HOST_WRITE_ONLY, |
| CL_BUFFER_CREATE_TYPE_REGION, |
| &rc, |
| &cl_err); cl_ok(cl_err); |
| |
| sc->host = clEnqueueMapBuffer(impl->cq, |
| sc->device, |
| CL_FALSE, |
| CL_MAP_WRITE_INVALIDATE_REGION, |
| 0,rc.size, |
| 0,NULL,&sc->map, |
| &cl_err); cl_ok(cl_err); |
| sb += 1; |
| sc += 1; |
| |
| rb.origin += rb.size; |
| rc.origin += rc.size; |
| } |
| |
| // |
| // initialize remaining members |
| // |
| impl->prev.from = 0; |
| impl->prev.to = 0; |
| impl->prev.rolling = 0; |
| |
| impl->curr.from = 0; |
| impl->curr.to = 0; |
| |
| impl->wip.to = 0; |
| |
| impl->wip.head = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE,config->block.bytes); |
| impl->wip.node = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE,config->block.bytes); |
| |
| impl->wip.rolling.one = SKC_BLOCK_ID_TAG_COUNT * config->block.subblocks; |
| impl->wip.rolling.next = 0; |
| |
| // for now, completely initialize builder before returning |
| cl(Finish(impl->cq)); |
| |
| return SKC_ERR_SUCCESS; |
| } |
| |
| // |
| // |
| // |
