| /* |
| * Copyright 2017 Google Inc. |
| * |
| * Use of this source code is governed by a BSD-style license that can be |
| * found in the LICENSE file. |
| */ |
| |
| #include "SkColorPriv.h" |
| #include "SkCpu.h" |
| #include "SkJumper.h" |
| #include "SkRasterPipeline.h" |
| #include "SkTemplates.h" |
| |
| // We'll use __has_feature(memory_sanitizer) to detect MSAN. |
| // SkJumper_generated.S is not compiled with MSAN, so MSAN would yell really loud. |
| #if !defined(__has_feature) |
| #define __has_feature(x) 0 |
| #endif |
| |
| // Stages expect these constants to be set to these values. |
| // It's fine to rearrange and add new ones if you update SkJumper_constants. |
| using K = const SkJumper_constants; |
| static K kConstants = { |
| {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f}, |
| }; |
| |
| // We can't express the real types of most stage functions portably, so we use a stand-in. |
| // We'll only ever call start_pipeline(), which then chains into the rest for us. |
| using StageFn = void(void); |
| |
| // Some platforms expect C "name" maps to asm "_name", others to "name". |
| #if defined(__APPLE__) |
| #define ASM(name, suffix) sk_##name##_##suffix |
| #else |
| #define ASM(name, suffix) _sk_##name##_##suffix |
| #endif |
| |
| extern "C" { |
| |
| #if __has_feature(memory_sanitizer) |
| // We'll just run portable code. |
| |
| #elif defined(__aarch64__) |
| size_t ASM(start_pipeline,aarch64)(size_t, void**, K*, size_t); |
| StageFn ASM(just_return,aarch64); |
| #define M(st) StageFn ASM(st,aarch64); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| |
| #elif defined(__arm__) |
| size_t ASM(start_pipeline,vfp4)(size_t, void**, K*, size_t); |
| StageFn ASM(just_return,vfp4); |
| #define M(st) StageFn ASM(st,vfp4); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| |
| #elif defined(__x86_64__) || defined(_M_X64) |
| size_t ASM(start_pipeline,hsw )(size_t, void**, K*, size_t); |
| size_t ASM(start_pipeline,avx )(size_t, void**, K*, size_t); |
| size_t ASM(start_pipeline,sse41)(size_t, void**, K*, size_t); |
| size_t ASM(start_pipeline,sse2 )(size_t, void**, K*, size_t); |
| |
| StageFn ASM(just_return,hsw), |
| ASM(just_return,avx), |
| ASM(just_return,sse41), |
| ASM(just_return,sse2); |
| |
| #define M(st) StageFn ASM(st,hsw); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| #define M(st) StageFn ASM(st,avx); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| #define M(st) StageFn ASM(st,sse41); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| #define M(st) StageFn ASM(st,sse2); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| #endif |
| |
| // Portable, single-pixel stages. |
| size_t sk_start_pipeline(size_t, void**, K*, size_t); |
| StageFn sk_just_return; |
| #define M(st) StageFn sk_##st; |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| } |
| |
| // Translate SkRasterPipeline's StockStage enum to StageFn function pointers. |
| |
| #if __has_feature(memory_sanitizer) |
| // We'll just run portable code. |
| |
| #elif defined(__aarch64__) |
| static StageFn* lookup_aarch64(SkRasterPipeline::StockStage st) { |
| switch (st) { |
| #define M(st) case SkRasterPipeline::st: return ASM(st,aarch64); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| } |
| return nullptr; |
| } |
| |
| #elif defined(__arm__) |
| static StageFn* lookup_vfp4(SkRasterPipeline::StockStage st) { |
| switch (st) { |
| #define M(st) case SkRasterPipeline::st: return ASM(st,vfp4); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| } |
| return nullptr; |
| } |
| |
| #elif defined(__x86_64__) || defined(_M_X64) |
| static StageFn* lookup_hsw(SkRasterPipeline::StockStage st) { |
| switch (st) { |
| #define M(st) case SkRasterPipeline::st: return ASM(st,hsw); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| } |
| return nullptr; |
| } |
| static StageFn* lookup_avx(SkRasterPipeline::StockStage st) { |
| switch (st) { |
| #define M(st) case SkRasterPipeline::st: return ASM(st,avx); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| } |
| return nullptr; |
| } |
| static StageFn* lookup_sse41(SkRasterPipeline::StockStage st) { |
| switch (st) { |
| #define M(st) case SkRasterPipeline::st: return ASM(st,sse41); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| } |
| return nullptr; |
| } |
| static StageFn* lookup_sse2(SkRasterPipeline::StockStage st) { |
| switch (st) { |
| #define M(st) case SkRasterPipeline::st: return ASM(st,sse2); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| } |
| return nullptr; |
| } |
| #endif |
| |
| static StageFn* lookup_portable(SkRasterPipeline::StockStage st) { |
| switch (st) { |
| #define M(st) case SkRasterPipeline::st: return sk_##st; |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| } |
| return nullptr; |
| } |
| |
| void SkRasterPipeline::run(size_t x, size_t n) const { |
| SkAutoSTMalloc<64, void*> program(2*fStages.size() + 1); |
| const size_t limit = x+n; |
| |
| auto build_and_run = [&](size_t min_stride, |
| StageFn* (*lookup)(SkRasterPipeline::StockStage), |
| StageFn* just_return, |
| size_t (*start_pipeline)(size_t, void**, K*, size_t)) { |
| if (x + min_stride <= limit) { |
| void** ip = program.get(); |
| for (auto&& st : fStages) { |
| auto fn = lookup(st.stage); |
| SkASSERT(fn); |
| *ip++ = (void*)fn; |
| if (st.ctx) { |
| *ip++ = st.ctx; |
| } |
| } |
| *ip = (void*)just_return; |
| |
| x = start_pipeline(x, program.get(), &kConstants, limit); |
| } |
| }; |
| |
| // While possible, build and run at full vector stride. |
| #if __has_feature(memory_sanitizer) |
| // We'll just run portable code. |
| |
| #elif defined(__aarch64__) |
| build_and_run(4, lookup_aarch64, ASM(just_return,aarch64), ASM(start_pipeline,aarch64)); |
| |
| #elif defined(__arm__) |
| if (1 && SkCpu::Supports(SkCpu::NEON|SkCpu::NEON_FMA|SkCpu::VFP_FP16)) { |
| build_and_run(2, lookup_vfp4, ASM(just_return,vfp4), ASM(start_pipeline,vfp4)); |
| } |
| |
| #elif defined(__x86_64__) || defined(_M_X64) |
| if (1 && SkCpu::Supports(SkCpu::HSW)) { |
| build_and_run(1, lookup_hsw, ASM(just_return,hsw), ASM(start_pipeline,hsw)); |
| } |
| if (1 && SkCpu::Supports(SkCpu::AVX)) { |
| build_and_run(1, lookup_avx, ASM(just_return,avx), ASM(start_pipeline,avx)); |
| } |
| if (1 && SkCpu::Supports(SkCpu::SSE41)) { |
| build_and_run(4, lookup_sse41, ASM(just_return,sse41), ASM(start_pipeline,sse41)); |
| } |
| if (1 && SkCpu::Supports(SkCpu::SSE2)) { |
| build_and_run(4, lookup_sse2, ASM(just_return,sse2), ASM(start_pipeline,sse2)); |
| } |
| #endif |
| |
| // Finish up any leftover with portable code one pixel at a time. |
| build_and_run(1, lookup_portable, sk_just_return, sk_start_pipeline); |
| } |
