| /* |
| * Copyright 2017 Google Inc. |
| * |
| * Use of this source code is governed by a BSD-style license that can be |
| * found in the LICENSE file. |
| */ |
| |
| #include "SkColorData.h" |
| #include "SkCpu.h" |
| #include "SkJumper.h" |
| #include "SkOnce.h" |
| #include "SkRasterPipeline.h" |
| #include "SkTemplates.h" |
| |
| #if !defined(SK_JUMPER_USE_ASSEMBLY) |
| // We'll use __has_feature(memory_sanitizer) to detect MSAN. |
| // SkJumper_generated.S is not compiled with MSAN, so MSAN would yell really loud. |
| #if !defined(__has_feature) |
| #define __has_feature(x) 0 |
| #endif |
| |
| #if 0 || __has_feature(memory_sanitizer) |
| #define SK_JUMPER_USE_ASSEMBLY 0 |
| #else |
| #define SK_JUMPER_USE_ASSEMBLY 1 |
| #endif |
| #endif |
| |
| #define M(st) +1 |
| static const int kNumStages = SK_RASTER_PIPELINE_STAGES(M); |
| #undef M |
| |
| #ifndef SK_JUMPER_DISABLE_8BIT |
| // Intentionally commented out; optional logging for local debugging. |
| #if 0 && SK_JUMPER_USE_ASSEMBLY && (defined(__x86_64__) || defined(_M_X64)) |
| #include <atomic> |
| |
| #define M(st) #st, |
| static const char* kStageNames[] = { SK_RASTER_PIPELINE_STAGES(M) }; |
| #undef M |
| |
| static std::atomic<int> gMissingStageCounters[kNumStages]; |
| |
| static void log_missing(SkRasterPipeline::StockStage st) { |
| static SkOnce once; |
| once([] { atexit([] { |
| int total = 0; |
| for (int i = 0; i < kNumStages; i++) { |
| if (int count = gMissingStageCounters[i].load()) { |
| SkDebugf("%7d\t%s\n", count, kStageNames[i]); |
| total += count; |
| } |
| } |
| SkDebugf("%7d total\n", total); |
| }); }); |
| |
| gMissingStageCounters[st]++; |
| } |
| #else |
| static void log_missing(SkRasterPipeline::StockStage) {} |
| #endif |
| #endif |
| |
| // We can't express the real types of most stage functions portably, so we use a stand-in. |
| // We'll only ever call start_pipeline(), which then chains into the rest. |
| using StageFn = void(void); |
| using StartPipelineFn = void(size_t,size_t,size_t,size_t, void**); |
| |
| // Some platforms expect C "name" maps to asm "_name", others to "name". |
| #if defined(__APPLE__) |
| #define ASM(name, suffix) sk_##name##_##suffix |
| #else |
| #define ASM(name, suffix) _sk_##name##_##suffix |
| #endif |
| |
| extern "C" { |
| |
| #if !SK_JUMPER_USE_ASSEMBLY |
| // We'll just run baseline code. |
| |
| #elif defined(__x86_64__) || defined(_M_X64) |
| StartPipelineFn ASM(start_pipeline, skx), |
| ASM(start_pipeline, hsw), |
| ASM(start_pipeline, avx), |
| ASM(start_pipeline, sse41), |
| ASM(start_pipeline, sse2), |
| ASM(start_pipeline, hsw_lowp), |
| ASM(start_pipeline,sse41_lowp), |
| ASM(start_pipeline, sse2_lowp); |
| |
| StageFn ASM(just_return, skx), |
| ASM(just_return, hsw), |
| ASM(just_return, avx), |
| ASM(just_return, sse41), |
| ASM(just_return, sse2), |
| ASM(just_return, hsw_lowp), |
| ASM(just_return,sse41_lowp), |
| ASM(just_return, sse2_lowp); |
| |
| #define M(st) StageFn ASM(st, skx), \ |
| ASM(st, hsw), \ |
| ASM(st, avx), \ |
| ASM(st,sse41), \ |
| ASM(st, sse2), \ |
| ASM(st, hsw_lowp), \ |
| ASM(st,sse41_lowp), \ |
| ASM(st, sse2_lowp); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| |
| #elif defined(__i386__) || defined(_M_IX86) |
| StartPipelineFn ASM(start_pipeline,sse2), |
| ASM(start_pipeline,sse2_lowp); |
| StageFn ASM(just_return,sse2), |
| ASM(just_return,sse2_lowp); |
| #define M(st) StageFn ASM(st,sse2), \ |
| ASM(st,sse2_lowp); |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| |
| #endif |
| |
| // Baseline code compiled as a normal part of Skia. |
| StartPipelineFn sk_start_pipeline; |
| StageFn sk_just_return; |
| #define M(st) StageFn sk_##st; |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| |
| #if defined(JUMPER_HAS_NEON_LOWP) |
| StartPipelineFn sk_start_pipeline_lowp; |
| StageFn sk_just_return_lowp; |
| #define M(st) StageFn sk_##st##_lowp; |
| SK_RASTER_PIPELINE_STAGES(M) |
| #undef M |
| #endif |
| |
| } |
| |
| #if SK_JUMPER_USE_ASSEMBLY |
| #if defined(__x86_64__) || defined(_M_X64) |
| template <SkRasterPipeline::StockStage st> |
| static constexpr StageFn* hsw_lowp(); |
| |
| template <SkRasterPipeline::StockStage st> |
| static constexpr StageFn* sse41_lowp(); |
| |
| template <SkRasterPipeline::StockStage st> |
| static constexpr StageFn* sse2_lowp(); |
| |
| #define LOWP(st) \ |
| template <> constexpr StageFn* hsw_lowp<SkRasterPipeline::st>() { \ |
| return ASM(st,hsw_lowp); \ |
| } \ |
| template <> constexpr StageFn* sse41_lowp<SkRasterPipeline::st>() { \ |
| return ASM(st,sse41_lowp); \ |
| } \ |
| template <> constexpr StageFn* sse2_lowp<SkRasterPipeline::st>() { \ |
| return ASM(st,sse2_lowp); \ |
| } |
| #define NOPE(st) \ |
| template <> constexpr StageFn* hsw_lowp<SkRasterPipeline::st>() { \ |
| return nullptr; \ |
| } \ |
| template <> constexpr StageFn* sse41_lowp<SkRasterPipeline::st>() { \ |
| return nullptr; \ |
| } \ |
| template <> constexpr StageFn* sse2_lowp<SkRasterPipeline::st>() { \ |
| return nullptr; \ |
| } |
| |
| #elif defined(__i386__) || defined(_M_IX86) |
| template <SkRasterPipeline::StockStage st> |
| static constexpr StageFn* sse2_lowp(); |
| |
| #define LOWP(st) \ |
| template <> constexpr StageFn* sse2_lowp<SkRasterPipeline::st>() { \ |
| return ASM(st,sse2_lowp); \ |
| } |
| #define NOPE(st) \ |
| template <> constexpr StageFn* sse2_lowp<SkRasterPipeline::st>() { \ |
| return nullptr; \ |
| } |
| |
| #elif defined(JUMPER_HAS_NEON_LOWP) |
| template <SkRasterPipeline::StockStage st> |
| static constexpr StageFn* neon_lowp(); |
| |
| #define LOWP(st) \ |
| template <> constexpr StageFn* neon_lowp<SkRasterPipeline::st>() { \ |
| return sk_##st##_lowp; \ |
| } |
| #define NOPE(st) \ |
| template <> constexpr StageFn* neon_lowp<SkRasterPipeline::st>() { \ |
| return nullptr; \ |
| } |
| |
| #else |
| #define LOWP(st) |
| #define NOPE(st) |
| |
| #endif |
| |
| #define TODO(st) NOPE(st) // stages that should be implemented in lowp, but aren't. |
| |
| NOPE(callback) |
| LOWP(move_src_dst) LOWP(move_dst_src) |
| NOPE(clamp_0) NOPE(clamp_1) LOWP(clamp_a) LOWP(clamp_a_dst) |
| NOPE(unpremul) LOWP(premul) LOWP(premul_dst) |
| LOWP(force_opaque) LOWP(force_opaque_dst) |
| LOWP(set_rgb) LOWP(swap_rb) LOWP(invert) |
| NOPE(from_srgb) NOPE(from_srgb_dst) NOPE(to_srgb) |
| LOWP(black_color) LOWP(white_color) LOWP(uniform_color) |
| LOWP(seed_shader) NOPE(dither) |
| LOWP(load_a8) LOWP(load_a8_dst) LOWP(store_a8) LOWP(gather_a8) |
| LOWP(load_g8) LOWP(load_g8_dst) LOWP(gather_g8) |
| LOWP(load_565) LOWP(load_565_dst) LOWP(store_565) LOWP(gather_565) |
| LOWP(load_4444) LOWP(load_4444_dst) LOWP(store_4444) LOWP(gather_4444) |
| NOPE(load_f16) NOPE(load_f16_dst) NOPE(store_f16) NOPE(gather_f16) |
| NOPE(load_f32) NOPE(load_f32_dst) NOPE(store_f32) |
| LOWP(load_8888) LOWP(load_8888_dst) LOWP(store_8888) LOWP(gather_8888) |
| LOWP(load_bgra) LOWP(load_bgra_dst) LOWP(store_bgra) LOWP(gather_bgra) |
| NOPE(load_1010102) NOPE(load_1010102_dst) NOPE(store_1010102) NOPE(gather_1010102) |
| TODO(bilerp_clamp_8888) |
| TODO(load_u16_be) TODO(load_rgb_u16_be) TODO(store_u16_be) |
| NOPE(load_tables_u16_be) NOPE(load_tables_rgb_u16_be) NOPE(load_tables) |
| NOPE(load_rgba) NOPE(store_rgba) |
| LOWP(scale_u8) LOWP(scale_565) LOWP(scale_1_float) |
| LOWP( lerp_u8) LOWP( lerp_565) LOWP( lerp_1_float) |
| LOWP(dstatop) LOWP(dstin) LOWP(dstout) LOWP(dstover) |
| LOWP(srcatop) LOWP(srcin) LOWP(srcout) LOWP(srcover) |
| LOWP(clear) LOWP(modulate) LOWP(multiply) LOWP(plus_) LOWP(screen) LOWP(xor_) |
| NOPE(colorburn) NOPE(colordodge) LOWP(darken) LOWP(difference) |
| LOWP(exclusion) LOWP(hardlight) LOWP(lighten) LOWP(overlay) NOPE(softlight) |
| NOPE(hue) NOPE(saturation) NOPE(color) NOPE(luminosity) |
| LOWP(srcover_rgba_8888) LOWP(srcover_bgra_8888) |
| LOWP(luminance_to_alpha) |
| LOWP(matrix_translate) LOWP(matrix_scale_translate) |
| LOWP(matrix_2x3) NOPE(matrix_3x4) TODO(matrix_4x5) TODO(matrix_4x3) |
| LOWP(matrix_perspective) |
| NOPE(parametric_r) NOPE(parametric_g) NOPE(parametric_b) |
| NOPE(parametric_a) NOPE(gamma) NOPE(gamma_dst) |
| NOPE(table_r) NOPE(table_g) NOPE(table_b) NOPE(table_a) |
| NOPE(lab_to_xyz) |
| TODO(mirror_x) TODO(repeat_x) |
| TODO(mirror_y) TODO(repeat_y) |
| LOWP(decal_x) LOWP(decal_y) LOWP(decal_x_and_y) |
| LOWP(check_decal_mask) |
| TODO(bilinear_nx) TODO(bilinear_px) TODO(bilinear_ny) TODO(bilinear_py) |
| TODO(bicubic_n3x) TODO(bicubic_n1x) TODO(bicubic_p1x) TODO(bicubic_p3x) |
| TODO(bicubic_n3y) TODO(bicubic_n1y) TODO(bicubic_p1y) TODO(bicubic_p3y) |
| TODO(save_xy) TODO(accumulate) |
| LOWP(clamp_x_1) LOWP(mirror_x_1) LOWP(repeat_x_1) |
| LOWP(evenly_spaced_gradient) |
| LOWP(gradient) |
| LOWP(evenly_spaced_2_stop_gradient) |
| LOWP(xy_to_unit_angle) |
| LOWP(xy_to_radius) |
| TODO(negate_x) |
| TODO(xy_to_2pt_conical_strip) |
| TODO(xy_to_2pt_conical_focal_on_circle) |
| TODO(xy_to_2pt_conical_well_behaved) |
| TODO(xy_to_2pt_conical_greater) |
| TODO(xy_to_2pt_conical_smaller) |
| TODO(alter_2pt_conical_compensate_focal) |
| TODO(alter_2pt_conical_unswap) |
| TODO(mask_2pt_conical_nan) TODO(mask_2pt_conical_degenerates) TODO(apply_vector_mask) |
| TODO(byte_tables) TODO(byte_tables_rgb) |
| NOPE(rgb_to_hsl) NOPE(hsl_to_rgb) |
| NOPE(clut_3D) NOPE(clut_4D) |
| NOPE(gauss_a_to_rgba) |
| |
| #undef LOWP |
| #undef TODO |
| #undef NOPE |
| #endif |
| |
| // Engines comprise everything we need to run SkRasterPipelines. |
| struct SkJumper_Engine { |
| StageFn* stages[kNumStages]; |
| StartPipelineFn* start_pipeline; |
| StageFn* just_return; |
| }; |
| |
| // We'll default to this baseline engine, but try to choose a better one at runtime. |
| static const SkJumper_Engine kBaseline = { |
| #define M(stage) sk_##stage, |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| #undef M |
| sk_start_pipeline, |
| sk_just_return, |
| }; |
| static SkJumper_Engine gEngine = kBaseline; |
| static SkOnce gChooseEngineOnce; |
| |
| static SkJumper_Engine choose_engine() { |
| #if !SK_JUMPER_USE_ASSEMBLY |
| // We'll just run baseline code. |
| |
| #elif defined(__x86_64__) || defined(_M_X64) |
| #if !defined(_MSC_VER) // No _skx stages for Windows yet. |
| if (1 && SkCpu::Supports(SkCpu::SKX)) { |
| return { |
| #define M(stage) ASM(stage, skx), |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| M(start_pipeline) |
| M(just_return) |
| #undef M |
| }; |
| } |
| #endif |
| if (1 && SkCpu::Supports(SkCpu::HSW)) { |
| return { |
| #define M(stage) ASM(stage, hsw), |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| M(start_pipeline) |
| M(just_return) |
| #undef M |
| }; |
| } |
| if (1 && SkCpu::Supports(SkCpu::AVX)) { |
| return { |
| #define M(stage) ASM(stage, avx), |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| M(start_pipeline) |
| M(just_return) |
| #undef M |
| }; |
| } |
| if (1 && SkCpu::Supports(SkCpu::SSE41)) { |
| return { |
| #define M(stage) ASM(stage, sse41), |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| M(start_pipeline) |
| M(just_return) |
| #undef M |
| }; |
| } |
| if (1 && SkCpu::Supports(SkCpu::SSE2)) { |
| return { |
| #define M(stage) ASM(stage, sse2), |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| M(start_pipeline) |
| M(just_return) |
| #undef M |
| }; |
| } |
| |
| #elif defined(__i386__) || defined(_M_IX86) |
| if (1 && SkCpu::Supports(SkCpu::SSE2)) { |
| return { |
| #define M(stage) ASM(stage, sse2), |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| M(start_pipeline) |
| M(just_return) |
| #undef M |
| }; |
| } |
| |
| #endif |
| return kBaseline; |
| } |
| |
| #ifndef SK_JUMPER_DISABLE_8BIT |
| static const SkJumper_Engine kNone = { |
| #define M(stage) nullptr, |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| #undef M |
| nullptr, |
| nullptr, |
| }; |
| static SkJumper_Engine gLowp = kNone; |
| static SkOnce gChooseLowpOnce; |
| |
| static SkJumper_Engine choose_lowp() { |
| #if SK_JUMPER_USE_ASSEMBLY |
| #if defined(__x86_64__) || defined(_M_X64) |
| if (1 && SkCpu::Supports(SkCpu::HSW)) { |
| return { |
| #define M(st) hsw_lowp<SkRasterPipeline::st>(), |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| ASM(start_pipeline,hsw_lowp), |
| ASM(just_return ,hsw_lowp), |
| #undef M |
| }; |
| } |
| if (1 && SkCpu::Supports(SkCpu::SSE41)) { |
| return { |
| #define M(st) sse41_lowp<SkRasterPipeline::st>(), |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| ASM(start_pipeline,sse41_lowp), |
| ASM(just_return ,sse41_lowp), |
| #undef M |
| }; |
| } |
| if (1 && SkCpu::Supports(SkCpu::SSE2)) { |
| return { |
| #define M(st) sse2_lowp<SkRasterPipeline::st>(), |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| ASM(start_pipeline,sse2_lowp), |
| ASM(just_return ,sse2_lowp), |
| #undef M |
| }; |
| } |
| #elif defined(__i386__) || defined(_M_IX86) |
| if (1 && SkCpu::Supports(SkCpu::SSE2)) { |
| return { |
| #define M(st) sse2_lowp<SkRasterPipeline::st>(), |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| ASM(start_pipeline,sse2_lowp), |
| ASM(just_return ,sse2_lowp), |
| #undef M |
| }; |
| } |
| |
| #elif defined(JUMPER_HAS_NEON_LOWP) |
| return { |
| #define M(st) neon_lowp<SkRasterPipeline::st>(), |
| { SK_RASTER_PIPELINE_STAGES(M) }, |
| sk_start_pipeline_lowp, |
| sk_just_return_lowp, |
| #undef M |
| }; |
| #endif |
| #endif |
| return kNone; |
| } |
| #endif |
| |
| const SkJumper_Engine& SkRasterPipeline::build_pipeline(void** ip) const { |
| #ifndef SK_JUMPER_DISABLE_8BIT |
| gChooseLowpOnce([]{ gLowp = choose_lowp(); }); |
| |
| // First try to build a lowp pipeline. If that fails, fall back to normal float gEngine. |
| void** reset_point = ip; |
| *--ip = (void*)gLowp.just_return; |
| for (const StageList* st = fStages; st; st = st->prev) { |
| if (st->stage == SkRasterPipeline::clamp_0 || |
| st->stage == SkRasterPipeline::clamp_1) { |
| continue; // No-ops in lowp. |
| } |
| if (StageFn* fn = gLowp.stages[st->stage]) { |
| if (st->ctx) { |
| *--ip = st->ctx; |
| } |
| *--ip = (void*)fn; |
| } else { |
| log_missing(st->stage); |
| ip = reset_point; |
| break; |
| } |
| } |
| if (ip != reset_point) { |
| return gLowp; |
| } |
| #endif |
| |
| gChooseEngineOnce([]{ gEngine = choose_engine(); }); |
| // We're building the pipeline backwards, so we start with the final stage just_return. |
| *--ip = (void*)gEngine.just_return; |
| |
| // Still going backwards, each stage's context pointer then its StageFn. |
| for (const StageList* st = fStages; st; st = st->prev) { |
| if (st->ctx) { |
| *--ip = st->ctx; |
| } |
| *--ip = (void*)gEngine.stages[st->stage]; |
| } |
| return gEngine; |
| } |
| |
| void SkRasterPipeline::run(size_t x, size_t y, size_t w, size_t h) const { |
| if (this->empty()) { |
| return; |
| } |
| |
| // Best to not use fAlloc here... we can't bound how often run() will be called. |
| SkAutoSTMalloc<64, void*> program(fSlotsNeeded); |
| |
| const SkJumper_Engine& engine = this->build_pipeline(program.get() + fSlotsNeeded); |
| engine.start_pipeline(x,y,x+w,y+h, program.get()); |
| } |
| |
| std::function<void(size_t, size_t, size_t, size_t)> SkRasterPipeline::compile() const { |
| if (this->empty()) { |
| return [](size_t, size_t, size_t, size_t) {}; |
| } |
| |
| void** program = fAlloc->makeArray<void*>(fSlotsNeeded); |
| const SkJumper_Engine& engine = this->build_pipeline(program + fSlotsNeeded); |
| |
| auto start_pipeline = engine.start_pipeline; |
| return [=](size_t x, size_t y, size_t w, size_t h) { |
| start_pipeline(x,y,x+w,y+h, program); |
| }; |
| } |
