| /* |
| * Copyright © 2009 Nokia Corporation |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the next |
| * paragraph) shall be included in all copies or substantial portions of the |
| * Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
| * DEALINGS IN THE SOFTWARE. |
| * |
| * Author: Siarhei Siamashka (siarhei.siamashka@nokia.com) |
| */ |
| |
| /* |
| * Copyright (c) 2018 RISC OS Open Ltd |
| * |
| * This software is provided 'as-is', without any express or implied |
| * warranty. In no event will the authors be held liable for any damages |
| * arising from the use of this software. |
| * |
| * Permission is granted to anyone to use this software for any purpose, |
| * including commercial applications, and to alter it and redistribute it |
| * freely, subject to the following restrictions: |
| * |
| * 1. The origin of this software must not be misrepresented; you must not |
| * claim that you wrote the original software. If you use this software |
| * in a product, an acknowledgment in the product documentation would be |
| * appreciated but is not required. |
| * 2. Altered source versions must be plainly marked as such, and must not be |
| * misrepresented as being the original software. |
| * 3. This notice may not be removed or altered from any source distribution. |
| */ |
| |
| /* Prevent the stack from becoming executable for no reason... */ |
| #if defined(__linux__) && defined(__ELF__) |
| .section .note.GNU-stack,"",%progbits |
| #endif |
| |
| .text |
| .fpu neon |
| .arch armv7a |
| .object_arch armv4 |
| .eabi_attribute 10, 0 /* suppress Tag_FP_arch */ |
| .eabi_attribute 12, 0 /* suppress Tag_Advanced_SIMD_arch */ |
| .arm |
| .altmacro |
| .p2align 2 |
| |
| #include "pixman-arm-asm.h" |
| #include "pixman-arm-neon-asm.h" |
| |
| /* Global configuration options and preferences */ |
| |
| /* |
| * The code can optionally make use of unaligned memory accesses to improve |
| * performance of handling leading/trailing pixels for each scanline. |
| * Configuration variable RESPECT_STRICT_ALIGNMENT can be set to 0 for |
| * example in linux if unaligned memory accesses are not configured to |
| * generate.exceptions. |
| */ |
| .set RESPECT_STRICT_ALIGNMENT, 1 |
| |
| /* |
| * Set default prefetch type. There is a choice between the following options: |
| * |
| * PREFETCH_TYPE_NONE (may be useful for the ARM cores where PLD is set to work |
| * as NOP to workaround some HW bugs or for whatever other reason) |
| * |
| * PREFETCH_TYPE_SIMPLE (may be useful for simple single-issue ARM cores where |
| * advanced prefetch intruduces heavy overhead) |
| * |
| * PREFETCH_TYPE_ADVANCED (useful for superscalar cores such as ARM Cortex-A8 |
| * which can run ARM and NEON instructions simultaneously so that extra ARM |
| * instructions do not add (many) extra cycles, but improve prefetch efficiency) |
| * |
| * Note: some types of function can't support advanced prefetch and fallback |
| * to simple one (those which handle 24bpp pixels) |
| */ |
| .set PREFETCH_TYPE_DEFAULT, PREFETCH_TYPE_ADVANCED |
| |
| /* Prefetch distance in pixels for simple prefetch */ |
| .set PREFETCH_DISTANCE_SIMPLE, 64 |
| |
| /******************************************************************************/ |
| |
| /* We can actually do significantly better than the Pixman macros, at least for |
| * the case of fills, by using a carefully scheduled inner loop. Cortex-A53 |
| * shows an improvement of up to 78% in ideal cases (large fills to L1 cache). |
| */ |
| |
| .macro generate_fillrect_function name, bpp, log2Bpp |
| /* |
| * void name(int32_t w, int32_t h, uint8_t *dst, int32_t dst_stride, uint8_t src); |
| * On entry: |
| * a1 = width, pixels |
| * a2 = height, rows |
| * a3 = pointer to top-left destination pixel |
| * a4 = stride, pixels |
| * [sp] = pixel value to fill with |
| * Within the function: |
| * v1 = width remaining |
| * v2 = vst offset |
| * v3 = alternate pointer |
| * ip = data ARM register |
| */ |
| pixman_asm_function name |
| vld1.\bpp {d0[],d1[]}, [sp] |
| sub a4, a1 |
| vld1.\bpp {d2[],d3[]}, [sp] |
| cmp a1, #(15+64) >> \log2Bpp |
| push {v1-v3,lr} |
| vmov ip, s0 |
| blo 51f |
| |
| /* Long-row case */ |
| mov v2, #64 |
| 1: mov v1, a1 |
| ands v3, a3, #15 |
| beq 2f |
| /* Leading pixels */ |
| rsb v3, v3, #16 /* number of leading bytes until 16-byte aligned */ |
| sub v1, v1, v3, lsr #\log2Bpp |
| rbit v3, v3 |
| .if bpp <= 16 |
| .if bpp == 8 |
| tst a3, #1 /* bit 0 unaffected by rsb so can avoid register interlock */ |
| strneb ip, [a3], #1 |
| tst v3, #1<<30 |
| .else |
| tst a3, #2 /* bit 1 unaffected by rsb (assuming halfword alignment) so can avoid register interlock */ |
| .endif |
| strneh ip, [a3], #2 |
| .endif |
| movs v3, v3, lsl #3 |
| vstmcs a3!, {s0} |
| vstmmi a3!, {d0} |
| 2: sub v1, v1, #64 >> \log2Bpp /* simplifies inner loop termination */ |
| add v3, a3, #32 |
| /* Inner loop */ |
| 3: vst1.\bpp {q0-q1}, [a3 :128], v2 |
| subs v1, v1, #64 >> \log2Bpp |
| vst1.\bpp {q0-q1}, [v3 :128], v2 |
| bhs 3b |
| /* Trailing pixels */ |
| 4: movs v1, v1, lsl #27 + \log2Bpp |
| bcc 5f |
| vst1.\bpp {q0-q1}, [a3 :128]! |
| 5: bpl 6f |
| vst1.\bpp {q0}, [a3 :128]! |
| 6: movs v1, v1, lsl #2 |
| vstmcs a3!, {d0} |
| vstmmi a3!, {s0} |
| .if bpp <= 16 |
| movs v1, v1, lsl #2 |
| strcsh ip, [a3], #2 |
| .if bpp == 8 |
| strmib ip, [a3], #1 |
| .endif |
| .endif |
| subs a2, a2, #1 |
| add a3, a3, a4, lsl #\log2Bpp |
| bhi 1b |
| pop {v1-v3,pc} |
| |
| /* Short-row case */ |
| 51: movs v1, a1 |
| .if bpp == 8 |
| tst a3, #3 |
| beq 53f |
| 52: subs v1, v1, #1 |
| blo 57f |
| strb ip, [a3], #1 |
| tst a3, #3 |
| bne 52b |
| .elseif bpp == 16 |
| tstne a3, #2 |
| subne v1, v1, #1 |
| strneh ip, [a3], #2 |
| .endif |
| 53: cmp v1, #32 >> \log2Bpp |
| bcc 54f |
| vst1.\bpp {q0-q1}, [a3]! |
| sub v1, v1, #32 >> \log2Bpp |
| /* Trailing pixels */ |
| 54: movs v1, v1, lsl #27 + \log2Bpp |
| bcc 55f |
| vst1.\bpp {q0-q1}, [a3]! |
| 55: bpl 56f |
| vst1.\bpp {q0}, [a3]! |
| 56: movs v1, v1, lsl #2 |
| vstmcs a3!, {d0} |
| vstmmi a3!, {s0} |
| .if bpp <= 16 |
| movs v1, v1, lsl #2 |
| strcsh ip, [a3], #2 |
| .if bpp == 8 |
| strmib ip, [a3], #1 |
| .endif |
| .endif |
| subs a2, a2, #1 |
| add a3, a3, a4, lsl #\log2Bpp |
| bhi 51b |
| 57: pop {v1-v3,pc} |
| |
| .endfunc |
| .endm |
| |
| generate_fillrect_function FillRect32ARMNEONAsm, 32, 2 |
| generate_fillrect_function FillRect16ARMNEONAsm, 16, 1 |
| generate_fillrect_function FillRect8ARMNEONAsm, 8, 0 |
| |
| /******************************************************************************/ |
| |
| .macro RGBtoRGBPixelAlpha_process_pixblock_head |
| vmvn d30, d3 /* get inverted source alpha */ |
| vmov d31, d7 /* dest alpha is always unchanged */ |
| vmull.u8 q14, d0, d3 |
| vmlal.u8 q14, d4, d30 |
| vmull.u8 q0, d1, d3 |
| vmlal.u8 q0, d5, d30 |
| vmull.u8 q1, d2, d3 |
| vmlal.u8 q1, d6, d30 |
| vrshr.u16 q2, q14, #8 |
| vrshr.u16 q3, q0, #8 |
| vraddhn.u16 d28, q14, q2 |
| vrshr.u16 q2, q1, #8 |
| vraddhn.u16 d29, q0, q3 |
| vraddhn.u16 d30, q1, q2 |
| .endm |
| |
| .macro RGBtoRGBPixelAlpha_process_pixblock_tail |
| /* nothing */ |
| .endm |
| |
| .macro RGBtoRGBPixelAlpha_process_pixblock_tail_head |
| vld4.8 {d0-d3}, [SRC]! |
| PF add PF_X, PF_X, #8 |
| vst4.8 {d28-d31}, [DST_W :128]! |
| PF tst PF_CTL, #0xF |
| vld4.8 {d4-d7}, [DST_R :128]! |
| PF addne PF_X, PF_X, #8 |
| vmvn d30, d3 /* get inverted source alpha */ |
| vmov d31, d7 /* dest alpha is always unchanged */ |
| vmull.u8 q14, d0, d3 |
| PF subne PF_CTL, PF_CTL, #1 |
| vmlal.u8 q14, d4, d30 |
| PF cmp PF_X, ORIG_W |
| vmull.u8 q0, d1, d3 |
| PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift] |
| vmlal.u8 q0, d5, d30 |
| PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift] |
| vmull.u8 q1, d2, d3 |
| PF subge PF_X, PF_X, ORIG_W |
| vmlal.u8 q1, d6, d30 |
| PF subges PF_CTL, PF_CTL, #0x10 |
| vrshr.u16 q2, q14, #8 |
| PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]! |
| vrshr.u16 q3, q0, #8 |
| PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]! |
| vraddhn.u16 d28, q14, q2 |
| vrshr.u16 q2, q1, #8 |
| vraddhn.u16 d29, q0, q3 |
| vraddhn.u16 d30, q1, q2 |
| .endm |
| |
| generate_composite_function \ |
| BlitRGBtoRGBPixelAlphaARMNEONAsm, 32, 0, 32, \ |
| FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ |
| 8, /* number of pixels, processed in a single block */ \ |
| 5, /* prefetch distance */ \ |
| default_init, \ |
| default_cleanup, \ |
| RGBtoRGBPixelAlpha_process_pixblock_head, \ |
| RGBtoRGBPixelAlpha_process_pixblock_tail, \ |
| RGBtoRGBPixelAlpha_process_pixblock_tail_head |
| |
| /******************************************************************************/ |
| |
| .macro ARGBto565PixelAlpha_process_pixblock_head |
| vmvn d6, d3 |
| vshr.u8 d1, #2 |
| vshr.u8 d3, #3 |
| vshr.u8 d0, #3 |
| vshrn.u16 d7, q2, #3 |
| vshrn.u16 d25, q2, #8 |
| vbic.i16 q2, #0xe0 |
| vshr.u8 d6, #3 |
| vshr.u8 d7, #2 |
| vshr.u8 d2, #3 |
| vmovn.u16 d24, q2 |
| vshr.u8 d25, #3 |
| vmull.u8 q13, d1, d3 |
| vmlal.u8 q13, d7, d6 |
| vmull.u8 q14, d0, d3 |
| vmlal.u8 q14, d24, d6 |
| vmull.u8 q15, d2, d3 |
| vmlal.u8 q15, d25, d6 |
| .endm |
| |
| .macro ARGBto565PixelAlpha_process_pixblock_tail |
| vsra.u16 q13, #5 |
| vsra.u16 q14, #5 |
| vsra.u16 q15, #5 |
| vrshr.u16 q13, #5 |
| vrshr.u16 q14, #5 |
| vrshr.u16 q15, #5 |
| vsli.u16 q14, q13, #5 |
| vsli.u16 q14, q15, #11 |
| .endm |
| |
| .macro ARGBto565PixelAlpha_process_pixblock_tail_head |
| vld4.8 {d0-d3}, [SRC]! |
| PF add PF_X, PF_X, #8 |
| vsra.u16 q13, #5 |
| PF tst PF_CTL, #0xF |
| vsra.u16 q14, #5 |
| PF addne PF_X, PF_X, #8 |
| vsra.u16 q15, #5 |
| PF subne PF_CTL, PF_CTL, #1 |
| vrshr.u16 q13, #5 |
| PF cmp PF_X, ORIG_W |
| vrshr.u16 q14, #5 |
| PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift] |
| vrshr.u16 q15, #5 |
| PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift] |
| vld1.8 {d4-d5}, [DST_R]! |
| PF subge PF_X, PF_X, ORIG_W |
| vsli.u16 q14, q13, #5 |
| PF subges PF_CTL, PF_CTL, #0x10 |
| vsli.u16 q14, q15, #11 |
| PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]! |
| vst1.8 {q14}, [DST_W :128]! |
| vmvn d6, d3 |
| vshr.u8 d1, #2 |
| vshr.u8 d3, #3 |
| vshr.u8 d0, #3 |
| vshrn.u16 d7, q2, #3 |
| vshrn.u16 d25, q2, #8 |
| vbic.i16 q2, #0xe0 |
| PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]! |
| vshr.u8 d6, #3 |
| vshr.u8 d7, #2 |
| vshr.u8 d2, #3 |
| vmovn.u16 d24, q2 |
| vshr.u8 d25, #3 |
| vmull.u8 q13, d1, d3 |
| vmlal.u8 q13, d7, d6 |
| vmull.u8 q14, d0, d3 |
| vmlal.u8 q14, d24, d6 |
| vmull.u8 q15, d2, d3 |
| vmlal.u8 q15, d25, d6 |
| .endm |
| |
| generate_composite_function \ |
| BlitARGBto565PixelAlphaARMNEONAsm, 32, 0, 16, \ |
| FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ |
| 8, /* number of pixels, processed in a single block */ \ |
| 6, /* prefetch distance */ \ |
| default_init, \ |
| default_cleanup, \ |
| ARGBto565PixelAlpha_process_pixblock_head, \ |
| ARGBto565PixelAlpha_process_pixblock_tail, \ |
| ARGBto565PixelAlpha_process_pixblock_tail_head |
