| |
| /* pnggccrd.c - mixed C/assembler version of utilities to read a PNG file |
| * |
| * For Intel x86 CPU (Pentium-MMX or later) and GNU C compiler. |
| * |
| * Last changed in libpng 1.2.19 June 20, 2007 |
| * For conditions of distribution and use, see copyright notice in png.h |
| * Copyright (c) 1998 Intel Corporation |
| * Copyright (c) 1999-2002,2007 Greg Roelofs |
| * Copyright (c) 1998-2007 Glenn Randers-Pehrson |
| * |
| * Based on MSVC code contributed by Nirav Chhatrapati, Intel Corp., 1998. |
| * Interface to libpng contributed by Gilles Vollant, 1999. |
| * GNU C port by Greg Roelofs, 1999-2001. |
| * |
| * References: |
| * |
| * http://www.intel.com/drg/pentiumII/appnotes/916/916.htm |
| * http://www.intel.com/drg/pentiumII/appnotes/923/923.htm |
| * [Intel's performance analysis of the MMX vs. non-MMX code; |
| * moved/deleted as of 2006, but text and some graphs still |
| * available via WayBack Machine at archive.org] |
| * |
| * http://www.ibiblio.org/gferg/ldp/GCC-Inline-Assembly-HOWTO.html |
| * http://sam.zoy.org/blog/2007-04-13-shlib-with-non-pic-code-have-inline-assembly-and-pic-mix-well |
| * http://gcc.gnu.org/onlinedocs/gcc/Variable-Attributes.html |
| * http://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html |
| * AMD x86-64 Architecture Programmer's Manual, volumes 1 and 5 |
| * |
| * png_read_filter_row_mmx_*() were converted in place with intel2gas 1.3.1: |
| * |
| * intel2gas -mdI pnggccrd.c.partially-msvc -o pnggccrd.c |
| * |
| * and then cleaned up by hand. See http://hermes.terminal.at/intel2gas/ . |
| * |
| * NOTE: A sufficiently recent version of GNU as (or as.exe under DOS/Windows) |
| * is required to assemble the newer asm instructions such as movq. (Version |
| * 2.5.2l.15 is definitely too old.) See ftp://ftp.gnu.org/pub/gnu/binutils/ . |
| */ |
| |
| /* |
| * PORTING NOTES AND CHANGELOG (mostly by Greg Roelofs) |
| * =========================== |
| * |
| * 19991006: |
| * - fixed sign error in post-MMX cleanup code (16- & 32-bit cases) |
| * |
| * 19991007: |
| * - additional optimizations (possible or definite): |
| * x [DONE] write MMX code for 64-bit case (pixel_bytes == 8) [not tested] |
| * - write MMX code for 48-bit case (pixel_bytes == 6) |
| * - figure out what's up with 24-bit case (pixel_bytes == 3): |
| * why subtract 8 from width_mmx in the pass 4/5 case? |
| * (only width_mmx case) (near line 1606) |
| * x [DONE] replace pixel_bytes within each block with the true |
| * constant value (or are compilers smart enough to do that?) |
| * - rewrite all MMX interlacing code so it's aligned with |
| * the *beginning* of the row buffer, not the end. This |
| * would not only allow one to eliminate half of the memory |
| * writes for odd passes (that is, pass == odd), it may also |
| * eliminate some unaligned-data-access exceptions (assuming |
| * there's a penalty for not aligning 64-bit accesses on |
| * 64-bit boundaries). The only catch is that the "leftover" |
| * pixel(s) at the end of the row would have to be saved, |
| * but there are enough unused MMX registers in every case, |
| * so this is not a problem. A further benefit is that the |
| * post-MMX cleanup code (C code) in at least some of the |
| * cases could be done within the assembler block. |
| * x [DONE] the "v3 v2 v1 v0 v7 v6 v5 v4" comments are confusing, |
| * inconsistent, and don't match the MMX Programmer's Reference |
| * Manual conventions anyway. They should be changed to |
| * "b7 b6 b5 b4 b3 b2 b1 b0," where b0 indicates the byte that |
| * was lowest in memory (e.g., corresponding to a left pixel) |
| * and b7 is the byte that was highest (e.g., a right pixel). |
| * |
| * 19991016: |
| * - Brennan's Guide notwithstanding, gcc under Linux does *not* |
| * want globals prefixed by underscores when referencing them-- |
| * i.e., if the variable is const4, then refer to it as const4, |
| * not _const4. This seems to be a djgpp-specific requirement. |
| * Also, such variables apparently *must* be declared outside |
| * of functions; neither static nor automatic variables work if |
| * defined within the scope of a single function, but both |
| * static and truly global (multi-module) variables work fine. |
| * |
| * 19991023: |
| * - fixed png_combine_row() non-MMX replication bug (odd passes only?) |
| * - switched from string-concatenation-with-macros to cleaner method of |
| * renaming global variables for djgpp--i.e., always use prefixes in |
| * inlined assembler code (== strings) and conditionally rename the |
| * variables, not the other way around. Hence _const4, _mask8_0, etc. |
| * |
| * 19991024: |
| * - fixed mmxsupport()/png_do_read_interlace() first-row bug |
| * This one was severely weird: even though mmxsupport() doesn't touch |
| * ebx (where "row" pointer was stored), it nevertheless managed to zero |
| * the register (even in static/non-fPIC code--see below), which in turn |
| * caused png_do_read_interlace() to return prematurely on the first row of |
| * interlaced images (i.e., without expanding the interlaced pixels). |
| * Inspection of the generated assembly code didn't turn up any clues, |
| * although it did point at a minor optimization (i.e., get rid of |
| * mmx_supported_local variable and just use eax). Possibly the CPUID |
| * instruction is more destructive than it looks? (Not yet checked.) |
| * - "info gcc" was next to useless, so compared fPIC and non-fPIC assembly |
| * listings... Apparently register spillage has to do with ebx, since |
| * it's used to index the global offset table. Commenting it out of the |
| * input-reg lists in png_combine_row() eliminated compiler barfage, so |
| * ifdef'd with __PIC__ macro: if defined, use a global for unmask |
| * |
| * 19991107: |
| * - verified CPUID clobberage: 12-char string constant ("GenuineIntel", |
| * "AuthenticAMD", etc.) placed in ebx:ecx:edx. Still need to polish. |
| * |
| * 19991120: |
| * - made "diff" variable (now "_dif") global to simplify conversion of |
| * filtering routines (running out of regs, sigh). "diff" is still used |
| * in interlacing routines, however. |
| * - fixed up both versions of mmxsupport() (ORIG_THAT_USED_TO_CLOBBER_EBX |
| * macro determines which is used); original not yet tested. |
| * |
| * 20000213: |
| * - when compiling with gcc, be sure to use -fomit-frame-pointer |
| * |
| * 20000319: |
| * - fixed a register-name typo in png_do_read_interlace(), default (MMX) case, |
| * pass == 4 or 5, that caused visible corruption of interlaced images |
| * |
| * 20000623: |
| * - Various problems were reported with gcc 2.95.2 in the Cygwin environment, |
| * many of the form "forbidden register 0 (ax) was spilled for class AREG." |
| * This is explained at http://gcc.gnu.org/fom_serv/cache/23.html, and |
| * Chuck Wilson supplied a patch involving dummy output registers. See |
| * http://sourceforge.net/bugs/?func=detailbug&bug_id=108741&group_id=5624 |
| * for the original (anonymous) SourceForge bug report. |
| * |
| * 20000706: |
| * - Chuck Wilson passed along these remaining gcc 2.95.2 errors: |
| * pnggccrd.c: In function `png_combine_row': |
| * pnggccrd.c:525: more than 10 operands in `asm' |
| * pnggccrd.c:669: more than 10 operands in `asm' |
| * pnggccrd.c:828: more than 10 operands in `asm' |
| * pnggccrd.c:994: more than 10 operands in `asm' |
| * pnggccrd.c:1177: more than 10 operands in `asm' |
| * They are all the same problem and can be worked around by using the |
| * global _unmask variable unconditionally, not just in the -fPIC case. |
| * Reportedly earlier versions of gcc also have the problem with more than |
| * 10 operands; they just don't report it. Much strangeness ensues, etc. |
| * |
| * 20000729: |
| * - enabled png_read_filter_row_mmx_up() (shortest remaining unconverted |
| * MMX routine); began converting png_read_filter_row_mmx_sub() |
| * - to finish remaining sections: |
| * - clean up indentation and comments |
| * - preload local variables |
| * - add output and input regs (order of former determines numerical |
| * mapping of latter) |
| * - avoid all usage of ebx (including bx, bh, bl) register [20000823] |
| * - remove "$" from addressing of Shift and Mask variables [20000823] |
| * |
| * 20000731: |
| * - global union vars causing segfaults in png_read_filter_row_mmx_sub()? |
| * |
| * 20000822: |
| * - ARGH, stupid png_read_filter_row_mmx_sub() segfault only happens with |
| * shared-library (-fPIC) version! Code works just fine as part of static |
| * library. Damn damn damn damn damn, should have tested that sooner. |
| * ebx is getting clobbered again (explicitly this time); need to save it |
| * on stack or rewrite asm code to avoid using it altogether. Blargh! |
| * |
| * 20000823: |
| * - first section was trickiest; all remaining sections have ebx -> edx now. |
| * (-fPIC works again.) Also added missing underscores to various Shift* |
| * and *Mask* globals and got rid of leading "$" signs. |
| * |
| * 20000826: |
| * - added visual separators to help navigate microscopic printed copies |
| * (http://pobox.com/~newt/code/gpr-latest.zip, mode 10); started working |
| * on png_read_filter_row_mmx_avg() |
| * |
| * 20000828: |
| * - finished png_read_filter_row_mmx_avg(): only Paeth left! (930 lines...) |
| * What the hell, did png_read_filter_row_mmx_paeth(), too. Comments not |
| * cleaned up/shortened in either routine, but functionality is complete |
| * and seems to be working fine. |
| * |
| * 20000829: |
| * - ahhh, figured out last(?) bit of gcc/gas asm-fu: if register is listed |
| * as an input reg (with dummy output variables, etc.), then it *cannot* |
| * also appear in the clobber list or gcc 2.95.2 will barf. The solution |
| * is simple enough... |
| * |
| * 20000914: |
| * - bug in png_read_filter_row_mmx_avg(): 16-bit grayscale not handled |
| * correctly (but 48-bit RGB just fine) |
| * |
| * 20000916: |
| * - fixed bug in png_read_filter_row_mmx_avg(), bpp == 2 case; three errors: |
| * - "_ShiftBpp.use = 24;" should have been "_ShiftBpp.use = 16;" |
| * - "_ShiftRem.use = 40;" should have been "_ShiftRem.use = 48;" |
| * - "psllq _ShiftRem, %%mm2" should have been "psrlq _ShiftRem, %%mm2" |
| * |
| * 20010101: |
| * - added new png_init_mmx_flags() function (here only because it needs to |
| * call mmxsupport(), which should probably become global png_mmxsupport()); |
| * modified other MMX routines to run conditionally (png_ptr->asm_flags) |
| * |
| * 20010103: |
| * - renamed mmxsupport() to png_mmx_support(), with auto-set of mmx_supported, |
| * and made it public; moved png_init_mmx_flags() to png.c as internal func |
| * |
| * 20010104: |
| * - removed dependency on png_read_filter_row_c() (C code already duplicated |
| * within MMX version of png_read_filter_row()) so no longer necessary to |
| * compile it into pngrutil.o |
| * |
| * 20010310: |
| * - fixed buffer-overrun bug in png_combine_row() C code (non-MMX) |
| * |
| * 20020304: |
| * - eliminated incorrect use of width_mmx in pixel_bytes == 8 case |
| * |
| * 20040724: |
| * - more tinkering with clobber list at lines 4529 and 5033 to get it to |
| * compile with gcc 3.4 |
| * |
| * 20070313: |
| * - finally applied Giuseppe Ghibò's 64-bit patch of 20060803 (completely |
| * overlooked Dylan Alex Simon's similar patch of 20060414, oops...) |
| * |
| * 20070524: |
| * - fixed link failure caused by asm-only variables being optimized out |
| * (identified by Dimitri of Trolltech) with __attribute__((used)), which |
| * also gets rid of warnings => nuked ugly png_squelch_warnings() hack |
| * - dropped redundant ifdef |
| * - moved png_mmx_support() back up where originally intended (as in |
| * pngvcrd.c), using __attribute__((noinline)) in extra prototype |
| * |
| * 20070527: |
| * - revised png_combine_row() to reuse mask in lieu of external _unmask |
| * - moved 32-bit (RGBA) case to top of png_combine_row(): most common |
| * - just about ready to give up on x86_64 -fPIC mode; can't even access 16 |
| * _mask*_* constants without triggering link error on shared library: |
| * /usr/bin/ld: pnggccrd.pic.o: relocation R_X86_64_32S against `a local |
| * symbol' can not be used when making a shared object; recompile with |
| * -fPIC |
| * pnggccrd.pic.o: could not read symbols: Bad value |
| * ("objdump -x pnggccrd.pic.o | grep rodata" to verify) |
| * [might be able to work around by doing within assembly code whatever |
| * -fPIC does, but given problems to date, seems like long shot...] |
| * [relevant ifdefs: __x86_64__ && __PIC__ => C code only] |
| * - changed #if 0 to #ifdef PNG_CLOBBER_MMX_REGS_SUPPORTED in case gcc ever |
| * supports MMX regs (%mm0, etc.) in clobber list (not supported by gcc |
| * 2.7.2.3, 2.91.66 (egcs 1.1.2), 3.x, or 4.1.2) |
| * |
| * 20070603: |
| * - revised png_combine_row() to use @GOTPCREL(%%rip) addressing on _c64 |
| * struct of _mask*_* constants for x86_64 -fPIC; see sam.zoy.org link |
| * above for details |
| * - moved _const4 and _const6 into _c64 struct, renamed to _amask5_3_0 and |
| * _amask7_1_0, respectively |
| * - can't figure out how to use _c64._mask*_* vars within asm code, so still |
| * need single variables for non-x86_64/-fPIC half :-( |
| * - replaced various __PIC__ ifdefs with *_GOT_ebx macros |
| * - moved _LBCarryMask and _HBClearMask into _c64 struct |
| * - conditionally replaced _p*temp variables with %r11d-%r13d (via p*_TEMP |
| * and CLOBBER_r1*d macros) |
| * |
| * 20070604: |
| * - replaced all _ActiveMask and _ActiveMaskEnd with new _amask*_*_* consts |
| * (_amask naming convention: numbers of 00-bytes, ff-bytes, 00-bytes) |
| * - _ActiveMask // (10) // avg/paeth/sub; read-only; consts; movq/pand |
| * 0x0000000000ffffffLL (bpp 3, avg) _amask5_3_0 |
| * 0xffffffffffffffffLL (bpp 4, 6, avg) _amask0_8_0 |
| * 0x000000000000ffffLL (bpp 2, avg) _amask6_2_0 |
| * 0x0000000000ffffffLL (bpp 3, paeth) _amask5_3_0 |
| * 0x00000000ffffffffLL (bpp 6, paeth) _amask4_4_0 |
| * 0x00000000ffffffffLL (bpp 4, paeth) _amask4_4_0 |
| * 0x00000000ffffffffLL (bpp 8, paeth) _amask4_4_0 |
| * 0x0000ffffff000000LL (bpp 3, sub) _amask2_3_3 |
| * 0x00000000ffff0000LL (bpp 2, sub) _amask4_2_2 |
| * - _ActiveMaskEnd // (1) // paeth only; read-only; const; pand |
| * 0xffff000000000000LL (bpp 3, paeth) _amask0_2_6 |
| * - changed all "#if defined(__x86_64__) // later // && defined(__PIC__)" |
| * lines to "#ifdef PNG_x86_64_USE_GOTPCREL" for easier/safer testing |
| * |
| * 20070605: |
| * - merged PNG_x86_64_USE_GOTPCREL, non-PNG_x86_64_USE_GOTPCREL code via |
| * *MASK* and LOAD/RESTORE macros |
| * |
| * 20070607: |
| * - replaced all constant instances of _ShiftBpp, _ShiftRem with immediates |
| * (still have two shared cases in avg, sub routines) |
| * |
| * 20070609: |
| * - replaced remaining instances of _ShiftBpp, _ShiftRem with immediates |
| * (split sub and avg 4/6-bpp cases into separate blocks) |
| * - fixed paeth bug due to clobbered r11/r12/r13 regs |
| * |
| * 20070610: |
| * - made global "_dif" variable (avg/paeth/sub routines) local again (now |
| * "diff"--see 19991120 entry above), using register constraints |
| * - note that %ebp in clobber list doesn't actually work, at least for 32-bit |
| * version and gcc 4.1.2; must save and restore manually. (Seems to work |
| * OK for 64-bit version and gcc 3.4.3, but gcc may not be using ebp/rbp |
| * in that case.) |
| * - started replacing direct _MMXLength accesses with register constraints |
| * |
| * 20070612: |
| * - continued replacing direct _MMXLength accesses with register constraints |
| * |
| * 20070613: |
| * - finished replacing direct _MMXLength accesses with register constraints; |
| * switched to local variable (and renamed back to MMXLength) |
| * |
| * 20070614: |
| * - fixed sub bpp = 1 bug |
| * - started replacing direct _FullLength accesses with register constraints |
| * |
| * 20070615: |
| * - fixed 64-bit paeth bpp 3 crash bug (misplaced LOAD_GOT_rbp) |
| * - fixed 64-bit paeth bpp 1/2 and cleanup-block crash bugs (misplaced |
| * RESTORE_r11_r12_r13) |
| * - slightly optimized avg/paeth cleanup blocks and paeth bpp 1/2 block |
| * (save/restore ebx only if needed) |
| * - continued replacing direct _FullLength accesses with register constraints |
| * |
| * 20070616: |
| * - finished replacing direct _FullLength accesses with register constraints |
| * (*ugly* conditional clobber-separator macros for avg and paeth, sigh) |
| * Changed all "ifdef *" to "if defined(*)" |
| * |
| * |
| * STILL TO DO: |
| * - fix final thread-unsafe code using stack vars and pointer? (paeth top, |
| * default, bottom only; default, bottom already 5 reg constraints; could |
| * replace bpp with pointer and group bpp/patemp/pbtemp/pctemp in array) |
| * - fix ebp/no-reg-constraint inefficiency (avg/paeth/sub top) |
| * - test png_do_read_interlace() 64-bit case (pixel_bytes == 8) |
| * - write MMX code for 48-bit case (pixel_bytes == 6) |
| * - figure out what's up with 24-bit case (pixel_bytes == 3): |
| * why subtract 8 from width_mmx in the pass 4/5 case? due to |
| * odd number of bytes? (only width_mmx case) (near line 1606) |
| * - rewrite all MMX interlacing code so it's aligned with beginning |
| * of the row buffer, not the end (see 19991007 for details) |
| * - add error messages to any remaining bogus default cases |
| * - enable pixel_depth == 8 cases in png_read_filter_row()? (test speed) |
| * - try =r, etc., as reg constraints? (would gcc use 64-bit ones on x86_64?) |
| * - need full, non-graphical, CRC-based test suite... maybe autogenerate |
| * random data of various height/width/depth, compute CRCs, write (C |
| * funcs), read (asm/MMX), recompute CRCs, and compare? |
| * - write true x86_64 version using 128-bit "media instructions", %xmm0-15, |
| * and extra general-purpose registers |
| */ |
| |
| #define PNG_INTERNAL |
| #include "png.h" |
| |
| #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_USE_PNGGCCRD) |
| |
| #if defined(__x86_64__) && defined(__PIC__) // optionally comment out __PIC__: |
| # define PNG_x86_64_USE_GOTPCREL // GOTPCREL => full thread-safety |
| # define PNG_CLOBBER_x86_64_REGS_SUPPORTED // works as of gcc 3.4.3 ... |
| #endif |
| |
| int PNGAPI png_mmx_support(void); |
| |
| #if defined(PNG_USE_LOCAL_ARRAYS) |
| static PNG_CONST int FARDATA png_pass_start[7] = {0, 4, 0, 2, 0, 1, 0}; |
| static PNG_CONST int FARDATA png_pass_inc[7] = {8, 8, 4, 4, 2, 2, 1}; |
| static PNG_CONST int FARDATA png_pass_width[7] = {8, 4, 4, 2, 2, 1, 1}; |
| #endif |
| |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| |
| /* djgpp, Win32, Cygwin, and OS2 add their own underscores to global variables, |
| * so define them without: */ |
| #if defined(__DJGPP__) || defined(WIN32) || defined(__CYGWIN__) || \ |
| defined(__OS2__) |
| # define _mmx_supported mmx_supported |
| # define _mask8_0 mask8_0 |
| # define _mask16_1 mask16_1 |
| # define _mask16_0 mask16_0 |
| # define _mask24_2 mask24_2 |
| # define _mask24_1 mask24_1 |
| # define _mask24_0 mask24_0 |
| # define _mask32_3 mask32_3 |
| # define _mask32_2 mask32_2 |
| # define _mask32_1 mask32_1 |
| # define _mask32_0 mask32_0 |
| # define _mask48_5 mask48_5 |
| # define _mask48_4 mask48_4 |
| # define _mask48_3 mask48_3 |
| # define _mask48_2 mask48_2 |
| # define _mask48_1 mask48_1 |
| # define _mask48_0 mask48_0 |
| # define _amask5_3_0 amask5_3_0 |
| # define _amask7_1_0 amask7_1_0 |
| # define _LBCarryMask LBCarryMask |
| # define _HBClearMask HBClearMask |
| # define _amask0_8_0 amask0_8_0 |
| # define _amask6_2_0 amask6_2_0 |
| # define _amask4_4_0 amask4_4_0 |
| # define _amask0_2_6 amask0_2_6 |
| # define _amask2_3_3 amask2_3_3 |
| # define _amask4_2_2 amask4_2_2 |
| # if defined(PNG_THREAD_UNSAFE_OK) |
| # define _patemp patemp |
| # define _pbtemp pbtemp |
| # define _pctemp pctemp |
| # endif |
| #endif // djgpp, Win32, Cygwin, OS2 |
| |
| |
| /* These constants are used in the inlined MMX assembly code. */ |
| |
| typedef unsigned long long ull; |
| |
| #if defined(PNG_x86_64_USE_GOTPCREL) |
| static PNG_CONST struct { |
| //ull _mask_array[26]; |
| |
| // png_combine_row() constants: |
| ull _mask8_0; |
| ull _mask16_0, _mask16_1; |
| ull _mask24_0, _mask24_1, _mask24_2; |
| ull _mask32_0, _mask32_1, _mask32_2, _mask32_3; |
| ull _mask48_0, _mask48_1, _mask48_2, _mask48_3, _mask48_4, _mask48_5; |
| |
| // png_do_read_interlace() constants: |
| ull _amask5_3_0, _amask7_1_0; // was _const4 and _const6, respectively |
| |
| // png_read_filter_row_mmx_avg() constants (also uses _amask5_3_0): |
| ull _LBCarryMask, _HBClearMask; |
| ull _amask0_8_0, _amask6_2_0; // was ActiveMask for bpp 4/6 and 2 cases |
| |
| // png_read_filter_row_mmx_paeth() constants (also uses _amask5_3_0): |
| ull _amask4_4_0, _amask0_2_6; // was ActiveMask{,End} for bpp 6/4/8 and 3 |
| |
| // png_read_filter_row_mmx_sub() constants: |
| ull _amask2_3_3, _amask4_2_2; // was ActiveMask for bpp 3 and 2 cases |
| |
| } _c64 __attribute__((used, aligned(8))) = { |
| |
| // png_combine_row() constants: |
| 0x0102040810204080LL, // _mask8_0 offset 0 |
| |
| 0x1010202040408080LL, // _mask16_0 offset 8 |
| 0x0101020204040808LL, // _mask16_1 offset 16 |
| |
| 0x2020404040808080LL, // _mask24_0 offset 24 |
| 0x0408080810101020LL, // _mask24_1 offset 32 |
| 0x0101010202020404LL, // _mask24_2 offset 40 |
| |
| 0x4040404080808080LL, // _mask32_0 offset 48 |
| 0x1010101020202020LL, // _mask32_1 offset 56 |
| 0x0404040408080808LL, // _mask32_2 offset 64 |
| 0x0101010102020202LL, // _mask32_3 offset 72 |
| |
| 0x4040808080808080LL, // _mask48_0 offset 80 |
| 0x2020202040404040LL, // _mask48_1 offset 88 |
| 0x1010101010102020LL, // _mask48_2 offset 96 |
| 0x0404080808080808LL, // _mask48_3 offset 104 |
| 0x0202020204040404LL, // _mask48_4 offset 112 |
| 0x0101010101010202LL, // _mask48_5 offset 120 |
| |
| // png_do_read_interlace() constants: |
| 0x0000000000FFFFFFLL, // _amask5_3_0 offset 128 (bpp 3, avg/paeth) const4 |
| 0x00000000000000FFLL, // _amask7_1_0 offset 136 const6 |
| |
| // png_read_filter_row_mmx_avg() constants: |
| 0x0101010101010101LL, // _LBCarryMask offset 144 |
| 0x7F7F7F7F7F7F7F7FLL, // _HBClearMask offset 152 |
| 0xFFFFFFFFFFFFFFFFLL, // _amask0_8_0 offset 160 (bpp 4/6, avg) |
| 0x000000000000FFFFLL, // _amask6_2_0 offset 168 (bpp 2, avg) |
| |
| // png_read_filter_row_mmx_paeth() constants: |
| 0x00000000FFFFFFFFLL, // _amask4_4_0 offset 176 (bpp 6/4/8, paeth) |
| 0xFFFF000000000000LL, // _amask0_2_6 offset 184 (bpp 3, paeth) A.M.End |
| |
| // png_read_filter_row_mmx_sub() constants: |
| 0x0000FFFFFF000000LL, // _amask2_3_3 offset 192 (bpp 3, sub) |
| 0x00000000FFFF0000LL, // _amask4_2_2 offset 200 (bpp 2, sub) |
| |
| }; |
| |
| #define MASK8_0 "(%%rbp)" |
| #define MASK16_0 "8(%%rbp)" |
| #define MASK16_1 "16(%%rbp)" |
| #define MASK24_0 "24(%%rbp)" |
| #define MASK24_1 "32(%%rbp)" |
| #define MASK24_2 "40(%%rbp)" |
| #define MASK32_0 "48(%%rbp)" |
| #define MASK32_1 "56(%%rbp)" |
| #define MASK32_2 "64(%%rbp)" |
| #define MASK32_3 "72(%%rbp)" |
| #define MASK48_0 "80(%%rbp)" |
| #define MASK48_1 "88(%%rbp)" |
| #define MASK48_2 "96(%%rbp)" |
| #define MASK48_3 "104(%%rbp)" |
| #define MASK48_4 "112(%%rbp)" |
| #define MASK48_5 "120(%%rbp)" |
| #define AMASK5_3_0 "128(%%rbp)" |
| #define AMASK7_1_0 "136(%%rbp)" |
| #define LB_CARRY_MASK "144(%%rbp)" |
| #define HB_CLEAR_MASK "152(%%rbp)" |
| #define AMASK0_8_0 "160(%%rbp)" |
| #define AMASK6_2_0 "168(%%rbp)" |
| #define AMASK4_4_0 "176(%%rbp)" |
| #define AMASK0_2_6 "184(%%rbp)" |
| #define AMASK2_3_3 "192(%%rbp)" |
| #define AMASK4_2_2 "200(%%rbp)" |
| |
| #else // !PNG_x86_64_USE_GOTPCREL |
| |
| static PNG_CONST ull _mask8_0 __attribute__((used, aligned(8))) = 0x0102040810204080LL; |
| |
| static PNG_CONST ull _mask16_1 __attribute__((used, aligned(8))) = 0x0101020204040808LL; |
| static PNG_CONST ull _mask16_0 __attribute__((used, aligned(8))) = 0x1010202040408080LL; |
| |
| static PNG_CONST ull _mask24_2 __attribute__((used, aligned(8))) = 0x0101010202020404LL; |
| static PNG_CONST ull _mask24_1 __attribute__((used, aligned(8))) = 0x0408080810101020LL; |
| static PNG_CONST ull _mask24_0 __attribute__((used, aligned(8))) = 0x2020404040808080LL; |
| |
| static PNG_CONST ull _mask32_3 __attribute__((used, aligned(8))) = 0x0101010102020202LL; |
| static PNG_CONST ull _mask32_2 __attribute__((used, aligned(8))) = 0x0404040408080808LL; |
| static PNG_CONST ull _mask32_1 __attribute__((used, aligned(8))) = 0x1010101020202020LL; |
| static PNG_CONST ull _mask32_0 __attribute__((used, aligned(8))) = 0x4040404080808080LL; |
| |
| static PNG_CONST ull _mask48_5 __attribute__((used, aligned(8))) = 0x0101010101010202LL; |
| static PNG_CONST ull _mask48_4 __attribute__((used, aligned(8))) = 0x0202020204040404LL; |
| static PNG_CONST ull _mask48_3 __attribute__((used, aligned(8))) = 0x0404080808080808LL; |
| static PNG_CONST ull _mask48_2 __attribute__((used, aligned(8))) = 0x1010101010102020LL; |
| static PNG_CONST ull _mask48_1 __attribute__((used, aligned(8))) = 0x2020202040404040LL; |
| static PNG_CONST ull _mask48_0 __attribute__((used, aligned(8))) = 0x4040808080808080LL; |
| |
| // png_do_read_interlace() constants: |
| static PNG_CONST ull _amask5_3_0 __attribute__((aligned(8))) = 0x0000000000FFFFFFLL; // was _const4 |
| static PNG_CONST ull _amask7_1_0 __attribute__((aligned(8))) = 0x00000000000000FFLL; // was _const6 |
| |
| // png_read_filter_row_mmx_avg() constants: |
| static PNG_CONST ull _LBCarryMask __attribute__((used, aligned(8))) = 0x0101010101010101LL; |
| static PNG_CONST ull _HBClearMask __attribute__((used, aligned(8))) = 0x7f7f7f7f7f7f7f7fLL; |
| static PNG_CONST ull _amask0_8_0 __attribute__((used, aligned(8))) = 0xFFFFFFFFFFFFFFFFLL; |
| static PNG_CONST ull _amask6_2_0 __attribute__((used, aligned(8))) = 0x000000000000FFFFLL; |
| |
| // png_read_filter_row_mmx_paeth() constants: |
| static PNG_CONST ull _amask4_4_0 __attribute__((used, aligned(8))) = 0x00000000FFFFFFFFLL; |
| static PNG_CONST ull _amask0_2_6 __attribute__((used, aligned(8))) = 0xFFFF000000000000LL; |
| |
| // png_read_filter_row_mmx_sub() constants: |
| static PNG_CONST ull _amask2_3_3 __attribute__((used, aligned(8))) = 0x0000FFFFFF000000LL; |
| static PNG_CONST ull _amask4_2_2 __attribute__((used, aligned(8))) = 0x00000000FFFF0000LL; |
| |
| #define MASK8_0 "_mask8_0" |
| #define MASK16_0 "_mask16_0" |
| #define MASK16_1 "_mask16_1" |
| #define MASK24_0 "_mask24_0" |
| #define MASK24_1 "_mask24_1" |
| #define MASK24_2 "_mask24_2" |
| #define MASK32_0 "_mask32_0" |
| #define MASK32_1 "_mask32_1" |
| #define MASK32_2 "_mask32_2" |
| #define MASK32_3 "_mask32_3" |
| #define MASK48_0 "_mask48_0" |
| #define MASK48_1 "_mask48_1" |
| #define MASK48_2 "_mask48_2" |
| #define MASK48_3 "_mask48_3" |
| #define MASK48_4 "_mask48_4" |
| #define MASK48_5 "_mask48_5" |
| #define AMASK5_3_0 "_amask5_3_0" |
| #define AMASK7_1_0 "_amask7_1_0" |
| #define LB_CARRY_MASK "_LBCarryMask" |
| #define HB_CLEAR_MASK "_HBClearMask" |
| #define AMASK0_8_0 "_amask0_8_0" |
| #define AMASK6_2_0 "_amask6_2_0" |
| #define AMASK4_4_0 "_amask4_4_0" |
| #define AMASK0_2_6 "_amask0_2_6" |
| #define AMASK2_3_3 "_amask2_3_3" |
| #define AMASK4_2_2 "_amask4_2_2" |
| |
| #endif // ?PNG_x86_64_USE_GOTPCREL |
| |
| |
| #if defined(PNG_HAVE_MMX_READ_FILTER_ROW) |
| |
| // this block is specific to png_read_filter_row_mmx_paeth() |
| #if defined(PNG_x86_64_USE_GOTPCREL) |
| # define pa_TEMP "%%r11d" |
| # define pb_TEMP "%%r12d" |
| # define pc_TEMP "%%r13d" |
| # if defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED) // works as of gcc 3.4.3 ... |
| # define SAVE_r11_r12_r13 |
| # define RESTORE_r11_r12_r13 |
| # define _CLOBBER_r11_r12_r13 ,"%r11", "%r12", "%r13" |
| # define CLOBBER_r11_r12_r13 "%r11", "%r12", "%r13" |
| # else // !PNG_CLOBBER_x86_64_REGS_SUPPORTED |
| # define SAVE_r11_r12_r13 "pushq %%r11 \n\t" \ |
| "pushq %%r12 \n\t" \ |
| "pushq %%r13 \n\t" // "normally 0-extended" |
| # define RESTORE_r11_r12_r13 "popq %%r13 \n\t" \ |
| "popq %%r12 \n\t" \ |
| "popq %%r11 \n\t" |
| # define _CLOBBER_r11_r12_r13 |
| # define CLOBBER_r11_r12_r13 |
| # endif |
| # define LOAD_GOT_rbp "pushq %%rbp \n\t" \ |
| "movq _c64@GOTPCREL(%%rip), %%rbp \n\t" |
| # define RESTORE_rbp "popq %%rbp \n\t" |
| #else // 32-bit and/or non-PIC |
| # if defined(PNG_THREAD_UNSAFE_OK) |
| // These variables are used in png_read_filter_row_mmx_paeth() and would be |
| // local variables if not for gcc-inline-assembly addressing limitations |
| // (some apparently related to ELF format, others to CPU type). |
| // |
| // WARNING: Their presence defeats the thread-safety of libpng. |
| static int _patemp __attribute__((used)); |
| static int _pbtemp __attribute__((used)); |
| static int _pctemp __attribute__((used)); |
| # define pa_TEMP "_patemp" |
| # define pb_TEMP "_pbtemp" // temp variables for |
| # define pc_TEMP "_pctemp" // Paeth routine |
| # define SAVE_r11_r12_r13 |
| # define RESTORE_r11_r12_r13 |
| # define _CLOBBER_r11_r12_r13 // not using regs => not clobbering |
| # define CLOBBER_r11_r12_r13 |
| # endif // PNG_THREAD_UNSAFE_OK |
| # define LOAD_GOT_rbp |
| # define RESTORE_rbp |
| #endif |
| |
| #if defined(__x86_64__) |
| # define SAVE_ebp |
| # define RESTORE_ebp |
| # define _CLOBBER_ebp ,"%ebp" |
| # define CLOBBER_ebp "%ebp" |
| # define SAVE_FullLength "movl %%eax, %%r15d \n\t" |
| # define RESTORE_FullLength "movl %%r15d, " // may go into eax or ecx |
| # if defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED) // works as of gcc 3.4.3 ... |
| # define SAVE_r15 |
| # define RESTORE_r15 |
| # define _CLOBBER_r15 ,"%r15" |
| # else |
| # define SAVE_r15 "pushq %%r15 \n\t" |
| # define RESTORE_r15 "popq %%r15 \n\t" |
| # define _CLOBBER_r15 |
| # endif |
| #else |
| # define SAVE_ebp "pushl %%ebp \n\t" // clobber list doesn't work |
| # define RESTORE_ebp "popl %%ebp \n\t" // for %ebp on 32-bit; not |
| # define _CLOBBER_ebp // clear why not |
| # define CLOBBER_ebp |
| # define SAVE_FullLength "pushl %%eax \n\t" |
| # define RESTORE_FullLength "popl " // eax (avg) or ecx (paeth) |
| # define SAVE_r15 |
| # define RESTORE_r15 |
| # define _CLOBBER_r15 |
| #endif |
| |
| // CLOB_COMMA_ebx_ebp: need comma ONLY if both CLOBBER_ebp and CLOBBER_GOT_ebx |
| // have values, i.e., only if __x86_64__ AND !__PIC__ |
| #if defined(__x86_64__) && !defined(__PIC__) |
| # define CLOB_COMMA_ebx_ebp , // clobbering both ebp and ebx => need comma |
| #else |
| # define CLOB_COMMA_ebx_ebp |
| #endif |
| |
| // CLOB_COMMA_ebX_r1X: need comma UNLESS both CLOBBER_ebp and CLOBBER_GOT_ebx |
| // are empty OR CLOBBER_r11_r12_r13 is empty--i.e., NO comma |
| // if (!__x86_64__ AND __PIC__) OR !(PNG_x86_64_USE_GOTPCREL |
| // AND PNG_CLOBBER_x86_64_REGS_SUPPORTED) (double sigh...) |
| #if (!defined(__x86_64__) && defined(__PIC__)) || \ |
| !defined(PNG_x86_64_USE_GOTPCREL) || \ |
| !defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED) |
| # define CLOB_COMMA_ebX_r1X |
| #else |
| # define CLOB_COMMA_ebX_r1X , // clobbering (ebp OR ebx) AND r11_r12_r13 |
| #endif |
| |
| // CLOB_COLON_ebx_ebp: need colon unless CLOBBER_ebp and CLOBBER_GOT_ebx are |
| // BOTH empty--i.e., NO colon if (!__x86_64__ AND __PIC__) |
| // CLOB_COLON_ebx_ebp_r1X: if, in addition, CLOBBER_r11_r12_r13 is empty, then |
| // no colon for Paeth blocks, either--i.e., NO colon |
| // if !(PNG_x86_64_USE_GOTPCREL AND |
| // PNG_CLOBBER_x86_64_REGS_SUPPORTED) |
| #if (!defined(__x86_64__) && defined(__PIC__)) |
| # define CLOB_COLON_ebx_ebp |
| # if !(defined(PNG_x86_64_USE_GOTPCREL) && \ |
| defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED)) |
| # define CLOB_COLON_ebx_ebp_r1X |
| # else |
| # define CLOB_COLON_ebx_ebp_r1X : // clobbering ebp OR ebx OR r11_r12_r13 |
| # endif |
| #else |
| # define CLOB_COLON_ebx_ebp : // clobbering ebp OR ebx |
| # define CLOB_COLON_ebx_ebp_r1X : // clobbering ebp OR ebx OR r11_r12_r13 |
| #endif |
| |
| #endif // PNG_HAVE_MMX_READ_FILTER_ROW |
| |
| #if defined(__PIC__) // macros to save, restore index to Global Offset Table |
| # if defined(__x86_64__) |
| # define SAVE_GOT_ebx "pushq %%rbx \n\t" |
| # define RESTORE_GOT_ebx "popq %%rbx \n\t" |
| # else |
| # define SAVE_GOT_ebx "pushl %%ebx \n\t" |
| # define RESTORE_GOT_ebx "popl %%ebx \n\t" |
| # endif |
| # define _CLOBBER_GOT_ebx // explicitly saved, restored => not clobbered |
| # define CLOBBER_GOT_ebx |
| #else |
| # define SAVE_GOT_ebx |
| # define RESTORE_GOT_ebx |
| # define _CLOBBER_GOT_ebx ,"%ebx" |
| # define CLOBBER_GOT_ebx "%ebx" |
| #endif |
| |
| #endif // PNG_MMX_CODE_SUPPORTED |
| |
| |
| |
| static int _mmx_supported = 2; // 0: no MMX; 1: MMX supported; 2: not tested |
| |
| /*===========================================================================*/ |
| /* */ |
| /* P N G _ M M X _ S U P P O R T */ |
| /* */ |
| /*===========================================================================*/ |
| |
| /* GRR NOTES: (1) the following code assumes 386 or better (pushfl/popfl) |
| * (2) all instructions compile with gcc 2.7.2.3 and later |
| * x (3) the function is moved down here to prevent gcc from |
| * x inlining it in multiple places and then barfing be- |
| * x cause the ".NOT_SUPPORTED" label is multiply defined |
| * [need to retest with gcc 2.7.2.3] |
| */ |
| |
| // GRR 20070524: This declaration apparently is compatible with but supersedes |
| // the one in png.h; in any case, the generated object file is slightly |
| // smaller. It is unnecessary with gcc 4.1.2, but gcc 2.x apparently |
| // replicated the ".NOT_SUPPORTED" label in each location the function was |
| // inlined, leading to compilation errors due to the "multiply defined" |
| // label. Old workaround was to leave the function at the end of this |
| // file; new one (still testing) is to use a gcc-specific function attribute |
| // to prevent inlining. |
| int PNGAPI |
| png_mmx_support(void) __attribute__((noinline)); |
| |
| int PNGAPI |
| png_mmx_support(void) |
| { |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| int result; |
| __asm__ __volatile__ ( |
| #if defined(__x86_64__) |
| "pushq %%rbx \n\t" // rbx gets clobbered by CPUID instruction |
| "pushq %%rcx \n\t" // so does rcx... |
| "pushq %%rdx \n\t" // ...and rdx (but rcx & rdx safe on Linux) |
| "pushfq \n\t" // save Eflag to stack |
| "popq %%rax \n\t" // get Eflag from stack into rax |
| "movq %%rax, %%rcx \n\t" // make another copy of Eflag in rcx |
| "xorl $0x200000, %%eax \n\t" // toggle ID bit in Eflag (i.e., bit 21) |
| "pushq %%rax \n\t" // save modified Eflag back to stack |
| "popfq \n\t" // restore modified value to Eflag reg |
| "pushfq \n\t" // save Eflag to stack |
| "popq %%rax \n\t" // get Eflag from stack |
| "pushq %%rcx \n\t" // save original Eflag to stack |
| "popfq \n\t" // restore original Eflag |
| #else |
| "pushl %%ebx \n\t" // ebx gets clobbered by CPUID instruction |
| "pushl %%ecx \n\t" // so does ecx... |
| "pushl %%edx \n\t" // ...and edx (but ecx & edx safe on Linux) |
| "pushfl \n\t" // save Eflag to stack |
| "popl %%eax \n\t" // get Eflag from stack into eax |
| "movl %%eax, %%ecx \n\t" // make another copy of Eflag in ecx |
| "xorl $0x200000, %%eax \n\t" // toggle ID bit in Eflag (i.e., bit 21) |
| "pushl %%eax \n\t" // save modified Eflag back to stack |
| "popfl \n\t" // restore modified value to Eflag reg |
| "pushfl \n\t" // save Eflag to stack |
| "popl %%eax \n\t" // get Eflag from stack |
| "pushl %%ecx \n\t" // save original Eflag to stack |
| "popfl \n\t" // restore original Eflag |
| #endif |
| "xorl %%ecx, %%eax \n\t" // compare new Eflag with original Eflag |
| "jz 0f \n\t" // if same, CPUID instr. is not supported |
| |
| "xorl %%eax, %%eax \n\t" // set eax to zero |
| // ".byte 0x0f, 0xa2 \n\t" // CPUID instruction (two-byte opcode) |
| "cpuid \n\t" // get the CPU identification info |
| "cmpl $1, %%eax \n\t" // make sure eax return non-zero value |
| "jl 0f \n\t" // if eax is zero, MMX is not supported |
| |
| "xorl %%eax, %%eax \n\t" // set eax to zero and... |
| "incl %%eax \n\t" // ...increment eax to 1. This pair is |
| // faster than the instruction "mov eax, 1" |
| "cpuid \n\t" // get the CPU identification info again |
| "andl $0x800000, %%edx \n\t" // mask out all bits but MMX bit (23) |
| "cmpl $0, %%edx \n\t" // 0 = MMX not supported |
| "jz 0f \n\t" // non-zero = yes, MMX IS supported |
| |
| "movl $1, %%eax \n\t" // set return value to 1 |
| "jmp 1f \n\t" // DONE: have MMX support |
| |
| "0: \n\t" // .NOT_SUPPORTED: target label for jump instructions |
| "movl $0, %%eax \n\t" // set return value to 0 |
| "1: \n\t" // .RETURN: target label for jump instructions |
| #if defined(__x86_64__) |
| "popq %%rdx \n\t" // restore rdx |
| "popq %%rcx \n\t" // restore rcx |
| "popq %%rbx \n\t" // restore rbx |
| #else |
| "popl %%edx \n\t" // restore edx |
| "popl %%ecx \n\t" // restore ecx |
| "popl %%ebx \n\t" // restore ebx |
| #endif |
| |
| // "ret \n\t" // DONE: no MMX support |
| // (fall through to standard C "ret") |
| |
| : "=a" (result) // output list |
| |
| : // any variables used on input (none) |
| |
| // no clobber list |
| // , "%ebx", "%ecx", "%edx" // GRR: we handle these manually |
| // , "memory" // if write to a variable gcc thought was in a reg |
| // , "cc" // "condition codes" (flag bits) |
| ); |
| _mmx_supported = result; |
| #else |
| _mmx_supported = 0; |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| |
| return _mmx_supported; |
| } |
| |
| |
| /*===========================================================================*/ |
| /* */ |
| /* P N G _ C O M B I N E _ R O W */ |
| /* */ |
| /*===========================================================================*/ |
| |
| #if defined(PNG_HAVE_MMX_COMBINE_ROW) |
| |
| #define BPP2 2 |
| #define BPP3 3 /* bytes per pixel (a.k.a. pixel_bytes) */ |
| #define BPP4 4 |
| #define BPP6 6 /* (defined only to help avoid cut-and-paste errors) */ |
| #define BPP8 8 |
| |
| /* Combines the row recently read in with the previous row. |
| This routine takes care of alpha and transparency if requested. |
| This routine also handles the two methods of progressive display |
| of interlaced images, depending on the mask value. |
| The mask value describes which pixels are to be combined with |
| the row. The pattern always repeats every 8 pixels, so just 8 |
| bits are needed. A one indicates the pixel is to be combined; a |
| zero indicates the pixel is to be skipped. This is in addition |
| to any alpha or transparency value associated with the pixel. |
| If you want all pixels to be combined, pass 0xff (255) in mask. */ |
| |
| /* Use this routine for the x86 platform - it uses a faster MMX routine |
| if the machine supports MMX. */ |
| |
| void /* PRIVATE */ |
| png_combine_row(png_structp png_ptr, png_bytep row, int mask) |
| { |
| png_debug(1, "in png_combine_row (pnggccrd.c)\n"); |
| |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| if (_mmx_supported == 2) { |
| #if !defined(PNG_1_0_X) |
| /* this should have happened in png_init_mmx_flags() already */ |
| png_warning(png_ptr, "asm_flags may not have been initialized"); |
| #endif |
| png_mmx_support(); |
| } |
| #endif |
| |
| if (mask == 0xff) |
| { |
| png_debug(2,"mask == 0xff: doing single png_memcpy()\n"); |
| png_memcpy(row, png_ptr->row_buf + 1, |
| (png_size_t)PNG_ROWBYTES(png_ptr->row_info.pixel_depth,png_ptr->width)); |
| } |
| else /* (png_combine_row() is never called with mask == 0) */ |
| { |
| switch (png_ptr->row_info.pixel_depth) |
| { |
| // most common case: combining 32-bit RGBA |
| case 32: /* png_ptr->row_info.pixel_depth */ |
| { |
| png_bytep srcptr; |
| png_bytep dstptr; |
| |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW) |
| /* && _mmx_supported */ ) |
| #else |
| if (_mmx_supported) |
| #endif |
| { |
| png_uint_32 len; |
| int diff; |
| int dummy_value_a; // fix 'forbidden register spilled' error |
| int dummy_value_d; |
| int dummy_value_c; |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| srcptr = png_ptr->row_buf + 1; |
| dstptr = row; |
| len = png_ptr->width & ~7; // reduce to multiple of 8 |
| diff = (int) (png_ptr->width & 7); // amount lost |
| |
| __asm__ __volatile__ ( |
| "not %%edx \n\t" // mask => unmask |
| "movd %%edx, %%mm7 \n\t" // load bit pattern |
| "not %%edx \n\t" // unmask => mask for later |
| "psubb %%mm6, %%mm6 \n\t" // zero mm6 |
| "punpcklbw %%mm7, %%mm7 \n\t" |
| "punpcklwd %%mm7, %%mm7 \n\t" |
| "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks |
| |
| LOAD_GOT_rbp |
| "movq " MASK32_0 ", %%mm0 \n\t" // _mask32_0 |
| "movq " MASK32_1 ", %%mm1 \n\t" // _mask32_1 |
| "movq " MASK32_2 ", %%mm2 \n\t" // _mask32_2 |
| "movq " MASK32_3 ", %%mm3 \n\t" // _mask32_3 |
| RESTORE_rbp |
| |
| "pand %%mm7, %%mm0 \n\t" |
| "pand %%mm7, %%mm1 \n\t" |
| "pand %%mm7, %%mm2 \n\t" |
| "pand %%mm7, %%mm3 \n\t" |
| |
| "pcmpeqb %%mm6, %%mm0 \n\t" |
| "pcmpeqb %%mm6, %%mm1 \n\t" |
| "pcmpeqb %%mm6, %%mm2 \n\t" |
| "pcmpeqb %%mm6, %%mm3 \n\t" |
| |
| // preload "movl len, %%ecx \n\t" // load length of line |
| // preload "movl srcptr, %%esi \n\t" // load source |
| // preload "movl dstptr, %%edi \n\t" // load dest |
| |
| "cmpl $0, %%ecx \n\t" // lcr |
| "jz mainloop32end \n\t" |
| |
| "mainloop32: \n\t" |
| "movq (%%esi), %%mm4 \n\t" |
| "pand %%mm0, %%mm4 \n\t" |
| "movq %%mm0, %%mm6 \n\t" |
| "movq (%%edi), %%mm7 \n\t" |
| "pandn %%mm7, %%mm6 \n\t" |
| "por %%mm6, %%mm4 \n\t" |
| "movq %%mm4, (%%edi) \n\t" |
| |
| "movq 8(%%esi), %%mm5 \n\t" |
| "pand %%mm1, %%mm5 \n\t" |
| "movq %%mm1, %%mm7 \n\t" |
| "movq 8(%%edi), %%mm6 \n\t" |
| "pandn %%mm6, %%mm7 \n\t" |
| "por %%mm7, %%mm5 \n\t" |
| "movq %%mm5, 8(%%edi) \n\t" |
| |
| "movq 16(%%esi), %%mm6 \n\t" |
| "pand %%mm2, %%mm6 \n\t" |
| "movq %%mm2, %%mm4 \n\t" |
| "movq 16(%%edi), %%mm7 \n\t" |
| "pandn %%mm7, %%mm4 \n\t" |
| "por %%mm4, %%mm6 \n\t" |
| "movq %%mm6, 16(%%edi) \n\t" |
| |
| "movq 24(%%esi), %%mm7 \n\t" |
| "pand %%mm3, %%mm7 \n\t" |
| "movq %%mm3, %%mm5 \n\t" |
| "movq 24(%%edi), %%mm4 \n\t" |
| "pandn %%mm4, %%mm5 \n\t" |
| "por %%mm5, %%mm7 \n\t" |
| "movq %%mm7, 24(%%edi) \n\t" |
| |
| "addl $32, %%esi \n\t" // inc by 32 bytes processed |
| "addl $32, %%edi \n\t" |
| "subl $8, %%ecx \n\t" // dec by 8 pixels processed |
| "ja mainloop32 \n\t" |
| |
| "mainloop32end: \n\t" |
| // preload "movl diff, %%ecx \n\t" // (diff is in eax) |
| "movl %%eax, %%ecx \n\t" |
| "cmpl $0, %%ecx \n\t" |
| "jz end32 \n\t" |
| // preload "movl mask, %%edx \n\t" |
| "sall $24, %%edx \n\t" // low byte => high byte |
| |
| "secondloop32: \n\t" |
| "sall %%edx \n\t" // move high bit to CF |
| "jnc skip32 \n\t" // if CF = 0 |
| "movl (%%esi), %%eax \n\t" |
| "movl %%eax, (%%edi) \n\t" |
| |
| "skip32: \n\t" |
| "addl $4, %%esi \n\t" |
| "addl $4, %%edi \n\t" |
| "decl %%ecx \n\t" |
| "jnz secondloop32 \n\t" |
| |
| "end32: \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=a" (dummy_value_a), // output regs (dummy) |
| "=d" (dummy_value_d), |
| "=c" (dummy_value_c), |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "3" (srcptr), // esi // input regs |
| "4" (dstptr), // edi |
| "0" (diff), // eax |
| // was (unmask) "b" RESERVED // ebx // Global Offset Table idx |
| "2" (len), // ecx |
| "1" (mask) // edx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| else /* mmx _not supported - Use modified C routine */ |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| { |
| register png_uint_32 i; |
| png_uint_32 initial_val = BPP4 * png_pass_start[png_ptr->pass]; |
| /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ |
| register int stride = BPP4 * png_pass_inc[png_ptr->pass]; |
| /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ |
| register int rep_bytes = BPP4 * png_pass_width[png_ptr->pass]; |
| /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ |
| png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ |
| int diff = (int) (png_ptr->width & 7); /* amount lost */ |
| register png_uint_32 final_val = BPP4 * len; /* GRR bugfix */ |
| |
| srcptr = png_ptr->row_buf + 1 + initial_val; |
| dstptr = row + initial_val; |
| |
| for (i = initial_val; i < final_val; i += stride) |
| { |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| if (diff) /* number of leftover pixels: 3 for pngtest */ |
| { |
| final_val += diff*BPP4; |
| for (; i < final_val; i += stride) |
| { |
| if (rep_bytes > (int)(final_val-i)) |
| rep_bytes = (int)(final_val-i); |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| } |
| } /* end of else (_mmx_supported) */ |
| |
| break; |
| } /* end 32 bpp */ |
| |
| case 1: /* png_ptr->row_info.pixel_depth */ |
| { |
| png_bytep sp; |
| png_bytep dp; |
| int s_inc, s_start, s_end; |
| int m; |
| int shift; |
| png_uint_32 i; |
| |
| sp = png_ptr->row_buf + 1; |
| dp = row; |
| m = 0x80; |
| #if defined(PNG_READ_PACKSWAP_SUPPORTED) |
| if (png_ptr->transformations & PNG_PACKSWAP) |
| { |
| s_start = 0; |
| s_end = 7; |
| s_inc = 1; |
| } |
| else |
| #endif |
| { |
| s_start = 7; |
| s_end = 0; |
| s_inc = -1; |
| } |
| |
| shift = s_start; |
| |
| for (i = 0; i < png_ptr->width; i++) |
| { |
| if (m & mask) |
| { |
| int value; |
| |
| value = (*sp >> shift) & 0x1; |
| *dp &= (png_byte)((0x7f7f >> (7 - shift)) & 0xff); |
| *dp |= (png_byte)(value << shift); |
| } |
| |
| if (shift == s_end) |
| { |
| shift = s_start; |
| sp++; |
| dp++; |
| } |
| else |
| shift += s_inc; |
| |
| if (m == 1) |
| m = 0x80; |
| else |
| m >>= 1; |
| } |
| break; |
| } /* end 1 bpp */ |
| |
| case 2: /* png_ptr->row_info.pixel_depth */ |
| { |
| png_bytep sp; |
| png_bytep dp; |
| int s_start, s_end, s_inc; |
| int m; |
| int shift; |
| png_uint_32 i; |
| int value; |
| |
| sp = png_ptr->row_buf + 1; |
| dp = row; |
| m = 0x80; |
| #if defined(PNG_READ_PACKSWAP_SUPPORTED) |
| if (png_ptr->transformations & PNG_PACKSWAP) |
| { |
| s_start = 0; |
| s_end = 6; |
| s_inc = 2; |
| } |
| else |
| #endif |
| { |
| s_start = 6; |
| s_end = 0; |
| s_inc = -2; |
| } |
| |
| shift = s_start; |
| |
| for (i = 0; i < png_ptr->width; i++) |
| { |
| if (m & mask) |
| { |
| value = (*sp >> shift) & 0x3; |
| *dp &= (png_byte)((0x3f3f >> (6 - shift)) & 0xff); |
| *dp |= (png_byte)(value << shift); |
| } |
| |
| if (shift == s_end) |
| { |
| shift = s_start; |
| sp++; |
| dp++; |
| } |
| else |
| shift += s_inc; |
| if (m == 1) |
| m = 0x80; |
| else |
| m >>= 1; |
| } |
| break; |
| } /* end 2 bpp */ |
| |
| case 4: /* png_ptr->row_info.pixel_depth */ |
| { |
| png_bytep sp; |
| png_bytep dp; |
| int s_start, s_end, s_inc; |
| int m; |
| int shift; |
| png_uint_32 i; |
| int value; |
| |
| sp = png_ptr->row_buf + 1; |
| dp = row; |
| m = 0x80; |
| #if defined(PNG_READ_PACKSWAP_SUPPORTED) |
| if (png_ptr->transformations & PNG_PACKSWAP) |
| { |
| s_start = 0; |
| s_end = 4; |
| s_inc = 4; |
| } |
| else |
| #endif |
| { |
| s_start = 4; |
| s_end = 0; |
| s_inc = -4; |
| } |
| shift = s_start; |
| |
| for (i = 0; i < png_ptr->width; i++) |
| { |
| if (m & mask) |
| { |
| value = (*sp >> shift) & 0xf; |
| *dp &= (png_byte)((0xf0f >> (4 - shift)) & 0xff); |
| *dp |= (png_byte)(value << shift); |
| } |
| |
| if (shift == s_end) |
| { |
| shift = s_start; |
| sp++; |
| dp++; |
| } |
| else |
| shift += s_inc; |
| if (m == 1) |
| m = 0x80; |
| else |
| m >>= 1; |
| } |
| break; |
| } /* end 4 bpp */ |
| |
| case 8: /* png_ptr->row_info.pixel_depth */ |
| { |
| png_bytep srcptr; |
| png_bytep dstptr; |
| |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW) |
| /* && _mmx_supported */ ) |
| #else |
| if (_mmx_supported) |
| #endif |
| { |
| png_uint_32 len; |
| int diff; |
| int dummy_value_a; // fix 'forbidden register spilled' error |
| int dummy_value_d; |
| int dummy_value_c; |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| srcptr = png_ptr->row_buf + 1; |
| dstptr = row; |
| len = png_ptr->width & ~7; // reduce to multiple of 8 |
| diff = (int) (png_ptr->width & 7); // amount lost |
| |
| __asm__ __volatile__ ( |
| "not %%edx \n\t" // mask => unmask |
| "movd %%edx, %%mm7 \n\t" // load bit pattern |
| "not %%edx \n\t" // unmask => mask for later |
| "psubb %%mm6, %%mm6 \n\t" // zero mm6 |
| "punpcklbw %%mm7, %%mm7 \n\t" |
| "punpcklwd %%mm7, %%mm7 \n\t" |
| "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks |
| |
| LOAD_GOT_rbp |
| "movq " MASK8_0 ", %%mm0 \n\t" // _mask8_0 -> mm0 |
| RESTORE_rbp |
| |
| "pand %%mm7, %%mm0 \n\t" // nonzero if keep byte |
| "pcmpeqb %%mm6, %%mm0 \n\t" // zeros->1s, v versa |
| |
| // preload "movl len, %%ecx \n\t" // load length of line |
| // preload "movl srcptr, %%esi \n\t" // load source |
| // preload "movl dstptr, %%edi \n\t" // load dest |
| |
| "cmpl $0, %%ecx \n\t" // len == 0 ? |
| "je mainloop8end \n\t" |
| |
| "mainloop8: \n\t" |
| "movq (%%esi), %%mm4 \n\t" // *srcptr |
| "pand %%mm0, %%mm4 \n\t" |
| "movq %%mm0, %%mm6 \n\t" |
| "pandn (%%edi), %%mm6 \n\t" // *dstptr |
| "por %%mm6, %%mm4 \n\t" |
| "movq %%mm4, (%%edi) \n\t" |
| "addl $8, %%esi \n\t" // inc by 8 bytes processed |
| "addl $8, %%edi \n\t" |
| "subl $8, %%ecx \n\t" // dec by 8 pixels processed |
| "ja mainloop8 \n\t" |
| |
| "mainloop8end: \n\t" |
| // preload "movl diff, %%ecx \n\t" // (diff is in eax) |
| "movl %%eax, %%ecx \n\t" |
| "cmpl $0, %%ecx \n\t" |
| "jz end8 \n\t" |
| // preload "movl mask, %%edx \n\t" |
| "sall $24, %%edx \n\t" // make low byte, high byte |
| |
| "secondloop8: \n\t" |
| "sall %%edx \n\t" // move high bit to CF |
| "jnc skip8 \n\t" // if CF = 0 |
| "movb (%%esi), %%al \n\t" |
| "movb %%al, (%%edi) \n\t" |
| |
| "skip8: \n\t" |
| "incl %%esi \n\t" |
| "incl %%edi \n\t" |
| "decl %%ecx \n\t" |
| "jnz secondloop8 \n\t" |
| |
| "end8: \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=a" (dummy_value_a), // output regs (dummy) |
| "=d" (dummy_value_d), |
| "=c" (dummy_value_c), |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "3" (srcptr), // esi // input regs |
| "4" (dstptr), // edi |
| "0" (diff), // eax |
| // was (unmask) "b" RESERVED // ebx // Global Offset Table idx |
| "2" (len), // ecx |
| "1" (mask) // edx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm4", "%mm6", "%mm7" // clobber list |
| #endif |
| ); |
| } |
| else /* mmx _not supported - Use modified C routine */ |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| { |
| register png_uint_32 i; |
| png_uint_32 initial_val = png_pass_start[png_ptr->pass]; |
| /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ |
| register int stride = png_pass_inc[png_ptr->pass]; |
| /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ |
| register int rep_bytes = png_pass_width[png_ptr->pass]; |
| /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ |
| png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ |
| int diff = (int) (png_ptr->width & 7); /* amount lost */ |
| register png_uint_32 final_val = len; /* GRR bugfix */ |
| |
| srcptr = png_ptr->row_buf + 1 + initial_val; |
| dstptr = row + initial_val; |
| |
| for (i = initial_val; i < final_val; i += stride) |
| { |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| if (diff) /* number of leftover pixels: 3 for pngtest */ |
| { |
| final_val += diff /* *BPP1 */ ; |
| for (; i < final_val; i += stride) |
| { |
| if (rep_bytes > (int)(final_val-i)) |
| rep_bytes = (int)(final_val-i); |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| } |
| |
| } /* end of else (_mmx_supported) */ |
| |
| break; |
| } /* end 8 bpp */ |
| |
| case 16: /* png_ptr->row_info.pixel_depth */ |
| { |
| png_bytep srcptr; |
| png_bytep dstptr; |
| |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW) |
| /* && _mmx_supported */ ) |
| #else |
| if (_mmx_supported) |
| #endif |
| { |
| png_uint_32 len; |
| int diff; |
| int dummy_value_a; // fix 'forbidden register spilled' error |
| int dummy_value_d; |
| int dummy_value_c; |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| srcptr = png_ptr->row_buf + 1; |
| dstptr = row; |
| len = png_ptr->width & ~7; // reduce to multiple of 8 |
| diff = (int) (png_ptr->width & 7); // amount lost |
| |
| __asm__ __volatile__ ( |
| "not %%edx \n\t" // mask => unmask |
| "movd %%edx, %%mm7 \n\t" // load bit pattern |
| "not %%edx \n\t" // unmask => mask for later |
| "psubb %%mm6, %%mm6 \n\t" // zero mm6 |
| "punpcklbw %%mm7, %%mm7 \n\t" |
| "punpcklwd %%mm7, %%mm7 \n\t" |
| "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks |
| |
| LOAD_GOT_rbp |
| "movq " MASK16_0 ", %%mm0 \n\t" // _mask16_0 -> mm0 |
| "movq " MASK16_1 ", %%mm1 \n\t" // _mask16_1 -> mm1 |
| RESTORE_rbp |
| |
| "pand %%mm7, %%mm0 \n\t" |
| "pand %%mm7, %%mm1 \n\t" |
| |
| "pcmpeqb %%mm6, %%mm0 \n\t" |
| "pcmpeqb %%mm6, %%mm1 \n\t" |
| |
| // preload "movl len, %%ecx \n\t" // load length of line |
| // preload "movl srcptr, %%esi \n\t" // load source |
| // preload "movl dstptr, %%edi \n\t" // load dest |
| |
| "cmpl $0, %%ecx \n\t" |
| "jz mainloop16end \n\t" |
| |
| "mainloop16: \n\t" |
| "movq (%%esi), %%mm4 \n\t" |
| "pand %%mm0, %%mm4 \n\t" |
| "movq %%mm0, %%mm6 \n\t" |
| "movq (%%edi), %%mm7 \n\t" |
| "pandn %%mm7, %%mm6 \n\t" |
| "por %%mm6, %%mm4 \n\t" |
| "movq %%mm4, (%%edi) \n\t" |
| |
| "movq 8(%%esi), %%mm5 \n\t" |
| "pand %%mm1, %%mm5 \n\t" |
| "movq %%mm1, %%mm7 \n\t" |
| "movq 8(%%edi), %%mm6 \n\t" |
| "pandn %%mm6, %%mm7 \n\t" |
| "por %%mm7, %%mm5 \n\t" |
| "movq %%mm5, 8(%%edi) \n\t" |
| |
| "addl $16, %%esi \n\t" // inc by 16 bytes processed |
| "addl $16, %%edi \n\t" |
| "subl $8, %%ecx \n\t" // dec by 8 pixels processed |
| "ja mainloop16 \n\t" |
| |
| "mainloop16end: \n\t" |
| // preload "movl diff, %%ecx \n\t" // (diff is in eax) |
| "movl %%eax, %%ecx \n\t" |
| "cmpl $0, %%ecx \n\t" |
| "jz end16 \n\t" |
| // preload "movl mask, %%edx \n\t" |
| "sall $24, %%edx \n\t" // make low byte, high byte |
| |
| "secondloop16: \n\t" |
| "sall %%edx \n\t" // move high bit to CF |
| "jnc skip16 \n\t" // if CF = 0 |
| "movw (%%esi), %%ax \n\t" |
| "movw %%ax, (%%edi) \n\t" |
| |
| "skip16: \n\t" |
| "addl $2, %%esi \n\t" |
| "addl $2, %%edi \n\t" |
| "decl %%ecx \n\t" |
| "jnz secondloop16 \n\t" |
| |
| "end16: \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=a" (dummy_value_a), // output regs (dummy) |
| "=c" (dummy_value_c), |
| "=d" (dummy_value_d), |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "0" (diff), // eax // input regs |
| // was (unmask) " " RESERVED // ebx // Global Offset Table idx |
| "1" (len), // ecx |
| "2" (mask), // edx |
| "3" (srcptr), // esi |
| "4" (dstptr) // edi |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm4" // clobber list |
| , "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| else /* mmx _not supported - Use modified C routine */ |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| { |
| register png_uint_32 i; |
| png_uint_32 initial_val = BPP2 * png_pass_start[png_ptr->pass]; |
| /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ |
| register int stride = BPP2 * png_pass_inc[png_ptr->pass]; |
| /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ |
| register int rep_bytes = BPP2 * png_pass_width[png_ptr->pass]; |
| /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ |
| png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ |
| int diff = (int) (png_ptr->width & 7); /* amount lost */ |
| register png_uint_32 final_val = BPP2 * len; /* GRR bugfix */ |
| |
| srcptr = png_ptr->row_buf + 1 + initial_val; |
| dstptr = row + initial_val; |
| |
| for (i = initial_val; i < final_val; i += stride) |
| { |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| if (diff) /* number of leftover pixels: 3 for pngtest */ |
| { |
| final_val += diff*BPP2; |
| for (; i < final_val; i += stride) |
| { |
| if (rep_bytes > (int)(final_val-i)) |
| rep_bytes = (int)(final_val-i); |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| } |
| } /* end of else (_mmx_supported) */ |
| |
| break; |
| } /* end 16 bpp */ |
| |
| case 24: /* png_ptr->row_info.pixel_depth */ |
| { |
| png_bytep srcptr; |
| png_bytep dstptr; |
| |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW) |
| /* && _mmx_supported */ ) |
| #else |
| if (_mmx_supported) |
| #endif |
| { |
| png_uint_32 len; |
| int diff; |
| int dummy_value_a; // fix 'forbidden register spilled' error |
| int dummy_value_d; |
| int dummy_value_c; |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| srcptr = png_ptr->row_buf + 1; |
| dstptr = row; |
| len = png_ptr->width & ~7; // reduce to multiple of 8 |
| diff = (int) (png_ptr->width & 7); // amount lost |
| |
| __asm__ __volatile__ ( |
| "not %%edx \n\t" // mask => unmask |
| "movd %%edx, %%mm7 \n\t" // load bit pattern |
| "not %%edx \n\t" // unmask => mask for later |
| "psubb %%mm6, %%mm6 \n\t" // zero mm6 |
| "punpcklbw %%mm7, %%mm7 \n\t" |
| "punpcklwd %%mm7, %%mm7 \n\t" |
| "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks |
| |
| LOAD_GOT_rbp |
| "movq " MASK24_0 ", %%mm0 \n\t" // _mask24_0 -> mm0 |
| "movq " MASK24_1 ", %%mm1 \n\t" // _mask24_1 -> mm1 |
| "movq " MASK24_2 ", %%mm2 \n\t" // _mask24_2 -> mm2 |
| RESTORE_rbp |
| |
| "pand %%mm7, %%mm0 \n\t" |
| "pand %%mm7, %%mm1 \n\t" |
| "pand %%mm7, %%mm2 \n\t" |
| |
| "pcmpeqb %%mm6, %%mm0 \n\t" |
| "pcmpeqb %%mm6, %%mm1 \n\t" |
| "pcmpeqb %%mm6, %%mm2 \n\t" |
| |
| // preload "movl len, %%ecx \n\t" // load length of line |
| // preload "movl srcptr, %%esi \n\t" // load source |
| // preload "movl dstptr, %%edi \n\t" // load dest |
| |
| "cmpl $0, %%ecx \n\t" |
| "jz mainloop24end \n\t" |
| |
| "mainloop24: \n\t" |
| "movq (%%esi), %%mm4 \n\t" |
| "pand %%mm0, %%mm4 \n\t" |
| "movq %%mm0, %%mm6 \n\t" |
| "movq (%%edi), %%mm7 \n\t" |
| "pandn %%mm7, %%mm6 \n\t" |
| "por %%mm6, %%mm4 \n\t" |
| "movq %%mm4, (%%edi) \n\t" |
| |
| "movq 8(%%esi), %%mm5 \n\t" |
| "pand %%mm1, %%mm5 \n\t" |
| "movq %%mm1, %%mm7 \n\t" |
| "movq 8(%%edi), %%mm6 \n\t" |
| "pandn %%mm6, %%mm7 \n\t" |
| "por %%mm7, %%mm5 \n\t" |
| "movq %%mm5, 8(%%edi) \n\t" |
| |
| "movq 16(%%esi), %%mm6 \n\t" |
| "pand %%mm2, %%mm6 \n\t" |
| "movq %%mm2, %%mm4 \n\t" |
| "movq 16(%%edi), %%mm7 \n\t" |
| "pandn %%mm7, %%mm4 \n\t" |
| "por %%mm4, %%mm6 \n\t" |
| "movq %%mm6, 16(%%edi) \n\t" |
| |
| "addl $24, %%esi \n\t" // inc by 24 bytes processed |
| "addl $24, %%edi \n\t" |
| "subl $8, %%ecx \n\t" // dec by 8 pixels processed |
| |
| "ja mainloop24 \n\t" |
| |
| "mainloop24end: \n\t" |
| // preload "movl diff, %%ecx \n\t" // (diff is in eax) |
| "movl %%eax, %%ecx \n\t" |
| "cmpl $0, %%ecx \n\t" |
| "jz end24 \n\t" |
| // preload "movl mask, %%edx \n\t" |
| "sall $24, %%edx \n\t" // make low byte, high byte |
| |
| "secondloop24: \n\t" |
| "sall %%edx \n\t" // move high bit to CF |
| "jnc skip24 \n\t" // if CF = 0 |
| "movw (%%esi), %%ax \n\t" |
| "movw %%ax, (%%edi) \n\t" |
| "xorl %%eax, %%eax \n\t" |
| "movb 2(%%esi), %%al \n\t" |
| "movb %%al, 2(%%edi) \n\t" |
| |
| "skip24: \n\t" |
| "addl $3, %%esi \n\t" |
| "addl $3, %%edi \n\t" |
| "decl %%ecx \n\t" |
| "jnz secondloop24 \n\t" |
| |
| "end24: \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=a" (dummy_value_a), // output regs (dummy) |
| "=d" (dummy_value_d), |
| "=c" (dummy_value_c), |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "3" (srcptr), // esi // input regs |
| "4" (dstptr), // edi |
| "0" (diff), // eax |
| // was (unmask) "b" RESERVED // ebx // Global Offset Table idx |
| "2" (len), // ecx |
| "1" (mask) // edx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2" // clobber list |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| else /* mmx _not supported - Use modified C routine */ |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| { |
| register png_uint_32 i; |
| png_uint_32 initial_val = BPP3 * png_pass_start[png_ptr->pass]; |
| /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ |
| register int stride = BPP3 * png_pass_inc[png_ptr->pass]; |
| /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ |
| register int rep_bytes = BPP3 * png_pass_width[png_ptr->pass]; |
| /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ |
| png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ |
| int diff = (int) (png_ptr->width & 7); /* amount lost */ |
| register png_uint_32 final_val = BPP3 * len; /* GRR bugfix */ |
| |
| srcptr = png_ptr->row_buf + 1 + initial_val; |
| dstptr = row + initial_val; |
| |
| for (i = initial_val; i < final_val; i += stride) |
| { |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| if (diff) /* number of leftover pixels: 3 for pngtest */ |
| { |
| final_val += diff*BPP3; |
| for (; i < final_val; i += stride) |
| { |
| if (rep_bytes > (int)(final_val-i)) |
| rep_bytes = (int)(final_val-i); |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| } |
| } /* end of else (_mmx_supported) */ |
| |
| break; |
| } /* end 24 bpp */ |
| |
| case 48: /* png_ptr->row_info.pixel_depth */ |
| { |
| png_bytep srcptr; |
| png_bytep dstptr; |
| |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW) |
| /* && _mmx_supported */ ) |
| #else |
| if (_mmx_supported) |
| #endif |
| { |
| png_uint_32 len; |
| int diff; |
| int dummy_value_a; // fix 'forbidden register spilled' error |
| int dummy_value_d; |
| int dummy_value_c; |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| srcptr = png_ptr->row_buf + 1; |
| dstptr = row; |
| len = png_ptr->width & ~7; // reduce to multiple of 8 |
| diff = (int) (png_ptr->width & 7); // amount lost |
| |
| __asm__ __volatile__ ( |
| "not %%edx \n\t" // mask => unmask |
| "movd %%edx, %%mm7 \n\t" // load bit pattern |
| "not %%edx \n\t" // unmask => mask for later |
| "psubb %%mm6, %%mm6 \n\t" // zero mm6 |
| "punpcklbw %%mm7, %%mm7 \n\t" |
| "punpcklwd %%mm7, %%mm7 \n\t" |
| "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks |
| |
| LOAD_GOT_rbp |
| "movq " MASK48_0 ", %%mm0 \n\t" // _mask48_0 -> mm0 |
| "movq " MASK48_1 ", %%mm1 \n\t" // _mask48_1 -> mm1 |
| "movq " MASK48_2 ", %%mm2 \n\t" // _mask48_2 -> mm2 |
| "movq " MASK48_3 ", %%mm3 \n\t" // _mask48_3 -> mm3 |
| "movq " MASK48_4 ", %%mm4 \n\t" // _mask48_4 -> mm4 |
| "movq " MASK48_5 ", %%mm5 \n\t" // _mask48_5 -> mm5 |
| RESTORE_rbp |
| |
| "pand %%mm7, %%mm0 \n\t" |
| "pand %%mm7, %%mm1 \n\t" |
| "pand %%mm7, %%mm2 \n\t" |
| "pand %%mm7, %%mm3 \n\t" |
| "pand %%mm7, %%mm4 \n\t" |
| "pand %%mm7, %%mm5 \n\t" |
| |
| "pcmpeqb %%mm6, %%mm0 \n\t" |
| "pcmpeqb %%mm6, %%mm1 \n\t" |
| "pcmpeqb %%mm6, %%mm2 \n\t" |
| "pcmpeqb %%mm6, %%mm3 \n\t" |
| "pcmpeqb %%mm6, %%mm4 \n\t" |
| "pcmpeqb %%mm6, %%mm5 \n\t" |
| |
| // preload "movl len, %%ecx \n\t" // load length of line |
| // preload "movl srcptr, %%esi \n\t" // load source |
| // preload "movl dstptr, %%edi \n\t" // load dest |
| |
| "cmpl $0, %%ecx \n\t" |
| "jz mainloop48end \n\t" |
| |
| "mainloop48: \n\t" |
| "movq (%%esi), %%mm7 \n\t" |
| "pand %%mm0, %%mm7 \n\t" |
| "movq %%mm0, %%mm6 \n\t" |
| "pandn (%%edi), %%mm6 \n\t" |
| "por %%mm6, %%mm7 \n\t" |
| "movq %%mm7, (%%edi) \n\t" |
| |
| "movq 8(%%esi), %%mm6 \n\t" |
| "pand %%mm1, %%mm6 \n\t" |
| "movq %%mm1, %%mm7 \n\t" |
| "pandn 8(%%edi), %%mm7 \n\t" |
| "por %%mm7, %%mm6 \n\t" |
| "movq %%mm6, 8(%%edi) \n\t" |
| |
| "movq 16(%%esi), %%mm6 \n\t" |
| "pand %%mm2, %%mm6 \n\t" |
| "movq %%mm2, %%mm7 \n\t" |
| "pandn 16(%%edi), %%mm7 \n\t" |
| "por %%mm7, %%mm6 \n\t" |
| "movq %%mm6, 16(%%edi) \n\t" |
| |
| "movq 24(%%esi), %%mm7 \n\t" |
| "pand %%mm3, %%mm7 \n\t" |
| "movq %%mm3, %%mm6 \n\t" |
| "pandn 24(%%edi), %%mm6 \n\t" |
| "por %%mm6, %%mm7 \n\t" |
| "movq %%mm7, 24(%%edi) \n\t" |
| |
| "movq 32(%%esi), %%mm6 \n\t" |
| "pand %%mm4, %%mm6 \n\t" |
| "movq %%mm4, %%mm7 \n\t" |
| "pandn 32(%%edi), %%mm7 \n\t" |
| "por %%mm7, %%mm6 \n\t" |
| "movq %%mm6, 32(%%edi) \n\t" |
| |
| "movq 40(%%esi), %%mm7 \n\t" |
| "pand %%mm5, %%mm7 \n\t" |
| "movq %%mm5, %%mm6 \n\t" |
| "pandn 40(%%edi), %%mm6 \n\t" |
| "por %%mm6, %%mm7 \n\t" |
| "movq %%mm7, 40(%%edi) \n\t" |
| |
| "addl $48, %%esi \n\t" // inc by 48 bytes processed |
| "addl $48, %%edi \n\t" |
| "subl $8, %%ecx \n\t" // dec by 8 pixels processed |
| |
| "ja mainloop48 \n\t" |
| |
| "mainloop48end: \n\t" |
| // preload "movl diff, %%ecx \n\t" // (diff is in eax) |
| "movl %%eax, %%ecx \n\t" |
| "cmpl $0, %%ecx \n\t" |
| "jz end48 \n\t" |
| // preload "movl mask, %%edx \n\t" |
| "sall $24, %%edx \n\t" // make low byte, high byte |
| |
| "secondloop48: \n\t" |
| "sall %%edx \n\t" // move high bit to CF |
| "jnc skip48 \n\t" // if CF = 0 |
| "movl (%%esi), %%eax \n\t" |
| "movl %%eax, (%%edi) \n\t" |
| |
| "skip48: \n\t" |
| "addl $4, %%esi \n\t" |
| "addl $4, %%edi \n\t" |
| "decl %%ecx \n\t" |
| "jnz secondloop48 \n\t" |
| |
| "end48: \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=a" (dummy_value_a), // output regs (dummy) |
| "=d" (dummy_value_d), |
| "=c" (dummy_value_c), |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "3" (srcptr), // esi // input regs |
| "4" (dstptr), // edi |
| "0" (diff), // eax |
| // was (unmask) "b" RESERVED // ebx // Global Offset Table idx |
| "2" (len), // ecx |
| "1" (mask) // edx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| else /* mmx _not supported - Use modified C routine */ |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| { |
| register png_uint_32 i; |
| png_uint_32 initial_val = BPP6 * png_pass_start[png_ptr->pass]; |
| /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ |
| register int stride = BPP6 * png_pass_inc[png_ptr->pass]; |
| /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ |
| register int rep_bytes = BPP6 * png_pass_width[png_ptr->pass]; |
| /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ |
| png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ |
| int diff = (int) (png_ptr->width & 7); /* amount lost */ |
| register png_uint_32 final_val = BPP6 * len; /* GRR bugfix */ |
| |
| srcptr = png_ptr->row_buf + 1 + initial_val; |
| dstptr = row + initial_val; |
| |
| for (i = initial_val; i < final_val; i += stride) |
| { |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| if (diff) /* number of leftover pixels: 3 for pngtest */ |
| { |
| final_val += diff*BPP6; |
| for (; i < final_val; i += stride) |
| { |
| if (rep_bytes > (int)(final_val-i)) |
| rep_bytes = (int)(final_val-i); |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| } |
| } /* end of else (_mmx_supported) */ |
| |
| break; |
| } /* end 48 bpp */ |
| |
| case 64: /* png_ptr->row_info.pixel_depth */ |
| { |
| png_bytep srcptr; |
| png_bytep dstptr; |
| register png_uint_32 i; |
| png_uint_32 initial_val = BPP8 * png_pass_start[png_ptr->pass]; |
| /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ |
| register int stride = BPP8 * png_pass_inc[png_ptr->pass]; |
| /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ |
| register int rep_bytes = BPP8 * png_pass_width[png_ptr->pass]; |
| /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ |
| png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ |
| int diff = (int) (png_ptr->width & 7); /* amount lost */ |
| register png_uint_32 final_val = BPP8 * len; /* GRR bugfix */ |
| |
| srcptr = png_ptr->row_buf + 1 + initial_val; |
| dstptr = row + initial_val; |
| |
| for (i = initial_val; i < final_val; i += stride) |
| { |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| if (diff) /* number of leftover pixels: 3 for pngtest */ |
| { |
| final_val += diff*BPP8; |
| for (; i < final_val; i += stride) |
| { |
| if (rep_bytes > (int)(final_val-i)) |
| rep_bytes = (int)(final_val-i); |
| png_memcpy(dstptr, srcptr, rep_bytes); |
| srcptr += stride; |
| dstptr += stride; |
| } |
| } |
| |
| break; |
| } /* end 64 bpp */ |
| |
| default: /* png_ptr->row_info.pixel_depth != 1,2,4,8,16,24,32,48,64 */ |
| { |
| /* this should never happen */ |
| png_warning(png_ptr, "Invalid row_info.pixel_depth in pnggccrd"); |
| break; |
| } |
| } /* end switch (png_ptr->row_info.pixel_depth) */ |
| |
| } /* end if (non-trivial mask) */ |
| |
| } /* end png_combine_row() */ |
| |
| #endif /* PNG_HAVE_MMX_COMBINE_ROW */ |
| |
| |
| |
| |
| /*===========================================================================*/ |
| /* */ |
| /* P N G _ D O _ R E A D _ I N T E R L A C E */ |
| /* */ |
| /*===========================================================================*/ |
| |
| #if defined(PNG_READ_INTERLACING_SUPPORTED) |
| #if defined(PNG_HAVE_MMX_READ_INTERLACE) |
| |
| /* png_do_read_interlace() is called after any 16-bit to 8-bit conversion |
| * has taken place. [GRR: what other steps come before and/or after?] |
| */ |
| |
| void /* PRIVATE */ |
| png_do_read_interlace(png_structp png_ptr) |
| { |
| png_row_infop row_info = &(png_ptr->row_info); |
| png_bytep row = png_ptr->row_buf + 1; |
| int pass = png_ptr->pass; |
| #if defined(PNG_READ_PACKSWAP_SUPPORTED) |
| png_uint_32 transformations = png_ptr->transformations; |
| #endif |
| |
| png_debug(1, "in png_do_read_interlace (pnggccrd.c)\n"); |
| |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| if (_mmx_supported == 2) { |
| #if !defined(PNG_1_0_X) |
| /* this should have happened in png_init_mmx_flags() already */ |
| png_warning(png_ptr, "asm_flags may not have been initialized"); |
| #endif |
| png_mmx_support(); |
| } |
| #endif |
| |
| if (row != NULL && row_info != NULL) |
| { |
| png_uint_32 final_width; |
| |
| final_width = row_info->width * png_pass_inc[pass]; |
| |
| switch (row_info->pixel_depth) |
| { |
| case 1: |
| { |
| png_bytep sp, dp; |
| int sshift, dshift; |
| int s_start, s_end, s_inc; |
| png_byte v; |
| png_uint_32 i; |
| int j; |
| |
| sp = row + (png_size_t)((row_info->width - 1) >> 3); |
| dp = row + (png_size_t)((final_width - 1) >> 3); |
| #if defined(PNG_READ_PACKSWAP_SUPPORTED) |
| if (transformations & PNG_PACKSWAP) |
| { |
| sshift = (int)((row_info->width + 7) & 7); |
| dshift = (int)((final_width + 7) & 7); |
| s_start = 7; |
| s_end = 0; |
| s_inc = -1; |
| } |
| else |
| #endif |
| { |
| sshift = 7 - (int)((row_info->width + 7) & 7); |
| dshift = 7 - (int)((final_width + 7) & 7); |
| s_start = 0; |
| s_end = 7; |
| s_inc = 1; |
| } |
| |
| for (i = row_info->width; i; i--) |
| { |
| v = (png_byte)((*sp >> sshift) & 0x1); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| *dp &= (png_byte)((0x7f7f >> (7 - dshift)) & 0xff); |
| *dp |= (png_byte)(v << dshift); |
| if (dshift == s_end) |
| { |
| dshift = s_start; |
| dp--; |
| } |
| else |
| dshift += s_inc; |
| } |
| if (sshift == s_end) |
| { |
| sshift = s_start; |
| sp--; |
| } |
| else |
| sshift += s_inc; |
| } |
| break; |
| } |
| |
| case 2: |
| { |
| png_bytep sp, dp; |
| int sshift, dshift; |
| int s_start, s_end, s_inc; |
| png_uint_32 i; |
| |
| sp = row + (png_size_t)((row_info->width - 1) >> 2); |
| dp = row + (png_size_t)((final_width - 1) >> 2); |
| #if defined(PNG_READ_PACKSWAP_SUPPORTED) |
| if (transformations & PNG_PACKSWAP) |
| { |
| sshift = (png_size_t)(((row_info->width + 3) & 3) << 1); |
| dshift = (png_size_t)(((final_width + 3) & 3) << 1); |
| s_start = 6; |
| s_end = 0; |
| s_inc = -2; |
| } |
| else |
| #endif |
| { |
| sshift = (png_size_t)((3 - ((row_info->width + 3) & 3)) << 1); |
| dshift = (png_size_t)((3 - ((final_width + 3) & 3)) << 1); |
| s_start = 0; |
| s_end = 6; |
| s_inc = 2; |
| } |
| |
| for (i = row_info->width; i; i--) |
| { |
| png_byte v; |
| int j; |
| |
| v = (png_byte)((*sp >> sshift) & 0x3); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| *dp &= (png_byte)((0x3f3f >> (6 - dshift)) & 0xff); |
| *dp |= (png_byte)(v << dshift); |
| if (dshift == s_end) |
| { |
| dshift = s_start; |
| dp--; |
| } |
| else |
| dshift += s_inc; |
| } |
| if (sshift == s_end) |
| { |
| sshift = s_start; |
| sp--; |
| } |
| else |
| sshift += s_inc; |
| } |
| break; |
| } |
| |
| case 4: |
| { |
| png_bytep sp, dp; |
| int sshift, dshift; |
| int s_start, s_end, s_inc; |
| png_uint_32 i; |
| |
| sp = row + (png_size_t)((row_info->width - 1) >> 1); |
| dp = row + (png_size_t)((final_width - 1) >> 1); |
| #if defined(PNG_READ_PACKSWAP_SUPPORTED) |
| if (transformations & PNG_PACKSWAP) |
| { |
| sshift = (png_size_t)(((row_info->width + 1) & 1) << 2); |
| dshift = (png_size_t)(((final_width + 1) & 1) << 2); |
| s_start = 4; |
| s_end = 0; |
| s_inc = -4; |
| } |
| else |
| #endif |
| { |
| sshift = (png_size_t)((1 - ((row_info->width + 1) & 1)) << 2); |
| dshift = (png_size_t)((1 - ((final_width + 1) & 1)) << 2); |
| s_start = 0; |
| s_end = 4; |
| s_inc = 4; |
| } |
| |
| for (i = row_info->width; i; i--) |
| { |
| png_byte v; |
| int j; |
| |
| v = (png_byte)((*sp >> sshift) & 0xf); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| *dp &= (png_byte)((0xf0f >> (4 - dshift)) & 0xff); |
| *dp |= (png_byte)(v << dshift); |
| if (dshift == s_end) |
| { |
| dshift = s_start; |
| dp--; |
| } |
| else |
| dshift += s_inc; |
| } |
| if (sshift == s_end) |
| { |
| sshift = s_start; |
| sp--; |
| } |
| else |
| sshift += s_inc; |
| } |
| break; |
| } |
| |
| /*====================================================================*/ |
| |
| default: /* 8-bit or larger (this is where the routine is modified) */ |
| { |
| png_bytep sptr, dp; |
| png_uint_32 i; |
| png_size_t pixel_bytes; |
| int width = (int)row_info->width; |
| |
| pixel_bytes = (row_info->pixel_depth >> 3); |
| |
| /* point sptr at the last pixel in the pre-expanded row: */ |
| sptr = row + (width - 1) * pixel_bytes; |
| |
| /* point dp at the last pixel position in the expanded row: */ |
| dp = row + (final_width - 1) * pixel_bytes; |
| |
| /* New code by Nirav Chhatrapati - Intel Corporation */ |
| |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_INTERLACE) |
| /* && _mmx_supported */ ) |
| #else |
| if (_mmx_supported) |
| #endif |
| { |
| //-------------------------------------------------------------- |
| if (pixel_bytes == 3) |
| { |
| if (((pass == 0) || (pass == 1)) && width) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| long dummy_value_a; |
| |
| __asm__ __volatile__ ( |
| "subl $21, %%edi \n\t" |
| // (png_pass_inc[pass] - 1)*pixel_bytes |
| |
| ".loop3_pass0: \n\t" |
| "movd (%%esi), %%mm0 \n\t" // x x x x x 2 1 0 |
| "pand (%3), %%mm0 \n\t" // z z z z z 2 1 0 |
| "movq %%mm0, %%mm1 \n\t" // z z z z z 2 1 0 |
| "psllq $16, %%mm0 \n\t" // z z z 2 1 0 z z |
| "movq %%mm0, %%mm2 \n\t" // z z z 2 1 0 z z |
| "psllq $24, %%mm0 \n\t" // 2 1 0 z z z z z |
| "psrlq $8, %%mm1 \n\t" // z z z z z z 2 1 |
| "por %%mm2, %%mm0 \n\t" // 2 1 0 2 1 0 z z |
| "por %%mm1, %%mm0 \n\t" // 2 1 0 2 1 0 2 1 |
| "movq %%mm0, %%mm3 \n\t" // 2 1 0 2 1 0 2 1 |
| "psllq $16, %%mm0 \n\t" // 0 2 1 0 2 1 z z |
| "movq %%mm3, %%mm4 \n\t" // 2 1 0 2 1 0 2 1 |
| "punpckhdq %%mm0, %%mm3 \n\t" // 0 2 1 0 2 1 0 2 |
| "movq %%mm4, 16(%%edi) \n\t" |
| "psrlq $32, %%mm0 \n\t" // z z z z 0 2 1 0 |
| "movq %%mm3, 8(%%edi) \n\t" |
| "punpckldq %%mm4, %%mm0 \n\t" // 1 0 2 1 0 2 1 0 |
| "subl $3, %%esi \n\t" |
| "movq %%mm0, (%%edi) \n\t" |
| "subl $24, %%edi \n\t" |
| "decl %%ecx \n\t" |
| "jnz .loop3_pass0 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D), |
| "=a" (dummy_value_a) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width), // ecx |
| #if defined(PNG_x86_64_USE_GOTPCREL) // formerly _const4: |
| "3" (&_c64._amask5_3_0) // (0x0000000000FFFFFFLL) |
| #else |
| "3" (&_amask5_3_0) // (0x0000000000FFFFFFLL) |
| #endif |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2" // clobber list |
| , "%mm3", "%mm4" |
| #endif |
| ); |
| } |
| else if (((pass == 2) || (pass == 3)) && width) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| long dummy_value_a; |
| |
| __asm__ __volatile__ ( |
| "subl $9, %%edi \n\t" |
| // (png_pass_inc[pass] - 1)*pixel_bytes |
| |
| ".loop3_pass2: \n\t" |
| "movd (%%esi), %%mm0 \n\t" // x x x x x 2 1 0 |
| "pand (%3), %%mm0 \n\t" // z z z z z 2 1 0 |
| "movq %%mm0, %%mm1 \n\t" // z z z z z 2 1 0 |
| "psllq $16, %%mm0 \n\t" // z z z 2 1 0 z z |
| "movq %%mm0, %%mm2 \n\t" // z z z 2 1 0 z z |
| "psllq $24, %%mm0 \n\t" // 2 1 0 z z z z z |
| "psrlq $8, %%mm1 \n\t" // z z z z z z 2 1 |
| "por %%mm2, %%mm0 \n\t" // 2 1 0 2 1 0 z z |
| "por %%mm1, %%mm0 \n\t" // 2 1 0 2 1 0 2 1 |
| "movq %%mm0, 4(%%edi) \n\t" |
| "psrlq $16, %%mm0 \n\t" // z z 2 1 0 2 1 0 |
| "subl $3, %%esi \n\t" |
| "movd %%mm0, (%%edi) \n\t" |
| "subl $12, %%edi \n\t" |
| "decl %%ecx \n\t" |
| "jnz .loop3_pass2 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D), |
| "=a" (dummy_value_a) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width), // ecx |
| #if defined(PNG_x86_64_USE_GOTPCREL) // formerly _const4: |
| "3" (&_c64._amask5_3_0) // (0x0000000000FFFFFFLL) |
| #else |
| "3" (&_amask5_3_0) // (0x0000000000FFFFFFLL) |
| #endif |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2" // clobber list |
| #endif |
| ); |
| } |
| else if (width) /* && ((pass == 4) || (pass == 5)) */ |
| { |
| int width_mmx = ((width >> 1) << 1) - 8; // GRR: huh? |
| if (width_mmx < 0) |
| width_mmx = 0; |
| width -= width_mmx; // 8 or 9 pix, 24 or 27 bytes |
| if (width_mmx) |
| { |
| // png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; |
| // sptr points at last pixel in pre-expanded row |
| // dp points at last pixel position in expanded row |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| long dummy_value_a; |
| long dummy_value_d; |
| |
| __asm__ __volatile__ ( |
| "subl $3, %%esi \n\t" |
| "subl $9, %%edi \n\t" |
| // (png_pass_inc[pass] + 1)*pixel_bytes |
| |
| ".loop3_pass4: \n\t" |
| "movq (%%esi), %%mm0 \n\t" // x x 5 4 3 2 1 0 |
| "movq %%mm0, %%mm1 \n\t" // x x 5 4 3 2 1 0 |
| "movq %%mm0, %%mm2 \n\t" // x x 5 4 3 2 1 0 |
| "psllq $24, %%mm0 \n\t" // 4 3 2 1 0 z z z |
| "pand (%3), %%mm1 \n\t" // z z z z z 2 1 0 |
| "psrlq $24, %%mm2 \n\t" // z z z x x 5 4 3 |
| "por %%mm1, %%mm0 \n\t" // 4 3 2 1 0 2 1 0 |
| "movq %%mm2, %%mm3 \n\t" // z z z x x 5 4 3 |
| "psllq $8, %%mm2 \n\t" // z z x x 5 4 3 z |
| "movq %%mm0, (%%edi) \n\t" |
| "psrlq $16, %%mm3 \n\t" // z z z z z x x 5 |
| "pand (%4), %%mm3 \n\t" // z z z z z z z 5 |
| "por %%mm3, %%mm2 \n\t" // z z x x 5 4 3 5 |
| "subl $6, %%esi \n\t" |
| "movd %%mm2, 8(%%edi) \n\t" |
| "subl $12, %%edi \n\t" |
| "subl $2, %%ecx \n\t" |
| "jnz .loop3_pass4 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D), |
| "=a" (dummy_value_a), |
| "=d" (dummy_value_d) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width_mmx), // ecx |
| #if defined(PNG_x86_64_USE_GOTPCREL) // formerly _const4 and _const6: |
| "3" (&_c64._amask5_3_0), // (0x0000000000FFFFFFLL) |
| "4" (&_c64._amask7_1_0) // (0x00000000000000FFLL) |
| #else |
| "3" (&_amask5_3_0), // (0x0000000000FFFFFFLL) |
| "4" (&_amask7_1_0) // (0x00000000000000FFLL) |
| #endif |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1" // clobber list |
| , "%mm2", "%mm3" |
| #endif |
| ); |
| } |
| |
| sptr -= width_mmx*3; |
| dp -= width_mmx*6; |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| |
| png_memcpy(v, sptr, 3); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| png_memcpy(dp, v, 3); |
| dp -= 3; |
| } |
| sptr -= 3; |
| } |
| } |
| } /* end of pixel_bytes == 3 */ |
| |
| //-------------------------------------------------------------- |
| else if (pixel_bytes == 1) |
| { |
| if (((pass == 0) || (pass == 1)) && width) |
| { |
| int width_mmx = ((width >> 2) << 2); |
| width -= width_mmx; // 0-3 pixels => 0-3 bytes |
| if (width_mmx) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| __asm__ __volatile__ ( |
| "subl $3, %%esi \n\t" |
| "subl $31, %%edi \n\t" |
| |
| ".loop1_pass0: \n\t" |
| "movd (%%esi), %%mm0 \n\t" // x x x x 3 2 1 0 |
| "movq %%mm0, %%mm1 \n\t" // x x x x 3 2 1 0 |
| "punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0 |
| "movq %%mm0, %%mm2 \n\t" // 3 3 2 2 1 1 0 0 |
| "punpcklwd %%mm0, %%mm0 \n\t" // 1 1 1 1 0 0 0 0 |
| "movq %%mm0, %%mm3 \n\t" // 1 1 1 1 0 0 0 0 |
| "punpckldq %%mm0, %%mm0 \n\t" // 0 0 0 0 0 0 0 0 |
| "punpckhdq %%mm3, %%mm3 \n\t" // 1 1 1 1 1 1 1 1 |
| "movq %%mm0, (%%edi) \n\t" |
| "punpckhwd %%mm2, %%mm2 \n\t" // 3 3 3 3 2 2 2 2 |
| "movq %%mm3, 8(%%edi) \n\t" |
| "movq %%mm2, %%mm4 \n\t" // 3 3 3 3 2 2 2 2 |
| "punpckldq %%mm2, %%mm2 \n\t" // 2 2 2 2 2 2 2 2 |
| "punpckhdq %%mm4, %%mm4 \n\t" // 3 3 3 3 3 3 3 3 |
| "movq %%mm2, 16(%%edi) \n\t" |
| "subl $4, %%esi \n\t" |
| "movq %%mm4, 24(%%edi) \n\t" |
| "subl $32, %%edi \n\t" |
| "subl $4, %%ecx \n\t" |
| "jnz .loop1_pass0 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width_mmx) // ecx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2" // clobber list |
| , "%mm3", "%mm4" |
| #endif |
| ); |
| } |
| |
| sptr -= width_mmx; |
| dp -= width_mmx*8; |
| for (i = width; i; i--) |
| { |
| int j; |
| |
| /* I simplified this part in version 1.0.4e |
| * here and in several other instances where |
| * pixel_bytes == 1 -- GR-P |
| * |
| * Original code: |
| * |
| * png_byte v[8]; |
| * png_memcpy(v, sptr, pixel_bytes); |
| * for (j = 0; j < png_pass_inc[pass]; j++) |
| * { |
| * png_memcpy(dp, v, pixel_bytes); |
| * dp -= pixel_bytes; |
| * } |
| * sptr -= pixel_bytes; |
| * |
| * Replacement code is in the next three lines: |
| */ |
| |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| *dp-- = *sptr; |
| } |
| --sptr; |
| } |
| } |
| else if (((pass == 2) || (pass == 3)) && width) |
| { |
| int width_mmx = ((width >> 2) << 2); |
| width -= width_mmx; // 0-3 pixels => 0-3 bytes |
| if (width_mmx) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| __asm__ __volatile__ ( |
| "subl $3, %%esi \n\t" |
| "subl $15, %%edi \n\t" |
| |
| ".loop1_pass2: \n\t" |
| "movd (%%esi), %%mm0 \n\t" // x x x x 3 2 1 0 |
| "punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0 |
| "movq %%mm0, %%mm1 \n\t" // 3 3 2 2 1 1 0 0 |
| "punpcklwd %%mm0, %%mm0 \n\t" // 1 1 1 1 0 0 0 0 |
| "punpckhwd %%mm1, %%mm1 \n\t" // 3 3 3 3 2 2 2 2 |
| "movq %%mm0, (%%edi) \n\t" |
| "subl $4, %%esi \n\t" |
| "movq %%mm1, 8(%%edi) \n\t" |
| "subl $16, %%edi \n\t" |
| "subl $4, %%ecx \n\t" |
| "jnz .loop1_pass2 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width_mmx) // ecx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1" // clobber list |
| #endif |
| ); |
| } |
| |
| sptr -= width_mmx; |
| dp -= width_mmx*4; |
| for (i = width; i; i--) |
| { |
| int j; |
| |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| *dp-- = *sptr; |
| } |
| --sptr; |
| } |
| } |
| else if (width) /* && ((pass == 4) || (pass == 5)) */ |
| { |
| int width_mmx = ((width >> 3) << 3); |
| width -= width_mmx; // 0-3 pixels => 0-3 bytes |
| if (width_mmx) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| __asm__ __volatile__ ( |
| "subl $7, %%esi \n\t" |
| "subl $15, %%edi \n\t" |
| |
| ".loop1_pass4: \n\t" |
| "movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 |
| "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0 |
| "punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0 |
| "punpckhbw %%mm1, %%mm1 \n\t" // 7 7 6 6 5 5 4 4 |
| "movq %%mm1, 8(%%edi) \n\t" |
| "subl $8, %%esi \n\t" |
| "movq %%mm0, (%%edi) \n\t" |
| "subl $16, %%edi \n\t" |
| "subl $8, %%ecx \n\t" |
| "jnz .loop1_pass4 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (none) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width_mmx) // ecx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1" // clobber list |
| #endif |
| ); |
| } |
| |
| sptr -= width_mmx; |
| dp -= width_mmx*2; |
| for (i = width; i; i--) |
| { |
| int j; |
| |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| *dp-- = *sptr; |
| } |
| --sptr; |
| } |
| } |
| } /* end of pixel_bytes == 1 */ |
| |
| //-------------------------------------------------------------- |
| else if (pixel_bytes == 2) |
| { |
| if (((pass == 0) || (pass == 1)) && width) |
| { |
| int width_mmx = ((width >> 1) << 1); |
| width -= width_mmx; // 0,1 pixels => 0,2 bytes |
| if (width_mmx) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| __asm__ __volatile__ ( |
| "subl $2, %%esi \n\t" |
| "subl $30, %%edi \n\t" |
| |
| ".loop2_pass0: \n\t" |
| "movd (%%esi), %%mm0 \n\t" // x x x x 3 2 1 0 |
| "punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0 |
| "movq %%mm0, %%mm1 \n\t" // 3 2 3 2 1 0 1 0 |
| "punpckldq %%mm0, %%mm0 \n\t" // 1 0 1 0 1 0 1 0 |
| "punpckhdq %%mm1, %%mm1 \n\t" // 3 2 3 2 3 2 3 2 |
| "movq %%mm0, (%%edi) \n\t" |
| "movq %%mm0, 8(%%edi) \n\t" |
| "movq %%mm1, 16(%%edi) \n\t" |
| "subl $4, %%esi \n\t" |
| "movq %%mm1, 24(%%edi) \n\t" |
| "subl $32, %%edi \n\t" |
| "subl $2, %%ecx \n\t" |
| "jnz .loop2_pass0 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width_mmx) // ecx |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1" // clobber list |
| #endif |
| ); |
| } |
| |
| sptr -= (width_mmx*2 - 2); // sign fixed |
| dp -= (width_mmx*16 - 2); // sign fixed |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| sptr -= 2; |
| png_memcpy(v, sptr, 2); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| dp -= 2; |
| png_memcpy(dp, v, 2); |
| } |
| } |
| } |
| else if (((pass == 2) || (pass == 3)) && width) |
| { |
| int width_mmx = ((width >> 1) << 1) ; |
| width -= width_mmx; // 0,1 pixels => 0,2 bytes |
| if (width_mmx) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| __asm__ __volatile__ ( |
| "subl $2, %%esi \n\t" |
| "subl $14, %%edi \n\t" |
| |
| ".loop2_pass2: \n\t" |
| "movd (%%esi), %%mm0 \n\t" // x x x x 3 2 1 0 |
| "punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0 |
| "movq %%mm0, %%mm1 \n\t" // 3 2 3 2 1 0 1 0 |
| "punpckldq %%mm0, %%mm0 \n\t" // 1 0 1 0 1 0 1 0 |
| "punpckhdq %%mm1, %%mm1 \n\t" // 3 2 3 2 3 2 3 2 |
| "movq %%mm0, (%%edi) \n\t" |
| "subl $4, %%esi \n\t" |
| "movq %%mm1, 8(%%edi) \n\t" |
| "subl $16, %%edi \n\t" |
| "subl $2, %%ecx \n\t" |
| "jnz .loop2_pass2 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width_mmx) // ecx |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1" // clobber list |
| #endif |
| ); |
| } |
| |
| sptr -= (width_mmx*2 - 2); // sign fixed |
| dp -= (width_mmx*8 - 2); // sign fixed |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| sptr -= 2; |
| png_memcpy(v, sptr, 2); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| dp -= 2; |
| png_memcpy(dp, v, 2); |
| } |
| } |
| } |
| else if (width) // pass == 4 or 5 |
| { |
| int width_mmx = ((width >> 1) << 1) ; |
| width -= width_mmx; // 0,1 pixels => 0,2 bytes |
| if (width_mmx) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| __asm__ __volatile__ ( |
| "subl $2, %%esi \n\t" |
| "subl $6, %%edi \n\t" |
| |
| ".loop2_pass4: \n\t" |
| "movd (%%esi), %%mm0 \n\t" // x x x x 3 2 1 0 |
| "punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0 |
| "subl $4, %%esi \n\t" |
| "movq %%mm0, (%%edi) \n\t" |
| "subl $8, %%edi \n\t" |
| "subl $2, %%ecx \n\t" |
| "jnz .loop2_pass4 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width_mmx) // ecx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0" // clobber list |
| #endif |
| ); |
| } |
| |
| sptr -= (width_mmx*2 - 2); // sign fixed |
| dp -= (width_mmx*4 - 2); // sign fixed |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| sptr -= 2; |
| png_memcpy(v, sptr, 2); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| dp -= 2; |
| png_memcpy(dp, v, 2); |
| } |
| } |
| } |
| } /* end of pixel_bytes == 2 */ |
| |
| //-------------------------------------------------------------- |
| else if (pixel_bytes == 4) |
| { |
| if (((pass == 0) || (pass == 1)) && width) |
| { |
| int width_mmx = ((width >> 1) << 1); |
| width -= width_mmx; // 0,1 pixels => 0,4 bytes |
| if (width_mmx) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| __asm__ __volatile__ ( |
| "subl $4, %%esi \n\t" |
| "subl $60, %%edi \n\t" |
| |
| ".loop4_pass0: \n\t" |
| "movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 |
| "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0 |
| "punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0 |
| "punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4 |
| "movq %%mm0, (%%edi) \n\t" |
| "movq %%mm0, 8(%%edi) \n\t" |
| "movq %%mm0, 16(%%edi) \n\t" |
| "movq %%mm0, 24(%%edi) \n\t" |
| "movq %%mm1, 32(%%edi) \n\t" |
| "movq %%mm1, 40(%%edi) \n\t" |
| "movq %%mm1, 48(%%edi) \n\t" |
| "subl $8, %%esi \n\t" |
| "movq %%mm1, 56(%%edi) \n\t" |
| "subl $64, %%edi \n\t" |
| "subl $2, %%ecx \n\t" |
| "jnz .loop4_pass0 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width_mmx) // ecx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1" // clobber list |
| #endif |
| ); |
| } |
| |
| sptr -= (width_mmx*4 - 4); // sign fixed |
| dp -= (width_mmx*32 - 4); // sign fixed |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| sptr -= 4; |
| png_memcpy(v, sptr, 4); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| dp -= 4; |
| png_memcpy(dp, v, 4); |
| } |
| } |
| } |
| else if (((pass == 2) || (pass == 3)) && width) |
| { |
| int width_mmx = ((width >> 1) << 1); |
| width -= width_mmx; // 0,1 pixels => 0,4 bytes |
| if (width_mmx) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| __asm__ __volatile__ ( |
| "subl $4, %%esi \n\t" |
| "subl $28, %%edi \n\t" |
| |
| ".loop4_pass2: \n\t" |
| "movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 |
| "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0 |
| "punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0 |
| "punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4 |
| "movq %%mm0, (%%edi) \n\t" |
| "movq %%mm0, 8(%%edi) \n\t" |
| "movq %%mm1, 16(%%edi) \n\t" |
| "movq %%mm1, 24(%%edi) \n\t" |
| "subl $8, %%esi \n\t" |
| "subl $32, %%edi \n\t" |
| "subl $2, %%ecx \n\t" |
| "jnz .loop4_pass2 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width_mmx) // ecx |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1" // clobber list |
| #endif |
| ); |
| } |
| |
| sptr -= (width_mmx*4 - 4); // sign fixed |
| dp -= (width_mmx*16 - 4); // sign fixed |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| sptr -= 4; |
| png_memcpy(v, sptr, 4); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| dp -= 4; |
| png_memcpy(dp, v, 4); |
| } |
| } |
| } |
| else if (width) // pass == 4 or 5 |
| { |
| int width_mmx = ((width >> 1) << 1) ; |
| width -= width_mmx; // 0,1 pixels => 0,4 bytes |
| if (width_mmx) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| __asm__ __volatile__ ( |
| "subl $4, %%esi \n\t" |
| "subl $12, %%edi \n\t" |
| |
| ".loop4_pass4: \n\t" |
| "movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 |
| "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0 |
| "punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0 |
| "punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4 |
| "movq %%mm0, (%%edi) \n\t" |
| "subl $8, %%esi \n\t" |
| "movq %%mm1, 8(%%edi) \n\t" |
| "subl $16, %%edi \n\t" |
| "subl $2, %%ecx \n\t" |
| "jnz .loop4_pass4 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width_mmx) // ecx |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1" // clobber list |
| #endif |
| ); |
| } |
| |
| sptr -= (width_mmx*4 - 4); // sign fixed |
| dp -= (width_mmx*8 - 4); // sign fixed |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| sptr -= 4; |
| png_memcpy(v, sptr, 4); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| dp -= 4; |
| png_memcpy(dp, v, 4); |
| } |
| } |
| } |
| } /* end of pixel_bytes == 4 */ |
| |
| //-------------------------------------------------------------- |
| else if (pixel_bytes == 8) |
| { |
| // GRR TEST: should work, but needs testing (special 64-bit version of rpng2?) |
| // GRR NOTE: no need to combine passes here! |
| if (((pass == 0) || (pass == 1)) && width) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| // source is 8-byte RRGGBBAA |
| // dest is 64-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA ... |
| __asm__ __volatile__ ( |
| "subl $56, %%edi \n\t" // start of last block |
| |
| ".loop8_pass0: \n\t" |
| "movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 |
| "movq %%mm0, (%%edi) \n\t" |
| "movq %%mm0, 8(%%edi) \n\t" |
| "movq %%mm0, 16(%%edi) \n\t" |
| "movq %%mm0, 24(%%edi) \n\t" |
| "movq %%mm0, 32(%%edi) \n\t" |
| "movq %%mm0, 40(%%edi) \n\t" |
| "movq %%mm0, 48(%%edi) \n\t" |
| "subl $8, %%esi \n\t" |
| "movq %%mm0, 56(%%edi) \n\t" |
| "subl $64, %%edi \n\t" |
| "decl %%ecx \n\t" |
| "jnz .loop8_pass0 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width) // ecx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0" // clobber list |
| #endif |
| ); |
| } |
| else if (((pass == 2) || (pass == 3)) && width) |
| { |
| // source is 8-byte RRGGBBAA |
| // dest is 32-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA |
| // (recall that expansion is _in place_: sptr and dp |
| // both point at locations within same row buffer) |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| __asm__ __volatile__ ( |
| "subl $24, %%edi \n\t" // start of last block |
| |
| ".loop8_pass2: \n\t" |
| "movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 |
| "movq %%mm0, (%%edi) \n\t" |
| "movq %%mm0, 8(%%edi) \n\t" |
| "movq %%mm0, 16(%%edi) \n\t" |
| "subl $8, %%esi \n\t" |
| "movq %%mm0, 24(%%edi) \n\t" |
| "subl $32, %%edi \n\t" |
| "decl %%ecx \n\t" |
| "jnz .loop8_pass2 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width) // ecx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0" // clobber list |
| #endif |
| ); |
| } |
| } |
| else if (width) // pass == 4 or 5 |
| { |
| // source is 8-byte RRGGBBAA |
| // dest is 16-byte RRGGBBAA RRGGBBAA |
| { |
| int dummy_value_c; // fix 'forbidden register spilled' |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| __asm__ __volatile__ ( |
| "subl $8, %%edi \n\t" // start of last block |
| |
| ".loop8_pass4: \n\t" |
| "movq (%%esi), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 |
| "movq %%mm0, (%%edi) \n\t" |
| "subl $8, %%esi \n\t" |
| "movq %%mm0, 8(%%edi) \n\t" |
| "subl $16, %%edi \n\t" |
| "decl %%ecx \n\t" |
| "jnz .loop8_pass4 \n\t" |
| "EMMS \n\t" // DONE |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D) |
| |
| : "1" (sptr), // esi // input regs |
| "2" (dp), // edi |
| "0" (width) // ecx |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0" // clobber list |
| #endif |
| ); |
| } |
| } |
| |
| } /* end of pixel_bytes == 8 */ |
| |
| //-------------------------------------------------------------- |
| else if (pixel_bytes == 6) |
| { |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| png_memcpy(v, sptr, 6); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| png_memcpy(dp, v, 6); |
| dp -= 6; |
| } |
| sptr -= 6; |
| } |
| } /* end of pixel_bytes == 6 */ |
| |
| //-------------------------------------------------------------- |
| else |
| { |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| png_memcpy(v, sptr, pixel_bytes); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| png_memcpy(dp, v, pixel_bytes); |
| dp -= pixel_bytes; |
| } |
| sptr-= pixel_bytes; |
| } |
| } |
| } // end of _mmx_supported ======================================== |
| |
| else /* MMX not supported: use modified C code - takes advantage |
| * of inlining of png_memcpy for a constant */ |
| /* GRR 19991007: does it? or should pixel_bytes in each |
| * block be replaced with immediate value (e.g., 1)? */ |
| /* GRR 19991017: replaced with constants in each case */ |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| { |
| if (pixel_bytes == 1) |
| { |
| for (i = width; i; i--) |
| { |
| int j; |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| *dp-- = *sptr; |
| } |
| --sptr; |
| } |
| } |
| else if (pixel_bytes == 3) |
| { |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| png_memcpy(v, sptr, 3); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| png_memcpy(dp, v, 3); |
| dp -= 3; |
| } |
| sptr -= 3; |
| } |
| } |
| else if (pixel_bytes == 2) |
| { |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| png_memcpy(v, sptr, 2); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| png_memcpy(dp, v, 2); |
| dp -= 2; |
| } |
| sptr -= 2; |
| } |
| } |
| else if (pixel_bytes == 4) |
| { |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| png_memcpy(v, sptr, 4); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| #if defined(PNG_DEBUG) |
| if (dp < row || dp+3 > row+png_ptr->row_buf_size) |
| { |
| printf("dp out of bounds: row=%d, dp=%d, rp=%d\n", |
| row, dp, row+png_ptr->row_buf_size); |
| printf("row_buf=%d\n",png_ptr->row_buf_size); |
| } |
| #endif |
| png_memcpy(dp, v, 4); |
| dp -= 4; |
| } |
| sptr -= 4; |
| } |
| } |
| else if (pixel_bytes == 6) |
| { |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| png_memcpy(v, sptr, 6); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| png_memcpy(dp, v, 6); |
| dp -= 6; |
| } |
| sptr -= 6; |
| } |
| } |
| else if (pixel_bytes == 8) |
| { |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| png_memcpy(v, sptr, 8); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| png_memcpy(dp, v, 8); |
| dp -= 8; |
| } |
| sptr -= 8; |
| } |
| } |
| else /* GRR: should never be reached */ |
| { |
| for (i = width; i; i--) |
| { |
| png_byte v[8]; |
| int j; |
| png_memcpy(v, sptr, pixel_bytes); |
| for (j = 0; j < png_pass_inc[pass]; j++) |
| { |
| png_memcpy(dp, v, pixel_bytes); |
| dp -= pixel_bytes; |
| } |
| sptr -= pixel_bytes; |
| } |
| } |
| |
| } /* end if (MMX not supported) */ |
| break; |
| } |
| } /* end switch (row_info->pixel_depth) */ |
| |
| row_info->width = final_width; |
| |
| row_info->rowbytes = PNG_ROWBYTES(row_info->pixel_depth,final_width); |
| } |
| |
| } /* end png_do_read_interlace() */ |
| |
| #endif /* PNG_HAVE_MMX_READ_INTERLACE */ |
| #endif /* PNG_READ_INTERLACING_SUPPORTED */ |
| |
| |
| |
| #if defined(PNG_HAVE_MMX_READ_FILTER_ROW) |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| |
| //===========================================================================// |
| // // |
| // P N G _ R E A D _ F I L T E R _ R O W _ M M X _ A V G // |
| // // |
| //===========================================================================// |
| |
| // Optimized code for PNG Average filter decoder |
| |
| static void /* PRIVATE */ |
| png_read_filter_row_mmx_avg(png_row_infop row_info, png_bytep row, |
| png_bytep prev_row) |
| { |
| png_uint_32 FullLength, MMXLength; |
| int bpp; |
| int dummy_value_a; |
| int dummy_value_c; // fix 'forbidden register 2 (cx) was spilled' error |
| int dummy_value_d; |
| int dummy_value_S; |
| int dummy_value_D; |
| int diff; // __attribute__((used)); |
| |
| bpp = (row_info->pixel_depth + 7) >> 3; // calc number of bytes per pixel |
| FullLength = row_info->rowbytes; // number of bytes to filter |
| |
| __asm__ __volatile__ ( |
| SAVE_GOT_ebx |
| SAVE_r15 |
| SAVE_ebp |
| // initialize address pointers and offset |
| //pre "movl row, %%edi \n\t" // edi: ptr to Avg(x) |
| "xorl %%ebx, %%ebx \n\t" // ebx: x |
| //pre "movl prev_row, %%esi \n\t" // esi: ptr to Prior(x) |
| "movl %%edi, %%edx \n\t" |
| //pre "subl bpp, %%edx \n\t" // (bpp is preloaded into ecx) |
| "subl %%ecx, %%edx \n\t" // edx: ptr to Raw(x-bpp) |
| //pre "movl FullLength, %%eax \n\t" // bring in via eax... |
| SAVE_FullLength // ...but store for later use |
| "xorl %%eax, %%eax \n\t" |
| |
| // Compute the Raw value for the first bpp bytes |
| // Raw(x) = Avg(x) + (Prior(x)/2) |
| "avg_rlp: \n\t" |
| "movb (%%esi,%%ebx,), %%al \n\t" // load al with Prior(x) |
| "incl %%ebx \n\t" |
| "shrb %%al \n\t" // divide by 2 |
| "addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset inc ebx |
| //pre "cmpl bpp, %%ebx \n\t" // (bpp is preloaded into ecx) |
| "cmpl %%ecx, %%ebx \n\t" |
| "movb %%al, -1(%%edi,%%ebx,) \n\t" // write Raw(x); -1 to offset inc ebx |
| "jb avg_rlp \n\t" // mov does not affect flags |
| |
| // get # of bytes to alignment |
| "movl %%edi, %%ebp \n\t" // take start of row |
| "addl %%ebx, %%ebp \n\t" // add bpp |
| "addl $0xf, %%ebp \n\t" // add 7+8 to incr past alignment bdry |
| "andl $0xfffffff8, %%ebp \n\t" // mask to alignment boundary |
| "subl %%edi, %%ebp \n\t" // subtract from start => value ebx at |
| "jz avg_go \n\t" // alignment |
| |
| // fix alignment |
| // Compute the Raw value for the bytes up to the alignment boundary |
| // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2) |
| "xorl %%ecx, %%ecx \n\t" |
| |
| "avg_lp1: \n\t" |
| "xorl %%eax, %%eax \n\t" |
| "movb (%%esi,%%ebx,), %%cl \n\t" // load cl with Prior(x) |
| "movb (%%edx,%%ebx,), %%al \n\t" // load al with Raw(x-bpp) |
| "addw %%cx, %%ax \n\t" |
| "incl %%ebx \n\t" |
| "shrw %%ax \n\t" // divide by 2 |
| "addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset inc ebx |
| "cmpl %%ebp, %%ebx \n\t" // check if at alignment boundary |
| "movb %%al, -1(%%edi,%%ebx,) \n\t" // write Raw(x); -1 to offset inc ebx |
| "jb avg_lp1 \n\t" // repeat until at alignment boundary |
| |
| "avg_go: \n\t" |
| RESTORE_FullLength "%%eax \n\t" // FullLength -> eax |
| "movl %%eax, %%ecx \n\t" |
| "subl %%ebx, %%eax \n\t" // subtract alignment fix |
| "andl $0x00000007, %%eax \n\t" // calc bytes over mult of 8 |
| "subl %%eax, %%ecx \n\t" // drop over bytes from original length |
| //out "movl %%ecx, MMXLength \n\t" |
| "movl %%ebp, %%edi \n\t" // ebp = diff, but no reg constraint(?) |
| RESTORE_ebp // (could swap ebp and ecx functions) |
| RESTORE_r15 |
| RESTORE_GOT_ebx |
| |
| : "=c" (MMXLength), // output regs |
| "=S" (dummy_value_S), |
| "=D" (diff), |
| "=a" (dummy_value_a) |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (prev_row), // esi |
| "2" (row), // edi |
| "3" (FullLength) // eax |
| |
| : "%edx" // clobber list |
| _CLOBBER_r15 |
| _CLOBBER_ebp |
| _CLOBBER_GOT_ebx |
| ); |
| |
| // now do the math for the rest of the row |
| switch (bpp) |
| { |
| case 3: |
| { |
| // _ShiftBpp = 24; // == 3 * 8 |
| // _ShiftRem = 40; // == 64 - 24 |
| |
| __asm__ __volatile__ ( |
| // re-init address pointers and offset |
| LOAD_GOT_rbp |
| "movq " AMASK5_3_0 ", %%mm7 \n\t" // _amask5_3_0 -> mm7 |
| // preload "movl diff, %%ecx \n\t" // ecx: x = offset to |
| // alignment boundary |
| "movq " LB_CARRY_MASK ", %%mm5 \n\t" // [interleave for parallel.?] |
| // preload "movl row, %%edi \n\t" // edi: Avg(x) |
| "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4 |
| // preload "movl prev_row, %%esi \n\t" // esi: Prior(x) |
| RESTORE_rbp |
| |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%ecx,), %%mm2 \n\t"// load previous aligned 8 bytes |
| // (correct pos. in loop below) |
| "avg_3lp: \n\t" |
| "movq (%%edi,%%ecx,), %%mm0 \n\t" // load mm0 with Avg(x) |
| "movq %%mm5, %%mm3 \n\t" |
| "psrlq $40, %%mm2 \n\t" // correct position Raw(x-bpp) |
| // data |
| "movq (%%esi,%%ecx,), %%mm1 \n\t" // load mm1 with Prior(x) |
| "movq %%mm7, %%mm6 \n\t" |
| "pand %%mm1, %%mm3 \n\t" // get lsb for each prevrow byte |
| "psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2 |
| "pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for |
| // each byte |
| // add 1st active group (Raw(x-bpp)/2) to average with LBCarry |
| "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting |
| // LBCarrys |
| "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte |
| // where both lsb's were == 1 |
| // (valid only for active group) |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to Raw(x-bpp)/2 |
| // for each byte |
| "pand %%mm6, %%mm2 \n\t" // leave only Active Group 1 |
| // bytes to add to Avg |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to |
| // Avg for each Active byte |
| // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry |
| "psllq $24, %%mm6 \n\t" // shift the mm6 mask to cover |
| // bytes 3-5 |
| "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 |
| "psllq $24, %%mm2 \n\t" // shift data to pos. correctly |
| "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting |
| // LBCarrys |
| "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte |
| // where both lsb's were == 1 |
| // (valid only for active group) |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to Raw(x-bpp)/2 |
| // for each byte |
| "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2 |
| // bytes to add to Avg |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to |
| // Avg for each Active byte |
| |
| // add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry |
| "psllq $24, %%mm6 \n\t" // shift mm6 mask to cover last |
| // two bytes |
| "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 |
| "psllq $24, %%mm2 \n\t" // shift data to pos. correctly |
| // Data need be shifted only once here to |
| // get the correct x-bpp offset. |
| "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting |
| // LBCarrys |
| "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte |
| // where both |
| // lsb's were == 1 (only valid for active group) |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to Raw(x-bpp)/2 |
| // for each byte |
| "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2 |
| // bytes to add to Avg |
| "addl $8, %%ecx \n\t" |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to |
| // Avg for each Active byte |
| // now ready to write back to memory |
| "movq %%mm0, -8(%%edi,%%ecx,) \n\t" |
| // move updated Raw(x) to use as Raw(x-bpp) for next loop |
| "cmpl %%eax, %%ecx \n\t" // MMXLength |
| "movq %%mm0, %%mm2 \n\t" // mov updated Raw(x) to mm2 |
| "jb avg_3lp \n\t" |
| |
| : "=S" (dummy_value_S), // output regs (dummy) |
| "=D" (dummy_value_D), |
| "=c" (dummy_value_c), |
| "=a" (dummy_value_a) |
| |
| : "0" (prev_row), // esi // input regs |
| "1" (row), // edi |
| "2" (diff), // ecx |
| "3" (MMXLength) // eax |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| break; // end 3 bpp |
| |
| case 4: // formerly shared with 6 bpp case via _ShiftBpp and _ShiftRem, |
| { // but loop uses all 8 MMX regs, and psrlq/psllq require 64-bit |
| // mem (PIC/.so problems), MMX reg (none avail.), or immediate |
| // _ShiftBpp = bpp << 3; // 32 (psllq) |
| // _ShiftRem = 64 - _ShiftBpp; // 32 (psrlq) |
| |
| __asm__ __volatile__ ( |
| LOAD_GOT_rbp |
| "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4 |
| "movq " LB_CARRY_MASK ", %%mm5 \n\t" // _LBCarryMask -> mm5 |
| // re-init address pointers and offset |
| // preload "movl diff, %%ecx \n\t" // ecx: x = offset to |
| // alignment boundary |
| "movq " AMASK0_8_0 ", %%mm7 \n\t" // _amask0_8_0 -> mm7 |
| RESTORE_rbp |
| |
| // ... and clear all bytes except for 1st active group |
| // preload "movl row, %%edi \n\t" // edi: Avg(x) |
| "psrlq $32, %%mm7 \n\t" // was _ShiftRem |
| // preload "movl prev_row, %%esi \n\t" // esi: Prior(x) |
| "movq %%mm7, %%mm6 \n\t" |
| "psllq $32, %%mm6 \n\t" // mask for 2nd active group |
| |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes |
| // (we correct pos. in loop below) |
| "avg_4lp: \n\t" |
| "movq (%%edi,%%ecx,), %%mm0 \n\t" |
| "psrlq $32, %%mm2 \n\t" // shift data to pos. correctly |
| "movq (%%esi,%%ecx,), %%mm1 \n\t" |
| // add (Prev_row/2) to average |
| "movq %%mm5, %%mm3 \n\t" |
| "pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte |
| "psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2 |
| "pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for |
| // each byte |
| // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry |
| "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting |
| // LBCarrys |
| "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte |
| // where both |
| // lsb's were == 1 (only valid for active group) |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2) |
| // for each byte |
| "pand %%mm7, %%mm2 \n\t" // leave only Active Group 1 |
| // bytes to add to Avg |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to Avg |
| // for each Active |
| // byte |
| // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry |
| "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 |
| "psllq $32, %%mm2 \n\t" // shift data to pos. correctly |
| "addl $8, %%ecx \n\t" |
| "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting |
| // LBCarrys |
| "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte |
| // where both |
| // lsb's were == 1 (only valid for active group) |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2) |
| // for each byte |
| "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2 |
| // bytes to add to Avg |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to |
| // Avg for each Active byte |
| "cmpl %%eax, %%ecx \n\t" // MMXLength |
| // now ready to write back to memory |
| "movq %%mm0, -8(%%edi,%%ecx,) \n\t" |
| // prep Raw(x-bpp) for next loop |
| "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 |
| "jb avg_4lp \n\t" |
| |
| : "=S" (dummy_value_S), // output regs (dummy) |
| "=D" (dummy_value_D), |
| "=c" (dummy_value_c), |
| "=a" (dummy_value_a) |
| |
| : "0" (prev_row), // esi // input regs |
| "1" (row), // edi |
| "2" (diff), // ecx |
| "3" (MMXLength) // eax |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| break; // end 4 bpp |
| |
| case 6: // formerly shared with 4 bpp case (see comments there) |
| { |
| // _ShiftBpp = bpp << 3; // 48 (psllq) |
| // _ShiftRem = 64 - _ShiftBpp; // 16 (psrlq) |
| |
| __asm__ __volatile__ ( |
| LOAD_GOT_rbp |
| "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4 |
| "movq " LB_CARRY_MASK ", %%mm5 \n\t" // _LBCarryMask -> mm5 |
| // re-init address pointers and offset |
| // preload "movl diff, %%ecx \n\t" // ecx: x = offset to |
| // alignment boundary |
| "movq " AMASK0_8_0 ", %%mm7 \n\t" // _amask0_8_0 -> mm7 |
| RESTORE_rbp |
| |
| // ... and clear all bytes except for 1st active group |
| // preload "movl row, %%edi \n\t" // edi: Avg(x) |
| "psrlq $16, %%mm7 \n\t" |
| // preload "movl prev_row, %%esi \n\t" // esi: Prior(x) |
| "movq %%mm7, %%mm6 \n\t" |
| "psllq $48, %%mm6 \n\t" // mask for 2nd active group |
| |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes |
| // (we correct pos. in loop below) |
| "avg_6lp: \n\t" |
| "movq (%%edi,%%ecx,), %%mm0 \n\t" |
| "psrlq $16, %%mm2 \n\t" // shift data to pos. correctly |
| "movq (%%esi,%%ecx,), %%mm1 \n\t" |
| // add (Prev_row/2) to average |
| "movq %%mm5, %%mm3 \n\t" |
| "pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte |
| "psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2 |
| "pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for |
| // each byte |
| // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry |
| "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting |
| // LBCarrys |
| "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte |
| // where both |
| // lsb's were == 1 (only valid for active group) |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2) |
| // for each byte |
| "pand %%mm7, %%mm2 \n\t" // leave only Active Group 1 |
| // bytes to add to Avg |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to Avg |
| // for each Active |
| // byte |
| // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry |
| "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 |
| "psllq $48, %%mm2 \n\t" // shift data to pos. correctly |
| "addl $8, %%ecx \n\t" |
| "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting |
| // LBCarrys |
| "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte |
| // where both |
| // lsb's were == 1 (only valid for active group) |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2) |
| // for each byte |
| "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2 |
| // bytes to add to Avg |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to |
| // Avg for each Active byte |
| "cmpl %%eax, %%ecx \n\t" // MMXLength |
| // now ready to write back to memory |
| "movq %%mm0, -8(%%edi,%%ecx,) \n\t" |
| // prep Raw(x-bpp) for next loop |
| "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 |
| "jb avg_6lp \n\t" |
| |
| : "=S" (dummy_value_S), // output regs (dummy) |
| "=D" (dummy_value_D), |
| "=c" (dummy_value_c), |
| "=a" (dummy_value_a) |
| |
| : "0" (prev_row), // esi // input regs |
| "1" (row), // edi |
| "2" (diff), // ecx |
| "3" (MMXLength) // eax |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| break; // end 6 bpp |
| |
| case 2: |
| { |
| // _ShiftBpp = 16; // == 2 * 8 |
| // _ShiftRem = 48; // == 64 - _ShiftBpp |
| |
| __asm__ __volatile__ ( |
| LOAD_GOT_rbp |
| // load (former) _ActiveMask |
| "movq " AMASK6_2_0 ", %%mm7 \n\t" // _amask6_2_0 -> mm7 |
| // re-init address pointers and offset |
| // preload "movl diff, %%ecx \n\t" // ecx: x = offset to |
| // alignment boundary |
| "movq " LB_CARRY_MASK ", %%mm5 \n\t" // _LBCarryMask -> mm5 |
| // preload "movl row, %%edi \n\t" // edi: Avg(x) |
| "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4 |
| // preload "movl prev_row, %%esi \n\t" // esi: Prior(x) |
| RESTORE_rbp |
| |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes |
| // (we correct pos. in loop below) |
| "avg_2lp: \n\t" |
| "movq (%%edi,%%ecx,), %%mm0 \n\t" |
| "psrlq $48, %%mm2 \n\t" // shift data to pos. correctly |
| "movq (%%esi,%%ecx,), %%mm1 \n\t" // (GRR BUGFIX: was psllq) |
| // add (Prev_row/2) to average |
| "movq %%mm5, %%mm3 \n\t" |
| "pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte |
| "psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2 |
| "pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each |
| // byte |
| "movq %%mm7, %%mm6 \n\t" |
| "paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for |
| // each byte |
| |
| // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry |
| "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting |
| // LBCarrys |
| "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte |
| // where both |
| // lsb's were == 1 (only valid |
| // for active group) |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2) |
| // for each byte |
| "pand %%mm6, %%mm2 \n\t" // leave only Active Group 1 |
| // bytes to add to Avg |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to Avg |
| // for each Active byte |
| |
| // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry |
| "psllq $16, %%mm6 \n\t" // shift the mm6 mask to cover |
| // bytes 2 & 3 |
| "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 |
| "psllq $16, %%mm2 \n\t" // shift data to pos. correctly |
| "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting |
| // LBCarrys |
| "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte |
| // where both |
| // lsb's were == 1 (only valid |
| // for active group) |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2) |
| // for each byte |
| "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2 |
| // bytes to add to Avg |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to |
| // Avg for each Active byte |
| |
| // add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry |
| "psllq $16, %%mm6 \n\t" // shift the mm6 mask to cover |
| // bytes 4 & 5 |
| "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 |
| "psllq $16, %%mm2 \n\t" // shift data to pos. correctly |
| "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting |
| // LBCarrys |
| "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte |
| // where both lsb's were == 1 |
| // (only valid for active group) |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2) |
| // for each byte |
| "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2 |
| // bytes to add to Avg |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to |
| // Avg for each Active byte |
| |
| // add 4th active group (Raw(x-bpp)/2) to average with _LBCarry |
| "psllq $16, %%mm6 \n\t" // shift the mm6 mask to cover |
| // bytes 6 & 7 |
| "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 |
| "psllq $16, %%mm2 \n\t" // shift data to pos. correctly |
| "addl $8, %%ecx \n\t" |
| "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting |
| // LBCarrys |
| "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte |
| // where both |
| // lsb's were == 1 (only valid |
| // for active group) |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each |
| // byte |
| "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2) |
| // for each byte |
| "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2 |
| // bytes to add to Avg |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to |
| // Avg for each Active byte |
| "cmpl %%eax, %%ecx \n\t" // MMXLength |
| // now ready to write back to memory |
| "movq %%mm0, -8(%%edi,%%ecx,) \n\t" |
| // prep Raw(x-bpp) for next loop |
| "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 |
| "jb avg_2lp \n\t" |
| |
| : "=S" (dummy_value_S), // output regs (dummy) |
| "=D" (dummy_value_D), |
| "=c" (dummy_value_c), |
| "=a" (dummy_value_a) |
| |
| : "0" (prev_row), // esi // input regs |
| "1" (row), // edi |
| "2" (diff), // ecx |
| "3" (MMXLength) // eax |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| break; // end 2 bpp |
| |
| case 1: |
| { |
| __asm__ __volatile__ ( |
| // re-init address pointers and offset |
| // preload "movl diff, %%eax \n\t" // eax: x = offset to align. bdry |
| // preload "movl row, %%edi \n\t" // edi: Avg(x) |
| // preload "movl FullLength, %%edx \n\t" |
| "cmpl %%edx, %%eax \n\t" // test if offset at end of array |
| "jnb avg_1end \n\t" |
| |
| SAVE_ebp |
| |
| // do Avg decode for remaining bytes |
| // preload "movl prev_row, %%esi \n\t" // esi: Prior(x) |
| "movl %%edi, %%ebp \n\t" |
| // preload "subl bpp, %%ebp \n\t" // (bpp is preloaded into ecx) |
| "subl %%ecx, %%ebp \n\t" // ebp: Raw(x-bpp) |
| "xorl %%ecx, %%ecx \n\t" // zero ecx before using cl & cx |
| // in loop below |
| SAVE_GOT_ebx |
| |
| "avg_1lp: \n\t" |
| // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2) |
| "xorl %%ebx, %%ebx \n\t" |
| "movb (%%esi,%%eax,), %%cl \n\t" // load cl with Prior(x) |
| "movb (%%ebp,%%eax,), %%bl \n\t" // load bl with Raw(x-bpp) |
| "addw %%cx, %%bx \n\t" |
| "incl %%eax \n\t" |
| "shrw %%bx \n\t" // divide by 2 |
| "addb -1(%%edi,%%eax,), %%bl \n\t" // add Avg(x); -1 to offset |
| // inc eax |
| "cmpl %%edx, %%eax \n\t" // check if at end of array |
| "movb %%bl, -1(%%edi,%%eax,) \n\t" // write back Raw(x); |
| // mov does not affect flags; -1 to offset inc eax |
| "jb avg_1lp \n\t" |
| |
| RESTORE_GOT_ebx |
| RESTORE_ebp |
| |
| "avg_1end: \n\t" |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D), |
| "=a" (dummy_value_a), |
| "=d" (dummy_value_d) |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (prev_row), // esi |
| "2" (row), // edi |
| "3" (diff), // eax |
| "4" (FullLength) // edx |
| |
| CLOB_COLON_ebx_ebp // clobber list |
| CLOBBER_GOT_ebx |
| CLOB_COMMA_ebx_ebp |
| CLOBBER_ebp |
| ); |
| } |
| return; // end 1 bpp |
| |
| case 8: |
| { |
| __asm__ __volatile__ ( |
| // re-init address pointers and offset |
| // preload "movl diff, %%ecx \n\t" // ecx: x = offset to |
| // alignment boundary |
| LOAD_GOT_rbp |
| "movq " LB_CARRY_MASK ", %%mm5 \n\t" // [interleave for parallel.?] |
| // preload "movl row, %%edi \n\t" // edi: Avg(x) |
| "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4 |
| // preload "movl prev_row, %%esi \n\t" // esi: Prior(x) |
| RESTORE_rbp |
| |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes |
| // (NO NEED to correct pos. in loop below) |
| |
| "avg_8lp: \n\t" |
| "movq (%%edi,%%ecx,), %%mm0 \n\t" |
| "movq %%mm5, %%mm3 \n\t" |
| "movq (%%esi,%%ecx,), %%mm1 \n\t" |
| "addl $8, %%ecx \n\t" |
| "pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte |
| "psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2 |
| "pand %%mm2, %%mm3 \n\t" // get LBCarrys for each byte |
| // where both lsb's were == 1 |
| "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 |
| "pand %%mm4, %%mm1 \n\t" // clear invalid bit 7, each byte |
| "paddb %%mm3, %%mm0 \n\t" // add LBCarrys to Avg, each byte |
| "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7, each byte |
| "paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg, each |
| "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) to Avg for each |
| "cmpl %%eax, %%ecx \n\t" // MMXLength |
| "movq %%mm0, -8(%%edi,%%ecx,) \n\t" |
| "movq %%mm0, %%mm2 \n\t" // reuse as Raw(x-bpp) |
| "jb avg_8lp \n\t" |
| |
| : "=S" (dummy_value_S), // output regs (dummy) |
| "=D" (dummy_value_D), |
| "=c" (dummy_value_c), |
| "=a" (dummy_value_a) |
| |
| : "0" (prev_row), // esi // input regs |
| "1" (row), // edi |
| "2" (diff), // ecx |
| "3" (MMXLength) // eax |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2" // clobber list |
| , "%mm3", "%mm4", "%mm5" |
| #endif |
| ); |
| } |
| break; // end 8 bpp |
| |
| default: // bpp != 1,2,3,4,6,8: doesn't exist |
| { |
| #if defined(PNG_DEBUG) |
| // ERROR: SHOULD NEVER BE REACHED |
| png_debug(1, |
| "Internal libpng logic error (GCC png_read_filter_row_mmx_avg())\n"); |
| #endif |
| } |
| break; |
| |
| } // end switch (bpp) |
| |
| __asm__ __volatile__ ( |
| // MMX acceleration complete; now do clean-up |
| // check if any remaining bytes left to decode |
| //pre "movl FullLength, %%edx \n\t" |
| //pre "movl MMXLength, %%eax \n\t" // eax: x == offset bytes after MMX |
| //pre "movl row, %%edi \n\t" // edi: Avg(x) |
| "cmpl %%edx, %%eax \n\t" // test if offset at end of array |
| "jnb avg_end \n\t" |
| |
| SAVE_ebp |
| |
| // do Avg decode for remaining bytes |
| //pre "movl prev_row, %%esi \n\t" // esi: Prior(x) |
| "movl %%edi, %%ebp \n\t" |
| //pre "subl bpp, %%ebp \n\t" // (bpp is preloaded into ecx) |
| "subl %%ecx, %%ebp \n\t" // ebp: Raw(x-bpp) |
| "xorl %%ecx, %%ecx \n\t" // zero ecx before using cl & cx below |
| |
| SAVE_GOT_ebx |
| |
| "avg_lp2: \n\t" |
| // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2) |
| "xorl %%ebx, %%ebx \n\t" |
| "movb (%%esi,%%eax,), %%cl \n\t" // load cl with Prior(x) |
| "movb (%%ebp,%%eax,), %%bl \n\t" // load bl with Raw(x-bpp) |
| "addw %%cx, %%bx \n\t" |
| "incl %%eax \n\t" |
| "shrw %%bx \n\t" // divide by 2 |
| "addb -1(%%edi,%%eax,), %%bl \n\t" // add Avg(x); -1 to offset inc eax |
| "cmpl %%edx, %%eax \n\t" // check if at end of array |
| "movb %%bl, -1(%%edi,%%eax,) \n\t" // write back Raw(x) [mov does not |
| "jb avg_lp2 \n\t" // affect flags; -1 to offset inc eax] |
| |
| RESTORE_GOT_ebx |
| RESTORE_ebp |
| |
| "avg_end: \n\t" |
| "EMMS \n\t" // end MMX; prep for poss. FP instrs. |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D), |
| "=a" (dummy_value_a), |
| "=d" (dummy_value_d) |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (prev_row), // esi |
| "2" (row), // edi |
| "3" (MMXLength), // eax |
| "4" (FullLength) // edx |
| |
| CLOB_COLON_ebx_ebp // clobber list |
| CLOBBER_GOT_ebx |
| CLOB_COMMA_ebx_ebp |
| CLOBBER_ebp |
| ); |
| |
| } /* end png_read_filter_row_mmx_avg() */ |
| |
| |
| |
| #if defined(PNG_x86_64_USE_GOTPCREL) || defined(PNG_THREAD_UNSAFE_OK) |
| |
| //===========================================================================// |
| // // |
| // P N G _ R E A D _ F I L T E R _ R O W _ M M X _ P A E T H // |
| // // |
| //===========================================================================// |
| |
| // Optimized code for PNG Paeth filter decoder |
| |
| static void /* PRIVATE */ |
| png_read_filter_row_mmx_paeth(png_row_infop row_info, png_bytep row, |
| png_bytep prev_row) |
| { |
| png_uint_32 FullLength, MMXLength; |
| int bpp; |
| int dummy_value_a; |
| int dummy_value_c; // fix 'forbidden register 2 (cx) was spilled' error |
| int dummy_value_d; |
| int dummy_value_S; |
| int dummy_value_D; |
| int diff; // __attribute__((used)); |
| |
| bpp = (row_info->pixel_depth + 7) >> 3; // calc number of bytes per pixel |
| FullLength = row_info->rowbytes; // number of bytes to filter |
| |
| __asm__ __volatile__ ( |
| SAVE_GOT_ebx |
| SAVE_r15 |
| SAVE_ebp |
| //pre "movl row, %%edi \n\t" |
| "xorl %%ebx, %%ebx \n\t" // ebx: x offset |
| //pre "movl prev_row, %%esi \n\t" |
| "xorl %%edx, %%edx \n\t" // edx: x-bpp offset |
| //pre "movl FullLength, %%eax \n\t" // bring in via eax... |
| SAVE_FullLength // ...but store for later use |
| "xorl %%eax, %%eax \n\t" |
| |
| // Compute the Raw value for the first bpp bytes |
| // Note: the formula works out to be always |
| // Paeth(x) = Raw(x) + Prior(x) where x < bpp |
| "paeth_rlp: \n\t" |
| "movb (%%edi,%%ebx,), %%al \n\t" |
| "addb (%%esi,%%ebx,), %%al \n\t" |
| "incl %%ebx \n\t" |
| //pre "cmpl bpp, %%ebx \n\t" (bpp is preloaded into ecx) |
| "cmpl %%ecx, %%ebx \n\t" |
| "movb %%al, -1(%%edi,%%ebx,) \n\t" |
| "jb paeth_rlp \n\t" |
| |
| // get # of bytes to alignment |
| "movl %%edi, %%ebp \n\t" // take start of row |
| "addl %%ebx, %%ebp \n\t" // add bpp |
| "xorl %%ecx, %%ecx \n\t" |
| "addl $0xf, %%ebp \n\t" // add 7+8 to incr past alignment bdry |
| "andl $0xfffffff8, %%ebp \n\t" // mask to alignment boundary |
| "subl %%edi, %%ebp \n\t" // subtract from start ==> value ebx |
| // at alignment |
| "jz paeth_go \n\t" |
| // fix alignment |
| |
| SAVE_r11_r12_r13 |
| |
| "paeth_lp1: \n\t" |
| "xorl %%eax, %%eax \n\t" |
| // pav = p - a = (a + b - c) - a = b - c |
| "movb (%%esi,%%ebx,), %%al \n\t" // load Prior(x) into al |
| "movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl |
| "subl %%ecx, %%eax \n\t" // subtract Prior(x-bpp) |
| "movl %%eax, " pa_TEMP " \n\t" // Save pav for later use |
| "xorl %%eax, %%eax \n\t" |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movb (%%edi,%%edx,), %%al \n\t" // load Raw(x-bpp) into al |
| "subl %%ecx, %%eax \n\t" // subtract Prior(x-bpp) |
| "movl %%eax, %%ecx \n\t" |
| // pcv = p - c = (a + b - c) - c = (a - c) + (b - c) = pav + pbv |
| "addl " pa_TEMP ", %%eax \n\t" // pcv = pav + pbv |
| // pc = abs(pcv) |
| "testl $0x80000000, %%eax \n\t" |
| "jz paeth_pca \n\t" |
| "negl %%eax \n\t" // reverse sign of neg values |
| |
| "paeth_pca: \n\t" |
| "movl %%eax, " pc_TEMP " \n\t" // save pc for later use |
| // pb = abs(pbv) |
| "testl $0x80000000, %%ecx \n\t" |
| "jz paeth_pba \n\t" |
| "negl %%ecx \n\t" // reverse sign of neg values |
| |
| "paeth_pba: \n\t" |
| "movl %%ecx, " pb_TEMP " \n\t" // save pb for later use |
| // pa = abs(pav) |
| "movl " pa_TEMP ", %%eax \n\t" |
| "testl $0x80000000, %%eax \n\t" |
| "jz paeth_paa \n\t" |
| "negl %%eax \n\t" // reverse sign of neg values |
| |
| "paeth_paa: \n\t" |
| "movl %%eax, " pa_TEMP " \n\t" // save pa for later use |
| // test if pa <= pb |
| "cmpl %%ecx, %%eax \n\t" |
| "jna paeth_abb \n\t" |
| // pa > pb; now test if pb <= pc |
| "cmpl " pc_TEMP ", %%ecx \n\t" |
| "jna paeth_bbc \n\t" |
| // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp) |
| "movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl |
| "jmp paeth_paeth \n\t" |
| |
| "paeth_bbc: \n\t" |
| // pb <= pc; Raw(x) = Paeth(x) + Prior(x) |
| "movb (%%esi,%%ebx,), %%cl \n\t" // load Prior(x) into cl |
| "jmp paeth_paeth \n\t" |
| |
| "paeth_abb: \n\t" |
| // pa <= pb; now test if pa <= pc |
| "cmpl " pc_TEMP ", %%eax \n\t" |
| "jna paeth_abc \n\t" |
| // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp) |
| "movb (%%esi,%%edx,), %%cl \n\t" // load Prior(x-bpp) into cl |
| "jmp paeth_paeth \n\t" |
| |
| "paeth_abc: \n\t" |
| // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp) |
| "movb (%%edi,%%edx,), %%cl \n\t" // load Raw(x-bpp) into cl |
| |
| "paeth_paeth: \n\t" |
| "incl %%ebx \n\t" |
| "incl %%edx \n\t" |
| // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256 |
| "addb %%cl, -1(%%edi,%%ebx,) \n\t" |
| "cmpl %%ebp, %%ebx \n\t" |
| "jb paeth_lp1 \n\t" |
| |
| RESTORE_r11_r12_r13 |
| |
| "paeth_go: \n\t" |
| RESTORE_FullLength "%%ecx \n\t" // FullLength -> ecx |
| "movl %%ecx, %%eax \n\t" |
| "subl %%ebx, %%eax \n\t" // subtract alignment fix |
| "andl $0x00000007, %%eax \n\t" // calc bytes over mult of 8 |
| "subl %%eax, %%ecx \n\t" // drop over bytes from original length |
| //out "movl %%ecx, MMXLength \n\t" |
| "movl %%ebp, %%edi \n\t" // ebp = diff, but no reg constraint(?) |
| RESTORE_ebp // (could swap ebp and ecx functions) |
| RESTORE_r15 |
| RESTORE_GOT_ebx |
| |
| : "=c" (MMXLength), // output regs |
| "=S" (dummy_value_S), |
| "=D" (diff), |
| "=a" (dummy_value_a) |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (prev_row), // esi |
| "2" (row), // edi |
| "3" (FullLength) // eax |
| |
| : "%edx" // clobber list |
| _CLOBBER_r11_r12_r13 |
| _CLOBBER_r15 |
| _CLOBBER_ebp |
| _CLOBBER_GOT_ebx |
| ); |
| |
| // now do the math for the rest of the row |
| switch (bpp) |
| { |
| case 3: |
| { |
| // _ShiftBpp = 24; // == bpp * 8 |
| // _ShiftRem = 40; // == 64 - _ShiftBpp |
| |
| __asm__ __volatile__ ( |
| LOAD_GOT_rbp |
| // preload "movl diff, %%ecx \n\t" |
| // preload "movl row, %%edi \n\t" |
| // preload "movl prev_row, %%esi \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%ecx,), %%mm1 \n\t" |
| "paeth_3lp: \n\t" |
| "psrlq $40, %%mm1 \n\t" // shift last 3 bytes to 1st |
| // 3 bytes |
| "movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x) |
| "punpcklbw %%mm0, %%mm1 \n\t" // unpack High bytes of a |
| "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // prep c=Prior(x-bpp) bytes |
| "punpcklbw %%mm0, %%mm2 \n\t" // unpack High bytes of b |
| "psrlq $40, %%mm3 \n\t" // shift last 3 bytes to 1st |
| // 3 bytes |
| // pav = p - a = (a + b - c) - a = b - c |
| "movq %%mm2, %%mm4 \n\t" |
| "punpcklbw %%mm0, %%mm3 \n\t" // unpack High bytes of c |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movq %%mm1, %%mm5 \n\t" |
| "psubw %%mm3, %%mm4 \n\t" |
| "pxor %%mm7, %%mm7 \n\t" |
| // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv |
| "movq %%mm4, %%mm6 \n\t" |
| "psubw %%mm3, %%mm5 \n\t" |
| |
| // pa = abs(p-a) = abs(pav) |
| // pb = abs(p-b) = abs(pbv) |
| // pc = abs(p-c) = abs(pcv) |
| "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0 |
| "paddw %%mm5, %%mm6 \n\t" |
| "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "psubw %%mm7, %%mm5 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0 |
| "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "psubw %%mm7, %%mm5 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| // test pa <= pb |
| "movq %%mm4, %%mm7 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb? |
| "movq %%mm7, %%mm0 \n\t" |
| // use mm7 mask to merge pa & pb |
| "pand %%mm7, %%mm5 \n\t" |
| // use mm0 mask copy to merge a & b |
| "pand %%mm0, %%mm2 \n\t" |
| "pandn %%mm4, %%mm7 \n\t" |
| "pandn %%mm1, %%mm0 \n\t" |
| "paddw %%mm5, %%mm7 \n\t" |
| "paddw %%mm2, %%mm0 \n\t" |
| // test ((pa <= pb)? pa:pb) <= pc |
| "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc? |
| "pxor %%mm1, %%mm1 \n\t" |
| "pand %%mm7, %%mm3 \n\t" |
| "pandn %%mm0, %%mm7 \n\t" |
| "paddw %%mm3, %%mm7 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "packuswb %%mm1, %%mm7 \n\t" |
| "movq (%%esi,%%ecx,), %%mm3 \n\t" // load c=Prior(x-bpp) |
| "pand " AMASK5_3_0 ", %%mm7 \n\t" // _amask5_3_0 (was _ActiveMask) |
| "movq %%mm3, %%mm2 \n\t" // load b=Prior(x) step 1 |
| "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor + Raw(x) |
| "punpcklbw %%mm0, %%mm3 \n\t" // unpack High bytes of c |
| "movq %%mm7, (%%edi,%%ecx,) \n\t" // write back updated value |
| "movq %%mm7, %%mm1 \n\t" // now mm1 will be used as |
| // Raw(x-bpp) |
| // now do Paeth for 2nd set of bytes (3-5) |
| "psrlq $24, %%mm2 \n\t" // load b=Prior(x) step 2 |
| "punpcklbw %%mm0, %%mm1 \n\t" // unpack High bytes of a |
| "pxor %%mm7, %%mm7 \n\t" |
| "punpcklbw %%mm0, %%mm2 \n\t" // unpack High bytes of b |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movq %%mm1, %%mm5 \n\t" |
| // pav = p - a = (a + b - c) - a = b - c |
| "movq %%mm2, %%mm4 \n\t" |
| "psubw %%mm3, %%mm5 \n\t" |
| "psubw %%mm3, %%mm4 \n\t" |
| // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = |
| // pav + pbv = pbv + pav |
| "movq %%mm5, %%mm6 \n\t" |
| "paddw %%mm4, %%mm6 \n\t" |
| |
| // pa = abs(p-a) = abs(pav) |
| // pb = abs(p-b) = abs(pbv) |
| // pc = abs(p-c) = abs(pcv) |
| "pcmpgtw %%mm5, %%mm0 \n\t" // create mask pbv bytes < 0 |
| "pcmpgtw %%mm4, %%mm7 \n\t" // create mask pav bytes < 0 |
| "pand %%mm5, %%mm0 \n\t" // only pbv bytes < 0 in mm0 |
| "pand %%mm4, %%mm7 \n\t" // only pav bytes < 0 in mm7 |
| "psubw %%mm0, %%mm5 \n\t" |
| "psubw %%mm7, %%mm4 \n\t" |
| "psubw %%mm0, %%mm5 \n\t" |
| "psubw %%mm7, %%mm4 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0 |
| "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "psubw %%mm0, %%mm6 \n\t" |
| // test pa <= pb |
| "movq %%mm4, %%mm7 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb? |
| "movq %%mm7, %%mm0 \n\t" |
| // use mm7 mask to merge pa & pb |
| "pand %%mm7, %%mm5 \n\t" |
| // use mm0 mask copy to merge a & b |
| "pand %%mm0, %%mm2 \n\t" |
| "pandn %%mm4, %%mm7 \n\t" |
| "pandn %%mm1, %%mm0 \n\t" |
| "paddw %%mm5, %%mm7 \n\t" |
| "paddw %%mm2, %%mm0 \n\t" |
| // test ((pa <= pb)? pa:pb) <= pc |
| "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc? |
| "movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x) |
| "pand %%mm7, %%mm3 \n\t" |
| "pandn %%mm0, %%mm7 \n\t" |
| "pxor %%mm1, %%mm1 \n\t" |
| "paddw %%mm3, %%mm7 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "packuswb %%mm1, %%mm7 \n\t" |
| "movq %%mm2, %%mm3 \n\t" // load c=Prior(x-bpp) step 1 |
| "pand " AMASK5_3_0 ", %%mm7 \n\t" // _amask5_3_0 (was _ActiveMask) |
| "punpckhbw %%mm0, %%mm2 \n\t" // unpack High bytes of b |
| "psllq $24, %%mm7 \n\t" // shift bytes to 2nd group of |
| // 3 bytes |
| // pav = p - a = (a + b - c) - a = b - c |
| "movq %%mm2, %%mm4 \n\t" |
| "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor + Raw(x) |
| "psllq $24, %%mm3 \n\t" // load c=Prior(x-bpp) step 2 |
| "movq %%mm7, (%%edi,%%ecx,) \n\t" // write back updated value |
| "movq %%mm7, %%mm1 \n\t" |
| "punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c |
| "psllq $24, %%mm1 \n\t" // shift bytes (was _ShiftBpp) |
| // now mm1 will be used as Raw(x-bpp) |
| // now do Paeth for 3rd, and final, set of bytes (6-7) |
| "pxor %%mm7, %%mm7 \n\t" |
| "punpckhbw %%mm0, %%mm1 \n\t" // unpack High bytes of a |
| "psubw %%mm3, %%mm4 \n\t" |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movq %%mm1, %%mm5 \n\t" |
| // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv |
| "movq %%mm4, %%mm6 \n\t" |
| "psubw %%mm3, %%mm5 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "paddw %%mm5, %%mm6 \n\t" |
| |
| // pa = abs(p-a) = abs(pav) |
| // pb = abs(p-b) = abs(pbv) |
| // pc = abs(p-c) = abs(pcv) |
| "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0 |
| "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0 |
| "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "psubw %%mm7, %%mm5 \n\t" |
| "psubw %%mm0, %%mm4 \n\t" |
| "psubw %%mm7, %%mm5 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0 |
| "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "psubw %%mm0, %%mm6 \n\t" |
| // test pa <= pb |
| "movq %%mm4, %%mm7 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb? |
| "movq %%mm7, %%mm0 \n\t" |
| // use mm0 mask copy to merge a & b |
| "pand %%mm0, %%mm2 \n\t" |
| // use mm7 mask to merge pa & pb |
| "pand %%mm7, %%mm5 \n\t" |
| "pandn %%mm1, %%mm0 \n\t" |
| "pandn %%mm4, %%mm7 \n\t" |
| "paddw %%mm2, %%mm0 \n\t" |
| "paddw %%mm5, %%mm7 \n\t" |
| // test ((pa <= pb)? pa:pb) <= pc |
| "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc? |
| "pand %%mm7, %%mm3 \n\t" |
| "pandn %%mm0, %%mm7 \n\t" |
| "paddw %%mm3, %%mm7 \n\t" |
| "pxor %%mm1, %%mm1 \n\t" |
| "packuswb %%mm7, %%mm1 \n\t" |
| // step ecx to next set of 8 bytes and repeat loop til done |
| "addl $8, %%ecx \n\t" |
| "pand " AMASK0_2_6 ", %%mm1 \n\t" // _amask0_2_6 (_ActiveMaskEnd) |
| "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor + Raw(x) |
| |
| "cmpl %%eax, %%ecx \n\t" // MMXLength |
| "pxor %%mm0, %%mm0 \n\t" // pxor does not affect flags |
| "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value |
| // mm1 will be used as Raw(x-bpp) next loop |
| // mm3 ready to be used as Prior(x-bpp) next loop |
| "jb paeth_3lp \n\t" |
| RESTORE_rbp |
| |
| : "=S" (dummy_value_S), // output regs (dummy) |
| "=D" (dummy_value_D), |
| "=c" (dummy_value_c), |
| "=a" (dummy_value_a) |
| |
| : "0" (prev_row), // esi // input regs |
| "1" (row), // edi |
| "2" (diff), // ecx |
| "3" (MMXLength) // eax |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| break; // end 3 bpp |
| |
| case 6: |
| { |
| // _ActiveMask2 = 0xffffffff00000000LL; // NOT USED ("_amask_0_4_4") |
| // _ShiftBpp = 48; // bpp << 3 == bpp * 8 |
| // _ShiftRem = 16; // 64 - _ShiftBpp |
| |
| __asm__ __volatile__ ( |
| // preload "movl diff, %%ecx \n\t" |
| // preload "movl row, %%edi \n\t" |
| // preload "movl prev_row, %%esi \n\t" |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%ecx,), %%mm1 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| |
| "paeth_6lp: \n\t" |
| // must shift to position Raw(x-bpp) data |
| "psrlq $16, %%mm1 \n\t" // was _ShiftRem |
| // do first set of 4 bytes |
| "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes |
| "punpcklbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a |
| "movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x) |
| "punpcklbw %%mm0, %%mm2 \n\t" // unpack Low bytes of b |
| // must shift to position Prior(x-bpp) data |
| "psrlq $16, %%mm3 \n\t" // was _ShiftRem |
| // pav = p - a = (a + b - c) - a = b - c |
| "movq %%mm2, %%mm4 \n\t" |
| "punpcklbw %%mm0, %%mm3 \n\t" // unpack Low bytes of c |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movq %%mm1, %%mm5 \n\t" |
| "psubw %%mm3, %%mm4 \n\t" |
| "pxor %%mm7, %%mm7 \n\t" |
| // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv |
| "movq %%mm4, %%mm6 \n\t" |
| "psubw %%mm3, %%mm5 \n\t" |
| // pa = abs(p-a) = abs(pav) |
| // pb = abs(p-b) = abs(pbv) |
| // pc = abs(p-c) = abs(pcv) |
| "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0 |
| "paddw %%mm5, %%mm6 \n\t" |
| "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "psubw %%mm7, %%mm5 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0 |
| "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "psubw %%mm7, %%mm5 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| // test pa <= pb |
| "movq %%mm4, %%mm7 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb? |
| "movq %%mm7, %%mm0 \n\t" |
| // use mm7 mask to merge pa & pb |
| "pand %%mm7, %%mm5 \n\t" |
| // use mm0 mask copy to merge a & b |
| "pand %%mm0, %%mm2 \n\t" |
| "pandn %%mm4, %%mm7 \n\t" |
| "pandn %%mm1, %%mm0 \n\t" |
| "paddw %%mm5, %%mm7 \n\t" |
| "paddw %%mm2, %%mm0 \n\t" |
| // test ((pa <= pb)? pa:pb) <= pc |
| "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc? |
| "pxor %%mm1, %%mm1 \n\t" |
| "pand %%mm7, %%mm3 \n\t" |
| "pandn %%mm0, %%mm7 \n\t" |
| "paddw %%mm3, %%mm7 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "packuswb %%mm1, %%mm7 \n\t" |
| "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // load c=Prior(x-bpp) |
| LOAD_GOT_rbp |
| "pand " AMASK4_4_0 ", %%mm7 \n\t" // _amask4_4_0 (was _ActiveMask) |
| RESTORE_rbp |
| "psrlq $16, %%mm3 \n\t" |
| "movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x) step 1 |
| "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor + Raw(x) |
| "movq %%mm2, %%mm6 \n\t" |
| "movq %%mm7, (%%edi,%%ecx,) \n\t" // write back updated value |
| "movq -8(%%edi,%%ecx,), %%mm1 \n\t" |
| "psllq $48, %%mm6 \n\t" // bpp * 8 = bits per pixel |
| "movq %%mm7, %%mm5 \n\t" |
| "psrlq $16, %%mm1 \n\t" // 64 - (bpp * 8) = remainder |
| "por %%mm6, %%mm3 \n\t" |
| "psllq $48, %%mm5 \n\t" // was _ShiftBpp |
| "punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c |
| "por %%mm5, %%mm1 \n\t" |
| // do second set of 4 bytes |
| "punpckhbw %%mm0, %%mm2 \n\t" // unpack High bytes of b |
| "punpckhbw %%mm0, %%mm1 \n\t" // unpack High bytes of a |
| // pav = p - a = (a + b - c) - a = b - c |
| "movq %%mm2, %%mm4 \n\t" |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movq %%mm1, %%mm5 \n\t" |
| "psubw %%mm3, %%mm4 \n\t" |
| "pxor %%mm7, %%mm7 \n\t" |
| // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv |
| "movq %%mm4, %%mm6 \n\t" |
| "psubw %%mm3, %%mm5 \n\t" |
| // pa = abs(p-a) = abs(pav) |
| // pb = abs(p-b) = abs(pbv) |
| // pc = abs(p-c) = abs(pcv) |
| "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0 |
| "paddw %%mm5, %%mm6 \n\t" |
| "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "psubw %%mm7, %%mm5 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0 |
| "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "psubw %%mm7, %%mm5 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| // test pa <= pb |
| "movq %%mm4, %%mm7 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb? |
| "movq %%mm7, %%mm0 \n\t" |
| // use mm7 mask to merge pa & pb |
| "pand %%mm7, %%mm5 \n\t" |
| // use mm0 mask copy to merge a & b |
| "pand %%mm0, %%mm2 \n\t" |
| "pandn %%mm4, %%mm7 \n\t" |
| "pandn %%mm1, %%mm0 \n\t" |
| "paddw %%mm5, %%mm7 \n\t" |
| "paddw %%mm2, %%mm0 \n\t" |
| // test ((pa <= pb)? pa:pb) <= pc |
| "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc? |
| "pxor %%mm1, %%mm1 \n\t" |
| "pand %%mm7, %%mm3 \n\t" |
| "pandn %%mm0, %%mm7 \n\t" |
| "pxor %%mm1, %%mm1 \n\t" |
| "paddw %%mm3, %%mm7 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| // step ecx to next set of 8 bytes and repeat loop til done |
| "addl $8, %%ecx \n\t" |
| "packuswb %%mm7, %%mm1 \n\t" |
| "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor + Raw(x) |
| "cmpl %%eax, %%ecx \n\t" // MMXLength |
| "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value |
| // mm1 will be used as Raw(x-bpp) next loop |
| "jb paeth_6lp \n\t" |
| |
| : "=S" (dummy_value_S), // output regs (dummy) |
| "=D" (dummy_value_D), |
| "=c" (dummy_value_c), |
| "=a" (dummy_value_a) |
| |
| : "0" (prev_row), // esi // input regs |
| "1" (row), // edi |
| "2" (diff), // ecx |
| "3" (MMXLength) // eax |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| break; // end 6 bpp |
| |
| case 4: |
| { |
| __asm__ __volatile__ ( |
| // preload "movl diff, %%ecx \n\t" |
| // preload "movl row, %%edi \n\t" |
| // preload "movl prev_row, %%esi \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // only time should need to read |
| // a=Raw(x-bpp) bytes |
| "paeth_4lp: \n\t" |
| // do first set of 4 bytes |
| "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes |
| "punpckhbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a |
| "movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x) |
| "punpcklbw %%mm0, %%mm2 \n\t" // unpack High bytes of b |
| // pav = p - a = (a + b - c) - a = b - c |
| "movq %%mm2, %%mm4 \n\t" |
| "punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movq %%mm1, %%mm5 \n\t" |
| "psubw %%mm3, %%mm4 \n\t" |
| "pxor %%mm7, %%mm7 \n\t" |
| // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv |
| "movq %%mm4, %%mm6 \n\t" |
| "psubw %%mm3, %%mm5 \n\t" |
| // pa = abs(p-a) = abs(pav) |
| // pb = abs(p-b) = abs(pbv) |
| // pc = abs(p-c) = abs(pcv) |
| "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0 |
| "paddw %%mm5, %%mm6 \n\t" |
| "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "psubw %%mm7, %%mm5 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0 |
| "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "psubw %%mm7, %%mm5 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| // test pa <= pb |
| "movq %%mm4, %%mm7 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb? |
| "movq %%mm7, %%mm0 \n\t" |
| // use mm7 mask to merge pa & pb |
| "pand %%mm7, %%mm5 \n\t" |
| // use mm0 mask copy to merge a & b |
| "pand %%mm0, %%mm2 \n\t" |
| "pandn %%mm4, %%mm7 \n\t" |
| "pandn %%mm1, %%mm0 \n\t" |
| "paddw %%mm5, %%mm7 \n\t" |
| "paddw %%mm2, %%mm0 \n\t" |
| // test ((pa <= pb)? pa:pb) <= pc |
| "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc? |
| "pxor %%mm1, %%mm1 \n\t" |
| "pand %%mm7, %%mm3 \n\t" |
| "pandn %%mm0, %%mm7 \n\t" |
| "paddw %%mm3, %%mm7 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "packuswb %%mm1, %%mm7 \n\t" |
| "movq (%%esi,%%ecx,), %%mm3 \n\t" // load c=Prior(x-bpp) |
| LOAD_GOT_rbp |
| "pand " AMASK4_4_0 ", %%mm7 \n\t" // _amask4_4_0 (was _ActiveMask) |
| RESTORE_rbp |
| "movq %%mm3, %%mm2 \n\t" // load b=Prior(x) step 1 |
| "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor + Raw(x) |
| "punpcklbw %%mm0, %%mm3 \n\t" // unpack High bytes of c |
| "movq %%mm7, (%%edi,%%ecx,) \n\t" // write back updated value |
| "movq %%mm7, %%mm1 \n\t" // now mm1 will be used as |
| // Raw(x-bpp) |
| // do second set of 4 bytes |
| "punpckhbw %%mm0, %%mm2 \n\t" // unpack Low bytes of b |
| "punpcklbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a |
| // pav = p - a = (a + b - c) - a = b - c |
| "movq %%mm2, %%mm4 \n\t" |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movq %%mm1, %%mm5 \n\t" |
| "psubw %%mm3, %%mm4 \n\t" |
| "pxor %%mm7, %%mm7 \n\t" |
| // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv |
| "movq %%mm4, %%mm6 \n\t" |
| "psubw %%mm3, %%mm5 \n\t" |
| // pa = abs(p-a) = abs(pav) |
| // pb = abs(p-b) = abs(pbv) |
| // pc = abs(p-c) = abs(pcv) |
| "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0 |
| "paddw %%mm5, %%mm6 \n\t" |
| "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "psubw %%mm7, %%mm5 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0 |
| "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "psubw %%mm7, %%mm5 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| // test pa <= pb |
| "movq %%mm4, %%mm7 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb? |
| "movq %%mm7, %%mm0 \n\t" |
| // use mm7 mask to merge pa & pb |
| "pand %%mm7, %%mm5 \n\t" |
| // use mm0 mask copy to merge a & b |
| "pand %%mm0, %%mm2 \n\t" |
| "pandn %%mm4, %%mm7 \n\t" |
| "pandn %%mm1, %%mm0 \n\t" |
| "paddw %%mm5, %%mm7 \n\t" |
| "paddw %%mm2, %%mm0 \n\t" |
| // test ((pa <= pb)? pa:pb) <= pc |
| "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc? |
| "pxor %%mm1, %%mm1 \n\t" |
| "pand %%mm7, %%mm3 \n\t" |
| "pandn %%mm0, %%mm7 \n\t" |
| "pxor %%mm1, %%mm1 \n\t" |
| "paddw %%mm3, %%mm7 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| // step ecx to next set of 8 bytes and repeat loop til done |
| "addl $8, %%ecx \n\t" |
| "packuswb %%mm7, %%mm1 \n\t" |
| "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add predictor with Raw(x) |
| "cmpl %%eax, %%ecx \n\t" // MMXLength |
| "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value |
| // mm1 will be used as Raw(x-bpp) next loop |
| "jb paeth_4lp \n\t" |
| |
| : "=S" (dummy_value_S), // output regs (dummy) |
| "=D" (dummy_value_D), |
| "=c" (dummy_value_c), |
| "=a" (dummy_value_a) |
| |
| : "0" (prev_row), // esi // input regs |
| "1" (row), // edi |
| "2" (diff), // ecx |
| "3" (MMXLength) // eax |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| break; // end 4 bpp |
| |
| case 8: // bpp == 8 |
| { |
| __asm__ __volatile__ ( |
| // preload "movl diff, %%ecx \n\t" |
| // preload "movl row, %%edi \n\t" |
| // preload "movl prev_row, %%esi \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // only time should need to read |
| // a=Raw(x-bpp) bytes |
| "paeth_8lp: \n\t" |
| // do first set of 4 bytes |
| "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes |
| "punpcklbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a |
| "movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x) |
| "punpcklbw %%mm0, %%mm2 \n\t" // unpack Low bytes of b |
| // pav = p - a = (a + b - c) - a = b - c |
| "movq %%mm2, %%mm4 \n\t" |
| "punpcklbw %%mm0, %%mm3 \n\t" // unpack Low bytes of c |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movq %%mm1, %%mm5 \n\t" |
| "psubw %%mm3, %%mm4 \n\t" |
| "pxor %%mm7, %%mm7 \n\t" |
| // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv |
| "movq %%mm4, %%mm6 \n\t" |
| "psubw %%mm3, %%mm5 \n\t" |
| // pa = abs(p-a) = abs(pav) |
| // pb = abs(p-b) = abs(pbv) |
| // pc = abs(p-c) = abs(pcv) |
| "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0 |
| "paddw %%mm5, %%mm6 \n\t" |
| "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "psubw %%mm7, %%mm5 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0 |
| "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "psubw %%mm7, %%mm5 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| // test pa <= pb |
| "movq %%mm4, %%mm7 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb? |
| "movq %%mm7, %%mm0 \n\t" |
| // use mm7 mask to merge pa & pb |
| "pand %%mm7, %%mm5 \n\t" |
| // use mm0 mask copy to merge a & b |
| "pand %%mm0, %%mm2 \n\t" |
| "pandn %%mm4, %%mm7 \n\t" |
| "pandn %%mm1, %%mm0 \n\t" |
| "paddw %%mm5, %%mm7 \n\t" |
| "paddw %%mm2, %%mm0 \n\t" |
| // test ((pa <= pb)? pa:pb) <= pc |
| "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc? |
| "pxor %%mm1, %%mm1 \n\t" |
| "pand %%mm7, %%mm3 \n\t" |
| "pandn %%mm0, %%mm7 \n\t" |
| "paddw %%mm3, %%mm7 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "packuswb %%mm1, %%mm7 \n\t" |
| "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes |
| LOAD_GOT_rbp |
| "pand " AMASK4_4_0 ", %%mm7 \n\t" // _amask4_4_0 (was _ActiveMask) |
| RESTORE_rbp |
| "movq (%%esi,%%ecx,), %%mm2 \n\t" // load b=Prior(x) |
| "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor + Raw(x) |
| "punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c |
| "movq %%mm7, (%%edi,%%ecx,) \n\t" // write back updated value |
| "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // read a=Raw(x-bpp) bytes |
| |
| // do second set of 4 bytes |
| "punpckhbw %%mm0, %%mm2 \n\t" // unpack High bytes of b |
| "punpckhbw %%mm0, %%mm1 \n\t" // unpack High bytes of a |
| // pav = p - a = (a + b - c) - a = b - c |
| "movq %%mm2, %%mm4 \n\t" |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movq %%mm1, %%mm5 \n\t" |
| "psubw %%mm3, %%mm4 \n\t" |
| "pxor %%mm7, %%mm7 \n\t" |
| // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv |
| "movq %%mm4, %%mm6 \n\t" |
| "psubw %%mm3, %%mm5 \n\t" |
| // pa = abs(p-a) = abs(pav) |
| // pb = abs(p-b) = abs(pbv) |
| // pc = abs(p-c) = abs(pcv) |
| "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0 |
| "paddw %%mm5, %%mm6 \n\t" |
| "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0 |
| "psubw %%mm0, %%mm4 \n\t" |
| "psubw %%mm7, %%mm5 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0 |
| "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7 |
| "psubw %%mm7, %%mm5 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| // test pa <= pb |
| "movq %%mm4, %%mm7 \n\t" |
| "psubw %%mm0, %%mm6 \n\t" |
| "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb? |
| "movq %%mm7, %%mm0 \n\t" |
| // use mm7 mask to merge pa & pb |
| "pand %%mm7, %%mm5 \n\t" |
| // use mm0 mask copy to merge a & b |
| "pand %%mm0, %%mm2 \n\t" |
| "pandn %%mm4, %%mm7 \n\t" |
| "pandn %%mm1, %%mm0 \n\t" |
| "paddw %%mm5, %%mm7 \n\t" |
| "paddw %%mm2, %%mm0 \n\t" |
| // test ((pa <= pb)? pa:pb) <= pc |
| "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc? |
| "pxor %%mm1, %%mm1 \n\t" |
| "pand %%mm7, %%mm3 \n\t" |
| "pandn %%mm0, %%mm7 \n\t" |
| "pxor %%mm1, %%mm1 \n\t" |
| "paddw %%mm3, %%mm7 \n\t" |
| "pxor %%mm0, %%mm0 \n\t" |
| // step ecx to next set of 8 bytes and repeat loop til done |
| "addl $8, %%ecx \n\t" |
| "packuswb %%mm7, %%mm1 \n\t" |
| "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor + Raw(x) |
| "cmpl %%eax, %%ecx \n\t" // MMXLength |
| "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value |
| // mm1 will be used as Raw(x-bpp) next loop |
| "jb paeth_8lp \n\t" |
| |
| : "=S" (dummy_value_S), // output regs (dummy) |
| "=D" (dummy_value_D), |
| "=c" (dummy_value_c), |
| "=a" (dummy_value_a) |
| |
| : "0" (prev_row), // esi // input regs |
| "1" (row), // edi |
| "2" (diff), // ecx |
| "3" (MMXLength) // eax |
| |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| break; // end 8 bpp |
| |
| default: // bpp = 1 or 2 |
| { |
| __asm__ __volatile__ ( |
| // preload "movl diff, %%eax \n\t" // eax: x = offset to align. bdry |
| // preload "movl FullLength, %%edx \n\t" |
| "cmpl %%edx, %%eax \n\t" |
| "jnb paeth_dend \n\t" |
| |
| SAVE_ebp |
| |
| // preload "movl row, %%edi \n\t" |
| // do Paeth decode for remaining bytes |
| // preload "movl prev_row, %%esi \n\t" |
| "movl %%eax, %%ebp \n\t" |
| // preload "subl bpp, %%ebp \n\t" // (bpp is preloaded into ecx) |
| "subl %%ecx, %%ebp \n\t" // ebp = eax - bpp |
| "xorl %%ecx, %%ecx \n\t" // zero ecx before using cl & cx |
| |
| SAVE_GOT_ebx |
| SAVE_r11_r12_r13 |
| |
| "paeth_dlp: \n\t" |
| "xorl %%ebx, %%ebx \n\t" |
| // pav = p - a = (a + b - c) - a = b - c |
| "movb (%%esi,%%eax,), %%bl \n\t" // load Prior(x) into bl |
| "movb (%%esi,%%ebp,), %%cl \n\t" // load Prior(x-bpp) into cl |
| "subl %%ecx, %%ebx \n\t" // subtract Prior(x-bpp) |
| "movl %%ebx, " pa_TEMP " \n\t" // Save pav for later use |
| "xorl %%ebx, %%ebx \n\t" |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movb (%%edi,%%ebp,), %%bl \n\t" // load Raw(x-bpp) into bl |
| "subl %%ecx, %%ebx \n\t" // subtract Prior(x-bpp) |
| "movl %%ebx, %%ecx \n\t" |
| // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv |
| "addl " pa_TEMP ", %%ebx \n\t" // pcv = pav + pbv |
| // pc = abs(pcv) |
| "testl $0x80000000, %%ebx \n\t" |
| "jz paeth_dpca \n\t" |
| "negl %%ebx \n\t" // reverse sign of neg values |
| |
| "paeth_dpca: \n\t" |
| "movl %%ebx, " pc_TEMP " \n\t" // save pc for later use |
| // pb = abs(pbv) |
| "testl $0x80000000, %%ecx \n\t" |
| "jz paeth_dpba \n\t" |
| "negl %%ecx \n\t" // reverse sign of neg values |
| |
| "paeth_dpba: \n\t" |
| "movl %%ecx, " pb_TEMP " \n\t" // save pb for later use |
| // pa = abs(pav) |
| "movl " pa_TEMP ", %%ebx \n\t" |
| "testl $0x80000000, %%ebx \n\t" |
| "jz paeth_dpaa \n\t" |
| "negl %%ebx \n\t" // reverse sign of neg values |
| |
| "paeth_dpaa: \n\t" |
| "movl %%ebx, " pa_TEMP " \n\t" // save pa for later use |
| // test if pa <= pb |
| "cmpl %%ecx, %%ebx \n\t" |
| "jna paeth_dabb \n\t" |
| // pa > pb; now test if pb <= pc |
| "cmpl " pc_TEMP ", %%ecx \n\t" |
| "jna paeth_dbbc \n\t" |
| // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp) |
| "movb (%%esi,%%ebp,), %%cl \n\t" // load Prior(x-bpp) into cl |
| "jmp paeth_dpaeth \n\t" |
| |
| "paeth_dbbc: \n\t" |
| // pb <= pc; Raw(x) = Paeth(x) + Prior(x) |
| "movb (%%esi,%%eax,), %%cl \n\t" // load Prior(x) into cl |
| "jmp paeth_dpaeth \n\t" |
| |
| "paeth_dabb: \n\t" |
| // pa <= pb; now test if pa <= pc |
| "cmpl " pc_TEMP ", %%ebx \n\t" |
| "jna paeth_dabc \n\t" |
| // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp) |
| "movb (%%esi,%%ebp,), %%cl \n\t" // load Prior(x-bpp) into cl |
| "jmp paeth_dpaeth \n\t" |
| |
| "paeth_dabc: \n\t" |
| // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp) |
| "movb (%%edi,%%ebp,), %%cl \n\t" // load Raw(x-bpp) into cl |
| |
| "paeth_dpaeth: \n\t" |
| "incl %%eax \n\t" |
| "incl %%ebp \n\t" |
| // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256 |
| "addb %%cl, -1(%%edi,%%eax,) \n\t" |
| "cmpl %%edx, %%eax \n\t" // check against FullLength |
| "jb paeth_dlp \n\t" |
| |
| RESTORE_r11_r12_r13 |
| RESTORE_GOT_ebx |
| RESTORE_ebp |
| |
| "paeth_dend: \n\t" |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D), |
| "=a" (dummy_value_a), |
| "=d" (dummy_value_d) |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (prev_row), // esi |
| "2" (row), // edi |
| "3" (diff), // eax |
| "4" (FullLength) // edx |
| |
| CLOB_COLON_ebx_ebp_r1X // clobber list |
| CLOBBER_GOT_ebx |
| CLOB_COMMA_ebx_ebp |
| CLOBBER_ebp |
| CLOB_COMMA_ebX_r1X |
| CLOBBER_r11_r12_r13 |
| ); |
| } |
| return; // No need to go further with this one |
| |
| } // end switch (bpp) |
| |
| __asm__ __volatile__ ( |
| // MMX acceleration complete; now do clean-up |
| // check if any remaining bytes left to decode |
| //pre "movl FullLength, %%edx \n\t" |
| //pre "movl MMXLength, %%eax \n\t" |
| "cmpl %%edx, %%eax \n\t" |
| "jnb paeth_end \n\t" |
| |
| SAVE_ebp |
| |
| //pre "movl row, %%edi \n\t" |
| //pre "movl prev_row, %%esi \n\t" |
| // do Paeth decode for remaining bytes |
| "movl %%eax, %%ebp \n\t" |
| //pre "subl bpp, %%ebp \n\t" // (bpp is preloaded into ecx) |
| "subl %%ecx, %%ebp \n\t" // ebp = eax - bpp |
| "xorl %%ecx, %%ecx \n\t" // zero ecx before using cl & cx below |
| |
| SAVE_GOT_ebx |
| SAVE_r11_r12_r13 |
| |
| "paeth_lp2: \n\t" |
| "xorl %%ebx, %%ebx \n\t" |
| // pav = p - a = (a + b - c) - a = b - c |
| "movb (%%esi,%%eax,), %%bl \n\t" // load Prior(x) into bl |
| "movb (%%esi,%%ebp,), %%cl \n\t" // load Prior(x-bpp) into cl |
| "subl %%ecx, %%ebx \n\t" // subtract Prior(x-bpp) |
| "movl %%ebx, " pa_TEMP " \n\t" // Save pav for later use |
| "xorl %%ebx, %%ebx \n\t" |
| // pbv = p - b = (a + b - c) - b = a - c |
| "movb (%%edi,%%ebp,), %%bl \n\t" // load Raw(x-bpp) into bl |
| "subl %%ecx, %%ebx \n\t" // subtract Prior(x-bpp) |
| "movl %%ebx, %%ecx \n\t" |
| // pcv = p - c = (a + b - c) - c = (a - c) + (b - c) = pav + pbv |
| "addl " pa_TEMP ", %%ebx \n\t" // pcv = pav + pbv |
| // pc = abs(pcv) |
| "testl $0x80000000, %%ebx \n\t" |
| "jz paeth_pca2 \n\t" |
| "negl %%ebx \n\t" // reverse sign of neg values |
| |
| "paeth_pca2: \n\t" |
| "movl %%ebx, " pc_TEMP " \n\t" // save pc for later use |
| // pb = abs(pbv) |
| "testl $0x80000000, %%ecx \n\t" |
| "jz paeth_pba2 \n\t" |
| "negl %%ecx \n\t" // reverse sign of neg values |
| |
| "paeth_pba2: \n\t" |
| "movl %%ecx, " pb_TEMP " \n\t" // save pb for later use |
| // pa = abs(pav) |
| "movl " pa_TEMP ", %%ebx \n\t" |
| "testl $0x80000000, %%ebx \n\t" |
| "jz paeth_paa2 \n\t" |
| "negl %%ebx \n\t" // reverse sign of neg values |
| |
| "paeth_paa2: \n\t" |
| "movl %%ebx, " pa_TEMP " \n\t" // save pa for later use |
| // test if pa <= pb |
| "cmpl %%ecx, %%ebx \n\t" |
| "jna paeth_abb2 \n\t" |
| // pa > pb; now test if pb <= pc |
| "cmpl " pc_TEMP ", %%ecx \n\t" |
| "jna paeth_bbc2 \n\t" |
| // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp) |
| "movb (%%esi,%%ebp,), %%cl \n\t" // load Prior(x-bpp) into cl |
| "jmp paeth_paeth2 \n\t" |
| |
| "paeth_bbc2: \n\t" |
| // pb <= pc; Raw(x) = Paeth(x) + Prior(x) |
| "movb (%%esi,%%eax,), %%cl \n\t" // load Prior(x) into cl |
| "jmp paeth_paeth2 \n\t" |
| |
| "paeth_abb2: \n\t" |
| // pa <= pb; now test if pa <= pc |
| "cmpl " pc_TEMP ", %%ebx \n\t" |
| "jna paeth_abc2 \n\t" |
| // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp) |
| "movb (%%esi,%%ebp,), %%cl \n\t" // load Prior(x-bpp) into cl |
| "jmp paeth_paeth2 \n\t" |
| |
| "paeth_abc2: \n\t" |
| // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp) |
| "movb (%%edi,%%ebp,), %%cl \n\t" // load Raw(x-bpp) into cl |
| |
| "paeth_paeth2: \n\t" |
| "incl %%eax \n\t" |
| "incl %%ebp \n\t" |
| // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256 |
| "addb %%cl, -1(%%edi,%%eax,) \n\t" |
| "cmpl %%edx, %%eax \n\t" // check against FullLength |
| "jb paeth_lp2 \n\t" |
| |
| RESTORE_r11_r12_r13 |
| RESTORE_GOT_ebx |
| RESTORE_ebp |
| |
| "paeth_end: \n\t" |
| "EMMS \n\t" // end MMX; prep for poss. FP instrs. |
| |
| : "=c" (dummy_value_c), // output regs (dummy) |
| "=S" (dummy_value_S), |
| "=D" (dummy_value_D), |
| "=a" (dummy_value_a), |
| "=d" (dummy_value_d) |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (prev_row), // esi |
| "2" (row), // edi |
| "3" (MMXLength), // eax |
| "4" (FullLength) // edx |
| |
| CLOB_COLON_ebx_ebp_r1X // clobber list |
| CLOBBER_GOT_ebx |
| CLOB_COMMA_ebx_ebp |
| CLOBBER_ebp |
| CLOB_COMMA_ebX_r1X |
| CLOBBER_r11_r12_r13 |
| ); |
| |
| } /* end png_read_filter_row_mmx_paeth() */ |
| |
| #endif // PNG_x86_64_USE_GOTPCREL || PNG_THREAD_UNSAFE_OK |
| |
| |
| |
| |
| //===========================================================================// |
| // // |
| // P N G _ R E A D _ F I L T E R _ R O W _ M M X _ S U B // |
| // // |
| //===========================================================================// |
| |
| // Optimized code for PNG Sub filter decoder |
| |
| static void /* PRIVATE */ |
| png_read_filter_row_mmx_sub(png_row_infop row_info, png_bytep row) |
| { |
| png_uint_32 FullLength, MMXLength; |
| int bpp; |
| int dummy_value_a; |
| int dummy_value_c; |
| int dummy_value_d; |
| int dummy_value_D; |
| int diff; // __attribute__((used)); |
| |
| bpp = (row_info->pixel_depth + 7) >> 3; // calc number of bytes per pixel |
| FullLength = row_info->rowbytes - bpp; // number of bytes to filter |
| // (why do we subtract off bpp? not so in avg or paeth...) |
| |
| __asm__ __volatile__ ( |
| SAVE_r15 |
| SAVE_ebp |
| //pre "movl row, %%edi \n\t" |
| "movl %%edi, %%esi \n\t" // lp = row |
| //pre "movl bpp, %%ecx \n\t" |
| "addl %%ecx, %%edi \n\t" // rp = row + bpp |
| //pre "movl FullLength, %%eax \n\t" // bring in via eax... |
| SAVE_FullLength // ...but store for later use |
| |
| // get # of bytes to alignment |
| "movl %%edi, %%ebp \n\t" // take start of row |
| "addl $0xf, %%ebp \n\t" // add 7+8 to incr past alignment bdry |
| "xorl %%eax, %%eax \n\t" |
| "andl $0xfffffff8, %%ebp \n\t" // mask to alignment boundary |
| "subl %%edi, %%ebp \n\t" // subtract from start ==> value |
| "jz sub_go \n\t" // eax at alignment |
| |
| "sub_lp1: \n\t" // fix alignment |
| "movb (%%esi,%%eax,), %%cl \n\t" |
| "addb %%cl, (%%edi,%%eax,) \n\t" |
| "incl %%eax \n\t" |
| "cmpl %%ebp, %%eax \n\t" |
| "jb sub_lp1 \n\t" |
| |
| "sub_go: \n\t" |
| RESTORE_FullLength "%%ecx \n\t" // FullLength -> ecx |
| "movl %%ecx, %%edx \n\t" |
| "subl %%eax, %%edx \n\t" // subtract alignment fix |
| "andl $0x00000007, %%edx \n\t" // calc bytes over mult of 8 |
| "subl %%edx, %%ecx \n\t" // drop over bytes from length |
| //out "movl %%ecx, MMXLength \n\t" |
| "movl %%ebp, %%edi \n\t" // ebp = diff, but no reg constraint(?) |
| RESTORE_ebp // (could swap ebp and ecx functions) |
| RESTORE_r15 |
| |
| : "=c" (MMXLength), // 0 // output regs |
| "=D" (diff), // 1 |
| "=a" (dummy_value_a) // 2 |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (row), // edi |
| "2" (FullLength) // eax |
| |
| : "%esi", "%edx" // clobber list |
| _CLOBBER_r15 |
| _CLOBBER_ebp |
| ); |
| |
| // now do the math for the rest of the row |
| switch (bpp) |
| { |
| case 3: |
| { |
| // _ShiftBpp = 24; // == 3 * 8 |
| // _ShiftRem = 40; // == 64 - 24 |
| |
| __asm__ __volatile__ ( |
| // preload "movl row, %%edi \n\t" |
| LOAD_GOT_rbp |
| // load (former) _ActiveMask for 2nd active byte group |
| "movq " AMASK2_3_3 ", %%mm7 \n\t" // _amask2_3_3 |
| RESTORE_rbp |
| |
| "movl %%edi, %%esi \n\t" // lp = row |
| // preload "movl bpp, %%ecx \n\t" |
| "addl %%ecx, %%edi \n\t" // rp = row + bpp |
| "movq %%mm7, %%mm6 \n\t" |
| // preload "movl diff, %%edx \n\t" |
| "psllq $24, %%mm6 \n\t" // move mask in mm6 to cover |
| // 3rd active byte group |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%edx,), %%mm1 \n\t" |
| |
| "sub_3lp: \n\t" // shift data for adding first |
| "psrlq $40, %%mm1 \n\t" // bpp bytes (no need for mask; |
| // shift clears inactive bytes) |
| // add 1st active group |
| "movq (%%edi,%%edx,), %%mm0 \n\t" |
| "paddb %%mm1, %%mm0 \n\t" |
| |
| // add 2nd active group |
| "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1 |
| "psllq $24, %%mm1 \n\t" // shift data to pos. correctly |
| "pand %%mm7, %%mm1 \n\t" // mask to use 2nd active group |
| "paddb %%mm1, %%mm0 \n\t" |
| |
| // add 3rd active group |
| "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1 |
| "psllq $24, %%mm1 \n\t" // shift data to pos. correctly |
| "pand %%mm6, %%mm1 \n\t" // mask to use 3rd active group |
| "addl $8, %%edx \n\t" |
| "paddb %%mm1, %%mm0 \n\t" |
| |
| "cmpl %%eax, %%edx \n\t" // MMXLength |
| "movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array |
| "movq %%mm0, %%mm1 \n\t" // prep 1st add at top of loop |
| "jb sub_3lp \n\t" |
| |
| : "=c" (dummy_value_c), // 0 // output regs (dummy) |
| "=D" (dummy_value_D), // 1 |
| "=d" (dummy_value_d), // 2 |
| "=a" (dummy_value_a) // 3 |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (row), // edi |
| "2" (diff), // edx |
| "3" (MMXLength) // eax |
| |
| : "%esi" // clobber list |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| , "%mm0", "%mm1", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| break; // end 3 bpp |
| |
| case 4: // formerly shared with 6 bpp case via _ShiftBpp and _ShiftRem, |
| { // but 64-bit PIC/.so problems (could still share, moving vars |
| // into unused MMX regs via ecx/edx, but kludgy) |
| // _ShiftBpp = bpp << 3; // 32 (psllq) |
| // _ShiftRem = 64 - _ShiftBpp; // 32 (psrlq) |
| |
| __asm__ __volatile__ ( |
| // preload "movl row, %%edi \n\t" |
| // preload "movl diff, %%edx \n\t" |
| "movl %%edi, %%esi \n\t" // lp = row |
| // preload "movl bpp, %%ecx \n\t" |
| "addl %%ecx, %%edi \n\t" // rp = row + bpp |
| |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%edx,), %%mm1 \n\t" |
| |
| "sub_4lp: \n\t" // shift data for adding first |
| "psrlq $32, %%mm1 \n\t" // bpp bytes (no need for mask; |
| // shift clears inactive bytes) |
| "movq (%%edi,%%edx,), %%mm0 \n\t" |
| "paddb %%mm1, %%mm0 \n\t" |
| |
| // add 2nd active group |
| "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1 |
| "psllq $32, %%mm1 \n\t" // shift data to pos. correctly |
| "addl $8, %%edx \n\t" |
| "paddb %%mm1, %%mm0 \n\t" |
| |
| "cmpl %%eax, %%edx \n\t" // MMXLength |
| "movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array |
| "movq %%mm0, %%mm1 \n\t" // prep 1st add at top of loop |
| "jb sub_4lp \n\t" |
| |
| : "=c" (dummy_value_c), // 0 // output regs (dummy) |
| "=D" (dummy_value_D), // 1 |
| "=d" (dummy_value_d), // 2 |
| "=a" (dummy_value_a) // 3 |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (row), // edi |
| "2" (diff), // edx |
| "3" (MMXLength) // eax |
| |
| : "%esi" // clobber list |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| , "%mm0", "%mm1" |
| #endif |
| ); |
| } |
| break; // end 4 bpp |
| |
| case 6: // formerly shared with 4 bpp case (see comments there) |
| { |
| // _ShiftBpp = bpp << 3; // 48 (psllq) |
| // _ShiftRem = 64 - _ShiftBpp; // 16 (psrlq) |
| |
| __asm__ __volatile__ ( |
| // preload "movl row, %%edi \n\t" |
| // preload "movl diff, %%edx \n\t" |
| "movl %%edi, %%esi \n\t" // lp = row |
| // preload "movl bpp, %%ecx \n\t" |
| "addl %%ecx, %%edi \n\t" // rp = row + bpp |
| |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%edx,), %%mm1 \n\t" |
| |
| "sub_6lp: \n\t" // shift data for adding first |
| "psrlq $16, %%mm1 \n\t" // bpp bytes (no need for mask; |
| // shift clears inactive bytes) |
| "movq (%%edi,%%edx,), %%mm0 \n\t" |
| "paddb %%mm1, %%mm0 \n\t" |
| |
| // add 2nd active group |
| "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1 |
| "psllq $48, %%mm1 \n\t" // shift data to pos. correctly |
| "addl $8, %%edx \n\t" |
| "paddb %%mm1, %%mm0 \n\t" |
| |
| "cmpl %%eax, %%edx \n\t" // MMXLength |
| "movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array |
| "movq %%mm0, %%mm1 \n\t" // prep 1st add at top of loop |
| "jb sub_6lp \n\t" |
| |
| : "=c" (dummy_value_c), // 0 // output regs (dummy) |
| "=D" (dummy_value_D), // 1 |
| "=d" (dummy_value_d), // 2 |
| "=a" (dummy_value_a) // 3 |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (row), // edi |
| "2" (diff), // edx |
| "3" (MMXLength) // eax |
| |
| : "%esi" // clobber list |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| , "%mm0", "%mm1" |
| #endif |
| ); |
| } |
| break; // end 6 bpp |
| |
| case 1: |
| { |
| __asm__ __volatile__ ( |
| // preload "movl diff, %%edx \n\t" |
| // preload "movl row, %%edi \n\t" |
| // preload "cmpl FullLength, %%edx \n\t" |
| "cmpl %%eax, %%edx \n\t" |
| "jnb sub_1end \n\t" |
| "movl %%edi, %%esi \n\t" // lp = row |
| // irrel. "xorl %%ecx, %%ecx \n\t" // (actually bug with preload) |
| // preload "movl bpp, %%ecx \n\t" |
| "addl %%ecx, %%edi \n\t" // rp = row + bpp |
| |
| "sub_1lp: \n\t" |
| "movb (%%esi,%%edx,), %%cl \n\t" |
| "addb %%cl, (%%edi,%%edx,) \n\t" |
| "incl %%edx \n\t" |
| "cmpl %%eax, %%edx \n\t" // compare with FullLength |
| "jb sub_1lp \n\t" |
| |
| "sub_1end: \n\t" |
| |
| : "=c" (dummy_value_c), // 0 // output regs (dummy) |
| "=D" (dummy_value_D), // 1 |
| "=d" (dummy_value_d), // 2 |
| "=a" (dummy_value_a) // 3 |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (row), // edi |
| "2" (diff), // edx |
| "3" (FullLength) // eax |
| |
| : "%esi" // clobber list |
| ); |
| } |
| return; // end 1 bpp (bypassing cleanup block!) |
| |
| case 2: |
| { |
| // _ShiftBpp = 16; // == 2 * 8 |
| // _ShiftRem = 48; // == 64 - 16 |
| |
| __asm__ __volatile__ ( |
| LOAD_GOT_rbp |
| // load (former) _ActiveMask for 2nd active byte group |
| "movq " AMASK4_2_2 ", %%mm7 \n\t" // _amask4_2_2 |
| RESTORE_rbp |
| // preload "movl diff, %%edx \n\t" |
| "movq %%mm7, %%mm6 \n\t" |
| // preload "movl row, %%edi \n\t" |
| "psllq $16, %%mm6 \n\t" // move mask in mm6 to cover |
| // 3rd active byte group |
| "movl %%edi, %%esi \n\t" // lp = row |
| "movq %%mm6, %%mm5 \n\t" |
| // preload "movl bpp, %%ecx \n\t" |
| "addl %%ecx, %%edi \n\t" // rp = row + bpp |
| "psllq $16, %%mm5 \n\t" // move mask in mm5 to cover |
| // 4th active byte group |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%edx,), %%mm1 \n\t" |
| |
| "sub_2lp: \n\t" // shift data for adding first |
| "psrlq $48, %%mm1 \n\t" // bpp bytes (no need for mask; |
| // shift clears inactive bytes) |
| // add 1st active group |
| "movq (%%edi,%%edx,), %%mm0 \n\t" |
| "paddb %%mm1, %%mm0 \n\t" |
| |
| // add 2nd active group |
| "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1 |
| "psllq $16, %%mm1 \n\t" // shift data to pos. correctly |
| "pand %%mm7, %%mm1 \n\t" // mask to use 2nd active group |
| "paddb %%mm1, %%mm0 \n\t" |
| |
| // add 3rd active group |
| "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1 |
| "psllq $16, %%mm1 \n\t" // shift data to pos. correctly |
| "pand %%mm6, %%mm1 \n\t" // mask to use 3rd active group |
| "paddb %%mm1, %%mm0 \n\t" |
| |
| // add 4th active group |
| "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1 |
| "psllq $16, %%mm1 \n\t" // shift data to pos. correctly |
| "pand %%mm5, %%mm1 \n\t" // mask to use 4th active group |
| "addl $8, %%edx \n\t" |
| "paddb %%mm1, %%mm0 \n\t" |
| "cmpl %%eax, %%edx \n\t" // MMXLength |
| "movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array |
| "movq %%mm0, %%mm1 \n\t" // prep 1st add at top of loop |
| "jb sub_2lp \n\t" |
| |
| : "=c" (dummy_value_c), // 0 // output regs (dummy) |
| "=D" (dummy_value_D), // 1 |
| "=d" (dummy_value_d), // 2 |
| "=a" (dummy_value_a) // 3 |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (row), // edi |
| "2" (diff), // edx |
| "3" (MMXLength) // eax |
| |
| : "%esi" // clobber list |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| , "%mm0", "%mm1", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| break; // end 2 bpp |
| |
| case 8: |
| { |
| __asm__ __volatile__ ( |
| SAVE_ebp |
| // preload "movl row, %%edi \n\t" |
| // preload "movl diff, %%edx \n\t" |
| "movl %%edi, %%esi \n\t" // lp = row |
| // preload "movl bpp, %%ecx \n\t" |
| "addl %%ecx, %%edi \n\t" // rp = row + bpp |
| // preload "movl MMXLength, %%eax \n\t" |
| |
| // prime the pump: load the first Raw(x-bpp) data set |
| "movq -8(%%edi,%%edx,), %%mm7 \n\t" |
| "movl %%eax, %%ebp \n\t" // copy of MMXLength -> ebp |
| "andl $0x0000003f, %%ebp \n\t" // calc bytes over mult of 64 |
| |
| "sub_8lp: \n\t" |
| "movq (%%edi,%%edx,), %%mm0 \n\t" // load Sub(x) for 1st 8 bytes |
| "paddb %%mm7, %%mm0 \n\t" |
| "movq 8(%%edi,%%edx,), %%mm1 \n\t" // load Sub(x) for 2nd 8 bytes |
| "movq %%mm0, (%%edi,%%edx,) \n\t" // write Raw(x) for 1st 8 bytes |
| |
| // Now mm0 will be used as Raw(x-bpp) for the 2nd group of 8 bytes. |
| // This will be repeated for each group of 8 bytes with the 8th |
| // group being used as the Raw(x-bpp) for the 1st group of the |
| // next loop. |
| |
| "paddb %%mm0, %%mm1 \n\t" |
| "movq 16(%%edi,%%edx,), %%mm2 \n\t" // load Sub(x) for 3rd 8 bytes |
| "movq %%mm1, 8(%%edi,%%edx,) \n\t" // write Raw(x) for 2nd 8 bytes |
| "paddb %%mm1, %%mm2 \n\t" |
| "movq 24(%%edi,%%edx,), %%mm3 \n\t" // load Sub(x) for 4th 8 bytes |
| "movq %%mm2, 16(%%edi,%%edx,) \n\t" // write Raw(x) for 3rd 8 bytes |
| "paddb %%mm2, %%mm3 \n\t" |
| "movq 32(%%edi,%%edx,), %%mm4 \n\t" // load Sub(x) for 5th 8 bytes |
| "movq %%mm3, 24(%%edi,%%edx,) \n\t" // write Raw(x) for 4th 8 bytes |
| "paddb %%mm3, %%mm4 \n\t" |
| "movq 40(%%edi,%%edx,), %%mm5 \n\t" // load Sub(x) for 6th 8 bytes |
| "movq %%mm4, 32(%%edi,%%edx,) \n\t" // write Raw(x) for 5th 8 bytes |
| "paddb %%mm4, %%mm5 \n\t" |
| "movq 48(%%edi,%%edx,), %%mm6 \n\t" // load Sub(x) for 7th 8 bytes |
| "movq %%mm5, 40(%%edi,%%edx,) \n\t" // write Raw(x) for 6th 8 bytes |
| "paddb %%mm5, %%mm6 \n\t" |
| "movq 56(%%edi,%%edx,), %%mm7 \n\t" // load Sub(x) for 8th 8 bytes |
| "movq %%mm6, 48(%%edi,%%edx,) \n\t" // write Raw(x) for 7th 8 bytes |
| "addl $64, %%edx \n\t" |
| "paddb %%mm6, %%mm7 \n\t" |
| "cmpl %%ebp, %%edx \n\t" // cmp to bytes over mult of 64 |
| "movq %%mm7, -8(%%edi,%%edx,) \n\t" // write Raw(x) for 8th 8 bytes |
| "jb sub_8lp \n\t" |
| |
| "cmpl %%eax, %%edx \n\t" // compare to MMXLength |
| "jnb sub_8lt8 \n\t" |
| |
| "sub_8lpA: \n\t" |
| "movq (%%edi,%%edx,), %%mm0 \n\t" |
| "addl $8, %%edx \n\t" |
| "paddb %%mm7, %%mm0 \n\t" |
| "cmpl %%eax, %%edx \n\t" // compare to MMXLength |
| "movq %%mm0, -8(%%edi,%%edx,) \n\t" // -8 to offset early addl edx |
| "movq %%mm0, %%mm7 \n\t" // move calculated Raw(x) data |
| "jb sub_8lpA \n\t" // to mm7 to be new Raw(x-bpp) |
| // for next loop |
| "sub_8lt8: \n\t" |
| RESTORE_ebp |
| |
| : "=c" (dummy_value_c), // 0 // output regs (dummy) |
| "=D" (dummy_value_D), // 1 |
| "=d" (dummy_value_d), // 2 |
| "=a" (dummy_value_a) // 3 |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (row), // edi |
| "2" (diff), // edx |
| "3" (MMXLength) // eax |
| |
| : "%esi" // clobber list |
| _CLOBBER_ebp |
| #if defined(CLOBBER_MMX_REGS_SUPPORTED) |
| , "%mm0", "%mm1", "%mm2", "%mm3" |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| } |
| break; // end 8 bpp |
| |
| default: // bpp != 1,2,3,4,6,8: doesn't exist |
| { |
| #if defined(PNG_DEBUG) |
| // ERROR: SHOULD NEVER BE REACHED |
| png_debug(1, |
| "Internal libpng logic error (GCC png_read_filter_row_mmx_sub())\n"); |
| #endif |
| } |
| break; |
| |
| } // end switch (bpp) |
| |
| __asm__ __volatile__ ( |
| //pre "movl MMXLength, %%eax \n\t" |
| //pre "movl row, %%edi \n\t" |
| //pre "cmpl FullLength, %%eax \n\t" |
| "cmpl %%edx, %%eax \n\t" |
| "jnb sub_end \n\t" |
| |
| "movl %%edi, %%esi \n\t" // lp = row |
| //pre "movl bpp, %%ecx \n\t" |
| "addl %%ecx, %%edi \n\t" // rp = row + bpp |
| "xorl %%ecx, %%ecx \n\t" |
| |
| "sub_lp2: \n\t" |
| "movb (%%esi,%%eax,), %%cl \n\t" |
| "addb %%cl, (%%edi,%%eax,) \n\t" |
| "incl %%eax \n\t" |
| "cmpl %%edx, %%eax \n\t" // FullLength |
| "jb sub_lp2 \n\t" |
| |
| "sub_end: \n\t" |
| "EMMS \n\t" // end MMX instructions |
| |
| : "=c" (dummy_value_c), // 0 // output regs (dummy) |
| "=D" (dummy_value_D), // 1 |
| "=a" (dummy_value_a), // 2 |
| "=d" (dummy_value_d) // 3 |
| |
| : "0" (bpp), // ecx // input regs |
| "1" (row), // edi |
| "2" (MMXLength), // eax |
| "3" (FullLength) // edx |
| |
| : "%esi" // clobber list |
| ); |
| |
| } // end of png_read_filter_row_mmx_sub() |
| |
| |
| |
| |
| //===========================================================================// |
| // // |
| // P N G _ R E A D _ F I L T E R _ R O W _ M M X _ U P // |
| // // |
| //===========================================================================// |
| |
| // Optimized code for PNG Up filter decoder |
| |
| static void /* PRIVATE */ |
| png_read_filter_row_mmx_up(png_row_infop row_info, png_bytep row, |
| png_bytep prev_row) |
| { |
| png_uint_32 len; |
| int dummy_value_d; // fix 'forbidden register 3 (dx) was spilled' error |
| int dummy_value_S; |
| int dummy_value_D; |
| |
| len = row_info->rowbytes; // number of bytes to filter |
| |
| __asm__ __volatile__ ( |
| SAVE_GOT_ebx |
| //pre "movl row, %%edi \n\t" |
| // get # of bytes to alignment |
| "movl %%edi, %%ecx \n\t" |
| "xorl %%ebx, %%ebx \n\t" |
| "addl $0x7, %%ecx \n\t" |
| "xorl %%eax, %%eax \n\t" |
| "andl $0xfffffff8, %%ecx \n\t" |
| //pre "movl prev_row, %%esi \n\t" |
| "subl %%edi, %%ecx \n\t" |
| "jz up_go \n\t" |
| |
| "up_lp1: \n\t" // fix alignment |
| "movb (%%edi,%%ebx,), %%al \n\t" |
| "addb (%%esi,%%ebx,), %%al \n\t" |
| "incl %%ebx \n\t" |
| "cmpl %%ecx, %%ebx \n\t" |
| "movb %%al, -1(%%edi,%%ebx,) \n\t" // mov does not affect flags; -1 to |
| "jb up_lp1 \n\t" // offset incl ebx |
| |
| "up_go: \n\t" |
| //pre "movl len, %%edx \n\t" |
| "movl %%edx, %%ecx \n\t" |
| "subl %%ebx, %%edx \n\t" // subtract alignment fix |
| "andl $0x0000003f, %%edx \n\t" // calc bytes over mult of 64 |
| "subl %%edx, %%ecx \n\t" // drop over bytes from length |
| |
| // unrolled loop - use all MMX registers and interleave to reduce |
| // number of branch instructions (loops) and reduce partial stalls |
| "up_loop: \n\t" |
| "movq (%%esi,%%ebx,), %%mm1 \n\t" |
| "movq (%%edi,%%ebx,), %%mm0 \n\t" |
| "movq 8(%%esi,%%ebx,), %%mm3 \n\t" |
| "paddb %%mm1, %%mm0 \n\t" |
| "movq 8(%%edi,%%ebx,), %%mm2 \n\t" |
| "movq %%mm0, (%%edi,%%ebx,) \n\t" |
| "paddb %%mm3, %%mm2 \n\t" |
| "movq 16(%%esi,%%ebx,), %%mm5 \n\t" |
| "movq %%mm2, 8(%%edi,%%ebx,) \n\t" |
| "movq 16(%%edi,%%ebx,), %%mm4 \n\t" |
| "movq 24(%%esi,%%ebx,), %%mm7 \n\t" |
| "paddb %%mm5, %%mm4 \n\t" |
| "movq 24(%%edi,%%ebx,), %%mm6 \n\t" |
| "movq %%mm4, 16(%%edi,%%ebx,) \n\t" |
| "paddb %%mm7, %%mm6 \n\t" |
| "movq 32(%%esi,%%ebx,), %%mm1 \n\t" |
| "movq %%mm6, 24(%%edi,%%ebx,) \n\t" |
| "movq 32(%%edi,%%ebx,), %%mm0 \n\t" |
| "movq 40(%%esi,%%ebx,), %%mm3 \n\t" |
| "paddb %%mm1, %%mm0 \n\t" |
| "movq 40(%%edi,%%ebx,), %%mm2 \n\t" |
| "movq %%mm0, 32(%%edi,%%ebx,) \n\t" |
| "paddb %%mm3, %%mm2 \n\t" |
| "movq 48(%%esi,%%ebx,), %%mm5 \n\t" |
| "movq %%mm2, 40(%%edi,%%ebx,) \n\t" |
| "movq 48(%%edi,%%ebx,), %%mm4 \n\t" |
| "movq 56(%%esi,%%ebx,), %%mm7 \n\t" |
| "paddb %%mm5, %%mm4 \n\t" |
| "movq 56(%%edi,%%ebx,), %%mm6 \n\t" |
| "movq %%mm4, 48(%%edi,%%ebx,) \n\t" |
| "addl $64, %%ebx \n\t" |
| "paddb %%mm7, %%mm6 \n\t" |
| "cmpl %%ecx, %%ebx \n\t" |
| "movq %%mm6, -8(%%edi,%%ebx,) \n\t" // (+56)movq does not affect flags; |
| "jb up_loop \n\t" // -8 to offset addl ebx |
| |
| "cmpl $0, %%edx \n\t" // test for bytes over mult of 64 |
| "jz up_end \n\t" |
| |
| "cmpl $8, %%edx \n\t" // test for less than 8 bytes |
| "jb up_lt8 \n\t" // [added by lcreeve at netins.net] |
| |
| "addl %%edx, %%ecx \n\t" |
| "andl $0x00000007, %%edx \n\t" // calc bytes over mult of 8 |
| "subl %%edx, %%ecx \n\t" // drop over bytes from length |
| "jz up_lt8 \n\t" |
| |
| "up_lpA: \n\t" // use MMX regs to update 8 bytes sim. |
| "movq (%%esi,%%ebx,), %%mm1 \n\t" |
| "movq (%%edi,%%ebx,), %%mm0 \n\t" |
| "addl $8, %%ebx \n\t" |
| "paddb %%mm1, %%mm0 \n\t" |
| "cmpl %%ecx, %%ebx \n\t" |
| "movq %%mm0, -8(%%edi,%%ebx,) \n\t" // movq does not affect flags; -8 to |
| "jb up_lpA \n\t" // offset add ebx |
| "cmpl $0, %%edx \n\t" // test for bytes over mult of 8 |
| "jz up_end \n\t" |
| |
| "up_lt8: \n\t" |
| "xorl %%eax, %%eax \n\t" |
| "addl %%edx, %%ecx \n\t" // move over byte count into counter |
| |
| "up_lp2: \n\t" // use x86 regs for remaining bytes |
| "movb (%%edi,%%ebx,), %%al \n\t" |
| "addb (%%esi,%%ebx,), %%al \n\t" |
| "incl %%ebx \n\t" |
| "cmpl %%ecx, %%ebx \n\t" |
| "movb %%al, -1(%%edi,%%ebx,) \n\t" // mov does not affect flags; -1 to |
| "jb up_lp2 \n\t" // offset inc ebx |
| |
| "up_end: \n\t" |
| "EMMS \n\t" // conversion of filtered row complete |
| RESTORE_GOT_ebx |
| |
| : "=d" (dummy_value_d), // 0 // output regs (dummy) |
| "=S" (dummy_value_S), // 1 |
| "=D" (dummy_value_D) // 2 |
| |
| : "0" (len), // edx // input regs |
| "1" (prev_row), // esi |
| "2" (row) // edi |
| |
| : "%eax", "%ecx" // clobber list (no input regs!) |
| _CLOBBER_GOT_ebx |
| |
| #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) |
| , "%mm0", "%mm1", "%mm2", "%mm3" |
| , "%mm4", "%mm5", "%mm6", "%mm7" |
| #endif |
| ); |
| |
| } // end of png_read_filter_row_mmx_up() |
| |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| |
| |
| |
| |
| /*===========================================================================*/ |
| /* */ |
| /* P N G _ R E A D _ F I L T E R _ R O W */ |
| /* */ |
| /*===========================================================================*/ |
| |
| |
| /* Optimized png_read_filter_row routines */ |
| |
| void /* PRIVATE */ |
| png_read_filter_row(png_structp png_ptr, png_row_infop row_info, png_bytep |
| row, png_bytep prev_row, int filter) |
| { |
| #if defined(PNG_DEBUG) |
| char filnm[10]; |
| #endif |
| |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| if (_mmx_supported == 2) { |
| #if !defined(PNG_1_0_X) |
| /* this should have happened in png_init_mmx_flags() already */ |
| png_warning(png_ptr, "asm_flags may not have been initialized"); |
| #endif |
| png_mmx_support(); |
| } |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| |
| #if defined(PNG_DEBUG) |
| png_debug(1, "in png_read_filter_row (pnggccrd.c)\n"); |
| switch (filter) |
| { |
| case 0: |
| png_snprintf(filnm, 10, "none"); |
| break; |
| |
| case 1: |
| png_snprintf(filnm, 10, "sub-%s", |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB) && |
| (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) && |
| (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold)) |
| #else |
| _mmx_supported |
| #endif |
| ? "MMX" : |
| #endif |
| "x86"); |
| break; |
| |
| case 2: |
| png_snprintf(filnm, 10, "up-%s", |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP) && |
| (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) && |
| (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold)) |
| #else |
| _mmx_supported |
| #endif |
| ? "MMX" : |
| #endif |
| "x86"); |
| break; |
| |
| case 3: |
| png_snprintf(filnm, 10, "avg-%s", |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG) && |
| (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) && |
| (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold)) |
| #else |
| _mmx_supported |
| #endif |
| ? "MMX" : |
| #endif |
| "x86"); |
| break; |
| |
| case 4: |
| png_snprintf(filnm, 10, "Paeth-%s", |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH) && |
| (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) && |
| (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold)) |
| #else |
| _mmx_supported |
| #endif |
| ? "MMX" : |
| #endif |
| "x86"); |
| break; |
| |
| default: |
| png_snprintf(filnm, 10, "unknown"); |
| break; |
| } |
| png_debug2(0, "row_number=%5ld, %10s, ", png_ptr->row_number, filnm); |
| png_debug1(0, "row=0x%08lx, ", (unsigned long)row); |
| png_debug2(0, "pixdepth=%2d, bytes=%d, ", (int)row_info->pixel_depth, |
| (int)((row_info->pixel_depth + 7) >> 3)); |
| png_debug1(0,"rowbytes=%8ld\n", row_info->rowbytes); |
| #endif /* PNG_DEBUG */ |
| |
| switch (filter) |
| { |
| case PNG_FILTER_VALUE_NONE: |
| break; |
| |
| case PNG_FILTER_VALUE_SUB: |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB) && |
| (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) && |
| (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold)) |
| #else |
| if (_mmx_supported) |
| #endif |
| { |
| png_read_filter_row_mmx_sub(row_info, row); |
| } |
| else |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| { |
| png_uint_32 i; |
| png_uint_32 istop = row_info->rowbytes; |
| png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3; |
| png_bytep rp = row + bpp; |
| png_bytep lp = row; |
| |
| for (i = bpp; i < istop; i++) |
| { |
| *rp = (png_byte)(((int)(*rp) + (int)(*lp++)) & 0xff); |
| rp++; |
| } |
| } /* end !UseMMX_sub */ |
| break; |
| |
| case PNG_FILTER_VALUE_UP: |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP) && |
| (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) && |
| (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold)) |
| #else |
| if (_mmx_supported) |
| #endif |
| { |
| png_read_filter_row_mmx_up(row_info, row, prev_row); |
| } |
| else |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| { |
| png_uint_32 i; |
| png_uint_32 istop = row_info->rowbytes; |
| png_bytep rp = row; |
| png_bytep pp = prev_row; |
| |
| for (i = 0; i < istop; ++i) |
| { |
| *rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff); |
| rp++; |
| } |
| } /* end !UseMMX_up */ |
| break; |
| |
| case PNG_FILTER_VALUE_AVG: |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if !defined(PNG_1_0_X) |
| if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG) && |
| (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) && |
| (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold)) |
| #else |
| if (_mmx_supported) |
| #endif |
| { |
| png_read_filter_row_mmx_avg(row_info, row, prev_row); |
| } |
| else |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| { |
| png_uint_32 i; |
| png_bytep rp = row; |
| png_bytep pp = prev_row; |
| png_bytep lp = row; |
| png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3; |
| png_uint_32 istop = row_info->rowbytes - bpp; |
| |
| for (i = 0; i < bpp; i++) |
| { |
| *rp = (png_byte)(((int)(*rp) + |
| ((int)(*pp++) >> 1)) & 0xff); |
| rp++; |
| } |
| |
| for (i = 0; i < istop; i++) |
| { |
| *rp = (png_byte)(((int)(*rp) + |
| ((int)(*pp++ + *lp++) >> 1)) & 0xff); |
| rp++; |
| } |
| } /* end !UseMMX_avg */ |
| break; |
| |
| case PNG_FILTER_VALUE_PAETH: |
| #if defined(PNG_MMX_CODE_SUPPORTED) |
| #if defined(PNG_x86_64_USE_GOTPCREL) || defined(PNG_THREAD_UNSAFE_OK) |
| #if !defined(PNG_1_0_X) |
| if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH) && |
| (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) && |
| (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold)) |
| #else |
| if (_mmx_supported) |
| #endif |
| { |
| png_read_filter_row_mmx_paeth(row_info, row, prev_row); |
| } |
| else |
| #endif /* PNG_x86_64_USE_GOTPCREL || PNG_THREAD_UNSAFE_OK */ |
| #endif /* PNG_MMX_CODE_SUPPORTED */ |
| { |
| png_uint_32 i; |
| png_bytep rp = row; |
| png_bytep pp = prev_row; |
| png_bytep lp = row; |
| png_bytep cp = prev_row; |
| png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3; |
| png_uint_32 istop = row_info->rowbytes - bpp; |
| |
| for (i = 0; i < bpp; i++) |
| { |
| *rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff); |
| rp++; |
| } |
| |
| for (i = 0; i < istop; i++) /* use leftover rp,pp */ |
| { |
| int a, b, c, pa, pb, pc, p; |
| |
| a = *lp++; |
| b = *pp++; |
| c = *cp++; |
| |
| p = b - c; |
| pc = a - c; |
| |
| #if defined(PNG_USE_ABS) |
| pa = abs(p); |
| pb = abs(pc); |
| pc = abs(p + pc); |
| #else |
| pa = p < 0 ? -p : p; |
| pb = pc < 0 ? -pc : pc; |
| pc = (p + pc) < 0 ? -(p + pc) : p + pc; |
| #endif |
| |
| /* |
| if (pa <= pb && pa <= pc) |
| p = a; |
| else if (pb <= pc) |
| p = b; |
| else |
| p = c; |
| */ |
| |
| p = (pa <= pb && pa <= pc) ? a : (pb <= pc) ? b : c; |
| |
| *rp = (png_byte)(((int)(*rp) + p) & 0xff); |
| rp++; |
| } |
| } /* end !UseMMX_paeth */ |
| break; |
| |
| default: |
| png_warning(png_ptr, "Ignoring bad row-filter type"); |
| *row=0; |
| break; |
| } |
| } |
| |
| #endif /* PNG_HAVE_MMX_READ_FILTER_ROW */ |
| |
| |
| #endif /* PNG_ASSEMBLER_CODE_SUPPORTED && PNG_USE_PNGGCCRD */ |
