gtkmm-osx/trunk/libpng-1.2.5/pnggccrd.c

   1 /* pnggccrd.c - mixed C/assembler version of utilities to read a PNG file
   2  *
   3  * For Intel x86 CPU (Pentium-MMX or later) and GNU C compiler.
   4  *
   5  *     See http://www.intel.com/drg/pentiumII/appnotes/916/916.htm
   6  *     and http://www.intel.com/drg/pentiumII/appnotes/923/923.htm
   7  *     for Intel's performance analysis of the MMX vs. non-MMX code.
   8  *
   9  * libpng version 1.2.5 - October 3, 2002
  10  * For conditions of distribution and use, see copyright notice in png.h
  11  * Copyright (c) 1998-2002 Glenn Randers-Pehrson
  12  * Copyright (c) 1998, Intel Corporation
  13  *
  14  * Based on MSVC code contributed by Nirav Chhatrapati, Intel Corp., 1998.
  15  * Interface to libpng contributed by Gilles Vollant, 1999.
  16  * GNU C port by Greg Roelofs, 1999-2001.
  17  *
  18  * Lines 2350-4300 converted in place with intel2gas 1.3.1:
  19  *
  20  *   intel2gas -mdI pnggccrd.c.partially-msvc -o pnggccrd.c
  21  *
  22  * and then cleaned up by hand.  See http://hermes.terminal.at/intel2gas/ .
  23  *
  24  * NOTE:  A sufficiently recent version of GNU as (or as.exe under DOS/Windows)
  25  *        is required to assemble the newer MMX instructions such as movq.
  26  *        For djgpp, see
  27  *
  28  *           ftp://ftp.simtel.net/pub/simtelnet/gnu/djgpp/v2gnu/bnu281b.zip
  29  *
  30  *        (or a later version in the same directory).  For Linux, check your
  31  *        distribution's web site(s) or try these links:
  32  *
  33  *           http://rufus.w3.org/linux/RPM/binutils.html
  34  *           http://www.debian.org/Packages/stable/devel/binutils.html
  35  *           ftp://ftp.slackware.com/pub/linux/slackware/slackware/slakware/d1/
  36  *             binutils.tgz
  37  *
  38  *        For other platforms, see the main GNU site:
  39  *
  40  *           ftp://ftp.gnu.org/pub/gnu/binutils/
  41  *
  42  *        Version 2.5.2l.15 is definitely too old...
  43  */
  44
  45 /*
  46  * TEMPORARY PORTING NOTES AND CHANGELOG (mostly by Greg Roelofs)
  47  * =====================================
  48  *
  49  * 19991006:
  50  *  - fixed sign error in post-MMX cleanup code (16- & 32-bit cases)
  51  *
  52  * 19991007:
  53  *  - additional optimizations (possible or definite):
  54  *     x [DONE] write MMX code for 64-bit case (pixel_bytes == 8) [not tested]
  55  *     - write MMX code for 48-bit case (pixel_bytes == 6)
  56  *     - figure out what's up with 24-bit case (pixel_bytes == 3):
  57  *        why subtract 8 from width_mmx in the pass 4/5 case?
  58  *        (only width_mmx case) (near line 1606)
  59  *     x [DONE] replace pixel_bytes within each block with the true
  60  *        constant value (or are compilers smart enough to do that?)
  61  *     - rewrite all MMX interlacing code so it's aligned with
  62  *        the *beginning* of the row buffer, not the end.  This
  63  *        would not only allow one to eliminate half of the memory
  64  *        writes for odd passes (that is, pass == odd), it may also
  65  *        eliminate some unaligned-data-access exceptions (assuming
  66  *        there's a penalty for not aligning 64-bit accesses on
  67  *        64-bit boundaries).  The only catch is that the "leftover"
  68  *        pixel(s) at the end of the row would have to be saved,
  69  *        but there are enough unused MMX registers in every case,
  70  *        so this is not a problem.  A further benefit is that the
  71  *        post-MMX cleanup code (C code) in at least some of the
  72  *        cases could be done within the assembler block.
  73  *  x [DONE] the "v3 v2 v1 v0 v7 v6 v5 v4" comments are confusing,
  74  *     inconsistent, and don't match the MMX Programmer's Reference
  75  *     Manual conventions anyway.  They should be changed to
  76  *     "b7 b6 b5 b4 b3 b2 b1 b0," where b0 indicates the byte that
  77  *     was lowest in memory (e.g., corresponding to a left pixel)
  78  *     and b7 is the byte that was highest (e.g., a right pixel).
  79  *
  80  * 19991016:
  81  *  - Brennan's Guide notwithstanding, gcc under Linux does *not*
  82  *     want globals prefixed by underscores when referencing them--
  83  *     i.e., if the variable is const4, then refer to it as const4,
  84  *     not _const4.  This seems to be a djgpp-specific requirement.
  85  *     Also, such variables apparently *must* be declared outside
  86  *     of functions; neither static nor automatic variables work if
  87  *     defined within the scope of a single function, but both
  88  *     static and truly global (multi-module) variables work fine.
  89  *
  90  * 19991023:
  91  *  - fixed png_combine_row() non-MMX replication bug (odd passes only?)
  92  *  - switched from string-concatenation-with-macros to cleaner method of
  93  *     renaming global variables for djgpp--i.e., always use prefixes in
  94  *     inlined assembler code (== strings) and conditionally rename the
  95  *     variables, not the other way around.  Hence _const4, _mask8_0, etc.
  96  *
  97  * 19991024:
  98  *  - fixed mmxsupport()/png_do_read_interlace() first-row bug
  99  *     This one was severely weird:  even though mmxsupport() doesn't touch
 100  *     ebx (where "row" pointer was stored), it nevertheless managed to zero
 101  *     the register (even in static/non-fPIC code--see below), which in turn
 102  *     caused png_do_read_interlace() to return prematurely on the first row of
 103  *     interlaced images (i.e., without expanding the interlaced pixels).
 104  *     Inspection of the generated assembly code didn't turn up any clues,
 105  *     although it did point at a minor optimization (i.e., get rid of
 106  *     mmx_supported_local variable and just use eax).  Possibly the CPUID
 107  *     instruction is more destructive than it looks?  (Not yet checked.)
 108  *  - "info gcc" was next to useless, so compared fPIC and non-fPIC assembly
 109  *     listings...  Apparently register spillage has to do with ebx, since
 110  *     it's used to index the global offset table.  Commenting it out of the
 111  *     input-reg lists in png_combine_row() eliminated compiler barfage, so
 112  *     ifdef'd with __PIC__ macro:  if defined, use a global for unmask
 113  *
 114  * 19991107:
 115  *  - verified CPUID clobberage:  12-char string constant ("GenuineIntel",
 116  *     "AuthenticAMD", etc.) placed in ebx:ecx:edx.  Still need to polish.
 117  *
 118  * 19991120:
 119  *  - made "diff" variable (now "_dif") global to simplify conversion of
 120  *     filtering routines (running out of regs, sigh).  "diff" is still used
 121  *     in interlacing routines, however.
 122  *  - fixed up both versions of mmxsupport() (ORIG_THAT_USED_TO_CLOBBER_EBX
 123  *     macro determines which is used); original not yet tested.
 124  *
 125  * 20000213:
 126  *  - when compiling with gcc, be sure to use  -fomit-frame-pointer
 127  *
 128  * 20000319:
 129  *  - fixed a register-name typo in png_do_read_interlace(), default (MMX) case,
 130  *     pass == 4 or 5, that caused visible corruption of interlaced images
 131  *
 132  * 20000623:
 133  *  - Various problems were reported with gcc 2.95.2 in the Cygwin environment,
 134  *     many of the form "forbidden register 0 (ax) was spilled for class AREG."
 135  *     This is explained at http://gcc.gnu.org/fom_serv/cache/23.html, and
 136  *     Chuck Wilson supplied a patch involving dummy output registers.  See
 137  *     http://sourceforge.net/bugs/?func=detailbug&bug_id=108741&group_id=5624
 138  *     for the original (anonymous) SourceForge bug report.
 139  *
 140  * 20000706:
 141  *  - Chuck Wilson passed along these remaining gcc 2.95.2 errors:
 142  *       pnggccrd.c: In function `png_combine_row':
 143  *       pnggccrd.c:525: more than 10 operands in `asm'
 144  *       pnggccrd.c:669: more than 10 operands in `asm'
 145  *       pnggccrd.c:828: more than 10 operands in `asm'
 146  *       pnggccrd.c:994: more than 10 operands in `asm'
 147  *       pnggccrd.c:1177: more than 10 operands in `asm'
 148  *     They are all the same problem and can be worked around by using the
 149  *     global _unmask variable unconditionally, not just in the -fPIC case.
 150  *     Reportedly earlier versions of gcc also have the problem with more than
 151  *     10 operands; they just don't report it.  Much strangeness ensues, etc.
 152  *
 153  * 20000729:
 154  *  - enabled png_read_filter_row_mmx_up() (shortest remaining unconverted
 155  *     MMX routine); began converting png_read_filter_row_mmx_sub()
 156  *  - to finish remaining sections:
 157  *     - clean up indentation and comments
 158  *     - preload local variables
 159  *     - add output and input regs (order of former determines numerical
 160  *        mapping of latter)
 161  *     - avoid all usage of ebx (including bx, bh, bl) register [20000823]
 162  *     - remove "$" from addressing of Shift and Mask variables [20000823]
 163  *
 164  * 20000731:
 165  *  - global union vars causing segfaults in png_read_filter_row_mmx_sub()?
 166  *
 167  * 20000822:
 168  *  - ARGH, stupid png_read_filter_row_mmx_sub() segfault only happens with
 169  *     shared-library (-fPIC) version!  Code works just fine as part of static
 170  *     library.  Damn damn damn damn damn, should have tested that sooner.
 171  *     ebx is getting clobbered again (explicitly this time); need to save it
 172  *     on stack or rewrite asm code to avoid using it altogether.  Blargh!
 173  *
 174  * 20000823:
 175  *  - first section was trickiest; all remaining sections have ebx -> edx now.
 176  *     (-fPIC works again.)  Also added missing underscores to various Shift*
 177  *     and *Mask* globals and got rid of leading "$" signs.
 178  *
 179  * 20000826:
 180  *  - added visual separators to help navigate microscopic printed copies
 181  *     (http://pobox.com/~newt/code/gpr-latest.zip, mode 10); started working
 182  *     on png_read_filter_row_mmx_avg()
 183  *
 184  * 20000828:
 185  *  - finished png_read_filter_row_mmx_avg():  only Paeth left! (930 lines...)
 186  *     What the hell, did png_read_filter_row_mmx_paeth(), too.  Comments not
 187  *     cleaned up/shortened in either routine, but functionality is complete
 188  *     and seems to be working fine.
 189  *
 190  * 20000829:
 191  *  - ahhh, figured out last(?) bit of gcc/gas asm-fu:  if register is listed
 192  *     as an input reg (with dummy output variables, etc.), then it *cannot*
 193  *     also appear in the clobber list or gcc 2.95.2 will barf.  The solution
 194  *     is simple enough...
 195  *
 196  * 20000914:
 197  *  - bug in png_read_filter_row_mmx_avg():  16-bit grayscale not handled
 198  *     correctly (but 48-bit RGB just fine)
 199  *
 200  * 20000916:
 201  *  - fixed bug in png_read_filter_row_mmx_avg(), bpp == 2 case; three errors:
 202  *     - "_ShiftBpp.use = 24;"      should have been   "_ShiftBpp.use = 16;"
 203  *     - "_ShiftRem.use = 40;"      should have been   "_ShiftRem.use = 48;"
 204  *     - "psllq _ShiftRem, %%mm2"   should have been   "psrlq _ShiftRem, %%mm2"
 205  *
 206  * 20010101:
 207  *  - added new png_init_mmx_flags() function (here only because it needs to
 208  *     call mmxsupport(), which should probably become global png_mmxsupport());
 209  *     modified other MMX routines to run conditionally (png_ptr->asm_flags)
 210  *
 211  * 20010103:
 212  *  - renamed mmxsupport() to png_mmx_support(), with auto-set of mmx_supported,
 213  *     and made it public; moved png_init_mmx_flags() to png.c as internal func
 214  *
 215  * 20010104:
 216  *  - removed dependency on png_read_filter_row_c() (C code already duplicated
 217  *     within MMX version of png_read_filter_row()) so no longer necessary to
 218  *     compile it into pngrutil.o
 219  *
 220  * 20010310:
 221  *  - fixed buffer-overrun bug in png_combine_row() C code (non-MMX)
 222  *
 223  * 20020304:
 224  *  - eliminated incorrect use of width_mmx in pixel_bytes == 8 case
 225  *
 226  * STILL TO DO:
 227  *     - test png_do_read_interlace() 64-bit case (pixel_bytes == 8)
 228  *     - write MMX code for 48-bit case (pixel_bytes == 6)
 229  *     - figure out what's up with 24-bit case (pixel_bytes == 3):
 230  *        why subtract 8 from width_mmx in the pass 4/5 case?
 231  *        (only width_mmx case) (near line 1606)
 232  *     - rewrite all MMX interlacing code so it's aligned with beginning
 233  *        of the row buffer, not the end (see 19991007 for details)
 234  *     x pick one version of mmxsupport() and get rid of the other
 235  *     - add error messages to any remaining bogus default cases
 236  *     - enable pixel_depth == 8 cases in png_read_filter_row()? (test speed)
 237  *     x add support for runtime enable/disable/query of various MMX routines
 238  */
 239
 240 #define PNG_INTERNAL
 241 #include "png.h"
 242
 243 #if defined(PNG_USE_PNGGCCRD)
 244
 245 int PNGAPI png_mmx_support(void);
 246
 247 #ifdef PNG_USE_LOCAL_ARRAYS
 248 static const int FARDATA png_pass_start[7] = {0, 4, 0, 2, 0, 1, 0};
 249 static const int FARDATA png_pass_inc[7]   = {8, 8, 4, 4, 2, 2, 1};
 250 static const int FARDATA png_pass_width[7] = {8, 4, 4, 2, 2, 1, 1};
 251 #endif
 252
 253 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
 254 /* djgpp, Win32, and Cygwin add their own underscores to global variables,
 255  * so define them without: */
 256 #if defined(__DJGPP__) || defined(WIN32) || defined(__CYGWIN__)
 257 #  define _mmx_supported  mmx_supported
 258 #  define _const4         const4
 259 #  define _const6         const6
 260 #  define _mask8_0        mask8_0
 261 #  define _mask16_1       mask16_1
 262 #  define _mask16_0       mask16_0
 263 #  define _mask24_2       mask24_2
 264 #  define _mask24_1       mask24_1
 265 #  define _mask24_0       mask24_0
 266 #  define _mask32_3       mask32_3
 267 #  define _mask32_2       mask32_2
 268 #  define _mask32_1       mask32_1
 269 #  define _mask32_0       mask32_0
 270 #  define _mask48_5       mask48_5
 271 #  define _mask48_4       mask48_4
 272 #  define _mask48_3       mask48_3
 273 #  define _mask48_2       mask48_2
 274 #  define _mask48_1       mask48_1
 275 #  define _mask48_0       mask48_0
 276 #  define _LBCarryMask    LBCarryMask
 277 #  define _HBClearMask    HBClearMask
 278 #  define _ActiveMask     ActiveMask
 279 #  define _ActiveMask2    ActiveMask2
 280 #  define _ActiveMaskEnd  ActiveMaskEnd
 281 #  define _ShiftBpp       ShiftBpp
 282 #  define _ShiftRem       ShiftRem
 283 #ifdef PNG_THREAD_UNSAFE_OK
 284 #  define _unmask         unmask
 285 #  define _FullLength     FullLength
 286 #  define _MMXLength      MMXLength
 287 #  define _dif            dif
 288 #  define _patemp         patemp
 289 #  define _pbtemp         pbtemp
 290 #  define _pctemp         pctemp
 291 #endif
 292 #endif
 293
 294
 295 /* These constants are used in the inlined MMX assembly code.
 296    Ignore gcc's "At top level: defined but not used" warnings. */
 297
 298 /* GRR 20000706:  originally _unmask was needed only when compiling with -fPIC,
 299  *  since that case uses the %ebx register for indexing the Global Offset Table
 300  *  and there were no other registers available.  But gcc 2.95 and later emit
 301  *  "more than 10 operands in `asm'" errors when %ebx is used to preload unmask
 302  *  in the non-PIC case, so we'll just use the global unconditionally now.
 303  */
 304 #ifdef PNG_THREAD_UNSAFE_OK
 305 static int _unmask;
 306 #endif
 307
 308 static unsigned long long _mask8_0  = 0x0102040810204080LL;
 309
 310 static unsigned long long _mask16_1 = 0x0101020204040808LL;
 311 static unsigned long long _mask16_0 = 0x1010202040408080LL;
 312
 313 static unsigned long long _mask24_2 = 0x0101010202020404LL;
 314 static unsigned long long _mask24_1 = 0x0408080810101020LL;
 315 static unsigned long long _mask24_0 = 0x2020404040808080LL;
 316
 317 static unsigned long long _mask32_3 = 0x0101010102020202LL;
 318 static unsigned long long _mask32_2 = 0x0404040408080808LL;
 319 static unsigned long long _mask32_1 = 0x1010101020202020LL;
 320 static unsigned long long _mask32_0 = 0x4040404080808080LL;
 321
 322 static unsigned long long _mask48_5 = 0x0101010101010202LL;
 323 static unsigned long long _mask48_4 = 0x0202020204040404LL;
 324 static unsigned long long _mask48_3 = 0x0404080808080808LL;
 325 static unsigned long long _mask48_2 = 0x1010101010102020LL;
 326 static unsigned long long _mask48_1 = 0x2020202040404040LL;
 327 static unsigned long long _mask48_0 = 0x4040808080808080LL;
 328
 329 static unsigned long long _const4   = 0x0000000000FFFFFFLL;
 330 //static unsigned long long _const5 = 0x000000FFFFFF0000LL;     // NOT USED
 331 static unsigned long long _const6   = 0x00000000000000FFLL;
 332
 333 // These are used in the row-filter routines and should/would be local
 334 //  variables if not for gcc addressing limitations.
 335 // WARNING: Their presence probably defeats the thread safety of libpng.
 336
 337 #ifdef PNG_THREAD_UNSAFE_OK
 338 static png_uint_32  _FullLength;
 339 static png_uint_32  _MMXLength;
 340 static int          _dif;
 341 static int          _patemp; // temp variables for Paeth routine
 342 static int          _pbtemp;
 343 static int          _pctemp;
 344 #endif
 345
 346 void /* PRIVATE */
 347 png_squelch_warnings(void)
 348 {
 349 #ifdef PNG_THREAD_UNSAFE_OK
 350    _dif = _dif;
 351    _patemp = _patemp;
 352    _pbtemp = _pbtemp;
 353    _pctemp = _pctemp;
 354    _MMXLength = _MMXLength;
 355 #endif
 356    _const4  = _const4;
 357    _const6  = _const6;
 358    _mask8_0  = _mask8_0;
 359    _mask16_1 = _mask16_1;
 360    _mask16_0 = _mask16_0;
 361    _mask24_2 = _mask24_2;
 362    _mask24_1 = _mask24_1;
 363    _mask24_0 = _mask24_0;
 364    _mask32_3 = _mask32_3;
 365    _mask32_2 = _mask32_2;
 366    _mask32_1 = _mask32_1;
 367    _mask32_0 = _mask32_0;
 368    _mask48_5 = _mask48_5;
 369    _mask48_4 = _mask48_4;
 370    _mask48_3 = _mask48_3;
 371    _mask48_2 = _mask48_2;
 372    _mask48_1 = _mask48_1;
 373    _mask48_0 = _mask48_0;
 374 }
 375 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
 376
 377
 378 static int _mmx_supported = 2;
 379
 380 /*===========================================================================*/
 381 /*                                                                           */
 382 /*                       P N G _ C O M B I N E _ R O W                       */
 383 /*                                                                           */
 384 /*===========================================================================*/
 385
 386 #if defined(PNG_HAVE_ASSEMBLER_COMBINE_ROW)
 387
 388 #define BPP2  2
 389 #define BPP3  3 /* bytes per pixel (a.k.a. pixel_bytes) */
 390 #define BPP4  4
 391 #define BPP6  6 /* (defined only to help avoid cut-and-paste errors) */
 392 #define BPP8  8
 393
 394 /* Combines the row recently read in with the previous row.
 395    This routine takes care of alpha and transparency if requested.
 396    This routine also handles the two methods of progressive display
 397    of interlaced images, depending on the mask value.
 398    The mask value describes which pixels are to be combined with
 399    the row.  The pattern always repeats every 8 pixels, so just 8
 400    bits are needed.  A one indicates the pixel is to be combined; a
 401    zero indicates the pixel is to be skipped.  This is in addition
 402    to any alpha or transparency value associated with the pixel.
 403    If you want all pixels to be combined, pass 0xff (255) in mask. */
 404
 405 /* Use this routine for the x86 platform - it uses a faster MMX routine
 406    if the machine supports MMX. */
 407
 408 void /* PRIVATE */
 409 png_combine_row(png_structp png_ptr, png_bytep row, int mask)
 410 {
 411    png_debug(1, "in png_combine_row (pnggccrd.c)\n");
 412
 413 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
 414    if (_mmx_supported == 2) {
 415        /* this should have happened in png_init_mmx_flags() already */
 416        png_warning(png_ptr, "asm_flags may not have been initialized");
 417        png_mmx_support();
 418    }
 419 #endif
 420
 421    if (mask == 0xff)
 422    {
 423       png_debug(2,"mask == 0xff:  doing single png_memcpy()\n");
 424       png_memcpy(row, png_ptr->row_buf + 1,
 425        (png_size_t)((png_ptr->width * png_ptr->row_info.pixel_depth + 7) >> 3));
 426    }
 427    else   /* (png_combine_row() is never called with mask == 0) */
 428    {
 429       switch (png_ptr->row_info.pixel_depth)
 430       {
 431          case 1:        /* png_ptr->row_info.pixel_depth */
 432          {
 433             png_bytep sp;
 434             png_bytep dp;
 435             int s_inc, s_start, s_end;
 436             int m;
 437             int shift;
 438             png_uint_32 i;
 439
 440             sp = png_ptr->row_buf + 1;
 441             dp = row;
 442             m = 0x80;
 443 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
 444             if (png_ptr->transformations & PNG_PACKSWAP)
 445             {
 446                 s_start = 0;
 447                 s_end = 7;
 448                 s_inc = 1;
 449             }
 450             else
 451 #endif
 452             {
 453                 s_start = 7;
 454                 s_end = 0;
 455                 s_inc = -1;
 456             }
 457
 458             shift = s_start;
 459
 460             for (i = 0; i < png_ptr->width; i++)
 461             {
 462                if (m & mask)
 463                {
 464                   int value;
 465
 466                   value = (*sp >> shift) & 0x1;
 467                   *dp &= (png_byte)((0x7f7f >> (7 - shift)) & 0xff);
 468                   *dp |= (png_byte)(value << shift);
 469                }
 470
 471                if (shift == s_end)
 472                {
 473                   shift = s_start;
 474                   sp++;
 475                   dp++;
 476                }
 477                else
 478                   shift += s_inc;
 479
 480                if (m == 1)
 481                   m = 0x80;
 482                else
 483                   m >>= 1;
 484             }
 485             break;
 486          }
 487
 488          case 2:        /* png_ptr->row_info.pixel_depth */
 489          {
 490             png_bytep sp;
 491             png_bytep dp;
 492             int s_start, s_end, s_inc;
 493             int m;
 494             int shift;
 495             png_uint_32 i;
 496             int value;
 497
 498             sp = png_ptr->row_buf + 1;
 499             dp = row;
 500             m = 0x80;
 501 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
 502             if (png_ptr->transformations & PNG_PACKSWAP)
 503             {
 504                s_start = 0;
 505                s_end = 6;
 506                s_inc = 2;
 507             }
 508             else
 509 #endif
 510             {
 511                s_start = 6;
 512                s_end = 0;
 513                s_inc = -2;
 514             }
 515
 516             shift = s_start;
 517
 518             for (i = 0; i < png_ptr->width; i++)
 519             {
 520                if (m & mask)
 521                {
 522                   value = (*sp >> shift) & 0x3;
 523                   *dp &= (png_byte)((0x3f3f >> (6 - shift)) & 0xff);
 524                   *dp |= (png_byte)(value << shift);
 525                }
 526
 527                if (shift == s_end)
 528                {
 529                   shift = s_start;
 530                   sp++;
 531                   dp++;
 532                }
 533                else
 534                   shift += s_inc;
 535                if (m == 1)
 536                   m = 0x80;
 537                else
 538                   m >>= 1;
 539             }
 540             break;
 541          }
 542
 543          case 4:        /* png_ptr->row_info.pixel_depth */
 544          {
 545             png_bytep sp;
 546             png_bytep dp;
 547             int s_start, s_end, s_inc;
 548             int m;
 549             int shift;
 550             png_uint_32 i;
 551             int value;
 552
 553             sp = png_ptr->row_buf + 1;
 554             dp = row;
 555             m = 0x80;
 556 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
 557             if (png_ptr->transformations & PNG_PACKSWAP)
 558             {
 559                s_start = 0;
 560                s_end = 4;
 561                s_inc = 4;
 562             }
 563             else
 564 #endif
 565             {
 566                s_start = 4;
 567                s_end = 0;
 568                s_inc = -4;
 569             }
 570             shift = s_start;
 571
 572             for (i = 0; i < png_ptr->width; i++)
 573             {
 574                if (m & mask)
 575                {
 576                   value = (*sp >> shift) & 0xf;
 577                   *dp &= (png_byte)((0xf0f >> (4 - shift)) & 0xff);
 578                   *dp |= (png_byte)(value << shift);
 579                }
 580
 581                if (shift == s_end)
 582                {
 583                   shift = s_start;
 584                   sp++;
 585                   dp++;
 586                }
 587                else
 588                   shift += s_inc;
 589                if (m == 1)
 590                   m = 0x80;
 591                else
 592                   m >>= 1;
 593             }
 594             break;
 595          }
 596
 597          case 8:        /* png_ptr->row_info.pixel_depth */
 598          {
 599             png_bytep srcptr;
 600             png_bytep dstptr;
 601
 602 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
 603 #if !defined(PNG_1_0_X)
 604             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
 605                 /* && _mmx_supported */ )
 606 #else
 607             if (_mmx_supported)
 608 #endif
 609             {
 610                png_uint_32 len;
 611                int diff;
 612                int dummy_value_a;   // fix 'forbidden register spilled' error
 613                int dummy_value_d;
 614                int dummy_value_c;
 615                int dummy_value_S;
 616                int dummy_value_D;
 617                _unmask = ~mask;            // global variable for -fPIC version
 618                srcptr = png_ptr->row_buf + 1;
 619                dstptr = row;
 620                len  = png_ptr->width &~7;  // reduce to multiple of 8
 621                diff = (int) (png_ptr->width & 7);  // amount lost
 622
 623                __asm__ __volatile__ (
 624                   "movd      _unmask, %%mm7  \n\t" // load bit pattern
 625                   "psubb     %%mm6, %%mm6    \n\t" // zero mm6
 626                   "punpcklbw %%mm7, %%mm7    \n\t"
 627                   "punpcklwd %%mm7, %%mm7    \n\t"
 628                   "punpckldq %%mm7, %%mm7    \n\t" // fill reg with 8 masks
 629
 630                   "movq      _mask8_0, %%mm0 \n\t"
 631                   "pand      %%mm7, %%mm0    \n\t" // nonzero if keep byte
 632                   "pcmpeqb   %%mm6, %%mm0    \n\t" // zeros->1s, v versa
 633
 634 // preload        "movl      len, %%ecx      \n\t" // load length of line
 635 // preload        "movl      srcptr, %%esi   \n\t" // load source
 636 // preload        "movl      dstptr, %%edi   \n\t" // load dest
 637
 638                   "cmpl      $0, %%ecx       \n\t" // len == 0 ?
 639                   "je        mainloop8end    \n\t"
 640
 641                 "mainloop8:                  \n\t"
 642                   "movq      (%%esi), %%mm4  \n\t" // *srcptr
 643                   "pand      %%mm0, %%mm4    \n\t"
 644                   "movq      %%mm0, %%mm6    \n\t"
 645                   "pandn     (%%edi), %%mm6  \n\t" // *dstptr
 646                   "por       %%mm6, %%mm4    \n\t"
 647                   "movq      %%mm4, (%%edi)  \n\t"
 648                   "addl      $8, %%esi       \n\t" // inc by 8 bytes processed
 649                   "addl      $8, %%edi       \n\t"
 650                   "subl      $8, %%ecx       \n\t" // dec by 8 pixels processed
 651                   "ja        mainloop8       \n\t"
 652
 653                 "mainloop8end:               \n\t"
 654 // preload        "movl      diff, %%ecx     \n\t" // (diff is in eax)
 655                   "movl      %%eax, %%ecx    \n\t"
 656                   "cmpl      $0, %%ecx       \n\t"
 657                   "jz        end8            \n\t"
 658 // preload        "movl      mask, %%edx     \n\t"
 659                   "sall      $24, %%edx      \n\t" // make low byte, high byte
 660
 661                 "secondloop8:                \n\t"
 662                   "sall      %%edx           \n\t" // move high bit to CF
 663                   "jnc       skip8           \n\t" // if CF = 0
 664                   "movb      (%%esi), %%al   \n\t"
 665                   "movb      %%al, (%%edi)   \n\t"
 666
 667                 "skip8:                      \n\t"
 668                   "incl      %%esi           \n\t"
 669                   "incl      %%edi           \n\t"
 670                   "decl      %%ecx           \n\t"
 671                   "jnz       secondloop8     \n\t"
 672
 673                 "end8:                       \n\t"
 674                   "EMMS                      \n\t"  // DONE
 675
 676                   : "=a" (dummy_value_a),           // output regs (dummy)
 677                     "=d" (dummy_value_d),
 678                     "=c" (dummy_value_c),
 679                     "=S" (dummy_value_S),
 680                     "=D" (dummy_value_D)
 681
 682                   : "3" (srcptr),      // esi       // input regs
 683                     "4" (dstptr),      // edi
 684                     "0" (diff),        // eax
 685 // was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
 686                     "2" (len),         // ecx
 687                     "1" (mask)         // edx
 688
 689 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
 690                   : "%mm0", "%mm4", "%mm6", "%mm7"  // clobber list
 691 #endif
 692                );
 693             }
 694             else /* mmx _not supported - Use modified C routine */
 695 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
 696             {
 697                register png_uint_32 i;
 698                png_uint_32 initial_val = png_pass_start[png_ptr->pass];
 699                  /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
 700                register int stride = png_pass_inc[png_ptr->pass];
 701                  /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
 702                register int rep_bytes = png_pass_width[png_ptr->pass];
 703                  /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
 704                png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
 705                int diff = (int) (png_ptr->width & 7); /* amount lost */
 706                register png_uint_32 final_val = len;  /* GRR bugfix */
 707
 708                srcptr = png_ptr->row_buf + 1 + initial_val;
 709                dstptr = row + initial_val;
 710
 711                for (i = initial_val; i < final_val; i += stride)
 712                {
 713                   png_memcpy(dstptr, srcptr, rep_bytes);
 714                   srcptr += stride;
 715                   dstptr += stride;
 716                }
 717                if (diff)  /* number of leftover pixels:  3 for pngtest */
 718                {
 719                   final_val+=diff /* *BPP1 */ ;
 720                   for (; i < final_val; i += stride)
 721                   {
 722                      if (rep_bytes > (int)(final_val-i))
 723                         rep_bytes = (int)(final_val-i);
 724                      png_memcpy(dstptr, srcptr, rep_bytes);
 725                      srcptr += stride;
 726                      dstptr += stride;
 727                   }
 728                }
 729
 730             } /* end of else (_mmx_supported) */
 731
 732             break;
 733          }       /* end 8 bpp */
 734
 735          case 16:       /* png_ptr->row_info.pixel_depth */
 736          {
 737             png_bytep srcptr;
 738             png_bytep dstptr;
 739
 740 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
 741 #if !defined(PNG_1_0_X)
 742             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
 743                 /* && _mmx_supported */ )
 744 #else
 745             if (_mmx_supported)
 746 #endif
 747             {
 748                png_uint_32 len;
 749                int diff;
 750                int dummy_value_a;   // fix 'forbidden register spilled' error
 751                int dummy_value_d;
 752                int dummy_value_c;
 753                int dummy_value_S;
 754                int dummy_value_D;
 755                _unmask = ~mask;            // global variable for -fPIC version
 756                srcptr = png_ptr->row_buf + 1;
 757                dstptr = row;
 758                len  = png_ptr->width &~7;  // reduce to multiple of 8
 759                diff = (int) (png_ptr->width & 7); // amount lost //
 760
 761                __asm__ __volatile__ (
 762                   "movd      _unmask, %%mm7   \n\t" // load bit pattern
 763                   "psubb     %%mm6, %%mm6     \n\t" // zero mm6
 764                   "punpcklbw %%mm7, %%mm7     \n\t"
 765                   "punpcklwd %%mm7, %%mm7     \n\t"
 766                   "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
 767
 768                   "movq      _mask16_0, %%mm0 \n\t"
 769                   "movq      _mask16_1, %%mm1 \n\t"
 770
 771                   "pand      %%mm7, %%mm0     \n\t"
 772                   "pand      %%mm7, %%mm1     \n\t"
 773
 774                   "pcmpeqb   %%mm6, %%mm0     \n\t"
 775                   "pcmpeqb   %%mm6, %%mm1     \n\t"
 776
 777 // preload        "movl      len, %%ecx       \n\t" // load length of line
 778 // preload        "movl      srcptr, %%esi    \n\t" // load source
 779 // preload        "movl      dstptr, %%edi    \n\t" // load dest
 780
 781                   "cmpl      $0, %%ecx        \n\t"
 782                   "jz        mainloop16end    \n\t"
 783
 784                 "mainloop16:                  \n\t"
 785                   "movq      (%%esi), %%mm4   \n\t"
 786                   "pand      %%mm0, %%mm4     \n\t"
 787                   "movq      %%mm0, %%mm6     \n\t"
 788                   "movq      (%%edi), %%mm7   \n\t"
 789                   "pandn     %%mm7, %%mm6     \n\t"
 790                   "por       %%mm6, %%mm4     \n\t"
 791                   "movq      %%mm4, (%%edi)   \n\t"
 792
 793                   "movq      8(%%esi), %%mm5  \n\t"
 794                   "pand      %%mm1, %%mm5     \n\t"
 795                   "movq      %%mm1, %%mm7     \n\t"
 796                   "movq      8(%%edi), %%mm6  \n\t"
 797                   "pandn     %%mm6, %%mm7     \n\t"
 798                   "por       %%mm7, %%mm5     \n\t"
 799                   "movq      %%mm5, 8(%%edi)  \n\t"
 800
 801                   "addl      $16, %%esi       \n\t" // inc by 16 bytes processed
 802                   "addl      $16, %%edi       \n\t"
 803                   "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
 804                   "ja        mainloop16       \n\t"
 805
 806                 "mainloop16end:               \n\t"
 807 // preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
 808                   "movl      %%eax, %%ecx     \n\t"
 809                   "cmpl      $0, %%ecx        \n\t"
 810                   "jz        end16            \n\t"
 811 // preload        "movl      mask, %%edx      \n\t"
 812                   "sall      $24, %%edx       \n\t" // make low byte, high byte
 813
 814                 "secondloop16:                \n\t"
 815                   "sall      %%edx            \n\t" // move high bit to CF
 816                   "jnc       skip16           \n\t" // if CF = 0
 817                   "movw      (%%esi), %%ax    \n\t"
 818                   "movw      %%ax, (%%edi)    \n\t"
 819
 820                 "skip16:                      \n\t"
 821                   "addl      $2, %%esi        \n\t"
 822                   "addl      $2, %%edi        \n\t"
 823                   "decl      %%ecx            \n\t"
 824                   "jnz       secondloop16     \n\t"
 825
 826                 "end16:                       \n\t"
 827                   "EMMS                       \n\t" // DONE
 828
 829                   : "=a" (dummy_value_a),           // output regs (dummy)
 830                     "=c" (dummy_value_c),
 831                     "=d" (dummy_value_d),
 832                     "=S" (dummy_value_S),
 833                     "=D" (dummy_value_D)
 834
 835                   : "0" (diff),        // eax       // input regs
 836 // was (unmask)     " "    RESERVED    // ebx       // Global Offset Table idx
 837                     "1" (len),         // ecx
 838                     "2" (mask),        // edx
 839                     "3" (srcptr),      // esi
 840                     "4" (dstptr)       // edi
 841
 842 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
 843                   : "%mm0", "%mm1", "%mm4"          // clobber list
 844                   , "%mm5", "%mm6", "%mm7"
 845 #endif
 846                );
 847             }
 848             else /* mmx _not supported - Use modified C routine */
 849 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
 850             {
 851                register png_uint_32 i;
 852                png_uint_32 initial_val = BPP2 * png_pass_start[png_ptr->pass];
 853                  /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
 854                register int stride = BPP2 * png_pass_inc[png_ptr->pass];
 855                  /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
 856                register int rep_bytes = BPP2 * png_pass_width[png_ptr->pass];
 857                  /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
 858                png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
 859                int diff = (int) (png_ptr->width & 7); /* amount lost */
 860                register png_uint_32 final_val = BPP2 * len;   /* GRR bugfix */
 861
 862                srcptr = png_ptr->row_buf + 1 + initial_val;
 863                dstptr = row + initial_val;
 864
 865                for (i = initial_val; i < final_val; i += stride)
 866                {
 867                   png_memcpy(dstptr, srcptr, rep_bytes);
 868                   srcptr += stride;
 869                   dstptr += stride;
 870                }
 871                if (diff)  /* number of leftover pixels:  3 for pngtest */
 872                {
 873                   final_val+=diff*BPP2;
 874                   for (; i < final_val; i += stride)
 875                   {
 876                      if (rep_bytes > (int)(final_val-i))
 877                         rep_bytes = (int)(final_val-i);
 878                      png_memcpy(dstptr, srcptr, rep_bytes);
 879                      srcptr += stride;
 880                      dstptr += stride;
 881                   }
 882                }
 883             } /* end of else (_mmx_supported) */
 884
 885             break;
 886          }       /* end 16 bpp */
 887
 888          case 24:       /* png_ptr->row_info.pixel_depth */
 889          {
 890             png_bytep srcptr;
 891             png_bytep dstptr;
 892
 893 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
 894 #if !defined(PNG_1_0_X)
 895             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
 896                 /* && _mmx_supported */ )
 897 #else
 898             if (_mmx_supported)
 899 #endif
 900             {
 901                png_uint_32 len;
 902                int diff;
 903                int dummy_value_a;   // fix 'forbidden register spilled' error
 904                int dummy_value_d;
 905                int dummy_value_c;
 906                int dummy_value_S;
 907                int dummy_value_D;
 908                _unmask = ~mask;            // global variable for -fPIC version
 909                srcptr = png_ptr->row_buf + 1;
 910                dstptr = row;
 911                len  = png_ptr->width &~7;  // reduce to multiple of 8
 912                diff = (int) (png_ptr->width & 7); // amount lost //
 913
 914                __asm__ __volatile__ (
 915                   "movd      _unmask, %%mm7   \n\t" // load bit pattern
 916                   "psubb     %%mm6, %%mm6     \n\t" // zero mm6
 917                   "punpcklbw %%mm7, %%mm7     \n\t"
 918                   "punpcklwd %%mm7, %%mm7     \n\t"
 919                   "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
 920
 921                   "movq      _mask24_0, %%mm0 \n\t"
 922                   "movq      _mask24_1, %%mm1 \n\t"
 923                   "movq      _mask24_2, %%mm2 \n\t"
 924
 925                   "pand      %%mm7, %%mm0     \n\t"
 926                   "pand      %%mm7, %%mm1     \n\t"
 927                   "pand      %%mm7, %%mm2     \n\t"
 928
 929                   "pcmpeqb   %%mm6, %%mm0     \n\t"
 930                   "pcmpeqb   %%mm6, %%mm1     \n\t"
 931                   "pcmpeqb   %%mm6, %%mm2     \n\t"
 932
 933 // preload        "movl      len, %%ecx       \n\t" // load length of line
 934 // preload        "movl      srcptr, %%esi    \n\t" // load source
 935 // preload        "movl      dstptr, %%edi    \n\t" // load dest
 936
 937                   "cmpl      $0, %%ecx        \n\t"
 938                   "jz        mainloop24end    \n\t"
 939
 940                 "mainloop24:                  \n\t"
 941                   "movq      (%%esi), %%mm4   \n\t"
 942                   "pand      %%mm0, %%mm4     \n\t"
 943                   "movq      %%mm0, %%mm6     \n\t"
 944                   "movq      (%%edi), %%mm7   \n\t"
 945                   "pandn     %%mm7, %%mm6     \n\t"
 946                   "por       %%mm6, %%mm4     \n\t"
 947                   "movq      %%mm4, (%%edi)   \n\t"
 948
 949                   "movq      8(%%esi), %%mm5  \n\t"
 950                   "pand      %%mm1, %%mm5     \n\t"
 951                   "movq      %%mm1, %%mm7     \n\t"
 952                   "movq      8(%%edi), %%mm6  \n\t"
 953                   "pandn     %%mm6, %%mm7     \n\t"
 954                   "por       %%mm7, %%mm5     \n\t"
 955                   "movq      %%mm5, 8(%%edi)  \n\t"
 956
 957                   "movq      16(%%esi), %%mm6 \n\t"
 958                   "pand      %%mm2, %%mm6     \n\t"
 959                   "movq      %%mm2, %%mm4     \n\t"
 960                   "movq      16(%%edi), %%mm7 \n\t"
 961                   "pandn     %%mm7, %%mm4     \n\t"
 962                   "por       %%mm4, %%mm6     \n\t"
 963                   "movq      %%mm6, 16(%%edi) \n\t"
 964
 965                   "addl      $24, %%esi       \n\t" // inc by 24 bytes processed
 966                   "addl      $24, %%edi       \n\t"
 967                   "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
 968
 969                   "ja        mainloop24       \n\t"
 970
 971                 "mainloop24end:               \n\t"
 972 // preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
 973                   "movl      %%eax, %%ecx     \n\t"
 974                   "cmpl      $0, %%ecx        \n\t"
 975                   "jz        end24            \n\t"
 976 // preload        "movl      mask, %%edx      \n\t"
 977                   "sall      $24, %%edx       \n\t" // make low byte, high byte
 978
 979                 "secondloop24:                \n\t"
 980                   "sall      %%edx            \n\t" // move high bit to CF
 981                   "jnc       skip24           \n\t" // if CF = 0
 982                   "movw      (%%esi), %%ax    \n\t"
 983                   "movw      %%ax, (%%edi)    \n\t"
 984                   "xorl      %%eax, %%eax     \n\t"
 985                   "movb      2(%%esi), %%al   \n\t"
 986                   "movb      %%al, 2(%%edi)   \n\t"
 987
 988                 "skip24:                      \n\t"
 989                   "addl      $3, %%esi        \n\t"
 990                   "addl      $3, %%edi        \n\t"
 991                   "decl      %%ecx            \n\t"
 992                   "jnz       secondloop24     \n\t"
 993
 994                 "end24:                       \n\t"
 995                   "EMMS                       \n\t" // DONE
 996
 997                   : "=a" (dummy_value_a),           // output regs (dummy)
 998                     "=d" (dummy_value_d),
 999                     "=c" (dummy_value_c),
1000                     "=S" (dummy_value_S),
1001                     "=D" (dummy_value_D)
1002
1003                   : "3" (srcptr),      // esi       // input regs
1004                     "4" (dstptr),      // edi
1005                     "0" (diff),        // eax
1006 // was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
1007                     "2" (len),         // ecx
1008                     "1" (mask)         // edx
1009
1010 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
1011                   : "%mm0", "%mm1", "%mm2"          // clobber list
1012                   , "%mm4", "%mm5", "%mm6", "%mm7"
1013 #endif
1014                );
1015             }
1016             else /* mmx _not supported - Use modified C routine */
1017 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
1018             {
1019                register png_uint_32 i;
1020                png_uint_32 initial_val = BPP3 * png_pass_start[png_ptr->pass];
1021                  /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1022                register int stride = BPP3 * png_pass_inc[png_ptr->pass];
1023                  /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1024                register int rep_bytes = BPP3 * png_pass_width[png_ptr->pass];
1025                  /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1026                png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
1027                int diff = (int) (png_ptr->width & 7); /* amount lost */
1028                register png_uint_32 final_val = BPP3 * len;   /* GRR bugfix */
1029
1030                srcptr = png_ptr->row_buf + 1 + initial_val;
1031                dstptr = row + initial_val;
1032
1033                for (i = initial_val; i < final_val; i += stride)
1034                {
1035                   png_memcpy(dstptr, srcptr, rep_bytes);
1036                   srcptr += stride;
1037                   dstptr += stride;
1038                }
1039                if (diff)  /* number of leftover pixels:  3 for pngtest */
1040                {
1041                   final_val+=diff*BPP3;
1042                   for (; i < final_val; i += stride)
1043                   {
1044                      if (rep_bytes > (int)(final_val-i))
1045                         rep_bytes = (int)(final_val-i);
1046                      png_memcpy(dstptr, srcptr, rep_bytes);
1047                      srcptr += stride;
1048                      dstptr += stride;
1049                   }
1050                }
1051             } /* end of else (_mmx_supported) */
1052
1053             break;
1054          }       /* end 24 bpp */
1055
1056          case 32:       /* png_ptr->row_info.pixel_depth */
1057          {
1058             png_bytep srcptr;
1059             png_bytep dstptr;
1060
1061 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
1062 #if !defined(PNG_1_0_X)
1063             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
1064                 /* && _mmx_supported */ )
1065 #else
1066             if (_mmx_supported)
1067 #endif
1068             {
1069                png_uint_32 len;
1070                int diff;
1071                int dummy_value_a;   // fix 'forbidden register spilled' error
1072                int dummy_value_d;
1073                int dummy_value_c;
1074                int dummy_value_S;
1075                int dummy_value_D;
1076                _unmask = ~mask;            // global variable for -fPIC version
1077                srcptr = png_ptr->row_buf + 1;
1078                dstptr = row;
1079                len  = png_ptr->width &~7;  // reduce to multiple of 8
1080                diff = (int) (png_ptr->width & 7); // amount lost //
1081
1082                __asm__ __volatile__ (
1083                   "movd      _unmask, %%mm7   \n\t" // load bit pattern
1084                   "psubb     %%mm6, %%mm6     \n\t" // zero mm6
1085                   "punpcklbw %%mm7, %%mm7     \n\t"
1086                   "punpcklwd %%mm7, %%mm7     \n\t"
1087                   "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
1088
1089                   "movq      _mask32_0, %%mm0 \n\t"
1090                   "movq      _mask32_1, %%mm1 \n\t"
1091                   "movq      _mask32_2, %%mm2 \n\t"
1092                   "movq      _mask32_3, %%mm3 \n\t"
1093
1094                   "pand      %%mm7, %%mm0     \n\t"
1095                   "pand      %%mm7, %%mm1     \n\t"
1096                   "pand      %%mm7, %%mm2     \n\t"
1097                   "pand      %%mm7, %%mm3     \n\t"
1098
1099                   "pcmpeqb   %%mm6, %%mm0     \n\t"
1100                   "pcmpeqb   %%mm6, %%mm1     \n\t"
1101                   "pcmpeqb   %%mm6, %%mm2     \n\t"
1102                   "pcmpeqb   %%mm6, %%mm3     \n\t"
1103
1104 // preload        "movl      len, %%ecx       \n\t" // load length of line
1105 // preload        "movl      srcptr, %%esi    \n\t" // load source
1106 // preload        "movl      dstptr, %%edi    \n\t" // load dest
1107
1108                   "cmpl      $0, %%ecx        \n\t" // lcr
1109                   "jz        mainloop32end    \n\t"
1110
1111                 "mainloop32:                  \n\t"
1112                   "movq      (%%esi), %%mm4   \n\t"
1113                   "pand      %%mm0, %%mm4     \n\t"
1114                   "movq      %%mm0, %%mm6     \n\t"
1115                   "movq      (%%edi), %%mm7   \n\t"
1116                   "pandn     %%mm7, %%mm6     \n\t"
1117                   "por       %%mm6, %%mm4     \n\t"
1118                   "movq      %%mm4, (%%edi)   \n\t"
1119
1120                   "movq      8(%%esi), %%mm5  \n\t"
1121                   "pand      %%mm1, %%mm5     \n\t"
1122                   "movq      %%mm1, %%mm7     \n\t"
1123                   "movq      8(%%edi), %%mm6  \n\t"
1124                   "pandn     %%mm6, %%mm7     \n\t"
1125                   "por       %%mm7, %%mm5     \n\t"
1126                   "movq      %%mm5, 8(%%edi)  \n\t"
1127
1128                   "movq      16(%%esi), %%mm6 \n\t"
1129                   "pand      %%mm2, %%mm6     \n\t"
1130                   "movq      %%mm2, %%mm4     \n\t"
1131                   "movq      16(%%edi), %%mm7 \n\t"
1132                   "pandn     %%mm7, %%mm4     \n\t"
1133                   "por       %%mm4, %%mm6     \n\t"
1134                   "movq      %%mm6, 16(%%edi) \n\t"
1135
1136                   "movq      24(%%esi), %%mm7 \n\t"
1137                   "pand      %%mm3, %%mm7     \n\t"
1138                   "movq      %%mm3, %%mm5     \n\t"
1139                   "movq      24(%%edi), %%mm4 \n\t"
1140                   "pandn     %%mm4, %%mm5     \n\t"
1141                   "por       %%mm5, %%mm7     \n\t"
1142                   "movq      %%mm7, 24(%%edi) \n\t"
1143
1144                   "addl      $32, %%esi       \n\t" // inc by 32 bytes processed
1145                   "addl      $32, %%edi       \n\t"
1146                   "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
1147                   "ja        mainloop32       \n\t"
1148
1149                 "mainloop32end:               \n\t"
1150 // preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
1151                   "movl      %%eax, %%ecx     \n\t"
1152                   "cmpl      $0, %%ecx        \n\t"
1153                   "jz        end32            \n\t"
1154 // preload        "movl      mask, %%edx      \n\t"
1155                   "sall      $24, %%edx       \n\t" // low byte => high byte
1156
1157                 "secondloop32:                \n\t"
1158                   "sall      %%edx            \n\t" // move high bit to CF
1159                   "jnc       skip32           \n\t" // if CF = 0
1160                   "movl      (%%esi), %%eax   \n\t"
1161                   "movl      %%eax, (%%edi)   \n\t"
1162
1163                 "skip32:                      \n\t"
1164                   "addl      $4, %%esi        \n\t"
1165                   "addl      $4, %%edi        \n\t"
1166                   "decl      %%ecx            \n\t"
1167                   "jnz       secondloop32     \n\t"
1168
1169                 "end32:                       \n\t"
1170                   "EMMS                       \n\t" // DONE
1171
1172                   : "=a" (dummy_value_a),           // output regs (dummy)
1173                     "=d" (dummy_value_d),
1174                     "=c" (dummy_value_c),
1175                     "=S" (dummy_value_S),
1176                     "=D" (dummy_value_D)
1177
1178                   : "3" (srcptr),      // esi       // input regs
1179                     "4" (dstptr),      // edi
1180                     "0" (diff),        // eax
1181 // was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
1182                     "2" (len),         // ecx
1183                     "1" (mask)         // edx
1184
1185 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
1186                   : "%mm0", "%mm1", "%mm2", "%mm3"  // clobber list
1187                   , "%mm4", "%mm5", "%mm6", "%mm7"
1188 #endif
1189                );
1190             }
1191             else /* mmx _not supported - Use modified C routine */
1192 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
1193             {
1194                register png_uint_32 i;
1195                png_uint_32 initial_val = BPP4 * png_pass_start[png_ptr->pass];
1196                  /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1197                register int stride = BPP4 * png_pass_inc[png_ptr->pass];
1198                  /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1199                register int rep_bytes = BPP4 * png_pass_width[png_ptr->pass];
1200                  /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1201                png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
1202                int diff = (int) (png_ptr->width & 7); /* amount lost */
1203                register png_uint_32 final_val = BPP4 * len;   /* GRR bugfix */
1204
1205                srcptr = png_ptr->row_buf + 1 + initial_val;
1206                dstptr = row + initial_val;
1207
1208                for (i = initial_val; i < final_val; i += stride)
1209                {
1210                   png_memcpy(dstptr, srcptr, rep_bytes);
1211                   srcptr += stride;
1212                   dstptr += stride;
1213                }
1214                if (diff)  /* number of leftover pixels:  3 for pngtest */
1215                {
1216                   final_val+=diff*BPP4;
1217                   for (; i < final_val; i += stride)
1218                   {
1219                      if (rep_bytes > (int)(final_val-i))
1220                         rep_bytes = (int)(final_val-i);
1221                      png_memcpy(dstptr, srcptr, rep_bytes);
1222                      srcptr += stride;
1223                      dstptr += stride;
1224                   }
1225                }
1226             } /* end of else (_mmx_supported) */
1227
1228             break;
1229          }       /* end 32 bpp */
1230
1231          case 48:       /* png_ptr->row_info.pixel_depth */
1232          {
1233             png_bytep srcptr;
1234             png_bytep dstptr;
1235
1236 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
1237 #if !defined(PNG_1_0_X)
1238             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
1239                 /* && _mmx_supported */ )
1240 #else
1241             if (_mmx_supported)
1242 #endif
1243             {
1244                png_uint_32 len;
1245                int diff;
1246                int dummy_value_a;   // fix 'forbidden register spilled' error
1247                int dummy_value_d;
1248                int dummy_value_c;
1249                int dummy_value_S;
1250                int dummy_value_D;
1251                _unmask = ~mask;            // global variable for -fPIC version
1252                srcptr = png_ptr->row_buf + 1;
1253                dstptr = row;
1254                len  = png_ptr->width &~7;  // reduce to multiple of 8
1255                diff = (int) (png_ptr->width & 7); // amount lost //
1256
1257                __asm__ __volatile__ (
1258                   "movd      _unmask, %%mm7   \n\t" // load bit pattern
1259                   "psubb     %%mm6, %%mm6     \n\t" // zero mm6
1260                   "punpcklbw %%mm7, %%mm7     \n\t"
1261                   "punpcklwd %%mm7, %%mm7     \n\t"
1262                   "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
1263
1264                   "movq      _mask48_0, %%mm0 \n\t"
1265                   "movq      _mask48_1, %%mm1 \n\t"
1266                   "movq      _mask48_2, %%mm2 \n\t"
1267                   "movq      _mask48_3, %%mm3 \n\t"
1268                   "movq      _mask48_4, %%mm4 \n\t"
1269                   "movq      _mask48_5, %%mm5 \n\t"
1270
1271                   "pand      %%mm7, %%mm0     \n\t"
1272                   "pand      %%mm7, %%mm1     \n\t"
1273                   "pand      %%mm7, %%mm2     \n\t"
1274                   "pand      %%mm7, %%mm3     \n\t"
1275                   "pand      %%mm7, %%mm4     \n\t"
1276                   "pand      %%mm7, %%mm5     \n\t"
1277
1278                   "pcmpeqb   %%mm6, %%mm0     \n\t"
1279                   "pcmpeqb   %%mm6, %%mm1     \n\t"
1280                   "pcmpeqb   %%mm6, %%mm2     \n\t"
1281                   "pcmpeqb   %%mm6, %%mm3     \n\t"
1282                   "pcmpeqb   %%mm6, %%mm4     \n\t"
1283                   "pcmpeqb   %%mm6, %%mm5     \n\t"
1284
1285 // preload        "movl      len, %%ecx       \n\t" // load length of line
1286 // preload        "movl      srcptr, %%esi    \n\t" // load source
1287 // preload        "movl      dstptr, %%edi    \n\t" // load dest
1288
1289                   "cmpl      $0, %%ecx        \n\t"
1290                   "jz        mainloop48end    \n\t"
1291
1292                 "mainloop48:                  \n\t"
1293                   "movq      (%%esi), %%mm7   \n\t"
1294                   "pand      %%mm0, %%mm7     \n\t"
1295                   "movq      %%mm0, %%mm6     \n\t"
1296                   "pandn     (%%edi), %%mm6   \n\t"
1297                   "por       %%mm6, %%mm7     \n\t"
1298                   "movq      %%mm7, (%%edi)   \n\t"
1299
1300                   "movq      8(%%esi), %%mm6  \n\t"
1301                   "pand      %%mm1, %%mm6     \n\t"
1302                   "movq      %%mm1, %%mm7     \n\t"
1303                   "pandn     8(%%edi), %%mm7  \n\t"
1304                   "por       %%mm7, %%mm6     \n\t"
1305                   "movq      %%mm6, 8(%%edi)  \n\t"
1306
1307                   "movq      16(%%esi), %%mm6 \n\t"
1308                   "pand      %%mm2, %%mm6     \n\t"
1309                   "movq      %%mm2, %%mm7     \n\t"
1310                   "pandn     16(%%edi), %%mm7 \n\t"
1311                   "por       %%mm7, %%mm6     \n\t"
1312                   "movq      %%mm6, 16(%%edi) \n\t"
1313
1314                   "movq      24(%%esi), %%mm7 \n\t"
1315                   "pand      %%mm3, %%mm7     \n\t"
1316                   "movq      %%mm3, %%mm6     \n\t"
1317                   "pandn     24(%%edi), %%mm6 \n\t"
1318                   "por       %%mm6, %%mm7     \n\t"
1319                   "movq      %%mm7, 24(%%edi) \n\t"
1320
1321                   "movq      32(%%esi), %%mm6 \n\t"
1322                   "pand      %%mm4, %%mm6     \n\t"
1323                   "movq      %%mm4, %%mm7     \n\t"
1324                   "pandn     32(%%edi), %%mm7 \n\t"
1325                   "por       %%mm7, %%mm6     \n\t"
1326                   "movq      %%mm6, 32(%%edi) \n\t"
1327
1328                   "movq      40(%%esi), %%mm7 \n\t"
1329                   "pand      %%mm5, %%mm7     \n\t"
1330                   "movq      %%mm5, %%mm6     \n\t"
1331                   "pandn     40(%%edi), %%mm6 \n\t"
1332                   "por       %%mm6, %%mm7     \n\t"
1333                   "movq      %%mm7, 40(%%edi) \n\t"
1334
1335                   "addl      $48, %%esi       \n\t" // inc by 48 bytes processed
1336                   "addl      $48, %%edi       \n\t"
1337                   "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
1338
1339                   "ja        mainloop48       \n\t"
1340
1341                 "mainloop48end:               \n\t"
1342 // preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
1343                   "movl      %%eax, %%ecx     \n\t"
1344                   "cmpl      $0, %%ecx        \n\t"
1345                   "jz        end48            \n\t"
1346 // preload        "movl      mask, %%edx      \n\t"
1347                   "sall      $24, %%edx       \n\t" // make low byte, high byte
1348
1349                 "secondloop48:                \n\t"
1350                   "sall      %%edx            \n\t" // move high bit to CF
1351                   "jnc       skip48           \n\t" // if CF = 0
1352                   "movl      (%%esi), %%eax   \n\t"
1353                   "movl      %%eax, (%%edi)   \n\t"
1354
1355                 "skip48:                      \n\t"
1356                   "addl      $4, %%esi        \n\t"
1357                   "addl      $4, %%edi        \n\t"
1358                   "decl      %%ecx            \n\t"
1359                   "jnz       secondloop48     \n\t"
1360
1361                 "end48:                       \n\t"
1362                   "EMMS                       \n\t" // DONE
1363
1364                   : "=a" (dummy_value_a),           // output regs (dummy)
1365                     "=d" (dummy_value_d),
1366                     "=c" (dummy_value_c),
1367                     "=S" (dummy_value_S),
1368                     "=D" (dummy_value_D)
1369
1370                   : "3" (srcptr),      // esi       // input regs
1371                     "4" (dstptr),      // edi
1372                     "0" (diff),        // eax
1373 // was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
1374                     "2" (len),         // ecx
1375                     "1" (mask)         // edx
1376
1377 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
1378                   : "%mm0", "%mm1", "%mm2", "%mm3"  // clobber list
1379                   , "%mm4", "%mm5", "%mm6", "%mm7"
1380 #endif
1381                );
1382             }
1383             else /* mmx _not supported - Use modified C routine */
1384 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
1385             {
1386                register png_uint_32 i;
1387                png_uint_32 initial_val = BPP6 * png_pass_start[png_ptr->pass];
1388                  /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1389                register int stride = BPP6 * png_pass_inc[png_ptr->pass];
1390                  /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1391                register int rep_bytes = BPP6 * png_pass_width[png_ptr->pass];
1392                  /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1393                png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
1394                int diff = (int) (png_ptr->width & 7); /* amount lost */
1395                register png_uint_32 final_val = BPP6 * len;   /* GRR bugfix */
1396
1397                srcptr = png_ptr->row_buf + 1 + initial_val;
1398                dstptr = row + initial_val;
1399
1400                for (i = initial_val; i < final_val; i += stride)
1401                {
1402                   png_memcpy(dstptr, srcptr, rep_bytes);
1403                   srcptr += stride;
1404                   dstptr += stride;
1405                }
1406                if (diff)  /* number of leftover pixels:  3 for pngtest */
1407                {
1408                   final_val+=diff*BPP6;
1409                   for (; i < final_val; i += stride)
1410                   {
1411                      if (rep_bytes > (int)(final_val-i))
1412                         rep_bytes = (int)(final_val-i);
1413                      png_memcpy(dstptr, srcptr, rep_bytes);
1414                      srcptr += stride;
1415                      dstptr += stride;
1416                   }
1417                }
1418             } /* end of else (_mmx_supported) */
1419
1420             break;
1421          }       /* end 48 bpp */
1422
1423          case 64:       /* png_ptr->row_info.pixel_depth */
1424          {
1425             png_bytep srcptr;
1426             png_bytep dstptr;
1427             register png_uint_32 i;
1428             png_uint_32 initial_val = BPP8 * png_pass_start[png_ptr->pass];
1429               /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1430             register int stride = BPP8 * png_pass_inc[png_ptr->pass];
1431               /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1432             register int rep_bytes = BPP8 * png_pass_width[png_ptr->pass];
1433               /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1434             png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
1435             int diff = (int) (png_ptr->width & 7); /* amount lost */
1436             register png_uint_32 final_val = BPP8 * len;   /* GRR bugfix */
1437
1438             srcptr = png_ptr->row_buf + 1 + initial_val;
1439             dstptr = row + initial_val;
1440
1441             for (i = initial_val; i < final_val; i += stride)
1442             {
1443                png_memcpy(dstptr, srcptr, rep_bytes);
1444                srcptr += stride;
1445                dstptr += stride;
1446             }
1447             if (diff)  /* number of leftover pixels:  3 for pngtest */
1448             {
1449                final_val+=diff*BPP8;
1450                for (; i < final_val; i += stride)
1451                {
1452                   if (rep_bytes > (int)(final_val-i))
1453                      rep_bytes = (int)(final_val-i);
1454                   png_memcpy(dstptr, srcptr, rep_bytes);
1455                   srcptr += stride;
1456                   dstptr += stride;
1457                }
1458             }
1459
1460             break;
1461          }       /* end 64 bpp */
1462
1463          default: /* png_ptr->row_info.pixel_depth != 1,2,4,8,16,24,32,48,64 */
1464          {
1465             /* this should never happen */
1466             png_warning(png_ptr, "Invalid row_info.pixel_depth in pnggccrd");
1467             break;
1468          }
1469       } /* end switch (png_ptr->row_info.pixel_depth) */
1470
1471    } /* end if (non-trivial mask) */
1472
1473 } /* end png_combine_row() */
1474
1475 #endif /* PNG_HAVE_ASSEMBLER_COMBINE_ROW */
1476
1477
1478
1479
1480 /*===========================================================================*/
1481 /*                                                                           */
1482 /*                 P N G _ D O _ R E A D _ I N T E R L A C E                 */
1483 /*                                                                           */
1484 /*===========================================================================*/
1485
1486 #if defined(PNG_READ_INTERLACING_SUPPORTED)
1487 #if defined(PNG_HAVE_ASSEMBLER_READ_INTERLACE)
1488
1489 /* png_do_read_interlace() is called after any 16-bit to 8-bit conversion
1490  * has taken place.  [GRR: what other steps come before and/or after?]
1491  */
1492
1493 void /* PRIVATE */
1494 png_do_read_interlace(png_structp png_ptr)
1495 {
1496    png_row_infop row_info = &(png_ptr->row_info);
1497    png_bytep row = png_ptr->row_buf + 1;
1498    int pass = png_ptr->pass;
1499 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
1500    png_uint_32 transformations = png_ptr->transformations;
1501 #endif
1502
1503    png_debug(1, "in png_do_read_interlace (pnggccrd.c)\n");
1504
1505 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
1506    if (_mmx_supported == 2) {
1507 #if !defined(PNG_1_0_X)
1508        /* this should have happened in png_init_mmx_flags() already */
1509        png_warning(png_ptr, "asm_flags may not have been initialized");
1510 #endif
1511        png_mmx_support();
1512    }
1513 #endif
1514
1515    if (row != NULL && row_info != NULL)
1516    {
1517       png_uint_32 final_width;
1518
1519       final_width = row_info->width * png_pass_inc[pass];
1520
1521       switch (row_info->pixel_depth)
1522       {
1523          case 1:
1524          {
1525             png_bytep sp, dp;
1526             int sshift, dshift;
1527             int s_start, s_end, s_inc;
1528             png_byte v;
1529             png_uint_32 i;
1530             int j;
1531
1532             sp = row + (png_size_t)((row_info->width - 1) >> 3);
1533             dp = row + (png_size_t)((final_width - 1) >> 3);
1534 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
1535             if (transformations & PNG_PACKSWAP)
1536             {
1537                sshift = (int)((row_info->width + 7) & 7);
1538                dshift = (int)((final_width + 7) & 7);
1539                s_start = 7;
1540                s_end = 0;
1541                s_inc = -1;
1542             }
1543             else
1544 #endif
1545             {
1546                sshift = 7 - (int)((row_info->width + 7) & 7);
1547                dshift = 7 - (int)((final_width + 7) & 7);
1548                s_start = 0;
1549                s_end = 7;
1550                s_inc = 1;
1551             }
1552
1553             for (i = row_info->width; i; i--)
1554             {
1555                v = (png_byte)((*sp >> sshift) & 0x1);
1556                for (j = 0; j < png_pass_inc[pass]; j++)
1557                {
1558                   *dp &= (png_byte)((0x7f7f >> (7 - dshift)) & 0xff);
1559                   *dp |= (png_byte)(v << dshift);
1560                   if (dshift == s_end)
1561                   {
1562                      dshift = s_start;
1563                      dp--;
1564                   }
1565                   else
1566                      dshift += s_inc;
1567                }
1568                if (sshift == s_end)
1569                {
1570                   sshift = s_start;
1571                   sp--;
1572                }
1573                else
1574                   sshift += s_inc;
1575             }
1576             break;
1577          }
1578
1579          case 2:
1580          {
1581             png_bytep sp, dp;
1582             int sshift, dshift;
1583             int s_start, s_end, s_inc;
1584             png_uint_32 i;
1585
1586             sp = row + (png_size_t)((row_info->width - 1) >> 2);
1587             dp = row + (png_size_t)((final_width - 1) >> 2);
1588 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
1589             if (transformations & PNG_PACKSWAP)
1590             {
1591                sshift = (png_size_t)(((row_info->width + 3) & 3) << 1);
1592                dshift = (png_size_t)(((final_width + 3) & 3) << 1);
1593                s_start = 6;
1594                s_end = 0;
1595                s_inc = -2;
1596             }
1597             else
1598 #endif
1599             {
1600                sshift = (png_size_t)((3 - ((row_info->width + 3) & 3)) << 1);
1601                dshift = (png_size_t)((3 - ((final_width + 3) & 3)) << 1);
1602                s_start = 0;
1603                s_end = 6;
1604                s_inc = 2;
1605             }
1606
1607             for (i = row_info->width; i; i--)
1608             {
1609                png_byte v;
1610                int j;
1611
1612                v = (png_byte)((*sp >> sshift) & 0x3);
1613                for (j = 0; j < png_pass_inc[pass]; j++)
1614                {
1615                   *dp &= (png_byte)((0x3f3f >> (6 - dshift)) & 0xff);
1616                   *dp |= (png_byte)(v << dshift);
1617                   if (dshift == s_end)
1618                   {
1619                      dshift = s_start;
1620                      dp--;
1621                   }
1622                   else
1623                      dshift += s_inc;
1624                }
1625                if (sshift == s_end)
1626                {
1627                   sshift = s_start;
1628                   sp--;
1629                }
1630                else
1631                   sshift += s_inc;
1632             }
1633             break;
1634          }
1635
1636          case 4:
1637          {
1638             png_bytep sp, dp;
1639             int sshift, dshift;
1640             int s_start, s_end, s_inc;
1641             png_uint_32 i;
1642
1643             sp = row + (png_size_t)((row_info->width - 1) >> 1);
1644             dp = row + (png_size_t)((final_width - 1) >> 1);
1645 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
1646             if (transformations & PNG_PACKSWAP)
1647             {
1648                sshift = (png_size_t)(((row_info->width + 1) & 1) << 2);
1649                dshift = (png_size_t)(((final_width + 1) & 1) << 2);
1650                s_start = 4;
1651                s_end = 0;
1652                s_inc = -4;
1653             }
1654             else
1655 #endif
1656             {
1657                sshift = (png_size_t)((1 - ((row_info->width + 1) & 1)) << 2);
1658                dshift = (png_size_t)((1 - ((final_width + 1) & 1)) << 2);
1659                s_start = 0;
1660                s_end = 4;
1661                s_inc = 4;
1662             }
1663
1664             for (i = row_info->width; i; i--)
1665             {
1666                png_byte v;
1667                int j;
1668
1669                v = (png_byte)((*sp >> sshift) & 0xf);
1670                for (j = 0; j < png_pass_inc[pass]; j++)
1671                {
1672                   *dp &= (png_byte)((0xf0f >> (4 - dshift)) & 0xff);
1673                   *dp |= (png_byte)(v << dshift);
1674                   if (dshift == s_end)
1675                   {
1676                      dshift = s_start;
1677                      dp--;
1678                   }
1679                   else
1680                      dshift += s_inc;
1681                }
1682                if (sshift == s_end)
1683                {
1684                   sshift = s_start;
1685                   sp--;
1686                }
1687                else
1688                   sshift += s_inc;
1689             }
1690             break;
1691          }
1692
1693        /*====================================================================*/
1694
1695          default: /* 8-bit or larger (this is where the routine is modified) */
1696          {
1697 #if 0
1698 //          static unsigned long long _const4 = 0x0000000000FFFFFFLL;  no good
1699 //          static unsigned long long const4 = 0x0000000000FFFFFFLL;   no good
1700 //          unsigned long long _const4 = 0x0000000000FFFFFFLL;         no good
1701 //          unsigned long long const4 = 0x0000000000FFFFFFLL;          no good
1702 #endif
1703             png_bytep sptr, dp;
1704             png_uint_32 i;
1705             png_size_t pixel_bytes;
1706             int width = (int)row_info->width;
1707
1708             pixel_bytes = (row_info->pixel_depth >> 3);
1709
1710             /* point sptr at the last pixel in the pre-expanded row: */
1711             sptr = row + (width - 1) * pixel_bytes;
1712
1713             /* point dp at the last pixel position in the expanded row: */
1714             dp = row + (final_width - 1) * pixel_bytes;
1715
1716             /* New code by Nirav Chhatrapati - Intel Corporation */
1717
1718 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
1719 #if !defined(PNG_1_0_X)
1720             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_INTERLACE)
1721                 /* && _mmx_supported */ )
1722 #else
1723             if (_mmx_supported)
1724 #endif
1725             {
1726                //--------------------------------------------------------------
1727                if (pixel_bytes == 3)
1728                {
1729                   if (((pass == 0) || (pass == 1)) && width)
1730                   {
1731                      int dummy_value_c;   // fix 'forbidden register spilled'
1732                      int dummy_value_S;
1733                      int dummy_value_D;
1734
1735                      __asm__ __volatile__ (
1736                         "subl $21, %%edi         \n\t"
1737                                      // (png_pass_inc[pass] - 1)*pixel_bytes
1738
1739                      ".loop3_pass0:              \n\t"
1740                         "movd (%%esi), %%mm0     \n\t" // x x x x x 2 1 0
1741                         "pand _const4, %%mm0     \n\t" // z z z z z 2 1 0
1742                         "movq %%mm0, %%mm1       \n\t" // z z z z z 2 1 0
1743                         "psllq $16, %%mm0        \n\t" // z z z 2 1 0 z z
1744                         "movq %%mm0, %%mm2       \n\t" // z z z 2 1 0 z z
1745                         "psllq $24, %%mm0        \n\t" // 2 1 0 z z z z z
1746                         "psrlq $8, %%mm1         \n\t" // z z z z z z 2 1
1747                         "por %%mm2, %%mm0        \n\t" // 2 1 0 2 1 0 z z
1748                         "por %%mm1, %%mm0        \n\t" // 2 1 0 2 1 0 2 1
1749                         "movq %%mm0, %%mm3       \n\t" // 2 1 0 2 1 0 2 1
1750                         "psllq $16, %%mm0        \n\t" // 0 2 1 0 2 1 z z
1751                         "movq %%mm3, %%mm4       \n\t" // 2 1 0 2 1 0 2 1
1752                         "punpckhdq %%mm0, %%mm3  \n\t" // 0 2 1 0 2 1 0 2
1753                         "movq %%mm4, 16(%%edi)   \n\t"
1754                         "psrlq $32, %%mm0        \n\t" // z z z z 0 2 1 0
1755                         "movq %%mm3, 8(%%edi)    \n\t"
1756                         "punpckldq %%mm4, %%mm0  \n\t" // 1 0 2 1 0 2 1 0
1757                         "subl $3, %%esi          \n\t"
1758                         "movq %%mm0, (%%edi)     \n\t"
1759                         "subl $24, %%edi         \n\t"
1760                         "decl %%ecx              \n\t"
1761                         "jnz .loop3_pass0        \n\t"
1762                         "EMMS                    \n\t" // DONE
1763
1764                         : "=c" (dummy_value_c),        // output regs (dummy)
1765                           "=S" (dummy_value_S),
1766                           "=D" (dummy_value_D)
1767
1768                         : "1" (sptr),      // esi      // input regs
1769                           "2" (dp),        // edi
1770                           "0" (width)      // ecx
1771 // doesn't work           "i" (0x0000000000FFFFFFLL)   // %1 (a.k.a. _const4)
1772
1773 #if 0  /* %mm0, ..., %mm4 not supported by gcc 2.7.2.3 or egcs 1.1 */
1774                         : "%mm0", "%mm1", "%mm2"       // clobber list
1775                         , "%mm3", "%mm4"
1776 #endif
1777                      );
1778                   }
1779                   else if (((pass == 2) || (pass == 3)) && width)
1780                   {
1781                      int dummy_value_c;   // fix 'forbidden register spilled'
1782                      int dummy_value_S;
1783                      int dummy_value_D;
1784
1785                      __asm__ __volatile__ (
1786                         "subl $9, %%edi          \n\t"
1787                                      // (png_pass_inc[pass] - 1)*pixel_bytes
1788
1789                      ".loop3_pass2:              \n\t"
1790                         "movd (%%esi), %%mm0     \n\t" // x x x x x 2 1 0
1791                         "pand _const4, %%mm0     \n\t" // z z z z z 2 1 0
1792                         "movq %%mm0, %%mm1       \n\t" // z z z z z 2 1 0
1793                         "psllq $16, %%mm0        \n\t" // z z z 2 1 0 z z
1794                         "movq %%mm0, %%mm2       \n\t" // z z z 2 1 0 z z
1795                         "psllq $24, %%mm0        \n\t" // 2 1 0 z z z z z
1796                         "psrlq $8, %%mm1         \n\t" // z z z z z z 2 1
1797                         "por %%mm2, %%mm0        \n\t" // 2 1 0 2 1 0 z z
1798                         "por %%mm1, %%mm0        \n\t" // 2 1 0 2 1 0 2 1
1799                         "movq %%mm0, 4(%%edi)    \n\t"
1800                         "psrlq $16, %%mm0        \n\t" // z z 2 1 0 2 1 0
1801                         "subl $3, %%esi          \n\t"
1802                         "movd %%mm0, (%%edi)     \n\t"
1803                         "subl $12, %%edi         \n\t"
1804                         "decl %%ecx              \n\t"
1805                         "jnz .loop3_pass2        \n\t"
1806                         "EMMS                    \n\t" // DONE
1807
1808                         : "=c" (dummy_value_c),        // output regs (dummy)
1809                           "=S" (dummy_value_S),
1810                           "=D" (dummy_value_D)
1811
1812                         : "1" (sptr),      // esi      // input regs
1813                           "2" (dp),        // edi
1814                           "0" (width)      // ecx
1815
1816 #if 0  /* %mm0, ..., %mm2 not supported by gcc 2.7.2.3 or egcs 1.1 */
1817                         : "%mm0", "%mm1", "%mm2"       // clobber list
1818 #endif
1819                      );
1820                   }
1821                   else if (width) /* && ((pass == 4) || (pass == 5)) */
1822                   {
1823                      int width_mmx = ((width >> 1) << 1) - 8;   // GRR:  huh?
1824                      if (width_mmx < 0)
1825                          width_mmx = 0;
1826                      width -= width_mmx;        // 8 or 9 pix, 24 or 27 bytes
1827                      if (width_mmx)
1828                      {
1829                         // png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1};
1830                         // sptr points at last pixel in pre-expanded row
1831                         // dp points at last pixel position in expanded row
1832                         int dummy_value_c;  // fix 'forbidden register spilled'
1833                         int dummy_value_S;
1834                         int dummy_value_D;
1835
1836                         __asm__ __volatile__ (
1837                            "subl $3, %%esi          \n\t"
1838                            "subl $9, %%edi          \n\t"
1839                                         // (png_pass_inc[pass] + 1)*pixel_bytes
1840
1841                         ".loop3_pass4:              \n\t"
1842                            "movq (%%esi), %%mm0     \n\t" // x x 5 4 3 2 1 0
1843                            "movq %%mm0, %%mm1       \n\t" // x x 5 4 3 2 1 0
1844                            "movq %%mm0, %%mm2       \n\t" // x x 5 4 3 2 1 0
1845                            "psllq $24, %%mm0        \n\t" // 4 3 2 1 0 z z z
1846                            "pand _const4, %%mm1     \n\t" // z z z z z 2 1 0
1847                            "psrlq $24, %%mm2        \n\t" // z z z x x 5 4 3
1848                            "por %%mm1, %%mm0        \n\t" // 4 3 2 1 0 2 1 0
1849                            "movq %%mm2, %%mm3       \n\t" // z z z x x 5 4 3
1850                            "psllq $8, %%mm2         \n\t" // z z x x 5 4 3 z
1851                            "movq %%mm0, (%%edi)     \n\t"
1852                            "psrlq $16, %%mm3        \n\t" // z z z z z x x 5
1853                            "pand _const6, %%mm3     \n\t" // z z z z z z z 5
1854                            "por %%mm3, %%mm2        \n\t" // z z x x 5 4 3 5
1855                            "subl $6, %%esi          \n\t"
1856                            "movd %%mm2, 8(%%edi)    \n\t"
1857                            "subl $12, %%edi         \n\t"
1858                            "subl $2, %%ecx          \n\t"
1859                            "jnz .loop3_pass4        \n\t"
1860                            "EMMS                    \n\t" // DONE
1861
1862                            : "=c" (dummy_value_c),        // output regs (dummy)
1863                              "=S" (dummy_value_S),
1864                              "=D" (dummy_value_D)
1865
1866                            : "1" (sptr),      // esi      // input regs
1867                              "2" (dp),        // edi
1868                              "0" (width_mmx)  // ecx
1869
1870 #if 0  /* %mm0, ..., %mm3 not supported by gcc 2.7.2.3 or egcs 1.1 */
1871                            : "%mm0", "%mm1"               // clobber list
1872                            , "%mm2", "%mm3"
1873 #endif
1874                         );
1875                      }
1876
1877                      sptr -= width_mmx*3;
1878                      dp -= width_mmx*6;
1879                      for (i = width; i; i--)
1880                      {
1881                         png_byte v[8];
1882                         int j;
1883
1884                         png_memcpy(v, sptr, 3);
1885                         for (j = 0; j < png_pass_inc[pass]; j++)
1886                         {
1887                            png_memcpy(dp, v, 3);
1888                            dp -= 3;
1889                         }
1890                         sptr -= 3;
1891                      }
1892                   }
1893                } /* end of pixel_bytes == 3 */
1894
1895                //--------------------------------------------------------------
1896                else if (pixel_bytes == 1)
1897                {
1898                   if (((pass == 0) || (pass == 1)) && width)
1899                   {
1900                      int width_mmx = ((width >> 2) << 2);
1901                      width -= width_mmx;        // 0-3 pixels => 0-3 bytes
1902                      if (width_mmx)
1903                      {
1904                         int dummy_value_c;  // fix 'forbidden register spilled'
1905                         int dummy_value_S;
1906                         int dummy_value_D;
1907
1908                         __asm__ __volatile__ (
1909                            "subl $3, %%esi          \n\t"
1910                            "subl $31, %%edi         \n\t"
1911
1912                         ".loop1_pass0:              \n\t"
1913                            "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
1914                            "movq %%mm0, %%mm1       \n\t" // x x x x 3 2 1 0
1915                            "punpcklbw %%mm0, %%mm0  \n\t" // 3 3 2 2 1 1 0 0
1916                            "movq %%mm0, %%mm2       \n\t" // 3 3 2 2 1 1 0 0
1917                            "punpcklwd %%mm0, %%mm0  \n\t" // 1 1 1 1 0 0 0 0
1918                            "movq %%mm0, %%mm3       \n\t" // 1 1 1 1 0 0 0 0
1919                            "punpckldq %%mm0, %%mm0  \n\t" // 0 0 0 0 0 0 0 0
1920                            "punpckhdq %%mm3, %%mm3  \n\t" // 1 1 1 1 1 1 1 1
1921                            "movq %%mm0, (%%edi)     \n\t"
1922                            "punpckhwd %%mm2, %%mm2  \n\t" // 3 3 3 3 2 2 2 2
1923                            "movq %%mm3, 8(%%edi)    \n\t"
1924                            "movq %%mm2, %%mm4       \n\t" // 3 3 3 3 2 2 2 2
1925                            "punpckldq %%mm2, %%mm2  \n\t" // 2 2 2 2 2 2 2 2
1926                            "punpckhdq %%mm4, %%mm4  \n\t" // 3 3 3 3 3 3 3 3
1927                            "movq %%mm2, 16(%%edi)   \n\t"
1928                            "subl $4, %%esi          \n\t"
1929                            "movq %%mm4, 24(%%edi)   \n\t"
1930                            "subl $32, %%edi         \n\t"
1931                            "subl $4, %%ecx          \n\t"
1932                            "jnz .loop1_pass0        \n\t"
1933                            "EMMS                    \n\t" // DONE
1934
1935                            : "=c" (dummy_value_c),        // output regs (dummy)
1936                              "=S" (dummy_value_S),
1937                              "=D" (dummy_value_D)
1938
1939                            : "1" (sptr),      // esi      // input regs
1940                              "2" (dp),        // edi
1941                              "0" (width_mmx)  // ecx
1942
1943 #if 0  /* %mm0, ..., %mm4 not supported by gcc 2.7.2.3 or egcs 1.1 */
1944                            : "%mm0", "%mm1", "%mm2"       // clobber list
1945                            , "%mm3", "%mm4"
1946 #endif
1947                         );
1948                      }
1949
1950                      sptr -= width_mmx;
1951                      dp -= width_mmx*8;
1952                      for (i = width; i; i--)
1953                      {
1954                         int j;
1955
1956                        /* I simplified this part in version 1.0.4e
1957                         * here and in several other instances where
1958                         * pixel_bytes == 1  -- GR-P
1959                         *
1960                         * Original code:
1961                         *
1962                         * png_byte v[8];
1963                         * png_memcpy(v, sptr, pixel_bytes);
1964                         * for (j = 0; j < png_pass_inc[pass]; j++)
1965                         * {
1966                         *    png_memcpy(dp, v, pixel_bytes);
1967                         *    dp -= pixel_bytes;
1968                         * }
1969                         * sptr -= pixel_bytes;
1970                         *
1971                         * Replacement code is in the next three lines:
1972                         */
1973
1974                         for (j = 0; j < png_pass_inc[pass]; j++)
1975                         {
1976                            *dp-- = *sptr;
1977                         }
1978                         --sptr;
1979                      }
1980                   }
1981                   else if (((pass == 2) || (pass == 3)) && width)
1982                   {
1983                      int width_mmx = ((width >> 2) << 2);
1984                      width -= width_mmx;        // 0-3 pixels => 0-3 bytes
1985                      if (width_mmx)
1986                      {
1987                         int dummy_value_c;  // fix 'forbidden register spilled'
1988                         int dummy_value_S;
1989                         int dummy_value_D;
1990
1991                         __asm__ __volatile__ (
1992                            "subl $3, %%esi          \n\t"
1993                            "subl $15, %%edi         \n\t"
1994
1995                         ".loop1_pass2:              \n\t"
1996                            "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
1997                            "punpcklbw %%mm0, %%mm0  \n\t" // 3 3 2 2 1 1 0 0
1998                            "movq %%mm0, %%mm1       \n\t" // 3 3 2 2 1 1 0 0
1999                            "punpcklwd %%mm0, %%mm0  \n\t" // 1 1 1 1 0 0 0 0
2000                            "punpckhwd %%mm1, %%mm1  \n\t" // 3 3 3 3 2 2 2 2
2001                            "movq %%mm0, (%%edi)     \n\t"
2002                            "subl $4, %%esi          \n\t"
2003                            "movq %%mm1, 8(%%edi)    \n\t"
2004                            "subl $16, %%edi         \n\t"
2005                            "subl $4, %%ecx          \n\t"
2006                            "jnz .loop1_pass2        \n\t"
2007                            "EMMS                    \n\t" // DONE
2008
2009                            : "=c" (dummy_value_c),        // output regs (dummy)
2010                              "=S" (dummy_value_S),
2011                              "=D" (dummy_value_D)
2012
2013                            : "1" (sptr),      // esi      // input regs
2014                              "2" (dp),        // edi
2015                              "0" (width_mmx)  // ecx
2016
2017 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2018                            : "%mm0", "%mm1"               // clobber list
2019 #endif
2020                         );
2021                      }
2022
2023                      sptr -= width_mmx;
2024                      dp -= width_mmx*4;
2025                      for (i = width; i; i--)
2026                      {
2027                         int j;
2028
2029                         for (j = 0; j < png_pass_inc[pass]; j++)
2030                         {
2031                            *dp-- = *sptr;
2032                         }
2033                         --sptr;
2034                      }
2035                   }
2036                   else if (width)  /* && ((pass == 4) || (pass == 5)) */
2037                   {
2038                      int width_mmx = ((width >> 3) << 3);
2039                      width -= width_mmx;        // 0-3 pixels => 0-3 bytes
2040                      if (width_mmx)
2041                      {
2042                         int dummy_value_c;  // fix 'forbidden register spilled'
2043                         int dummy_value_S;
2044                         int dummy_value_D;
2045
2046                         __asm__ __volatile__ (
2047                            "subl $7, %%esi          \n\t"
2048                            "subl $15, %%edi         \n\t"
2049
2050                         ".loop1_pass4:              \n\t"
2051                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2052                            "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
2053                            "punpcklbw %%mm0, %%mm0  \n\t" // 3 3 2 2 1 1 0 0
2054                            "punpckhbw %%mm1, %%mm1  \n\t" // 7 7 6 6 5 5 4 4
2055                            "movq %%mm1, 8(%%edi)    \n\t"
2056                            "subl $8, %%esi          \n\t"
2057                            "movq %%mm0, (%%edi)     \n\t"
2058                            "subl $16, %%edi         \n\t"
2059                            "subl $8, %%ecx          \n\t"
2060                            "jnz .loop1_pass4        \n\t"
2061                            "EMMS                    \n\t" // DONE
2062
2063                            : "=c" (dummy_value_c),        // output regs (none)
2064                              "=S" (dummy_value_S),
2065                              "=D" (dummy_value_D)
2066
2067                            : "1" (sptr),      // esi      // input regs
2068                              "2" (dp),        // edi
2069                              "0" (width_mmx)  // ecx
2070
2071 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2072                            : "%mm0", "%mm1"               // clobber list
2073 #endif
2074                         );
2075                      }
2076
2077                      sptr -= width_mmx;
2078                      dp -= width_mmx*2;
2079                      for (i = width; i; i--)
2080                      {
2081                         int j;
2082
2083                         for (j = 0; j < png_pass_inc[pass]; j++)
2084                         {
2085                            *dp-- = *sptr;
2086                         }
2087                         --sptr;
2088                      }
2089                   }
2090                } /* end of pixel_bytes == 1 */
2091
2092                //--------------------------------------------------------------
2093                else if (pixel_bytes == 2)
2094                {
2095                   if (((pass == 0) || (pass == 1)) && width)
2096                   {
2097                      int width_mmx = ((width >> 1) << 1);
2098                      width -= width_mmx;        // 0,1 pixels => 0,2 bytes
2099                      if (width_mmx)
2100                      {
2101                         int dummy_value_c;  // fix 'forbidden register spilled'
2102                         int dummy_value_S;
2103                         int dummy_value_D;
2104
2105                         __asm__ __volatile__ (
2106                            "subl $2, %%esi          \n\t"
2107                            "subl $30, %%edi         \n\t"
2108
2109                         ".loop2_pass0:              \n\t"
2110                            "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
2111                            "punpcklwd %%mm0, %%mm0  \n\t" // 3 2 3 2 1 0 1 0
2112                            "movq %%mm0, %%mm1       \n\t" // 3 2 3 2 1 0 1 0
2113                            "punpckldq %%mm0, %%mm0  \n\t" // 1 0 1 0 1 0 1 0
2114                            "punpckhdq %%mm1, %%mm1  \n\t" // 3 2 3 2 3 2 3 2
2115                            "movq %%mm0, (%%edi)     \n\t"
2116                            "movq %%mm0, 8(%%edi)    \n\t"
2117                            "movq %%mm1, 16(%%edi)   \n\t"
2118                            "subl $4, %%esi          \n\t"
2119                            "movq %%mm1, 24(%%edi)   \n\t"
2120                            "subl $32, %%edi         \n\t"
2121                            "subl $2, %%ecx          \n\t"
2122                            "jnz .loop2_pass0        \n\t"
2123                            "EMMS                    \n\t" // DONE
2124
2125                            : "=c" (dummy_value_c),        // output regs (dummy)
2126                              "=S" (dummy_value_S),
2127                              "=D" (dummy_value_D)
2128
2129                            : "1" (sptr),      // esi      // input regs
2130                              "2" (dp),        // edi
2131                              "0" (width_mmx)  // ecx
2132
2133 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2134                            : "%mm0", "%mm1"               // clobber list
2135 #endif
2136                         );
2137                      }
2138
2139                      sptr -= (width_mmx*2 - 2); // sign fixed
2140                      dp -= (width_mmx*16 - 2);  // sign fixed
2141                      for (i = width; i; i--)
2142                      {
2143                         png_byte v[8];
2144                         int j;
2145                         sptr -= 2;
2146                         png_memcpy(v, sptr, 2);
2147                         for (j = 0; j < png_pass_inc[pass]; j++)
2148                         {
2149                            dp -= 2;
2150                            png_memcpy(dp, v, 2);
2151                         }
2152                      }
2153                   }
2154                   else if (((pass == 2) || (pass == 3)) && width)
2155                   {
2156                      int width_mmx = ((width >> 1) << 1) ;
2157                      width -= width_mmx;        // 0,1 pixels => 0,2 bytes
2158                      if (width_mmx)
2159                      {
2160                         int dummy_value_c;  // fix 'forbidden register spilled'
2161                         int dummy_value_S;
2162                         int dummy_value_D;
2163
2164                         __asm__ __volatile__ (
2165                            "subl $2, %%esi          \n\t"
2166                            "subl $14, %%edi         \n\t"
2167
2168                         ".loop2_pass2:              \n\t"
2169                            "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
2170                            "punpcklwd %%mm0, %%mm0  \n\t" // 3 2 3 2 1 0 1 0
2171                            "movq %%mm0, %%mm1       \n\t" // 3 2 3 2 1 0 1 0
2172                            "punpckldq %%mm0, %%mm0  \n\t" // 1 0 1 0 1 0 1 0
2173                            "punpckhdq %%mm1, %%mm1  \n\t" // 3 2 3 2 3 2 3 2
2174                            "movq %%mm0, (%%edi)     \n\t"
2175                            "subl $4, %%esi          \n\t"
2176                            "movq %%mm1, 8(%%edi)    \n\t"
2177                            "subl $16, %%edi         \n\t"
2178                            "subl $2, %%ecx          \n\t"
2179                            "jnz .loop2_pass2        \n\t"
2180                            "EMMS                    \n\t" // DONE
2181
2182                            : "=c" (dummy_value_c),        // output regs (dummy)
2183                              "=S" (dummy_value_S),
2184                              "=D" (dummy_value_D)
2185
2186                            : "1" (sptr),      // esi      // input regs
2187                              "2" (dp),        // edi
2188                              "0" (width_mmx)  // ecx
2189
2190 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2191                            : "%mm0", "%mm1"               // clobber list
2192 #endif
2193                         );
2194                      }
2195
2196                      sptr -= (width_mmx*2 - 2); // sign fixed
2197                      dp -= (width_mmx*8 - 2);   // sign fixed
2198                      for (i = width; i; i--)
2199                      {
2200                         png_byte v[8];
2201                         int j;
2202                         sptr -= 2;
2203                         png_memcpy(v, sptr, 2);
2204                         for (j = 0; j < png_pass_inc[pass]; j++)
2205                         {
2206                            dp -= 2;
2207                            png_memcpy(dp, v, 2);
2208                         }
2209                      }
2210                   }
2211                   else if (width)  // pass == 4 or 5
2212                   {
2213                      int width_mmx = ((width >> 1) << 1) ;
2214                      width -= width_mmx;        // 0,1 pixels => 0,2 bytes
2215                      if (width_mmx)
2216                      {
2217                         int dummy_value_c;  // fix 'forbidden register spilled'
2218                         int dummy_value_S;
2219                         int dummy_value_D;
2220
2221                         __asm__ __volatile__ (
2222                            "subl $2, %%esi          \n\t"
2223                            "subl $6, %%edi          \n\t"
2224
2225                         ".loop2_pass4:              \n\t"
2226                            "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
2227                            "punpcklwd %%mm0, %%mm0  \n\t" // 3 2 3 2 1 0 1 0
2228                            "subl $4, %%esi          \n\t"
2229                            "movq %%mm0, (%%edi)     \n\t"
2230                            "subl $8, %%edi          \n\t"
2231                            "subl $2, %%ecx          \n\t"
2232                            "jnz .loop2_pass4        \n\t"
2233                            "EMMS                    \n\t" // DONE
2234
2235                            : "=c" (dummy_value_c),        // output regs (dummy)
2236                              "=S" (dummy_value_S),
2237                              "=D" (dummy_value_D)
2238
2239                            : "1" (sptr),      // esi      // input regs
2240                              "2" (dp),        // edi
2241                              "0" (width_mmx)  // ecx
2242
2243 #if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2244                            : "%mm0"                       // clobber list
2245 #endif
2246                         );
2247                      }
2248
2249                      sptr -= (width_mmx*2 - 2); // sign fixed
2250                      dp -= (width_mmx*4 - 2);   // sign fixed
2251                      for (i = width; i; i--)
2252                      {
2253                         png_byte v[8];
2254                         int j;
2255                         sptr -= 2;
2256                         png_memcpy(v, sptr, 2);
2257                         for (j = 0; j < png_pass_inc[pass]; j++)
2258                         {
2259                            dp -= 2;
2260                            png_memcpy(dp, v, 2);
2261                         }
2262                      }
2263                   }
2264                } /* end of pixel_bytes == 2 */
2265
2266                //--------------------------------------------------------------
2267                else if (pixel_bytes == 4)
2268                {
2269                   if (((pass == 0) || (pass == 1)) && width)
2270                   {
2271                      int width_mmx = ((width >> 1) << 1);
2272                      width -= width_mmx;        // 0,1 pixels => 0,4 bytes
2273                      if (width_mmx)
2274                      {
2275                         int dummy_value_c;  // fix 'forbidden register spilled'
2276                         int dummy_value_S;
2277                         int dummy_value_D;
2278
2279                         __asm__ __volatile__ (
2280                            "subl $4, %%esi          \n\t"
2281                            "subl $60, %%edi         \n\t"
2282
2283                         ".loop4_pass0:              \n\t"
2284                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2285                            "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
2286                            "punpckldq %%mm0, %%mm0  \n\t" // 3 2 1 0 3 2 1 0
2287                            "punpckhdq %%mm1, %%mm1  \n\t" // 7 6 5 4 7 6 5 4
2288                            "movq %%mm0, (%%edi)     \n\t"
2289                            "movq %%mm0, 8(%%edi)    \n\t"
2290                            "movq %%mm0, 16(%%edi)   \n\t"
2291                            "movq %%mm0, 24(%%edi)   \n\t"
2292                            "movq %%mm1, 32(%%edi)   \n\t"
2293                            "movq %%mm1, 40(%%edi)   \n\t"
2294                            "movq %%mm1, 48(%%edi)   \n\t"
2295                            "subl $8, %%esi          \n\t"
2296                            "movq %%mm1, 56(%%edi)   \n\t"
2297                            "subl $64, %%edi         \n\t"
2298                            "subl $2, %%ecx          \n\t"
2299                            "jnz .loop4_pass0        \n\t"
2300                            "EMMS                    \n\t" // DONE
2301
2302                            : "=c" (dummy_value_c),        // output regs (dummy)
2303                              "=S" (dummy_value_S),
2304                              "=D" (dummy_value_D)
2305
2306                            : "1" (sptr),      // esi      // input regs
2307                              "2" (dp),        // edi
2308                              "0" (width_mmx)  // ecx
2309
2310 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2311                            : "%mm0", "%mm1"               // clobber list
2312 #endif
2313                         );
2314                      }
2315
2316                      sptr -= (width_mmx*4 - 4); // sign fixed
2317                      dp -= (width_mmx*32 - 4);  // sign fixed
2318                      for (i = width; i; i--)
2319                      {
2320                         png_byte v[8];
2321                         int j;
2322                         sptr -= 4;
2323                         png_memcpy(v, sptr, 4);
2324                         for (j = 0; j < png_pass_inc[pass]; j++)
2325                         {
2326                            dp -= 4;
2327                            png_memcpy(dp, v, 4);
2328                         }
2329                      }
2330                   }
2331                   else if (((pass == 2) || (pass == 3)) && width)
2332                   {
2333                      int width_mmx = ((width >> 1) << 1);
2334                      width -= width_mmx;        // 0,1 pixels => 0,4 bytes
2335                      if (width_mmx)
2336                      {
2337                         int dummy_value_c;  // fix 'forbidden register spilled'
2338                         int dummy_value_S;
2339                         int dummy_value_D;
2340
2341                         __asm__ __volatile__ (
2342                            "subl $4, %%esi          \n\t"
2343                            "subl $28, %%edi         \n\t"
2344
2345                         ".loop4_pass2:              \n\t"
2346                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2347                            "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
2348                            "punpckldq %%mm0, %%mm0  \n\t" // 3 2 1 0 3 2 1 0
2349                            "punpckhdq %%mm1, %%mm1  \n\t" // 7 6 5 4 7 6 5 4
2350                            "movq %%mm0, (%%edi)     \n\t"
2351                            "movq %%mm0, 8(%%edi)    \n\t"
2352                            "movq %%mm1, 16(%%edi)   \n\t"
2353                            "movq %%mm1, 24(%%edi)   \n\t"
2354                            "subl $8, %%esi          \n\t"
2355                            "subl $32, %%edi         \n\t"
2356                            "subl $2, %%ecx          \n\t"
2357                            "jnz .loop4_pass2        \n\t"
2358                            "EMMS                    \n\t" // DONE
2359
2360                            : "=c" (dummy_value_c),        // output regs (dummy)
2361                              "=S" (dummy_value_S),
2362                              "=D" (dummy_value_D)
2363
2364                            : "1" (sptr),      // esi      // input regs
2365                              "2" (dp),        // edi
2366                              "0" (width_mmx)  // ecx
2367
2368 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2369                            : "%mm0", "%mm1"               // clobber list
2370 #endif
2371                         );
2372                      }
2373
2374                      sptr -= (width_mmx*4 - 4); // sign fixed
2375                      dp -= (width_mmx*16 - 4);  // sign fixed
2376                      for (i = width; i; i--)
2377                      {
2378                         png_byte v[8];
2379                         int j;
2380                         sptr -= 4;
2381                         png_memcpy(v, sptr, 4);
2382                         for (j = 0; j < png_pass_inc[pass]; j++)
2383                         {
2384                            dp -= 4;
2385                            png_memcpy(dp, v, 4);
2386                         }
2387                      }
2388                   }
2389                   else if (width)  // pass == 4 or 5
2390                   {
2391                      int width_mmx = ((width >> 1) << 1) ;
2392                      width -= width_mmx;        // 0,1 pixels => 0,4 bytes
2393                      if (width_mmx)
2394                      {
2395                         int dummy_value_c;  // fix 'forbidden register spilled'
2396                         int dummy_value_S;
2397                         int dummy_value_D;
2398
2399                         __asm__ __volatile__ (
2400                            "subl $4, %%esi          \n\t"
2401                            "subl $12, %%edi         \n\t"
2402
2403                         ".loop4_pass4:              \n\t"
2404                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2405                            "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
2406                            "punpckldq %%mm0, %%mm0  \n\t" // 3 2 1 0 3 2 1 0
2407                            "punpckhdq %%mm1, %%mm1  \n\t" // 7 6 5 4 7 6 5 4
2408                            "movq %%mm0, (%%edi)     \n\t"
2409                            "subl $8, %%esi          \n\t"
2410                            "movq %%mm1, 8(%%edi)    \n\t"
2411                            "subl $16, %%edi         \n\t"
2412                            "subl $2, %%ecx          \n\t"
2413                            "jnz .loop4_pass4        \n\t"
2414                            "EMMS                    \n\t" // DONE
2415
2416                            : "=c" (dummy_value_c),        // output regs (dummy)
2417                              "=S" (dummy_value_S),
2418                              "=D" (dummy_value_D)
2419
2420                            : "1" (sptr),      // esi      // input regs
2421                              "2" (dp),        // edi
2422                              "0" (width_mmx)  // ecx
2423
2424 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2425                            : "%mm0", "%mm1"               // clobber list
2426 #endif
2427                         );
2428                      }
2429
2430                      sptr -= (width_mmx*4 - 4); // sign fixed
2431                      dp -= (width_mmx*8 - 4);   // sign fixed
2432                      for (i = width; i; i--)
2433                      {
2434                         png_byte v[8];
2435                         int j;
2436                         sptr -= 4;
2437                         png_memcpy(v, sptr, 4);
2438                         for (j = 0; j < png_pass_inc[pass]; j++)
2439                         {
2440                            dp -= 4;
2441                            png_memcpy(dp, v, 4);
2442                         }
2443                      }
2444                   }
2445                } /* end of pixel_bytes == 4 */
2446
2447                //--------------------------------------------------------------
2448                else if (pixel_bytes == 8)
2449                {
2450 // GRR TEST:  should work, but needs testing (special 64-bit version of rpng2?)
2451                   // GRR NOTE:  no need to combine passes here!
2452                   if (((pass == 0) || (pass == 1)) && width)
2453                   {
2454                      int dummy_value_c;  // fix 'forbidden register spilled'
2455                      int dummy_value_S;
2456                      int dummy_value_D;
2457
2458                      // source is 8-byte RRGGBBAA
2459                      // dest is 64-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA ...
2460                      __asm__ __volatile__ (
2461                         "subl $56, %%edi         \n\t" // start of last block
2462
2463                      ".loop8_pass0:              \n\t"
2464                         "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2465                         "movq %%mm0, (%%edi)     \n\t"
2466                         "movq %%mm0, 8(%%edi)    \n\t"
2467                         "movq %%mm0, 16(%%edi)   \n\t"
2468                         "movq %%mm0, 24(%%edi)   \n\t"
2469                         "movq %%mm0, 32(%%edi)   \n\t"
2470                         "movq %%mm0, 40(%%edi)   \n\t"
2471                         "movq %%mm0, 48(%%edi)   \n\t"
2472                         "subl $8, %%esi          \n\t"
2473                         "movq %%mm0, 56(%%edi)   \n\t"
2474                         "subl $64, %%edi         \n\t"
2475                         "decl %%ecx              \n\t"
2476                         "jnz .loop8_pass0        \n\t"
2477                         "EMMS                    \n\t" // DONE
2478
2479                         : "=c" (dummy_value_c),        // output regs (dummy)
2480                           "=S" (dummy_value_S),
2481                           "=D" (dummy_value_D)
2482
2483                         : "1" (sptr),      // esi      // input regs
2484                           "2" (dp),        // edi
2485                           "0" (width)      // ecx
2486
2487 #if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2488                         : "%mm0"                       // clobber list
2489 #endif
2490                      );
2491                   }
2492                   else if (((pass == 2) || (pass == 3)) && width)
2493                   {
2494                      // source is 8-byte RRGGBBAA
2495                      // dest is 32-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA
2496                      // (recall that expansion is _in place_:  sptr and dp
2497                      //  both point at locations within same row buffer)
2498                      {
2499                         int dummy_value_c;  // fix 'forbidden register spilled'
2500                         int dummy_value_S;
2501                         int dummy_value_D;
2502
2503                         __asm__ __volatile__ (
2504                            "subl $24, %%edi         \n\t" // start of last block
2505
2506                         ".loop8_pass2:              \n\t"
2507                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2508                            "movq %%mm0, (%%edi)     \n\t"
2509                            "movq %%mm0, 8(%%edi)    \n\t"
2510                            "movq %%mm0, 16(%%edi)   \n\t"
2511                            "subl $8, %%esi          \n\t"
2512                            "movq %%mm0, 24(%%edi)   \n\t"
2513                            "subl $32, %%edi         \n\t"
2514                            "decl %%ecx              \n\t"
2515                            "jnz .loop8_pass2        \n\t"
2516                            "EMMS                    \n\t" // DONE
2517
2518                            : "=c" (dummy_value_c),        // output regs (dummy)
2519                              "=S" (dummy_value_S),
2520                              "=D" (dummy_value_D)
2521
2522                            : "1" (sptr),      // esi      // input regs
2523                              "2" (dp),        // edi
2524                              "0" (width)      // ecx
2525
2526 #if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2527                            : "%mm0"                       // clobber list
2528 #endif
2529                         );
2530                      }
2531                   }
2532                   else if (width)  // pass == 4 or 5
2533                   {
2534                      // source is 8-byte RRGGBBAA
2535                      // dest is 16-byte RRGGBBAA RRGGBBAA
2536                      {
2537                         int dummy_value_c;  // fix 'forbidden register spilled'
2538                         int dummy_value_S;
2539                         int dummy_value_D;
2540
2541                         __asm__ __volatile__ (
2542                            "subl $8, %%edi          \n\t" // start of last block
2543
2544                         ".loop8_pass4:              \n\t"
2545                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2546                            "movq %%mm0, (%%edi)     \n\t"
2547                            "subl $8, %%esi          \n\t"
2548                            "movq %%mm0, 8(%%edi)    \n\t"
2549                            "subl $16, %%edi         \n\t"
2550                            "decl %%ecx              \n\t"
2551                            "jnz .loop8_pass4        \n\t"
2552                            "EMMS                    \n\t" // DONE
2553
2554                            : "=c" (dummy_value_c),        // output regs (dummy)
2555                              "=S" (dummy_value_S),
2556                              "=D" (dummy_value_D)
2557
2558                            : "1" (sptr),      // esi      // input regs
2559                              "2" (dp),        // edi
2560                              "0" (width)      // ecx
2561
2562 #if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2563                            : "%mm0"                       // clobber list
2564 #endif
2565                         );
2566                      }
2567                   }
2568
2569                } /* end of pixel_bytes == 8 */
2570
2571                //--------------------------------------------------------------
2572                else if (pixel_bytes == 6)
2573                {
2574                   for (i = width; i; i--)
2575                   {
2576                      png_byte v[8];
2577                      int j;
2578                      png_memcpy(v, sptr, 6);
2579                      for (j = 0; j < png_pass_inc[pass]; j++)
2580                      {
2581                         png_memcpy(dp, v, 6);
2582                         dp -= 6;
2583                      }
2584                      sptr -= 6;
2585                   }
2586                } /* end of pixel_bytes == 6 */
2587
2588                //--------------------------------------------------------------
2589                else
2590                {
2591                   for (i = width; i; i--)
2592                   {
2593                      png_byte v[8];
2594                      int j;
2595                      png_memcpy(v, sptr, pixel_bytes);
2596                      for (j = 0; j < png_pass_inc[pass]; j++)
2597                      {
2598                         png_memcpy(dp, v, pixel_bytes);
2599                         dp -= pixel_bytes;
2600                      }
2601                      sptr-= pixel_bytes;
2602                   }
2603                }
2604             } // end of _mmx_supported ========================================
2605
2606             else /* MMX not supported:  use modified C code - takes advantage
2607                   *   of inlining of png_memcpy for a constant */
2608                  /* GRR 19991007:  does it?  or should pixel_bytes in each
2609                   *   block be replaced with immediate value (e.g., 1)? */
2610                  /* GRR 19991017:  replaced with constants in each case */
2611 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
2612             {
2613                if (pixel_bytes == 1)
2614                {
2615                   for (i = width; i; i--)
2616                   {
2617                      int j;
2618                      for (j = 0; j < png_pass_inc[pass]; j++)
2619                      {
2620                         *dp-- = *sptr;
2621                      }
2622                      --sptr;
2623                   }
2624                }
2625                else if (pixel_bytes == 3)
2626                {
2627                   for (i = width; i; i--)
2628                   {
2629                      png_byte v[8];
2630                      int j;
2631                      png_memcpy(v, sptr, 3);
2632                      for (j = 0; j < png_pass_inc[pass]; j++)
2633                      {
2634                         png_memcpy(dp, v, 3);
2635                         dp -= 3;
2636                      }
2637                      sptr -= 3;
2638                   }
2639                }
2640                else if (pixel_bytes == 2)
2641                {
2642                   for (i = width; i; i--)
2643                   {
2644                      png_byte v[8];
2645                      int j;
2646                      png_memcpy(v, sptr, 2);
2647                      for (j = 0; j < png_pass_inc[pass]; j++)
2648                      {
2649                         png_memcpy(dp, v, 2);
2650                         dp -= 2;
2651                      }
2652                      sptr -= 2;
2653                   }
2654                }
2655                else if (pixel_bytes == 4)
2656                {
2657                   for (i = width; i; i--)
2658                   {
2659                      png_byte v[8];
2660                      int j;
2661                      png_memcpy(v, sptr, 4);
2662                      for (j = 0; j < png_pass_inc[pass]; j++)
2663                      {
2664 #ifdef PNG_DEBUG
2665                         if (dp < row || dp+3 > row+png_ptr->row_buf_size)
2666                         {
2667                            printf("dp out of bounds: row=%d, dp=%d, rp=%d\n",
2668                              row, dp, row+png_ptr->row_buf_size);
2669                            printf("row_buf=%d\n",png_ptr->row_buf_size);
2670                         }
2671 #endif
2672                         png_memcpy(dp, v, 4);
2673                         dp -= 4;
2674                      }
2675                      sptr -= 4;
2676                   }
2677                }
2678                else if (pixel_bytes == 6)
2679                {
2680                   for (i = width; i; i--)
2681                   {
2682                      png_byte v[8];
2683                      int j;
2684                      png_memcpy(v, sptr, 6);
2685                      for (j = 0; j < png_pass_inc[pass]; j++)
2686                      {
2687                         png_memcpy(dp, v, 6);
2688                         dp -= 6;
2689                      }
2690                      sptr -= 6;
2691                   }
2692                }
2693                else if (pixel_bytes == 8)
2694                {
2695                   for (i = width; i; i--)
2696                   {
2697                      png_byte v[8];
2698                      int j;
2699                      png_memcpy(v, sptr, 8);
2700                      for (j = 0; j < png_pass_inc[pass]; j++)
2701                      {
2702                         png_memcpy(dp, v, 8);
2703                         dp -= 8;
2704                      }
2705                      sptr -= 8;
2706                   }
2707                }
2708                else     /* GRR:  should never be reached */
2709                {
2710                   for (i = width; i; i--)
2711                   {
2712                      png_byte v[8];
2713                      int j;
2714                      png_memcpy(v, sptr, pixel_bytes);
2715                      for (j = 0; j < png_pass_inc[pass]; j++)
2716                      {
2717                         png_memcpy(dp, v, pixel_bytes);
2718                         dp -= pixel_bytes;
2719                      }
2720                      sptr -= pixel_bytes;
2721                   }
2722                }
2723
2724             } /* end if (MMX not supported) */
2725             break;
2726          }
2727       } /* end switch (row_info->pixel_depth) */
2728
2729       row_info->width = final_width;
2730       row_info->rowbytes = ((final_width *
2731          (png_uint_32)row_info->pixel_depth + 7) >> 3);
2732    }
2733
2734 } /* end png_do_read_interlace() */
2735
2736 #endif /* PNG_HAVE_ASSEMBLER_READ_INTERLACE */
2737 #endif /* PNG_READ_INTERLACING_SUPPORTED */
2738
2739
2740
2741 #if defined(PNG_HAVE_ASSEMBLER_READ_FILTER_ROW)
2742 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
2743
2744 // These variables are utilized in the functions below.  They are declared
2745 // globally here to ensure alignment on 8-byte boundaries.
2746
2747 union uAll {
2748    long long use;
2749    double  align;
2750 } _LBCarryMask = {0x0101010101010101LL},
2751   _HBClearMask = {0x7f7f7f7f7f7f7f7fLL},
2752   _ActiveMask, _ActiveMask2, _ActiveMaskEnd, _ShiftBpp, _ShiftRem;
2753
2754 #ifdef PNG_THREAD_UNSAFE_OK
2755 //===========================================================================//
2756 //                                                                           //
2757 //           P N G _ R E A D _ F I L T E R _ R O W _ M M X _ A V G           //
2758 //                                                                           //
2759 //===========================================================================//
2760
2761 // Optimized code for PNG Average filter decoder
2762
2763 static void /* PRIVATE */
2764 png_read_filter_row_mmx_avg(png_row_infop row_info, png_bytep row,
2765                             png_bytep prev_row)
2766 {
2767    int bpp;
2768    int dummy_value_c;   // fix 'forbidden register 2 (cx) was spilled' error
2769    int dummy_value_S;
2770    int dummy_value_D;
2771
2772    bpp = (row_info->pixel_depth + 7) >> 3;  // get # bytes per pixel
2773    _FullLength  = row_info->rowbytes;       // # of bytes to filter
2774
2775    __asm__ __volatile__ (
2776       // initialize address pointers and offset
2777 #ifdef __PIC__
2778       "pushl %%ebx                 \n\t" // save index to Global Offset Table
2779 #endif
2780 //pre "movl row, %%edi             \n\t" // edi:  Avg(x)
2781       "xorl %%ebx, %%ebx           \n\t" // ebx:  x
2782       "movl %%edi, %%edx           \n\t"
2783 //pre "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
2784 //pre "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
2785       "subl %%ecx, %%edx           \n\t" // edx:  Raw(x-bpp)
2786
2787       "xorl %%eax,%%eax            \n\t"
2788
2789       // Compute the Raw value for the first bpp bytes
2790       //    Raw(x) = Avg(x) + (Prior(x)/2)
2791    "avg_rlp:                       \n\t"
2792       "movb (%%esi,%%ebx,),%%al    \n\t" // load al with Prior(x)
2793       "incl %%ebx                  \n\t"
2794       "shrb %%al                   \n\t" // divide by 2
2795       "addb -1(%%edi,%%ebx,),%%al  \n\t" // add Avg(x); -1 to offset inc ebx
2796 //pre "cmpl bpp, %%ebx             \n\t" // (bpp is preloaded into ecx)
2797       "cmpl %%ecx, %%ebx           \n\t"
2798       "movb %%al,-1(%%edi,%%ebx,)  \n\t" // write Raw(x); -1 to offset inc ebx
2799       "jb avg_rlp                  \n\t" // mov does not affect flags
2800
2801       // get # of bytes to alignment
2802       "movl %%edi, _dif            \n\t" // take start of row
2803       "addl %%ebx, _dif            \n\t" // add bpp
2804       "addl $0xf, _dif             \n\t" // add 7+8 to incr past alignment bdry
2805       "andl $0xfffffff8, _dif      \n\t" // mask to alignment boundary
2806       "subl %%edi, _dif            \n\t" // subtract from start => value ebx at
2807       "jz avg_go                   \n\t" //  alignment
2808
2809       // fix alignment
2810       // Compute the Raw value for the bytes up to the alignment boundary
2811       //    Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
2812       "xorl %%ecx, %%ecx           \n\t"
2813
2814    "avg_lp1:                       \n\t"
2815       "xorl %%eax, %%eax           \n\t"
2816       "movb (%%esi,%%ebx,), %%cl   \n\t" // load cl with Prior(x)
2817       "movb (%%edx,%%ebx,), %%al   \n\t" // load al with Raw(x-bpp)
2818       "addw %%cx, %%ax             \n\t"
2819       "incl %%ebx                  \n\t"
2820       "shrw %%ax                   \n\t" // divide by 2
2821       "addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset inc ebx
2822       "cmpl _dif, %%ebx            \n\t" // check if at alignment boundary
2823       "movb %%al, -1(%%edi,%%ebx,) \n\t" // write Raw(x); -1 to offset inc ebx
2824       "jb avg_lp1                  \n\t" // repeat until at alignment boundary
2825
2826    "avg_go:                        \n\t"
2827       "movl _FullLength, %%eax     \n\t"
2828       "movl %%eax, %%ecx           \n\t"
2829       "subl %%ebx, %%eax           \n\t" // subtract alignment fix
2830       "andl $0x00000007, %%eax     \n\t" // calc bytes over mult of 8
2831       "subl %%eax, %%ecx           \n\t" // drop over bytes from original length
2832       "movl %%ecx, _MMXLength      \n\t"
2833 #ifdef __PIC__
2834       "popl %%ebx                  \n\t" // restore index to Global Offset Table
2835 #endif
2836
2837       : "=c" (dummy_value_c),            // output regs (dummy)
2838         "=S" (dummy_value_S),
2839         "=D" (dummy_value_D)
2840
2841       : "0" (bpp),       // ecx          // input regs
2842         "1" (prev_row),  // esi
2843         "2" (row)        // edi
2844
2845       : "%eax", "%edx"                   // clobber list
2846 #ifndef __PIC__
2847       , "%ebx"
2848 #endif
2849       // GRR: INCLUDE "memory" as clobbered? (_dif, _MMXLength)
2850       // (seems to work fine without...)
2851    );
2852
2853    // now do the math for the rest of the row
2854    switch (bpp)
2855    {
2856       case 3:
2857       {
2858          _ActiveMask.use  = 0x0000000000ffffffLL;
2859          _ShiftBpp.use = 24;    // == 3 * 8
2860          _ShiftRem.use = 40;    // == 64 - 24
2861
2862          __asm__ __volatile__ (
2863             // re-init address pointers and offset
2864             "movq _ActiveMask, %%mm7      \n\t"
2865             "movl _dif, %%ecx             \n\t" // ecx:  x = offset to
2866             "movq _LBCarryMask, %%mm5     \n\t" //  alignment boundary
2867 // preload  "movl row, %%edi              \n\t" // edi:  Avg(x)
2868             "movq _HBClearMask, %%mm4     \n\t"
2869 // preload  "movl prev_row, %%esi         \n\t" // esi:  Prior(x)
2870
2871             // prime the pump:  load the first Raw(x-bpp) data set
2872             "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
2873                                                 // (correct pos. in loop below)
2874          "avg_3lp:                        \n\t"
2875             "movq (%%edi,%%ecx,), %%mm0   \n\t" // load mm0 with Avg(x)
2876             "movq %%mm5, %%mm3            \n\t"
2877             "psrlq _ShiftRem, %%mm2       \n\t" // correct position Raw(x-bpp)
2878                                                 // data
2879             "movq (%%esi,%%ecx,), %%mm1   \n\t" // load mm1 with Prior(x)
2880             "movq %%mm7, %%mm6            \n\t"
2881             "pand %%mm1, %%mm3            \n\t" // get lsb for each prev_row byte
2882             "psrlq $1, %%mm1              \n\t" // divide prev_row bytes by 2
2883             "pand  %%mm4, %%mm1           \n\t" // clear invalid bit 7 of each
2884                                                 // byte
2885             "paddb %%mm1, %%mm0           \n\t" // add (Prev_row/2) to Avg for
2886                                                 // each byte
2887             // add 1st active group (Raw(x-bpp)/2) to average with LBCarry
2888             "movq %%mm3, %%mm1            \n\t" // now use mm1 for getting
2889                                                 // LBCarrys
2890             "pand %%mm2, %%mm1            \n\t" // get LBCarrys for each byte
2891                                                 // where both
2892                                // lsb's were == 1 (only valid for active group)
2893             "psrlq $1, %%mm2              \n\t" // divide raw bytes by 2
2894             "pand  %%mm4, %%mm2           \n\t" // clear invalid bit 7 of each
2895                                                 // byte
2896             "paddb %%mm1, %%mm2           \n\t" // add LBCarrys to (Raw(x-bpp)/2)
2897                                                 // for each byte
2898             "pand %%mm6, %%mm2            \n\t" // leave only Active Group 1
2899                                                 // bytes to add to Avg
2900             "paddb %%mm2, %%mm0           \n\t" // add (Raw/2) + LBCarrys to
2901                                                 // Avg for each Active
2902                                //  byte
2903             // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
2904             "psllq _ShiftBpp, %%mm6       \n\t" // shift the mm6 mask to cover
2905                                                 // bytes 3-5
2906             "movq %%mm0, %%mm2            \n\t" // mov updated Raws to mm2
2907             "psllq _ShiftBpp, %%mm2       \n\t" // shift data to pos. correctly
2908             "movq %%mm3, %%mm1            \n\t" // now use mm1 for getting
2909                                                 // LBCarrys
2910             "pand %%mm2, %%mm1            \n\t" // get LBCarrys for each byte
2911                                                 // where both
2912                                // lsb's were == 1 (only valid for active group)
2913             "psrlq $1, %%mm2              \n\t" // divide raw bytes by 2
2914             "pand  %%mm4, %%mm2           \n\t" // clear invalid bit 7 of each
2915                                                 // byte
2916             "paddb %%mm1, %%mm2           \n\t" // add LBCarrys to (Raw(x-bpp)/2)
2917                                                 // for each byte
2918             "pand %%mm6, %%mm2            \n\t" // leave only Active Group 2
2919                                                 // bytes to add to Avg
2920             "paddb %%mm2, %%mm0           \n\t" // add (Raw/2) + LBCarrys to
2921                                                 // Avg for each Active
2922                                //  byte
2923
2924             // add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry
2925             "psllq _ShiftBpp, %%mm6       \n\t" // shift mm6 mask to cover last
2926                                                 // two
2927                                  // bytes
2928             "movq %%mm0, %%mm2            \n\t" // mov updated Raws to mm2
2929             "psllq _ShiftBpp, %%mm2       \n\t" // shift data to pos. correctly
2930                               // Data only needs to be shifted once here to
2931                               // get the correct x-bpp offset.
2932             "movq %%mm3, %%mm1            \n\t" // now use mm1 for getting
2933                                                 // LBCarrys
2934             "pand %%mm2, %%mm1            \n\t" // get LBCarrys for each byte
2935                                                 // where both
2936                               // lsb's were == 1 (only valid for active group)
2937             "psrlq $1, %%mm2              \n\t" // divide raw bytes by 2
2938             "pand  %%mm4, %%mm2           \n\t" // clear invalid bit 7 of each
2939                                                 // byte
2940             "paddb %%mm1, %%mm2           \n\t" // add LBCarrys to (Raw(x-bpp)/2)
2941                                                 // for each byte
2942             "pand %%mm6, %%mm2            \n\t" // leave only Active Group 2
2943                                                 // bytes to add to Avg
2944             "addl $8, %%ecx               \n\t"
2945             "paddb %%mm2, %%mm0           \n\t" // add (Raw/2) + LBCarrys to
2946                                                 // Avg for each Active
2947                                                 // byte
2948             // now ready to write back to memory
2949             "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
2950             // move updated Raw(x) to use as Raw(x-bpp) for next loop
2951             "cmpl _MMXLength, %%ecx       \n\t"
2952             "movq %%mm0, %%mm2            \n\t" // mov updated Raw(x) to mm2
2953             "jb avg_3lp                   \n\t"
2954
2955             : "=S" (dummy_value_S),             // output regs (dummy)
2956               "=D" (dummy_value_D)
2957
2958             : "0" (prev_row),  // esi           // input regs
2959               "1" (row)        // edi
2960
2961             : "%ecx"                            // clobber list
2962 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
2963             , "%mm0", "%mm1", "%mm2", "%mm3"
2964             , "%mm4", "%mm5", "%mm6", "%mm7"
2965 #endif
2966          );
2967       }
2968       break;  // end 3 bpp
2969
2970       case 6:
2971       case 4:
2972       //case 7:   // who wrote this?  PNG doesn't support 5 or 7 bytes/pixel
2973       //case 5:   // GRR BOGUS
2974       {
2975          _ActiveMask.use  = 0xffffffffffffffffLL; // use shift below to clear
2976                                                   // appropriate inactive bytes
2977          _ShiftBpp.use = bpp << 3;
2978          _ShiftRem.use = 64 - _ShiftBpp.use;
2979
2980          __asm__ __volatile__ (
2981             "movq _HBClearMask, %%mm4    \n\t"
2982
2983             // re-init address pointers and offset
2984             "movl _dif, %%ecx            \n\t" // ecx:  x = offset to
2985                                                // alignment boundary
2986
2987             // load _ActiveMask and clear all bytes except for 1st active group
2988             "movq _ActiveMask, %%mm7     \n\t"
2989 // preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
2990             "psrlq _ShiftRem, %%mm7      \n\t"
2991 // preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
2992             "movq %%mm7, %%mm6           \n\t"
2993             "movq _LBCarryMask, %%mm5    \n\t"
2994             "psllq _ShiftBpp, %%mm6      \n\t" // create mask for 2nd active
2995                                                // group
2996
2997             // prime the pump:  load the first Raw(x-bpp) data set
2998             "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
2999                                           // (we correct pos. in loop below)
3000          "avg_4lp:                       \n\t"
3001             "movq (%%edi,%%ecx,), %%mm0  \n\t"
3002             "psrlq _ShiftRem, %%mm2      \n\t" // shift data to pos. correctly
3003             "movq (%%esi,%%ecx,), %%mm1  \n\t"
3004             // add (Prev_row/2) to average
3005             "movq %%mm5, %%mm3           \n\t"
3006             "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
3007             "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
3008             "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7 of each
3009                                                // byte
3010             "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg for
3011                                                // each byte
3012             // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry
3013             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3014                                                // LBCarrys
3015             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3016                                                // where both
3017                               // lsb's were == 1 (only valid for active group)
3018             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3019             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3020                                                // byte
3021             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3022                                                // for each byte
3023             "pand %%mm7, %%mm2           \n\t" // leave only Active Group 1
3024                                                // bytes to add to Avg
3025             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to Avg
3026                                                // for each Active
3027                               // byte
3028             // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
3029             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3030             "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
3031             "addl $8, %%ecx              \n\t"
3032             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3033                                                // LBCarrys
3034             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3035                                                // where both
3036                               // lsb's were == 1 (only valid for active group)
3037             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3038             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3039                                                // byte
3040             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3041                                                // for each byte
3042             "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
3043                                                // bytes to add to Avg
3044             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
3045                                                // Avg for each Active
3046                               // byte
3047             "cmpl _MMXLength, %%ecx      \n\t"
3048             // now ready to write back to memory
3049             "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
3050             // prep Raw(x-bpp) for next loop
3051             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3052             "jb avg_4lp                  \n\t"
3053
3054             : "=S" (dummy_value_S),            // output regs (dummy)
3055               "=D" (dummy_value_D)
3056
3057             : "0" (prev_row),  // esi          // input regs
3058               "1" (row)        // edi
3059
3060             : "%ecx"                           // clobber list
3061 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3062             , "%mm0", "%mm1", "%mm2", "%mm3"
3063             , "%mm4", "%mm5", "%mm6", "%mm7"
3064 #endif
3065          );
3066       }
3067       break;  // end 4,6 bpp
3068
3069       case 2:
3070       {
3071          _ActiveMask.use  = 0x000000000000ffffLL;
3072          _ShiftBpp.use = 16;   // == 2 * 8
3073          _ShiftRem.use = 48;   // == 64 - 16
3074
3075          __asm__ __volatile__ (
3076             // load _ActiveMask
3077             "movq _ActiveMask, %%mm7     \n\t"
3078             // re-init address pointers and offset
3079             "movl _dif, %%ecx            \n\t" // ecx:  x = offset to alignment
3080                                                // boundary
3081             "movq _LBCarryMask, %%mm5    \n\t"
3082 // preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
3083             "movq _HBClearMask, %%mm4    \n\t"
3084 // preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
3085
3086             // prime the pump:  load the first Raw(x-bpp) data set
3087             "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
3088                               // (we correct pos. in loop below)
3089          "avg_2lp:                       \n\t"
3090             "movq (%%edi,%%ecx,), %%mm0  \n\t"
3091             "psrlq _ShiftRem, %%mm2      \n\t" // shift data to pos. correctly
3092             "movq (%%esi,%%ecx,), %%mm1  \n\t" //  (GRR BUGFIX:  was psllq)
3093             // add (Prev_row/2) to average
3094             "movq %%mm5, %%mm3           \n\t"
3095             "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
3096             "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
3097             "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7 of each
3098                                                // byte
3099             "movq %%mm7, %%mm6           \n\t"
3100             "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg for
3101                                                // each byte
3102
3103             // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry
3104             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3105                                                // LBCarrys
3106             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3107                                                // where both
3108                                                // lsb's were == 1 (only valid
3109                                                // for active group)
3110             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3111             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3112                                                // byte
3113             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3114                                                // for each byte
3115             "pand %%mm6, %%mm2           \n\t" // leave only Active Group 1
3116                                                // bytes to add to Avg
3117             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to Avg
3118                                                // for each Active byte
3119
3120             // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
3121             "psllq _ShiftBpp, %%mm6      \n\t" // shift the mm6 mask to cover
3122                                                // bytes 2 & 3
3123             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3124             "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
3125             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3126                                                // LBCarrys
3127             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3128                                                // where both
3129                                                // lsb's were == 1 (only valid
3130                                                // for active group)
3131             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3132             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3133                                                // byte
3134             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3135                                                // for each byte
3136             "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
3137                                                // bytes to add to Avg
3138             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
3139                                                // Avg for each Active byte
3140
3141             // add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry
3142             "psllq _ShiftBpp, %%mm6      \n\t" // shift the mm6 mask to cover
3143                                                // bytes 4 & 5
3144             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3145             "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
3146             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3147                                                // LBCarrys
3148             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3149                                                // where both lsb's were == 1
3150                                                // (only valid for active group)
3151             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3152             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3153                                                // byte
3154             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3155                                                // for each byte
3156             "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
3157                                                // bytes to add to Avg
3158             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
3159                                                // Avg for each Active byte
3160
3161             // add 4th active group (Raw(x-bpp)/2) to average with _LBCarry
3162             "psllq _ShiftBpp, %%mm6      \n\t" // shift the mm6 mask to cover
3163                                                // bytes 6 & 7
3164             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3165             "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
3166             "addl $8, %%ecx              \n\t"
3167             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3168                                                // LBCarrys
3169             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3170                                                // where both
3171                                                // lsb's were == 1 (only valid
3172                                                // for active group)
3173             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3174             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3175                                                // byte
3176             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3177                                                // for each byte
3178             "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
3179                                                // bytes to add to Avg
3180             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
3181                                                // Avg for each Active byte
3182
3183             "cmpl _MMXLength, %%ecx      \n\t"
3184             // now ready to write back to memory
3185             "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
3186             // prep Raw(x-bpp) for next loop
3187             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3188             "jb avg_2lp                  \n\t"
3189
3190             : "=S" (dummy_value_S),            // output regs (dummy)
3191               "=D" (dummy_value_D)
3192
3193             : "0" (prev_row),  // esi          // input regs
3194               "1" (row)        // edi
3195
3196             : "%ecx"                           // clobber list
3197 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3198             , "%mm0", "%mm1", "%mm2", "%mm3"
3199             , "%mm4", "%mm5", "%mm6", "%mm7"
3200 #endif
3201          );
3202       }
3203       break;  // end 2 bpp
3204
3205       case 1:
3206       {
3207          __asm__ __volatile__ (
3208             // re-init address pointers and offset
3209 #ifdef __PIC__
3210             "pushl %%ebx                 \n\t" // save Global Offset Table index
3211 #endif
3212             "movl _dif, %%ebx            \n\t" // ebx:  x = offset to alignment
3213                                                // boundary
3214 // preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
3215             "cmpl _FullLength, %%ebx     \n\t" // test if offset at end of array
3216             "jnb avg_1end                \n\t"
3217             // do Paeth decode for remaining bytes
3218 // preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
3219             "movl %%edi, %%edx           \n\t"
3220 // preload  "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
3221             "subl %%ecx, %%edx           \n\t" // edx:  Raw(x-bpp)
3222             "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx
3223                                                //  in loop below
3224          "avg_1lp:                       \n\t"
3225             // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
3226             "xorl %%eax, %%eax           \n\t"
3227             "movb (%%esi,%%ebx,), %%cl   \n\t" // load cl with Prior(x)
3228             "movb (%%edx,%%ebx,), %%al   \n\t" // load al with Raw(x-bpp)
3229             "addw %%cx, %%ax             \n\t"
3230             "incl %%ebx                  \n\t"
3231             "shrw %%ax                   \n\t" // divide by 2
3232             "addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset
3233                                                // inc ebx
3234             "cmpl _FullLength, %%ebx     \n\t" // check if at end of array
3235             "movb %%al, -1(%%edi,%%ebx,) \n\t" // write back Raw(x);
3236                          // mov does not affect flags; -1 to offset inc ebx
3237             "jb avg_1lp                  \n\t"
3238
3239          "avg_1end:                      \n\t"
3240 #ifdef __PIC__
3241             "popl %%ebx                  \n\t" // Global Offset Table index
3242 #endif
3243
3244             : "=c" (dummy_value_c),            // output regs (dummy)
3245               "=S" (dummy_value_S),
3246               "=D" (dummy_value_D)
3247
3248             : "0" (bpp),       // ecx          // input regs
3249               "1" (prev_row),  // esi
3250               "2" (row)        // edi
3251
3252             : "%eax", "%edx"                   // clobber list
3253 #ifndef __PIC__
3254             , "%ebx"
3255 #endif
3256          );
3257       }
3258       return;  // end 1 bpp
3259
3260       case 8:
3261       {
3262          __asm__ __volatile__ (
3263             // re-init address pointers and offset
3264             "movl _dif, %%ecx            \n\t" // ecx:  x == offset to alignment
3265             "movq _LBCarryMask, %%mm5    \n\t" //            boundary
3266 // preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
3267             "movq _HBClearMask, %%mm4    \n\t"
3268 // preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
3269
3270             // prime the pump:  load the first Raw(x-bpp) data set
3271             "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
3272                                       // (NO NEED to correct pos. in loop below)
3273
3274          "avg_8lp:                       \n\t"
3275             "movq (%%edi,%%ecx,), %%mm0  \n\t"
3276             "movq %%mm5, %%mm3           \n\t"
3277             "movq (%%esi,%%ecx,), %%mm1  \n\t"
3278             "addl $8, %%ecx              \n\t"
3279             "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
3280             "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
3281             "pand %%mm2, %%mm3           \n\t" // get LBCarrys for each byte
3282                                                //  where both lsb's were == 1
3283             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3284             "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7, each byte
3285             "paddb %%mm3, %%mm0          \n\t" // add LBCarrys to Avg, each byte
3286             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7, each byte
3287             "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg, each
3288             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) to Avg for each
3289             "cmpl _MMXLength, %%ecx      \n\t"
3290             "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
3291             "movq %%mm0, %%mm2           \n\t" // reuse as Raw(x-bpp)
3292             "jb avg_8lp                  \n\t"
3293
3294             : "=S" (dummy_value_S),            // output regs (dummy)
3295               "=D" (dummy_value_D)
3296
3297             : "0" (prev_row),  // esi          // input regs
3298               "1" (row)        // edi
3299
3300             : "%ecx"                           // clobber list
3301 #if 0  /* %mm0, ..., %mm5 not supported by gcc 2.7.2.3 or egcs 1.1 */
3302             , "%mm0", "%mm1", "%mm2"
3303             , "%mm3", "%mm4", "%mm5"
3304 #endif
3305          );
3306       }
3307       break;  // end 8 bpp
3308
3309       default:                  // bpp greater than 8 (!= 1,2,3,4,[5],6,[7],8)
3310       {
3311
3312 #ifdef PNG_DEBUG
3313          // GRR:  PRINT ERROR HERE:  SHOULD NEVER BE REACHED
3314         png_debug(1,
3315         "Internal logic error in pnggccrd (png_read_filter_row_mmx_avg())\n");
3316 #endif
3317
3318 #if 0
3319         __asm__ __volatile__ (
3320             "movq _LBCarryMask, %%mm5    \n\t"
3321             // re-init address pointers and offset
3322             "movl _dif, %%ebx            \n\t" // ebx:  x = offset to
3323                                                // alignment boundary
3324             "movl row, %%edi             \n\t" // edi:  Avg(x)
3325             "movq _HBClearMask, %%mm4    \n\t"
3326             "movl %%edi, %%edx           \n\t"
3327             "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
3328             "subl bpp, %%edx             \n\t" // edx:  Raw(x-bpp)
3329          "avg_Alp:                       \n\t"
3330             "movq (%%edi,%%ebx,), %%mm0  \n\t"
3331             "movq %%mm5, %%mm3           \n\t"
3332             "movq (%%esi,%%ebx,), %%mm1  \n\t"
3333             "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
3334             "movq (%%edx,%%ebx,), %%mm2  \n\t"
3335             "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
3336             "pand %%mm2, %%mm3           \n\t" // get LBCarrys for each byte
3337                                                // where both lsb's were == 1
3338             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3339             "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7 of each
3340                                                // byte
3341             "paddb %%mm3, %%mm0          \n\t" // add LBCarrys to Avg for each
3342                                                // byte
3343             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3344                                                // byte
3345             "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg for
3346                                                // each byte
3347             "addl $8, %%ebx              \n\t"
3348             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) to Avg for each
3349                                                // byte
3350             "cmpl _MMXLength, %%ebx      \n\t"
3351             "movq %%mm0, -8(%%edi,%%ebx,) \n\t"
3352             "jb avg_Alp                  \n\t"
3353
3354             : // FIXASM: output regs/vars go here, e.g.:  "=m" (memory_var)
3355
3356             : // FIXASM: input regs, e.g.:  "c" (count), "S" (src), "D" (dest)
3357
3358             : "%ebx", "%edx", "%edi", "%esi" // CHECKASM: clobber list
3359          );
3360 #endif /* 0 - NEVER REACHED */
3361       }
3362       break;
3363
3364    } // end switch (bpp)
3365
3366    __asm__ __volatile__ (
3367       // MMX acceleration complete; now do clean-up
3368       // check if any remaining bytes left to decode
3369 #ifdef __PIC__
3370       "pushl %%ebx                 \n\t" // save index to Global Offset Table
3371 #endif
3372       "movl _MMXLength, %%ebx      \n\t" // ebx:  x == offset bytes after MMX
3373 //pre "movl row, %%edi             \n\t" // edi:  Avg(x)
3374       "cmpl _FullLength, %%ebx     \n\t" // test if offset at end of array
3375       "jnb avg_end                 \n\t"
3376
3377       // do Avg decode for remaining bytes
3378 //pre "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
3379       "movl %%edi, %%edx           \n\t"
3380 //pre "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
3381       "subl %%ecx, %%edx           \n\t" // edx:  Raw(x-bpp)
3382       "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx below
3383
3384    "avg_lp2:                       \n\t"
3385       // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
3386       "xorl %%eax, %%eax           \n\t"
3387       "movb (%%esi,%%ebx,), %%cl   \n\t" // load cl with Prior(x)
3388       "movb (%%edx,%%ebx,), %%al   \n\t" // load al with Raw(x-bpp)
3389       "addw %%cx, %%ax             \n\t"
3390       "incl %%ebx                  \n\t"
3391       "shrw %%ax                   \n\t" // divide by 2
3392       "addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset inc ebx
3393       "cmpl _FullLength, %%ebx     \n\t" // check if at end of array
3394       "movb %%al, -1(%%edi,%%ebx,) \n\t" // write back Raw(x) [mov does not
3395       "jb avg_lp2                  \n\t" //  affect flags; -1 to offset inc ebx]
3396
3397    "avg_end:                       \n\t"
3398       "EMMS                        \n\t" // end MMX; prep for poss. FP instrs.
3399 #ifdef __PIC__
3400       "popl %%ebx                  \n\t" // restore index to Global Offset Table
3401 #endif
3402
3403       : "=c" (dummy_value_c),            // output regs (dummy)
3404         "=S" (dummy_value_S),
3405         "=D" (dummy_value_D)
3406
3407       : "0" (bpp),       // ecx          // input regs
3408         "1" (prev_row),  // esi
3409         "2" (row)        // edi
3410
3411       : "%eax", "%edx"                   // clobber list
3412 #ifndef __PIC__
3413       , "%ebx"
3414 #endif
3415    );
3416
3417 } /* end png_read_filter_row_mmx_avg() */
3418 #endif
3419
3420
3421
3422 #ifdef PNG_THREAD_UNSAFE_OK
3423 //===========================================================================//
3424 //                                                                           //
3425 //         P N G _ R E A D _ F I L T E R _ R O W _ M M X _ P A E T H         //
3426 //                                                                           //
3427 //===========================================================================//
3428
3429 // Optimized code for PNG Paeth filter decoder
3430
3431 static void /* PRIVATE */
3432 png_read_filter_row_mmx_paeth(png_row_infop row_info, png_bytep row,
3433                               png_bytep prev_row)
3434 {
3435    int bpp;
3436    int dummy_value_c;   // fix 'forbidden register 2 (cx) was spilled' error
3437    int dummy_value_S;
3438    int dummy_value_D;
3439
3440    bpp = (row_info->pixel_depth + 7) >> 3; // Get # bytes per pixel
3441    _FullLength  = row_info->rowbytes; // # of bytes to filter
3442
3443    __asm__ __volatile__ (
3444 #ifdef __PIC__
3445       "pushl %%ebx                 \n\t" // save index to Global Offset Table
3446 #endif
3447       "xorl %%ebx, %%ebx           \n\t" // ebx:  x offset
3448 //pre "movl row, %%edi             \n\t"
3449       "xorl %%edx, %%edx           \n\t" // edx:  x-bpp offset
3450 //pre "movl prev_row, %%esi        \n\t"
3451       "xorl %%eax, %%eax           \n\t"
3452
3453       // Compute the Raw value for the first bpp bytes
3454       // Note: the formula works out to be always
3455       //   Paeth(x) = Raw(x) + Prior(x)      where x < bpp
3456    "paeth_rlp:                     \n\t"
3457       "movb (%%edi,%%ebx,), %%al   \n\t"
3458       "addb (%%esi,%%ebx,), %%al   \n\t"
3459       "incl %%ebx                  \n\t"
3460 //pre "cmpl bpp, %%ebx             \n\t" (bpp is preloaded into ecx)
3461       "cmpl %%ecx, %%ebx           \n\t"
3462       "movb %%al, -1(%%edi,%%ebx,) \n\t"
3463       "jb paeth_rlp                \n\t"
3464       // get # of bytes to alignment
3465       "movl %%edi, _dif            \n\t" // take start of row
3466       "addl %%ebx, _dif            \n\t" // add bpp
3467       "xorl %%ecx, %%ecx           \n\t"
3468       "addl $0xf, _dif             \n\t" // add 7 + 8 to incr past alignment
3469                                          // boundary
3470       "andl $0xfffffff8, _dif      \n\t" // mask to alignment boundary
3471       "subl %%edi, _dif            \n\t" // subtract from start ==> value ebx
3472                                          // at alignment
3473       "jz paeth_go                 \n\t"
3474       // fix alignment
3475
3476    "paeth_lp1:                     \n\t"
3477       "xorl %%eax, %%eax           \n\t"
3478       // pav = p - a = (a + b - c) - a = b - c
3479       "movb (%%esi,%%ebx,), %%al   \n\t" // load Prior(x) into al
3480       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
3481       "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
3482       "movl %%eax, _patemp         \n\t" // Save pav for later use
3483       "xorl %%eax, %%eax           \n\t"
3484       // pbv = p - b = (a + b - c) - b = a - c
3485       "movb (%%edi,%%edx,), %%al   \n\t" // load Raw(x-bpp) into al
3486       "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
3487       "movl %%eax, %%ecx           \n\t"
3488       // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3489       "addl _patemp, %%eax         \n\t" // pcv = pav + pbv
3490       // pc = abs(pcv)
3491       "testl $0x80000000, %%eax    \n\t"
3492       "jz paeth_pca                \n\t"
3493       "negl %%eax                  \n\t" // reverse sign of neg values
3494
3495    "paeth_pca:                     \n\t"
3496       "movl %%eax, _pctemp         \n\t" // save pc for later use
3497       // pb = abs(pbv)
3498       "testl $0x80000000, %%ecx    \n\t"
3499       "jz paeth_pba                \n\t"
3500       "negl %%ecx                  \n\t" // reverse sign of neg values
3501
3502    "paeth_pba:                     \n\t"
3503       "movl %%ecx, _pbtemp         \n\t" // save pb for later use
3504       // pa = abs(pav)
3505       "movl _patemp, %%eax         \n\t"
3506       "testl $0x80000000, %%eax    \n\t"
3507       "jz paeth_paa                \n\t"
3508       "negl %%eax                  \n\t" // reverse sign of neg values
3509
3510    "paeth_paa:                     \n\t"
3511       "movl %%eax, _patemp         \n\t" // save pa for later use
3512       // test if pa <= pb
3513       "cmpl %%ecx, %%eax           \n\t"
3514       "jna paeth_abb               \n\t"
3515       // pa > pb; now test if pb <= pc
3516       "cmpl _pctemp, %%ecx         \n\t"
3517       "jna paeth_bbc               \n\t"
3518       // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
3519       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
3520       "jmp paeth_paeth             \n\t"
3521
3522    "paeth_bbc:                     \n\t"
3523       // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
3524       "movb (%%esi,%%ebx,), %%cl   \n\t" // load Prior(x) into cl
3525       "jmp paeth_paeth             \n\t"
3526
3527    "paeth_abb:                     \n\t"
3528       // pa <= pb; now test if pa <= pc
3529       "cmpl _pctemp, %%eax         \n\t"
3530       "jna paeth_abc               \n\t"
3531       // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
3532       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
3533       "jmp paeth_paeth             \n\t"
3534
3535    "paeth_abc:                     \n\t"
3536       // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
3537       "movb (%%edi,%%edx,), %%cl   \n\t" // load Raw(x-bpp) into cl
3538
3539    "paeth_paeth:                   \n\t"
3540       "incl %%ebx                  \n\t"
3541       "incl %%edx                  \n\t"
3542       // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
3543       "addb %%cl, -1(%%edi,%%ebx,) \n\t"
3544       "cmpl _dif, %%ebx            \n\t"
3545       "jb paeth_lp1                \n\t"
3546
3547    "paeth_go:                      \n\t"
3548       "movl _FullLength, %%ecx     \n\t"
3549       "movl %%ecx, %%eax           \n\t"
3550       "subl %%ebx, %%eax           \n\t" // subtract alignment fix
3551       "andl $0x00000007, %%eax     \n\t" // calc bytes over mult of 8
3552       "subl %%eax, %%ecx           \n\t" // drop over bytes from original length
3553       "movl %%ecx, _MMXLength      \n\t"
3554 #ifdef __PIC__
3555       "popl %%ebx                  \n\t" // restore index to Global Offset Table
3556 #endif
3557
3558       : "=c" (dummy_value_c),            // output regs (dummy)
3559         "=S" (dummy_value_S),
3560         "=D" (dummy_value_D)
3561
3562       : "0" (bpp),       // ecx          // input regs
3563         "1" (prev_row),  // esi
3564         "2" (row)        // edi
3565
3566       : "%eax", "%edx"                   // clobber list
3567 #ifndef __PIC__
3568       , "%ebx"
3569 #endif
3570    );
3571
3572    // now do the math for the rest of the row
3573    switch (bpp)
3574    {
3575       case 3:
3576       {
3577          _ActiveMask.use = 0x0000000000ffffffLL;
3578          _ActiveMaskEnd.use = 0xffff000000000000LL;
3579          _ShiftBpp.use = 24;    // == bpp(3) * 8
3580          _ShiftRem.use = 40;    // == 64 - 24
3581
3582          __asm__ __volatile__ (
3583             "movl _dif, %%ecx            \n\t"
3584 // preload  "movl row, %%edi             \n\t"
3585 // preload  "movl prev_row, %%esi        \n\t"
3586             "pxor %%mm0, %%mm0           \n\t"
3587             // prime the pump:  load the first Raw(x-bpp) data set
3588             "movq -8(%%edi,%%ecx,), %%mm1 \n\t"
3589          "paeth_3lp:                     \n\t"
3590             "psrlq _ShiftRem, %%mm1      \n\t" // shift last 3 bytes to 1st
3591                                                // 3 bytes
3592             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
3593             "punpcklbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
3594             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // prep c=Prior(x-bpp) bytes
3595             "punpcklbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
3596             "psrlq _ShiftRem, %%mm3      \n\t" // shift last 3 bytes to 1st
3597                                                // 3 bytes
3598             // pav = p - a = (a + b - c) - a = b - c
3599             "movq %%mm2, %%mm4           \n\t"
3600             "punpcklbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
3601             // pbv = p - b = (a + b - c) - b = a - c
3602             "movq %%mm1, %%mm5           \n\t"
3603             "psubw %%mm3, %%mm4          \n\t"
3604             "pxor %%mm7, %%mm7           \n\t"
3605             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3606             "movq %%mm4, %%mm6           \n\t"
3607             "psubw %%mm3, %%mm5          \n\t"
3608
3609             // pa = abs(p-a) = abs(pav)
3610             // pb = abs(p-b) = abs(pbv)
3611             // pc = abs(p-c) = abs(pcv)
3612             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
3613             "paddw %%mm5, %%mm6          \n\t"
3614             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
3615             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
3616             "psubw %%mm0, %%mm4          \n\t"
3617             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
3618             "psubw %%mm0, %%mm4          \n\t"
3619             "psubw %%mm7, %%mm5          \n\t"
3620             "pxor %%mm0, %%mm0           \n\t"
3621             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
3622             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
3623             "psubw %%mm7, %%mm5          \n\t"
3624             "psubw %%mm0, %%mm6          \n\t"
3625             //  test pa <= pb
3626             "movq %%mm4, %%mm7           \n\t"
3627             "psubw %%mm0, %%mm6          \n\t"
3628             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
3629             "movq %%mm7, %%mm0           \n\t"
3630             // use mm7 mask to merge pa & pb
3631             "pand %%mm7, %%mm5           \n\t"
3632             // use mm0 mask copy to merge a & b
3633             "pand %%mm0, %%mm2           \n\t"
3634             "pandn %%mm4, %%mm7          \n\t"
3635             "pandn %%mm1, %%mm0          \n\t"
3636             "paddw %%mm5, %%mm7          \n\t"
3637             "paddw %%mm2, %%mm0          \n\t"
3638             //  test  ((pa <= pb)? pa:pb) <= pc
3639             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
3640             "pxor %%mm1, %%mm1           \n\t"
3641             "pand %%mm7, %%mm3           \n\t"
3642             "pandn %%mm0, %%mm7          \n\t"
3643             "paddw %%mm3, %%mm7          \n\t"
3644             "pxor %%mm0, %%mm0           \n\t"
3645             "packuswb %%mm1, %%mm7       \n\t"
3646             "movq (%%esi,%%ecx,), %%mm3  \n\t" // load c=Prior(x-bpp)
3647             "pand _ActiveMask, %%mm7     \n\t"
3648             "movq %%mm3, %%mm2           \n\t" // load b=Prior(x) step 1
3649             "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
3650             "punpcklbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
3651             "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
3652             "movq %%mm7, %%mm1           \n\t" // now mm1 will be used as
3653                                                // Raw(x-bpp)
3654             // now do Paeth for 2nd set of bytes (3-5)
3655             "psrlq _ShiftBpp, %%mm2      \n\t" // load b=Prior(x) step 2
3656             "punpcklbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
3657             "pxor %%mm7, %%mm7           \n\t"
3658             "punpcklbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
3659             // pbv = p - b = (a + b - c) - b = a - c
3660             "movq %%mm1, %%mm5           \n\t"
3661             // pav = p - a = (a + b - c) - a = b - c
3662             "movq %%mm2, %%mm4           \n\t"
3663             "psubw %%mm3, %%mm5          \n\t"
3664             "psubw %%mm3, %%mm4          \n\t"
3665             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) =
3666             //       pav + pbv = pbv + pav
3667             "movq %%mm5, %%mm6           \n\t"
3668             "paddw %%mm4, %%mm6          \n\t"
3669
3670             // pa = abs(p-a) = abs(pav)
3671             // pb = abs(p-b) = abs(pbv)
3672             // pc = abs(p-c) = abs(pcv)
3673             "pcmpgtw %%mm5, %%mm0        \n\t" // create mask pbv bytes < 0
3674             "pcmpgtw %%mm4, %%mm7        \n\t" // create mask pav bytes < 0
3675             "pand %%mm5, %%mm0           \n\t" // only pbv bytes < 0 in mm0
3676             "pand %%mm4, %%mm7           \n\t" // only pav bytes < 0 in mm7
3677             "psubw %%mm0, %%mm5          \n\t"
3678             "psubw %%mm7, %%mm4          \n\t"
3679             "psubw %%mm0, %%mm5          \n\t"
3680             "psubw %%mm7, %%mm4          \n\t"
3681             "pxor %%mm0, %%mm0           \n\t"
3682             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
3683             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
3684             "psubw %%mm0, %%mm6          \n\t"
3685             //  test pa <= pb
3686             "movq %%mm4, %%mm7           \n\t"
3687             "psubw %%mm0, %%mm6          \n\t"
3688             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
3689             "movq %%mm7, %%mm0           \n\t"
3690             // use mm7 mask to merge pa & pb
3691             "pand %%mm7, %%mm5           \n\t"
3692             // use mm0 mask copy to merge a & b
3693             "pand %%mm0, %%mm2           \n\t"
3694             "pandn %%mm4, %%mm7          \n\t"
3695             "pandn %%mm1, %%mm0          \n\t"
3696             "paddw %%mm5, %%mm7          \n\t"
3697             "paddw %%mm2, %%mm0          \n\t"
3698             //  test  ((pa <= pb)? pa:pb) <= pc
3699             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
3700             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
3701             "pand %%mm7, %%mm3           \n\t"
3702             "pandn %%mm0, %%mm7          \n\t"
3703             "pxor %%mm1, %%mm1           \n\t"
3704             "paddw %%mm3, %%mm7          \n\t"
3705             "pxor %%mm0, %%mm0           \n\t"
3706             "packuswb %%mm1, %%mm7       \n\t"
3707             "movq %%mm2, %%mm3           \n\t" // load c=Prior(x-bpp) step 1
3708             "pand _ActiveMask, %%mm7     \n\t"
3709             "punpckhbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
3710             "psllq _ShiftBpp, %%mm7      \n\t" // shift bytes to 2nd group of
3711                                                // 3 bytes
3712              // pav = p - a = (a + b - c) - a = b - c
3713             "movq %%mm2, %%mm4           \n\t"
3714             "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
3715             "psllq _ShiftBpp, %%mm3      \n\t" // load c=Prior(x-bpp) step 2
3716             "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
3717             "movq %%mm7, %%mm1           \n\t"
3718             "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
3719             "psllq _ShiftBpp, %%mm1      \n\t" // shift bytes
3720                                     // now mm1 will be used as Raw(x-bpp)
3721             // now do Paeth for 3rd, and final, set of bytes (6-7)
3722             "pxor %%mm7, %%mm7           \n\t"
3723             "punpckhbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
3724             "psubw %%mm3, %%mm4          \n\t"
3725             // pbv = p - b = (a + b - c) - b = a - c
3726             "movq %%mm1, %%mm5           \n\t"
3727             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3728             "movq %%mm4, %%mm6           \n\t"
3729             "psubw %%mm3, %%mm5          \n\t"
3730             "pxor %%mm0, %%mm0           \n\t"
3731             "paddw %%mm5, %%mm6          \n\t"
3732
3733             // pa = abs(p-a) = abs(pav)
3734             // pb = abs(p-b) = abs(pbv)
3735             // pc = abs(p-c) = abs(pcv)
3736             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
3737             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
3738             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
3739             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
3740             "psubw %%mm0, %%mm4          \n\t"
3741             "psubw %%mm7, %%mm5          \n\t"
3742             "psubw %%mm0, %%mm4          \n\t"
3743             "psubw %%mm7, %%mm5          \n\t"
3744             "pxor %%mm0, %%mm0           \n\t"
3745             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
3746             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
3747             "psubw %%mm0, %%mm6          \n\t"
3748             //  test pa <= pb
3749             "movq %%mm4, %%mm7           \n\t"
3750             "psubw %%mm0, %%mm6          \n\t"
3751             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
3752             "movq %%mm7, %%mm0           \n\t"
3753             // use mm0 mask copy to merge a & b
3754             "pand %%mm0, %%mm2           \n\t"
3755             // use mm7 mask to merge pa & pb
3756             "pand %%mm7, %%mm5           \n\t"
3757             "pandn %%mm1, %%mm0          \n\t"
3758             "pandn %%mm4, %%mm7          \n\t"
3759             "paddw %%mm2, %%mm0          \n\t"
3760             "paddw %%mm5, %%mm7          \n\t"
3761             //  test  ((pa <= pb)? pa:pb) <= pc
3762             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
3763             "pand %%mm7, %%mm3           \n\t"
3764             "pandn %%mm0, %%mm7          \n\t"
3765             "paddw %%mm3, %%mm7          \n\t"
3766             "pxor %%mm1, %%mm1           \n\t"
3767             "packuswb %%mm7, %%mm1       \n\t"
3768             // step ecx to next set of 8 bytes and repeat loop til done
3769             "addl $8, %%ecx              \n\t"
3770             "pand _ActiveMaskEnd, %%mm1  \n\t"
3771             "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with
3772                                                  // Raw(x)
3773
3774             "cmpl _MMXLength, %%ecx      \n\t"
3775             "pxor %%mm0, %%mm0           \n\t" // pxor does not affect flags
3776             "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
3777                                  // mm1 will be used as Raw(x-bpp) next loop
3778                            // mm3 ready to be used as Prior(x-bpp) next loop
3779             "jb paeth_3lp                \n\t"
3780
3781             : "=S" (dummy_value_S),             // output regs (dummy)
3782               "=D" (dummy_value_D)
3783
3784             : "0" (prev_row),  // esi           // input regs
3785               "1" (row)        // edi
3786
3787             : "%ecx"                            // clobber list
3788 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3789             , "%mm0", "%mm1", "%mm2", "%mm3"
3790             , "%mm4", "%mm5", "%mm6", "%mm7"
3791 #endif
3792          );
3793       }
3794       break;  // end 3 bpp
3795
3796       case 6:
3797       //case 7:   // GRR BOGUS
3798       //case 5:   // GRR BOGUS
3799       {
3800          _ActiveMask.use  = 0x00000000ffffffffLL;
3801          _ActiveMask2.use = 0xffffffff00000000LL;
3802          _ShiftBpp.use = bpp << 3;    // == bpp * 8
3803          _ShiftRem.use = 64 - _ShiftBpp.use;
3804
3805          __asm__ __volatile__ (
3806             "movl _dif, %%ecx            \n\t"
3807 // preload  "movl row, %%edi             \n\t"
3808 // preload  "movl prev_row, %%esi        \n\t"
3809             // prime the pump:  load the first Raw(x-bpp) data set
3810             "movq -8(%%edi,%%ecx,), %%mm1 \n\t"
3811             "pxor %%mm0, %%mm0           \n\t"
3812
3813          "paeth_6lp:                     \n\t"
3814             // must shift to position Raw(x-bpp) data
3815             "psrlq _ShiftRem, %%mm1      \n\t"
3816             // do first set of 4 bytes
3817             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
3818             "punpcklbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
3819             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
3820             "punpcklbw %%mm0, %%mm2      \n\t" // unpack Low bytes of b
3821             // must shift to position Prior(x-bpp) data
3822             "psrlq _ShiftRem, %%mm3      \n\t"
3823             // pav = p - a = (a + b - c) - a = b - c
3824             "movq %%mm2, %%mm4           \n\t"
3825             "punpcklbw %%mm0, %%mm3      \n\t" // unpack Low bytes of c
3826             // pbv = p - b = (a + b - c) - b = a - c
3827             "movq %%mm1, %%mm5           \n\t"
3828             "psubw %%mm3, %%mm4          \n\t"
3829             "pxor %%mm7, %%mm7           \n\t"
3830             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3831             "movq %%mm4, %%mm6           \n\t"
3832             "psubw %%mm3, %%mm5          \n\t"
3833             // pa = abs(p-a) = abs(pav)
3834             // pb = abs(p-b) = abs(pbv)
3835             // pc = abs(p-c) = abs(pcv)
3836             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
3837             "paddw %%mm5, %%mm6          \n\t"
3838             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
3839             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
3840             "psubw %%mm0, %%mm4          \n\t"
3841             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
3842             "psubw %%mm0, %%mm4          \n\t"
3843             "psubw %%mm7, %%mm5          \n\t"
3844             "pxor %%mm0, %%mm0           \n\t"
3845             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
3846             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
3847             "psubw %%mm7, %%mm5          \n\t"
3848             "psubw %%mm0, %%mm6          \n\t"
3849             //  test pa <= pb
3850             "movq %%mm4, %%mm7           \n\t"
3851             "psubw %%mm0, %%mm6          \n\t"
3852             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
3853             "movq %%mm7, %%mm0           \n\t"
3854             // use mm7 mask to merge pa & pb
3855             "pand %%mm7, %%mm5           \n\t"
3856             // use mm0 mask copy to merge a & b
3857             "pand %%mm0, %%mm2           \n\t"
3858             "pandn %%mm4, %%mm7          \n\t"
3859             "pandn %%mm1, %%mm0          \n\t"
3860             "paddw %%mm5, %%mm7          \n\t"
3861             "paddw %%mm2, %%mm0          \n\t"
3862             //  test  ((pa <= pb)? pa:pb) <= pc
3863             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
3864             "pxor %%mm1, %%mm1           \n\t"
3865             "pand %%mm7, %%mm3           \n\t"
3866             "pandn %%mm0, %%mm7          \n\t"
3867             "paddw %%mm3, %%mm7          \n\t"
3868             "pxor %%mm0, %%mm0           \n\t"
3869             "packuswb %%mm1, %%mm7       \n\t"
3870             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // load c=Prior(x-bpp)
3871             "pand _ActiveMask, %%mm7     \n\t"
3872             "psrlq _ShiftRem, %%mm3      \n\t"
3873             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x) step 1
3874             "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor and Raw(x)
3875             "movq %%mm2, %%mm6           \n\t"
3876             "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
3877             "movq -8(%%edi,%%ecx,), %%mm1 \n\t"
3878             "psllq _ShiftBpp, %%mm6      \n\t"
3879             "movq %%mm7, %%mm5           \n\t"
3880             "psrlq _ShiftRem, %%mm1      \n\t"
3881             "por %%mm6, %%mm3            \n\t"
3882             "psllq _ShiftBpp, %%mm5      \n\t"
3883             "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
3884             "por %%mm5, %%mm1            \n\t"
3885             // do second set of 4 bytes
3886             "punpckhbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
3887             "punpckhbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
3888             // pav = p - a = (a + b - c) - a = b - c
3889             "movq %%mm2, %%mm4           \n\t"
3890             // pbv = p - b = (a + b - c) - b = a - c
3891             "movq %%mm1, %%mm5           \n\t"
3892             "psubw %%mm3, %%mm4          \n\t"
3893             "pxor %%mm7, %%mm7           \n\t"
3894             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3895             "movq %%mm4, %%mm6           \n\t"
3896             "psubw %%mm3, %%mm5          \n\t"
3897             // pa = abs(p-a) = abs(pav)
3898             // pb = abs(p-b) = abs(pbv)
3899             // pc = abs(p-c) = abs(pcv)
3900             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
3901             "paddw %%mm5, %%mm6          \n\t"
3902             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
3903             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
3904             "psubw %%mm0, %%mm4          \n\t"
3905             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
3906             "psubw %%mm0, %%mm4          \n\t"
3907             "psubw %%mm7, %%mm5          \n\t"
3908             "pxor %%mm0, %%mm0           \n\t"
3909             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
3910             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
3911             "psubw %%mm7, %%mm5          \n\t"
3912             "psubw %%mm0, %%mm6          \n\t"
3913             //  test pa <= pb
3914             "movq %%mm4, %%mm7           \n\t"
3915             "psubw %%mm0, %%mm6          \n\t"
3916             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
3917             "movq %%mm7, %%mm0           \n\t"
3918             // use mm7 mask to merge pa & pb
3919             "pand %%mm7, %%mm5           \n\t"
3920             // use mm0 mask copy to merge a & b
3921             "pand %%mm0, %%mm2           \n\t"
3922             "pandn %%mm4, %%mm7          \n\t"
3923             "pandn %%mm1, %%mm0          \n\t"
3924             "paddw %%mm5, %%mm7          \n\t"
3925             "paddw %%mm2, %%mm0          \n\t"
3926             //  test  ((pa <= pb)? pa:pb) <= pc
3927             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
3928             "pxor %%mm1, %%mm1           \n\t"
3929             "pand %%mm7, %%mm3           \n\t"
3930             "pandn %%mm0, %%mm7          \n\t"
3931             "pxor %%mm1, %%mm1           \n\t"
3932             "paddw %%mm3, %%mm7          \n\t"
3933             "pxor %%mm0, %%mm0           \n\t"
3934             // step ecx to next set of 8 bytes and repeat loop til done
3935             "addl $8, %%ecx              \n\t"
3936             "packuswb %%mm7, %%mm1       \n\t"
3937             "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with Raw(x)
3938             "cmpl _MMXLength, %%ecx      \n\t"
3939             "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
3940                                 // mm1 will be used as Raw(x-bpp) next loop
3941             "jb paeth_6lp                \n\t"
3942
3943             : "=S" (dummy_value_S),             // output regs (dummy)
3944               "=D" (dummy_value_D)
3945
3946             : "0" (prev_row),  // esi           // input regs
3947               "1" (row)        // edi
3948
3949             : "%ecx"                            // clobber list
3950 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3951             , "%mm0", "%mm1", "%mm2", "%mm3"
3952             , "%mm4", "%mm5", "%mm6", "%mm7"
3953 #endif
3954          );
3955       }
3956       break;  // end 6 bpp
3957
3958       case 4:
3959       {
3960          _ActiveMask.use  = 0x00000000ffffffffLL;
3961
3962          __asm__ __volatile__ (
3963             "movl _dif, %%ecx            \n\t"
3964 // preload  "movl row, %%edi             \n\t"
3965 // preload  "movl prev_row, %%esi        \n\t"
3966             "pxor %%mm0, %%mm0           \n\t"
3967             // prime the pump:  load the first Raw(x-bpp) data set
3968             "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // only time should need to read
3969                                      //  a=Raw(x-bpp) bytes
3970          "paeth_4lp:                     \n\t"
3971             // do first set of 4 bytes
3972             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
3973             "punpckhbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
3974             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
3975             "punpcklbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
3976             // pav = p - a = (a + b - c) - a = b - c
3977             "movq %%mm2, %%mm4           \n\t"
3978             "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
3979             // pbv = p - b = (a + b - c) - b = a - c
3980             "movq %%mm1, %%mm5           \n\t"
3981             "psubw %%mm3, %%mm4          \n\t"
3982             "pxor %%mm7, %%mm7           \n\t"
3983             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3984             "movq %%mm4, %%mm6           \n\t"
3985             "psubw %%mm3, %%mm5          \n\t"
3986             // pa = abs(p-a) = abs(pav)
3987             // pb = abs(p-b) = abs(pbv)
3988             // pc = abs(p-c) = abs(pcv)
3989             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
3990             "paddw %%mm5, %%mm6          \n\t"
3991             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
3992             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
3993             "psubw %%mm0, %%mm4          \n\t"
3994             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
3995             "psubw %%mm0, %%mm4          \n\t"
3996             "psubw %%mm7, %%mm5          \n\t"
3997             "pxor %%mm0, %%mm0           \n\t"
3998             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
3999             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
4000             "psubw %%mm7, %%mm5          \n\t"
4001             "psubw %%mm0, %%mm6          \n\t"
4002             //  test pa <= pb
4003             "movq %%mm4, %%mm7           \n\t"
4004             "psubw %%mm0, %%mm6          \n\t"
4005             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
4006             "movq %%mm7, %%mm0           \n\t"
4007             // use mm7 mask to merge pa & pb
4008             "pand %%mm7, %%mm5           \n\t"
4009             // use mm0 mask copy to merge a & b
4010             "pand %%mm0, %%mm2           \n\t"
4011             "pandn %%mm4, %%mm7          \n\t"
4012             "pandn %%mm1, %%mm0          \n\t"
4013             "paddw %%mm5, %%mm7          \n\t"
4014             "paddw %%mm2, %%mm0          \n\t"
4015             //  test  ((pa <= pb)? pa:pb) <= pc
4016             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
4017             "pxor %%mm1, %%mm1           \n\t"
4018             "pand %%mm7, %%mm3           \n\t"
4019             "pandn %%mm0, %%mm7          \n\t"
4020             "paddw %%mm3, %%mm7          \n\t"
4021             "pxor %%mm0, %%mm0           \n\t"
4022             "packuswb %%mm1, %%mm7       \n\t"
4023             "movq (%%esi,%%ecx,), %%mm3  \n\t" // load c=Prior(x-bpp)
4024             "pand _ActiveMask, %%mm7     \n\t"
4025             "movq %%mm3, %%mm2           \n\t" // load b=Prior(x) step 1
4026             "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
4027             "punpcklbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
4028             "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
4029             "movq %%mm7, %%mm1           \n\t" // now mm1 will be used as Raw(x-bpp)
4030             // do second set of 4 bytes
4031             "punpckhbw %%mm0, %%mm2      \n\t" // unpack Low bytes of b
4032             "punpcklbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
4033             // pav = p - a = (a + b - c) - a = b - c
4034             "movq %%mm2, %%mm4           \n\t"
4035             // pbv = p - b = (a + b - c) - b = a - c
4036             "movq %%mm1, %%mm5           \n\t"
4037             "psubw %%mm3, %%mm4          \n\t"
4038             "pxor %%mm7, %%mm7           \n\t"
4039             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4040             "movq %%mm4, %%mm6           \n\t"
4041             "psubw %%mm3, %%mm5          \n\t"
4042             // pa = abs(p-a) = abs(pav)
4043             // pb = abs(p-b) = abs(pbv)
4044             // pc = abs(p-c) = abs(pcv)
4045             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
4046             "paddw %%mm5, %%mm6          \n\t"
4047             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
4048             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
4049             "psubw %%mm0, %%mm4          \n\t"
4050             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
4051             "psubw %%mm0, %%mm4          \n\t"
4052             "psubw %%mm7, %%mm5          \n\t"
4053             "pxor %%mm0, %%mm0           \n\t"
4054             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
4055             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
4056             "psubw %%mm7, %%mm5          \n\t"
4057             "psubw %%mm0, %%mm6          \n\t"
4058             //  test pa <= pb
4059             "movq %%mm4, %%mm7           \n\t"
4060             "psubw %%mm0, %%mm6          \n\t"
4061             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
4062             "movq %%mm7, %%mm0           \n\t"
4063             // use mm7 mask to merge pa & pb
4064             "pand %%mm7, %%mm5           \n\t"
4065             // use mm0 mask copy to merge a & b
4066             "pand %%mm0, %%mm2           \n\t"
4067             "pandn %%mm4, %%mm7          \n\t"
4068             "pandn %%mm1, %%mm0          \n\t"
4069             "paddw %%mm5, %%mm7          \n\t"
4070             "paddw %%mm2, %%mm0          \n\t"
4071             //  test  ((pa <= pb)? pa:pb) <= pc
4072             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
4073             "pxor %%mm1, %%mm1           \n\t"
4074             "pand %%mm7, %%mm3           \n\t"
4075             "pandn %%mm0, %%mm7          \n\t"
4076             "pxor %%mm1, %%mm1           \n\t"
4077             "paddw %%mm3, %%mm7          \n\t"
4078             "pxor %%mm0, %%mm0           \n\t"
4079             // step ecx to next set of 8 bytes and repeat loop til done
4080             "addl $8, %%ecx              \n\t"
4081             "packuswb %%mm7, %%mm1       \n\t"
4082             "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add predictor with Raw(x)
4083             "cmpl _MMXLength, %%ecx      \n\t"
4084             "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
4085                                 // mm1 will be used as Raw(x-bpp) next loop
4086             "jb paeth_4lp                \n\t"
4087
4088             : "=S" (dummy_value_S),             // output regs (dummy)
4089               "=D" (dummy_value_D)
4090
4091             : "0" (prev_row),  // esi           // input regs
4092               "1" (row)        // edi
4093
4094             : "%ecx"                            // clobber list
4095 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
4096             , "%mm0", "%mm1", "%mm2", "%mm3"
4097             , "%mm4", "%mm5", "%mm6", "%mm7"
4098 #endif
4099          );
4100       }
4101       break;  // end 4 bpp
4102
4103       case 8:                          // bpp == 8
4104       {
4105          _ActiveMask.use  = 0x00000000ffffffffLL;
4106
4107          __asm__ __volatile__ (
4108             "movl _dif, %%ecx            \n\t"
4109 // preload  "movl row, %%edi             \n\t"
4110 // preload  "movl prev_row, %%esi        \n\t"
4111             "pxor %%mm0, %%mm0           \n\t"
4112             // prime the pump:  load the first Raw(x-bpp) data set
4113             "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // only time should need to read
4114                                        //  a=Raw(x-bpp) bytes
4115          "paeth_8lp:                     \n\t"
4116             // do first set of 4 bytes
4117             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
4118             "punpcklbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
4119             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
4120             "punpcklbw %%mm0, %%mm2      \n\t" // unpack Low bytes of b
4121             // pav = p - a = (a + b - c) - a = b - c
4122             "movq %%mm2, %%mm4           \n\t"
4123             "punpcklbw %%mm0, %%mm3      \n\t" // unpack Low bytes of c
4124             // pbv = p - b = (a + b - c) - b = a - c
4125             "movq %%mm1, %%mm5           \n\t"
4126             "psubw %%mm3, %%mm4          \n\t"
4127             "pxor %%mm7, %%mm7           \n\t"
4128             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4129             "movq %%mm4, %%mm6           \n\t"
4130             "psubw %%mm3, %%mm5          \n\t"
4131             // pa = abs(p-a) = abs(pav)
4132             // pb = abs(p-b) = abs(pbv)
4133             // pc = abs(p-c) = abs(pcv)
4134             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
4135             "paddw %%mm5, %%mm6          \n\t"
4136             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
4137             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
4138             "psubw %%mm0, %%mm4          \n\t"
4139             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
4140             "psubw %%mm0, %%mm4          \n\t"
4141             "psubw %%mm7, %%mm5          \n\t"
4142             "pxor %%mm0, %%mm0           \n\t"
4143             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
4144             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
4145             "psubw %%mm7, %%mm5          \n\t"
4146             "psubw %%mm0, %%mm6          \n\t"
4147             //  test pa <= pb
4148             "movq %%mm4, %%mm7           \n\t"
4149             "psubw %%mm0, %%mm6          \n\t"
4150             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
4151             "movq %%mm7, %%mm0           \n\t"
4152             // use mm7 mask to merge pa & pb
4153             "pand %%mm7, %%mm5           \n\t"
4154             // use mm0 mask copy to merge a & b
4155             "pand %%mm0, %%mm2           \n\t"
4156             "pandn %%mm4, %%mm7          \n\t"
4157             "pandn %%mm1, %%mm0          \n\t"
4158             "paddw %%mm5, %%mm7          \n\t"
4159             "paddw %%mm2, %%mm0          \n\t"
4160             //  test  ((pa <= pb)? pa:pb) <= pc
4161             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
4162             "pxor %%mm1, %%mm1           \n\t"
4163             "pand %%mm7, %%mm3           \n\t"
4164             "pandn %%mm0, %%mm7          \n\t"
4165             "paddw %%mm3, %%mm7          \n\t"
4166             "pxor %%mm0, %%mm0           \n\t"
4167             "packuswb %%mm1, %%mm7       \n\t"
4168             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
4169             "pand _ActiveMask, %%mm7     \n\t"
4170             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
4171             "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
4172             "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
4173             "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
4174             "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // read a=Raw(x-bpp) bytes
4175
4176             // do second set of 4 bytes
4177             "punpckhbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
4178             "punpckhbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
4179             // pav = p - a = (a + b - c) - a = b - c
4180             "movq %%mm2, %%mm4           \n\t"
4181             // pbv = p - b = (a + b - c) - b = a - c
4182             "movq %%mm1, %%mm5           \n\t"
4183             "psubw %%mm3, %%mm4          \n\t"
4184             "pxor %%mm7, %%mm7           \n\t"
4185             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4186             "movq %%mm4, %%mm6           \n\t"
4187             "psubw %%mm3, %%mm5          \n\t"
4188             // pa = abs(p-a) = abs(pav)
4189             // pb = abs(p-b) = abs(pbv)
4190             // pc = abs(p-c) = abs(pcv)
4191             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
4192             "paddw %%mm5, %%mm6          \n\t"
4193             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
4194             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
4195             "psubw %%mm0, %%mm4          \n\t"
4196             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
4197             "psubw %%mm0, %%mm4          \n\t"
4198             "psubw %%mm7, %%mm5          \n\t"
4199             "pxor %%mm0, %%mm0           \n\t"
4200             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
4201             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
4202             "psubw %%mm7, %%mm5          \n\t"
4203             "psubw %%mm0, %%mm6          \n\t"
4204             //  test pa <= pb
4205             "movq %%mm4, %%mm7           \n\t"
4206             "psubw %%mm0, %%mm6          \n\t"
4207             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
4208             "movq %%mm7, %%mm0           \n\t"
4209             // use mm7 mask to merge pa & pb
4210             "pand %%mm7, %%mm5           \n\t"
4211             // use mm0 mask copy to merge a & b
4212             "pand %%mm0, %%mm2           \n\t"
4213             "pandn %%mm4, %%mm7          \n\t"
4214             "pandn %%mm1, %%mm0          \n\t"
4215             "paddw %%mm5, %%mm7          \n\t"
4216             "paddw %%mm2, %%mm0          \n\t"
4217             //  test  ((pa <= pb)? pa:pb) <= pc
4218             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
4219             "pxor %%mm1, %%mm1           \n\t"
4220             "pand %%mm7, %%mm3           \n\t"
4221             "pandn %%mm0, %%mm7          \n\t"
4222             "pxor %%mm1, %%mm1           \n\t"
4223             "paddw %%mm3, %%mm7          \n\t"
4224             "pxor %%mm0, %%mm0           \n\t"
4225             // step ecx to next set of 8 bytes and repeat loop til done
4226             "addl $8, %%ecx              \n\t"
4227             "packuswb %%mm7, %%mm1       \n\t"
4228             "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with Raw(x)
4229             "cmpl _MMXLength, %%ecx      \n\t"
4230             "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
4231                             // mm1 will be used as Raw(x-bpp) next loop
4232             "jb paeth_8lp                \n\t"
4233
4234             : "=S" (dummy_value_S),             // output regs (dummy)
4235               "=D" (dummy_value_D)
4236
4237             : "0" (prev_row),  // esi           // input regs
4238               "1" (row)        // edi
4239
4240             : "%ecx"                            // clobber list
4241 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
4242             , "%mm0", "%mm1", "%mm2", "%mm3"
4243             , "%mm4", "%mm5", "%mm6", "%mm7"
4244 #endif
4245          );
4246       }
4247       break;  // end 8 bpp
4248
4249       case 1:                // bpp = 1
4250       case 2:                // bpp = 2
4251       default:               // bpp > 8
4252       {
4253          __asm__ __volatile__ (
4254 #ifdef __PIC__
4255             "pushl %%ebx                 \n\t" // save Global Offset Table index
4256 #endif
4257             "movl _dif, %%ebx            \n\t"
4258             "cmpl _FullLength, %%ebx     \n\t"
4259             "jnb paeth_dend              \n\t"
4260
4261 // preload  "movl row, %%edi             \n\t"
4262 // preload  "movl prev_row, %%esi        \n\t"
4263             // do Paeth decode for remaining bytes
4264             "movl %%ebx, %%edx           \n\t"
4265 // preload  "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
4266             "subl %%ecx, %%edx           \n\t" // edx = ebx - bpp
4267             "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx
4268
4269          "paeth_dlp:                     \n\t"
4270             "xorl %%eax, %%eax           \n\t"
4271             // pav = p - a = (a + b - c) - a = b - c
4272             "movb (%%esi,%%ebx,), %%al   \n\t" // load Prior(x) into al
4273             "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4274             "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
4275             "movl %%eax, _patemp         \n\t" // Save pav for later use
4276             "xorl %%eax, %%eax           \n\t"
4277             // pbv = p - b = (a + b - c) - b = a - c
4278             "movb (%%edi,%%edx,), %%al   \n\t" // load Raw(x-bpp) into al
4279             "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
4280             "movl %%eax, %%ecx           \n\t"
4281             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4282             "addl _patemp, %%eax         \n\t" // pcv = pav + pbv
4283             // pc = abs(pcv)
4284             "testl $0x80000000, %%eax    \n\t"
4285             "jz paeth_dpca               \n\t"
4286             "negl %%eax                  \n\t" // reverse sign of neg values
4287
4288          "paeth_dpca:                    \n\t"
4289             "movl %%eax, _pctemp         \n\t" // save pc for later use
4290             // pb = abs(pbv)
4291             "testl $0x80000000, %%ecx    \n\t"
4292             "jz paeth_dpba               \n\t"
4293             "negl %%ecx                  \n\t" // reverse sign of neg values
4294
4295          "paeth_dpba:                    \n\t"
4296             "movl %%ecx, _pbtemp         \n\t" // save pb for later use
4297             // pa = abs(pav)
4298             "movl _patemp, %%eax         \n\t"
4299             "testl $0x80000000, %%eax    \n\t"
4300             "jz paeth_dpaa               \n\t"
4301             "negl %%eax                  \n\t" // reverse sign of neg values
4302
4303          "paeth_dpaa:                    \n\t"
4304             "movl %%eax, _patemp         \n\t" // save pa for later use
4305             // test if pa <= pb
4306             "cmpl %%ecx, %%eax           \n\t"
4307             "jna paeth_dabb              \n\t"
4308             // pa > pb; now test if pb <= pc
4309             "cmpl _pctemp, %%ecx         \n\t"
4310             "jna paeth_dbbc              \n\t"
4311             // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4312             "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4313             "jmp paeth_dpaeth            \n\t"
4314
4315          "paeth_dbbc:                    \n\t"
4316             // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
4317             "movb (%%esi,%%ebx,), %%cl   \n\t" // load Prior(x) into cl
4318             "jmp paeth_dpaeth            \n\t"
4319
4320          "paeth_dabb:                    \n\t"
4321             // pa <= pb; now test if pa <= pc
4322             "cmpl _pctemp, %%eax         \n\t"
4323             "jna paeth_dabc              \n\t"
4324             // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4325             "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4326             "jmp paeth_dpaeth            \n\t"
4327
4328          "paeth_dabc:                    \n\t"
4329             // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
4330             "movb (%%edi,%%edx,), %%cl   \n\t" // load Raw(x-bpp) into cl
4331
4332          "paeth_dpaeth:                  \n\t"
4333             "incl %%ebx                  \n\t"
4334             "incl %%edx                  \n\t"
4335             // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
4336             "addb %%cl, -1(%%edi,%%ebx,) \n\t"
4337             "cmpl _FullLength, %%ebx     \n\t"
4338             "jb paeth_dlp                \n\t"
4339
4340          "paeth_dend:                    \n\t"
4341 #ifdef __PIC__
4342             "popl %%ebx                  \n\t" // index to Global Offset Table
4343 #endif
4344
4345             : "=c" (dummy_value_c),            // output regs (dummy)
4346               "=S" (dummy_value_S),
4347               "=D" (dummy_value_D)
4348
4349             : "0" (bpp),       // ecx          // input regs
4350               "1" (prev_row),  // esi
4351               "2" (row)        // edi
4352
4353             : "%eax", "%edx"                   // clobber list
4354 #ifndef __PIC__
4355             , "%ebx"
4356 #endif
4357          );
4358       }
4359       return;                   // No need to go further with this one
4360
4361    } // end switch (bpp)
4362
4363    __asm__ __volatile__ (
4364       // MMX acceleration complete; now do clean-up
4365       // check if any remaining bytes left to decode
4366 #ifdef __PIC__
4367       "pushl %%ebx                 \n\t" // save index to Global Offset Table
4368 #endif
4369       "movl _MMXLength, %%ebx      \n\t"
4370       "cmpl _FullLength, %%ebx     \n\t"
4371       "jnb paeth_end               \n\t"
4372 //pre "movl row, %%edi             \n\t"
4373 //pre "movl prev_row, %%esi        \n\t"
4374       // do Paeth decode for remaining bytes
4375       "movl %%ebx, %%edx           \n\t"
4376 //pre "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
4377       "subl %%ecx, %%edx           \n\t" // edx = ebx - bpp
4378       "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx below
4379
4380    "paeth_lp2:                     \n\t"
4381       "xorl %%eax, %%eax           \n\t"
4382       // pav = p - a = (a + b - c) - a = b - c
4383       "movb (%%esi,%%ebx,), %%al   \n\t" // load Prior(x) into al
4384       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4385       "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
4386       "movl %%eax, _patemp         \n\t" // Save pav for later use
4387       "xorl %%eax, %%eax           \n\t"
4388       // pbv = p - b = (a + b - c) - b = a - c
4389       "movb (%%edi,%%edx,), %%al   \n\t" // load Raw(x-bpp) into al
4390       "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
4391       "movl %%eax, %%ecx           \n\t"
4392       // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4393       "addl _patemp, %%eax         \n\t" // pcv = pav + pbv
4394       // pc = abs(pcv)
4395       "testl $0x80000000, %%eax    \n\t"
4396       "jz paeth_pca2               \n\t"
4397       "negl %%eax                  \n\t" // reverse sign of neg values
4398
4399    "paeth_pca2:                    \n\t"
4400       "movl %%eax, _pctemp         \n\t" // save pc for later use
4401       // pb = abs(pbv)
4402       "testl $0x80000000, %%ecx    \n\t"
4403       "jz paeth_pba2               \n\t"
4404       "negl %%ecx                  \n\t" // reverse sign of neg values
4405
4406    "paeth_pba2:                    \n\t"
4407       "movl %%ecx, _pbtemp         \n\t" // save pb for later use
4408       // pa = abs(pav)
4409       "movl _patemp, %%eax         \n\t"
4410       "testl $0x80000000, %%eax    \n\t"
4411       "jz paeth_paa2               \n\t"
4412       "negl %%eax                  \n\t" // reverse sign of neg values
4413
4414    "paeth_paa2:                    \n\t"
4415       "movl %%eax, _patemp         \n\t" // save pa for later use
4416       // test if pa <= pb
4417       "cmpl %%ecx, %%eax           \n\t"
4418       "jna paeth_abb2              \n\t"
4419       // pa > pb; now test if pb <= pc
4420       "cmpl _pctemp, %%ecx         \n\t"
4421       "jna paeth_bbc2              \n\t"
4422       // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4423       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4424       "jmp paeth_paeth2            \n\t"
4425
4426    "paeth_bbc2:                    \n\t"
4427       // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
4428       "movb (%%esi,%%ebx,), %%cl   \n\t" // load Prior(x) into cl
4429       "jmp paeth_paeth2            \n\t"
4430
4431    "paeth_abb2:                    \n\t"
4432       // pa <= pb; now test if pa <= pc
4433       "cmpl _pctemp, %%eax         \n\t"
4434       "jna paeth_abc2              \n\t"
4435       // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4436       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4437       "jmp paeth_paeth2            \n\t"
4438
4439    "paeth_abc2:                    \n\t"
4440       // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
4441       "movb (%%edi,%%edx,), %%cl   \n\t" // load Raw(x-bpp) into cl
4442
4443    "paeth_paeth2:                  \n\t"
4444       "incl %%ebx                  \n\t"
4445       "incl %%edx                  \n\t"
4446       // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
4447       "addb %%cl, -1(%%edi,%%ebx,) \n\t"
4448       "cmpl _FullLength, %%ebx     \n\t"
4449       "jb paeth_lp2                \n\t"
4450
4451    "paeth_end:                     \n\t"
4452       "EMMS                        \n\t" // end MMX; prep for poss. FP instrs.
4453 #ifdef __PIC__
4454       "popl %%ebx                  \n\t" // restore index to Global Offset Table
4455 #endif
4456
4457       : "=c" (dummy_value_c),            // output regs (dummy)
4458         "=S" (dummy_value_S),
4459         "=D" (dummy_value_D)
4460
4461       : "0" (bpp),       // ecx          // input regs
4462         "1" (prev_row),  // esi
4463         "2" (row)        // edi
4464
4465       : "%eax", "%edx"                   // clobber list (no input regs!)
4466 #ifndef __PIC__
4467       , "%ebx"
4468 #endif
4469    );
4470
4471 } /* end png_read_filter_row_mmx_paeth() */
4472 #endif
4473
4474
4475
4476
4477 #ifdef PNG_THREAD_UNSAFE_OK
4478 //===========================================================================//
4479 //                                                                           //
4480 //           P N G _ R E A D _ F I L T E R _ R O W _ M M X _ S U B           //
4481 //                                                                           //
4482 //===========================================================================//
4483
4484 // Optimized code for PNG Sub filter decoder
4485
4486 static void /* PRIVATE */
4487 png_read_filter_row_mmx_sub(png_row_infop row_info, png_bytep row)
4488 {
4489    int bpp;
4490    int dummy_value_a;
4491    int dummy_value_D;
4492
4493    bpp = (row_info->pixel_depth + 7) >> 3;   // calc number of bytes per pixel
4494    _FullLength = row_info->rowbytes - bpp;   // number of bytes to filter
4495
4496    __asm__ __volatile__ (
4497 //pre "movl row, %%edi             \n\t"
4498       "movl %%edi, %%esi           \n\t" // lp = row
4499 //pre "movl bpp, %%eax             \n\t"
4500       "addl %%eax, %%edi           \n\t" // rp = row + bpp
4501 //irr "xorl %%eax, %%eax           \n\t"
4502       // get # of bytes to alignment
4503       "movl %%edi, _dif            \n\t" // take start of row
4504       "addl $0xf, _dif             \n\t" // add 7 + 8 to incr past
4505                                          //  alignment boundary
4506       "xorl %%ecx, %%ecx           \n\t"
4507       "andl $0xfffffff8, _dif      \n\t" // mask to alignment boundary
4508       "subl %%edi, _dif            \n\t" // subtract from start ==> value
4509       "jz sub_go                   \n\t" //  ecx at alignment
4510
4511    "sub_lp1:                       \n\t" // fix alignment
4512       "movb (%%esi,%%ecx,), %%al   \n\t"
4513       "addb %%al, (%%edi,%%ecx,)   \n\t"
4514       "incl %%ecx                  \n\t"
4515       "cmpl _dif, %%ecx            \n\t"
4516       "jb sub_lp1                  \n\t"
4517
4518    "sub_go:                        \n\t"
4519       "movl _FullLength, %%eax     \n\t"
4520       "movl %%eax, %%edx           \n\t"
4521       "subl %%ecx, %%edx           \n\t" // subtract alignment fix
4522       "andl $0x00000007, %%edx     \n\t" // calc bytes over mult of 8
4523       "subl %%edx, %%eax           \n\t" // drop over bytes from length
4524       "movl %%eax, _MMXLength      \n\t"
4525
4526       : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4527         "=D" (dummy_value_D)    // 1
4528
4529       : "0" (bpp),              // eax    // input regs
4530         "1" (row)               // edi
4531
4532       : "%ebx", "%ecx", "%edx"            // clobber list
4533       , "%esi"
4534
4535 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4536       , "%mm0", "%mm1", "%mm2", "%mm3"
4537       , "%mm4", "%mm5", "%mm6", "%mm7"
4538 #endif
4539    );
4540
4541    // now do the math for the rest of the row
4542    switch (bpp)
4543    {
4544       case 3:
4545       {
4546          _ActiveMask.use  = 0x0000ffffff000000LL;
4547          _ShiftBpp.use = 24;       // == 3 * 8
4548          _ShiftRem.use  = 40;      // == 64 - 24
4549
4550          __asm__ __volatile__ (
4551 // preload  "movl row, %%edi              \n\t"
4552             "movq _ActiveMask, %%mm7       \n\t" // load _ActiveMask for 2nd
4553                                                 //  active byte group
4554             "movl %%edi, %%esi            \n\t" // lp = row
4555 // preload  "movl bpp, %%eax              \n\t"
4556             "addl %%eax, %%edi            \n\t" // rp = row + bpp
4557             "movq %%mm7, %%mm6            \n\t"
4558             "movl _dif, %%edx             \n\t"
4559             "psllq _ShiftBpp, %%mm6       \n\t" // move mask in mm6 to cover
4560                                                 //  3rd active byte group
4561             // prime the pump:  load the first Raw(x-bpp) data set
4562             "movq -8(%%edi,%%edx,), %%mm1 \n\t"
4563
4564          "sub_3lp:                        \n\t" // shift data for adding first
4565             "psrlq _ShiftRem, %%mm1       \n\t" //  bpp bytes (no need for mask;
4566                                                 //  shift clears inactive bytes)
4567             // add 1st active group
4568             "movq (%%edi,%%edx,), %%mm0   \n\t"
4569             "paddb %%mm1, %%mm0           \n\t"
4570
4571             // add 2nd active group
4572             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4573             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4574             "pand %%mm7, %%mm1            \n\t" // mask to use 2nd active group
4575             "paddb %%mm1, %%mm0           \n\t"
4576
4577             // add 3rd active group
4578             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4579             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4580             "pand %%mm6, %%mm1            \n\t" // mask to use 3rd active group
4581             "addl $8, %%edx               \n\t"
4582             "paddb %%mm1, %%mm0           \n\t"
4583
4584             "cmpl _MMXLength, %%edx       \n\t"
4585             "movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array
4586             "movq %%mm0, %%mm1            \n\t" // prep 1st add at top of loop
4587             "jb sub_3lp                   \n\t"
4588
4589             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4590               "=D" (dummy_value_D)    // 1
4591
4592             : "0" (bpp),              // eax    // input regs
4593               "1" (row)               // edi
4594
4595             : "%edx", "%esi"                    // clobber list
4596 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4597             , "%mm0", "%mm1", "%mm6", "%mm7"
4598 #endif
4599          );
4600       }
4601       break;
4602
4603       case 1:
4604       {
4605          __asm__ __volatile__ (
4606             "movl _dif, %%edx            \n\t"
4607 // preload  "movl row, %%edi             \n\t"
4608             "cmpl _FullLength, %%edx     \n\t"
4609             "jnb sub_1end                \n\t"
4610             "movl %%edi, %%esi           \n\t" // lp = row
4611             "xorl %%eax, %%eax           \n\t"
4612 // preload  "movl bpp, %%eax             \n\t"
4613             "addl %%eax, %%edi           \n\t" // rp = row + bpp
4614
4615          "sub_1lp:                       \n\t"
4616             "movb (%%esi,%%edx,), %%al   \n\t"
4617             "addb %%al, (%%edi,%%edx,)   \n\t"
4618             "incl %%edx                  \n\t"
4619             "cmpl _FullLength, %%edx     \n\t"
4620             "jb sub_1lp                  \n\t"
4621
4622          "sub_1end:                      \n\t"
4623
4624             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4625               "=D" (dummy_value_D)    // 1
4626
4627             : "0" (bpp),              // eax    // input regs
4628               "1" (row)               // edi
4629
4630             : "%edx", "%esi"                    // clobber list
4631          );
4632       }
4633       return;
4634
4635       case 6:
4636       case 4:
4637       //case 7:   // GRR BOGUS
4638       //case 5:   // GRR BOGUS
4639       {
4640          _ShiftBpp.use = bpp << 3;
4641          _ShiftRem.use = 64 - _ShiftBpp.use;
4642
4643          __asm__ __volatile__ (
4644 // preload  "movl row, %%edi              \n\t"
4645             "movl _dif, %%edx             \n\t"
4646             "movl %%edi, %%esi            \n\t" // lp = row
4647 // preload  "movl bpp, %%eax              \n\t"
4648             "addl %%eax, %%edi            \n\t" // rp = row + bpp
4649
4650             // prime the pump:  load the first Raw(x-bpp) data set
4651             "movq -8(%%edi,%%edx,), %%mm1 \n\t"
4652
4653          "sub_4lp:                        \n\t" // shift data for adding first
4654             "psrlq _ShiftRem, %%mm1       \n\t" //  bpp bytes (no need for mask;
4655                                                 //  shift clears inactive bytes)
4656             "movq (%%edi,%%edx,), %%mm0   \n\t"
4657             "paddb %%mm1, %%mm0           \n\t"
4658
4659             // add 2nd active group
4660             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4661             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4662             "addl $8, %%edx               \n\t"
4663             "paddb %%mm1, %%mm0           \n\t"
4664
4665             "cmpl _MMXLength, %%edx       \n\t"
4666             "movq %%mm0, -8(%%edi,%%edx,) \n\t"
4667             "movq %%mm0, %%mm1            \n\t" // prep 1st add at top of loop
4668             "jb sub_4lp                   \n\t"
4669
4670             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4671               "=D" (dummy_value_D)    // 1
4672
4673             : "0" (bpp),              // eax    // input regs
4674               "1" (row)               // edi
4675
4676             : "%edx", "%esi"                    // clobber list
4677 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4678             , "%mm0", "%mm1"
4679 #endif
4680          );
4681       }
4682       break;
4683
4684       case 2:
4685       {
4686          _ActiveMask.use = 0x00000000ffff0000LL;
4687          _ShiftBpp.use = 16;       // == 2 * 8
4688          _ShiftRem.use = 48;       // == 64 - 16
4689
4690          __asm__ __volatile__ (
4691             "movq _ActiveMask, %%mm7      \n\t" // load _ActiveMask for 2nd
4692                                                 //  active byte group
4693             "movl _dif, %%edx             \n\t"
4694             "movq %%mm7, %%mm6            \n\t"
4695 // preload  "movl row, %%edi              \n\t"
4696             "psllq _ShiftBpp, %%mm6       \n\t" // move mask in mm6 to cover
4697                                                 //  3rd active byte group
4698             "movl %%edi, %%esi            \n\t" // lp = row
4699             "movq %%mm6, %%mm5            \n\t"
4700 // preload  "movl bpp, %%eax              \n\t"
4701             "addl %%eax, %%edi            \n\t" // rp = row + bpp
4702             "psllq _ShiftBpp, %%mm5       \n\t" // move mask in mm5 to cover
4703                                                 //  4th active byte group
4704             // prime the pump:  load the first Raw(x-bpp) data set
4705             "movq -8(%%edi,%%edx,), %%mm1 \n\t"
4706
4707          "sub_2lp:                        \n\t" // shift data for adding first
4708             "psrlq _ShiftRem, %%mm1       \n\t" //  bpp bytes (no need for mask;
4709                                                 //  shift clears inactive bytes)
4710             // add 1st active group
4711             "movq (%%edi,%%edx,), %%mm0   \n\t"
4712             "paddb %%mm1, %%mm0           \n\t"
4713
4714             // add 2nd active group
4715             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4716             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4717             "pand %%mm7, %%mm1            \n\t" // mask to use 2nd active group
4718             "paddb %%mm1, %%mm0           \n\t"
4719
4720             // add 3rd active group
4721             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4722             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4723             "pand %%mm6, %%mm1            \n\t" // mask to use 3rd active group
4724             "paddb %%mm1, %%mm0           \n\t"
4725
4726             // add 4th active group
4727             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4728             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4729             "pand %%mm5, %%mm1            \n\t" // mask to use 4th active group
4730             "addl $8, %%edx               \n\t"
4731             "paddb %%mm1, %%mm0           \n\t"
4732             "cmpl _MMXLength, %%edx       \n\t"
4733             "movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array
4734             "movq %%mm0, %%mm1            \n\t" // prep 1st add at top of loop
4735             "jb sub_2lp                   \n\t"
4736
4737             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4738               "=D" (dummy_value_D)    // 1
4739
4740             : "0" (bpp),              // eax    // input regs
4741               "1" (row)               // edi
4742
4743             : "%edx", "%esi"                    // clobber list
4744 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4745             , "%mm0", "%mm1", "%mm5", "%mm6", "%mm7"
4746 #endif
4747          );
4748       }
4749       break;
4750
4751       case 8:
4752       {
4753          __asm__ __volatile__ (
4754 // preload  "movl row, %%edi              \n\t"
4755             "movl _dif, %%edx             \n\t"
4756             "movl %%edi, %%esi            \n\t" // lp = row
4757 // preload  "movl bpp, %%eax              \n\t"
4758             "addl %%eax, %%edi            \n\t" // rp = row + bpp
4759             "movl _MMXLength, %%ecx       \n\t"
4760
4761             // prime the pump:  load the first Raw(x-bpp) data set
4762             "movq -8(%%edi,%%edx,), %%mm7 \n\t"
4763             "andl $0x0000003f, %%ecx      \n\t" // calc bytes over mult of 64
4764
4765          "sub_8lp:                        \n\t"
4766             "movq (%%edi,%%edx,), %%mm0   \n\t" // load Sub(x) for 1st 8 bytes
4767             "paddb %%mm7, %%mm0           \n\t"
4768             "movq 8(%%edi,%%edx,), %%mm1  \n\t" // load Sub(x) for 2nd 8 bytes
4769             "movq %%mm0, (%%edi,%%edx,)   \n\t" // write Raw(x) for 1st 8 bytes
4770
4771             // Now mm0 will be used as Raw(x-bpp) for the 2nd group of 8 bytes.
4772             // This will be repeated for each group of 8 bytes with the 8th
4773             // group being used as the Raw(x-bpp) for the 1st group of the
4774             // next loop.
4775
4776             "paddb %%mm0, %%mm1           \n\t"
4777             "movq 16(%%edi,%%edx,), %%mm2 \n\t" // load Sub(x) for 3rd 8 bytes
4778             "movq %%mm1, 8(%%edi,%%edx,)  \n\t" // write Raw(x) for 2nd 8 bytes
4779             "paddb %%mm1, %%mm2           \n\t"
4780             "movq 24(%%edi,%%edx,), %%mm3 \n\t" // load Sub(x) for 4th 8 bytes
4781             "movq %%mm2, 16(%%edi,%%edx,) \n\t" // write Raw(x) for 3rd 8 bytes
4782             "paddb %%mm2, %%mm3           \n\t"
4783             "movq 32(%%edi,%%edx,), %%mm4 \n\t" // load Sub(x) for 5th 8 bytes
4784             "movq %%mm3, 24(%%edi,%%edx,) \n\t" // write Raw(x) for 4th 8 bytes
4785             "paddb %%mm3, %%mm4           \n\t"
4786             "movq 40(%%edi,%%edx,), %%mm5 \n\t" // load Sub(x) for 6th 8 bytes
4787             "movq %%mm4, 32(%%edi,%%edx,) \n\t" // write Raw(x) for 5th 8 bytes
4788             "paddb %%mm4, %%mm5           \n\t"
4789             "movq 48(%%edi,%%edx,), %%mm6 \n\t" // load Sub(x) for 7th 8 bytes
4790             "movq %%mm5, 40(%%edi,%%edx,) \n\t" // write Raw(x) for 6th 8 bytes
4791             "paddb %%mm5, %%mm6           \n\t"
4792             "movq 56(%%edi,%%edx,), %%mm7 \n\t" // load Sub(x) for 8th 8 bytes
4793             "movq %%mm6, 48(%%edi,%%edx,) \n\t" // write Raw(x) for 7th 8 bytes
4794             "addl $64, %%edx              \n\t"
4795             "paddb %%mm6, %%mm7           \n\t"
4796             "cmpl %%ecx, %%edx            \n\t"
4797             "movq %%mm7, -8(%%edi,%%edx,) \n\t" // write Raw(x) for 8th 8 bytes
4798             "jb sub_8lp                   \n\t"
4799
4800             "cmpl _MMXLength, %%edx       \n\t"
4801             "jnb sub_8lt8                 \n\t"
4802
4803          "sub_8lpA:                       \n\t"
4804             "movq (%%edi,%%edx,), %%mm0   \n\t"
4805             "addl $8, %%edx               \n\t"
4806             "paddb %%mm7, %%mm0           \n\t"
4807             "cmpl _MMXLength, %%edx       \n\t"
4808             "movq %%mm0, -8(%%edi,%%edx,) \n\t" // -8 to offset early addl edx
4809             "movq %%mm0, %%mm7            \n\t" // move calculated Raw(x) data
4810                                                 //  to mm1 to be new Raw(x-bpp)
4811                                                 //  for next loop
4812             "jb sub_8lpA                  \n\t"
4813
4814          "sub_8lt8:                       \n\t"
4815
4816             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4817               "=D" (dummy_value_D)    // 1
4818
4819             : "0" (bpp),              // eax    // input regs
4820               "1" (row)               // edi
4821
4822             : "%ecx", "%edx", "%esi"            // clobber list
4823 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4824             , "%mm0", "%mm1", "%mm2", "%mm3", "%mm4", "%mm5", "%mm6", "%mm7"
4825 #endif
4826          );
4827       }
4828       break;
4829
4830       default:                // bpp greater than 8 bytes   GRR BOGUS
4831       {
4832          __asm__ __volatile__ (
4833             "movl _dif, %%edx             \n\t"
4834 // preload  "movl row, %%edi              \n\t"
4835             "movl %%edi, %%esi            \n\t" // lp = row
4836 // preload  "movl bpp, %%eax              \n\t"
4837             "addl %%eax, %%edi            \n\t" // rp = row + bpp
4838
4839          "sub_Alp:                        \n\t"
4840             "movq (%%edi,%%edx,), %%mm0   \n\t"
4841             "movq (%%esi,%%edx,), %%mm1   \n\t"
4842             "addl $8, %%edx               \n\t"
4843             "paddb %%mm1, %%mm0           \n\t"
4844             "cmpl _MMXLength, %%edx       \n\t"
4845             "movq %%mm0, -8(%%edi,%%edx,) \n\t" // mov does not affect flags;
4846                                                 //  -8 to offset addl edx
4847             "jb sub_Alp                   \n\t"
4848
4849             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4850               "=D" (dummy_value_D)    // 1
4851
4852             : "0" (bpp),              // eax    // input regs
4853               "1" (row)               // edi
4854
4855             : "%edx", "%esi"                    // clobber list
4856 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4857             , "%mm0", "%mm1"
4858 #endif
4859          );
4860       }
4861       break;
4862
4863    } // end switch (bpp)
4864
4865    __asm__ __volatile__ (
4866       "movl _MMXLength, %%edx       \n\t"
4867 //pre "movl row, %%edi              \n\t"
4868       "cmpl _FullLength, %%edx      \n\t"
4869       "jnb sub_end                  \n\t"
4870
4871       "movl %%edi, %%esi            \n\t" // lp = row
4872 //pre "movl bpp, %%eax              \n\t"
4873       "addl %%eax, %%edi            \n\t" // rp = row + bpp
4874       "xorl %%eax, %%eax            \n\t"
4875
4876    "sub_lp2:                        \n\t"
4877       "movb (%%esi,%%edx,), %%al    \n\t"
4878       "addb %%al, (%%edi,%%edx,)    \n\t"
4879       "incl %%edx                   \n\t"
4880       "cmpl _FullLength, %%edx      \n\t"
4881       "jb sub_lp2                   \n\t"
4882
4883    "sub_end:                        \n\t"
4884       "EMMS                         \n\t" // end MMX instructions
4885
4886       : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4887         "=D" (dummy_value_D)    // 1
4888
4889       : "0" (bpp),              // eax    // input regs
4890         "1" (row)               // edi
4891
4892       : "%edx", "%esi"                    // clobber list
4893    );
4894
4895 } // end of png_read_filter_row_mmx_sub()
4896 #endif
4897
4898
4899
4900
4901 //===========================================================================//
4902 //                                                                           //
4903 //            P N G _ R E A D _ F I L T E R _ R O W _ M M X _ U P            //
4904 //                                                                           //
4905 //===========================================================================//
4906
4907 // Optimized code for PNG Up filter decoder
4908
4909 static void /* PRIVATE */
4910 png_read_filter_row_mmx_up(png_row_infop row_info, png_bytep row,
4911                            png_bytep prev_row)
4912 {
4913    png_uint_32 len;
4914    int dummy_value_d;   // fix 'forbidden register 3 (dx) was spilled' error
4915    int dummy_value_S;
4916    int dummy_value_D;
4917
4918    len = row_info->rowbytes;              // number of bytes to filter
4919
4920    __asm__ __volatile__ (
4921 //pre "movl row, %%edi              \n\t"
4922       // get # of bytes to alignment
4923 #ifdef __PIC__
4924       "pushl %%ebx                  \n\t"
4925 #endif
4926       "movl %%edi, %%ecx            \n\t"
4927       "xorl %%ebx, %%ebx            \n\t"
4928       "addl $0x7, %%ecx             \n\t"
4929       "xorl %%eax, %%eax            \n\t"
4930       "andl $0xfffffff8, %%ecx      \n\t"
4931 //pre "movl prev_row, %%esi         \n\t"
4932       "subl %%edi, %%ecx            \n\t"
4933       "jz up_go                     \n\t"
4934
4935    "up_lp1:                         \n\t" // fix alignment
4936       "movb (%%edi,%%ebx,), %%al    \n\t"
4937       "addb (%%esi,%%ebx,), %%al    \n\t"
4938       "incl %%ebx                   \n\t"
4939       "cmpl %%ecx, %%ebx            \n\t"
4940       "movb %%al, -1(%%edi,%%ebx,)  \n\t" // mov does not affect flags; -1 to
4941       "jb up_lp1                    \n\t" //  offset incl ebx
4942
4943    "up_go:                          \n\t"
4944 //pre "movl len, %%edx              \n\t"
4945       "movl %%edx, %%ecx            \n\t"
4946       "subl %%ebx, %%edx            \n\t" // subtract alignment fix
4947       "andl $0x0000003f, %%edx      \n\t" // calc bytes over mult of 64
4948       "subl %%edx, %%ecx            \n\t" // drop over bytes from length
4949
4950       // unrolled loop - use all MMX registers and interleave to reduce
4951       // number of branch instructions (loops) and reduce partial stalls
4952    "up_loop:                        \n\t"
4953       "movq (%%esi,%%ebx,), %%mm1   \n\t"
4954       "movq (%%edi,%%ebx,), %%mm0   \n\t"
4955       "movq 8(%%esi,%%ebx,), %%mm3  \n\t"
4956       "paddb %%mm1, %%mm0           \n\t"
4957       "movq 8(%%edi,%%ebx,), %%mm2  \n\t"
4958       "movq %%mm0, (%%edi,%%ebx,)   \n\t"
4959       "paddb %%mm3, %%mm2           \n\t"
4960       "movq 16(%%esi,%%ebx,), %%mm5 \n\t"
4961       "movq %%mm2, 8(%%edi,%%ebx,)  \n\t"
4962       "movq 16(%%edi,%%ebx,), %%mm4 \n\t"
4963       "movq 24(%%esi,%%ebx,), %%mm7 \n\t"
4964       "paddb %%mm5, %%mm4           \n\t"
4965       "movq 24(%%edi,%%ebx,), %%mm6 \n\t"
4966       "movq %%mm4, 16(%%edi,%%ebx,) \n\t"
4967       "paddb %%mm7, %%mm6           \n\t"
4968       "movq 32(%%esi,%%ebx,), %%mm1 \n\t"
4969       "movq %%mm6, 24(%%edi,%%ebx,) \n\t"
4970       "movq 32(%%edi,%%ebx,), %%mm0 \n\t"
4971       "movq 40(%%esi,%%ebx,), %%mm3 \n\t"
4972       "paddb %%mm1, %%mm0           \n\t"
4973       "movq 40(%%edi,%%ebx,), %%mm2 \n\t"
4974       "movq %%mm0, 32(%%edi,%%ebx,) \n\t"
4975       "paddb %%mm3, %%mm2           \n\t"
4976       "movq 48(%%esi,%%ebx,), %%mm5 \n\t"
4977       "movq %%mm2, 40(%%edi,%%ebx,) \n\t"
4978       "movq 48(%%edi,%%ebx,), %%mm4 \n\t"
4979       "movq 56(%%esi,%%ebx,), %%mm7 \n\t"
4980       "paddb %%mm5, %%mm4           \n\t"
4981       "movq 56(%%edi,%%ebx,), %%mm6 \n\t"
4982       "movq %%mm4, 48(%%edi,%%ebx,) \n\t"
4983       "addl $64, %%ebx              \n\t"
4984       "paddb %%mm7, %%mm6           \n\t"
4985       "cmpl %%ecx, %%ebx            \n\t"
4986       "movq %%mm6, -8(%%edi,%%ebx,) \n\t" // (+56)movq does not affect flags;
4987       "jb up_loop                   \n\t" //  -8 to offset addl ebx
4988
4989       "cmpl $0, %%edx               \n\t" // test for bytes over mult of 64
4990       "jz up_end                    \n\t"
4991
4992       "cmpl $8, %%edx               \n\t" // test for less than 8 bytes
4993       "jb up_lt8                    \n\t" //  [added by lcreeve@netins.net]
4994
4995       "addl %%edx, %%ecx            \n\t"
4996       "andl $0x00000007, %%edx      \n\t" // calc bytes over mult of 8
4997       "subl %%edx, %%ecx            \n\t" // drop over bytes from length
4998       "jz up_lt8                    \n\t"
4999
5000    "up_lpA:                         \n\t" // use MMX regs to update 8 bytes sim.
5001       "movq (%%esi,%%ebx,), %%mm1   \n\t"
5002       "movq (%%edi,%%ebx,), %%mm0   \n\t"
5003       "addl $8, %%ebx               \n\t"
5004       "paddb %%mm1, %%mm0           \n\t"
5005       "cmpl %%ecx, %%ebx            \n\t"
5006       "movq %%mm0, -8(%%edi,%%ebx,) \n\t" // movq does not affect flags; -8 to
5007       "jb up_lpA                    \n\t" //  offset add ebx
5008       "cmpl $0, %%edx               \n\t" // test for bytes over mult of 8
5009       "jz up_end                    \n\t"
5010
5011    "up_lt8:                         \n\t"
5012       "xorl %%eax, %%eax            \n\t"
5013       "addl %%edx, %%ecx            \n\t" // move over byte count into counter
5014
5015    "up_lp2:                         \n\t" // use x86 regs for remaining bytes
5016       "movb (%%edi,%%ebx,), %%al    \n\t"
5017       "addb (%%esi,%%ebx,), %%al    \n\t"
5018       "incl %%ebx                   \n\t"
5019       "cmpl %%ecx, %%ebx            \n\t"
5020       "movb %%al, -1(%%edi,%%ebx,)  \n\t" // mov does not affect flags; -1 to
5021       "jb up_lp2                    \n\t" //  offset inc ebx
5022
5023    "up_end:                         \n\t"
5024       "EMMS                         \n\t" // conversion of filtered row complete
5025 #ifdef __PIC__
5026       "popl %%ebx                   \n\t"
5027 #endif
5028
5029       : "=d" (dummy_value_d),   // 0      // output regs (dummy)
5030         "=S" (dummy_value_S),   // 1
5031         "=D" (dummy_value_D)    // 2
5032
5033       : "0" (len),              // edx    // input regs
5034         "1" (prev_row),         // esi
5035         "2" (row)               // edi
5036
5037       : "%eax", "%ebx", "%ecx"            // clobber list (no input regs!)
5038
5039 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
5040       , "%mm0", "%mm1", "%mm2", "%mm3"
5041       , "%mm4", "%mm5", "%mm6", "%mm7"
5042 #endif
5043    );
5044
5045 } // end of png_read_filter_row_mmx_up()
5046
5047 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5048
5049
5050
5051
5052 /*===========================================================================*/
5053 /*                                                                           */
5054 /*                   P N G _ R E A D _ F I L T E R _ R O W                   */
5055 /*                                                                           */
5056 /*===========================================================================*/
5057
5058
5059 /* Optimized png_read_filter_row routines */
5060
5061 void /* PRIVATE */
5062 png_read_filter_row(png_structp png_ptr, png_row_infop row_info, png_bytep
5063    row, png_bytep prev_row, int filter)
5064 {
5065 #ifdef PNG_DEBUG
5066    char filnm[10];
5067 #endif
5068
5069 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
5070 /* GRR:  these are superseded by png_ptr->asm_flags: */
5071 #define UseMMX_sub    1   // GRR:  converted 20000730
5072 #define UseMMX_up     1   // GRR:  converted 20000729
5073 #define UseMMX_avg    1   // GRR:  converted 20000828 (+ 16-bit bugfix 20000916)
5074 #define UseMMX_paeth  1   // GRR:  converted 20000828
5075
5076    if (_mmx_supported == 2) {
5077        /* this should have happened in png_init_mmx_flags() already */
5078 #if !defined(PNG_1_0_X)
5079        png_warning(png_ptr, "asm_flags may not have been initialized");
5080 #endif
5081        png_mmx_support();
5082    }
5083 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5084
5085 #ifdef PNG_DEBUG
5086    png_debug(1, "in png_read_filter_row (pnggccrd.c)\n");
5087    switch (filter)
5088    {
5089       case 0: sprintf(filnm, "none");
5090          break;
5091       case 1: sprintf(filnm, "sub-%s",
5092 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5093 #if !defined(PNG_1_0_X)
5094         (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB)? "MMX" :
5095 #endif
5096 #endif
5097 "x86");
5098          break;
5099       case 2: sprintf(filnm, "up-%s",
5100 #ifdef PNG_ASSEMBLER_CODE_SUPPORTED
5101 #if !defined(PNG_1_0_X)
5102         (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP)? "MMX" :
5103 #endif
5104 #endif
5105  "x86");
5106          break;
5107       case 3: sprintf(filnm, "avg-%s",
5108 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5109 #if !defined(PNG_1_0_X)
5110         (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG)? "MMX" :
5111 #endif
5112 #endif
5113  "x86");
5114          break;
5115       case 4: sprintf(filnm, "Paeth-%s",
5116 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5117 #if !defined(PNG_1_0_X)
5118         (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH)? "MMX":
5119 #endif
5120 #endif
5121 "x86");
5122          break;
5123       default: sprintf(filnm, "unknw");
5124          break;
5125    }
5126    png_debug2(0, "row_number=%5ld, %5s, ", png_ptr->row_number, filnm);
5127    png_debug1(0, "row=0x%08lx, ", (unsigned long)row);
5128    png_debug2(0, "pixdepth=%2d, bytes=%d, ", (int)row_info->pixel_depth,
5129       (int)((row_info->pixel_depth + 7) >> 3));
5130    png_debug1(0,"rowbytes=%8ld\n", row_info->rowbytes);
5131 #endif /* PNG_DEBUG */
5132
5133    switch (filter)
5134    {
5135       case PNG_FILTER_VALUE_NONE:
5136          break;
5137
5138       case PNG_FILTER_VALUE_SUB:
5139 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5140 #if !defined(PNG_1_0_X)
5141          if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB) &&
5142              (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5143              (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5144 #else
5145          if (_mmx_supported)
5146 #endif
5147          {
5148             png_read_filter_row_mmx_sub(row_info, row);
5149          }
5150          else
5151 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5152          {
5153             png_uint_32 i;
5154             png_uint_32 istop = row_info->rowbytes;
5155             png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5156             png_bytep rp = row + bpp;
5157             png_bytep lp = row;
5158
5159             for (i = bpp; i < istop; i++)
5160             {
5161                *rp = (png_byte)(((int)(*rp) + (int)(*lp++)) & 0xff);
5162                rp++;
5163             }
5164          }  /* end !UseMMX_sub */
5165          break;
5166
5167       case PNG_FILTER_VALUE_UP:
5168 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
5169 #if !defined(PNG_1_0_X)
5170          if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP) &&
5171              (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5172              (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5173 #else
5174          if (_mmx_supported)
5175 #endif
5176          {
5177             png_read_filter_row_mmx_up(row_info, row, prev_row);
5178          }
5179           else
5180 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5181          {
5182             png_uint_32 i;
5183             png_uint_32 istop = row_info->rowbytes;
5184             png_bytep rp = row;
5185             png_bytep pp = prev_row;
5186
5187             for (i = 0; i < istop; ++i)
5188             {
5189                *rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff);
5190                rp++;
5191             }
5192          }  /* end !UseMMX_up */
5193          break;
5194
5195       case PNG_FILTER_VALUE_AVG:
5196 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5197 #if !defined(PNG_1_0_X)
5198          if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG) &&
5199              (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5200              (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5201 #else
5202          if (_mmx_supported)
5203 #endif
5204          {
5205             png_read_filter_row_mmx_avg(row_info, row, prev_row);
5206          }
5207          else
5208 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5209          {
5210             png_uint_32 i;
5211             png_bytep rp = row;
5212             png_bytep pp = prev_row;
5213             png_bytep lp = row;
5214             png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5215             png_uint_32 istop = row_info->rowbytes - bpp;
5216
5217             for (i = 0; i < bpp; i++)
5218             {
5219                *rp = (png_byte)(((int)(*rp) +
5220                   ((int)(*pp++) >> 1)) & 0xff);
5221                rp++;
5222             }
5223
5224             for (i = 0; i < istop; i++)
5225             {
5226                *rp = (png_byte)(((int)(*rp) +
5227                   ((int)(*pp++ + *lp++) >> 1)) & 0xff);
5228                rp++;
5229             }
5230          }  /* end !UseMMX_avg */
5231          break;
5232
5233       case PNG_FILTER_VALUE_PAETH:
5234 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5235 #if !defined(PNG_1_0_X)
5236          if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH) &&
5237              (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5238              (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5239 #else
5240          if (_mmx_supported)
5241 #endif
5242          {
5243             png_read_filter_row_mmx_paeth(row_info, row, prev_row);
5244          }
5245          else
5246 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5247          {
5248             png_uint_32 i;
5249             png_bytep rp = row;
5250             png_bytep pp = prev_row;
5251             png_bytep lp = row;
5252             png_bytep cp = prev_row;
5253             png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5254             png_uint_32 istop = row_info->rowbytes - bpp;
5255
5256             for (i = 0; i < bpp; i++)
5257             {
5258                *rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff);
5259                rp++;
5260             }
5261
5262             for (i = 0; i < istop; i++)   /* use leftover rp,pp */
5263             {
5264                int a, b, c, pa, pb, pc, p;
5265
5266                a = *lp++;
5267                b = *pp++;
5268                c = *cp++;
5269
5270                p = b - c;
5271                pc = a - c;
5272
5273 #ifdef PNG_USE_ABS
5274                pa = abs(p);
5275                pb = abs(pc);
5276                pc = abs(p + pc);
5277 #else
5278                pa = p < 0 ? -p : p;
5279                pb = pc < 0 ? -pc : pc;
5280                pc = (p + pc) < 0 ? -(p + pc) : p + pc;
5281 #endif
5282
5283                /*
5284                   if (pa <= pb && pa <= pc)
5285                      p = a;
5286                   else if (pb <= pc)
5287                      p = b;
5288                   else
5289                      p = c;
5290                 */
5291
5292                p = (pa <= pb && pa <= pc) ? a : (pb <= pc) ? b : c;
5293
5294                *rp = (png_byte)(((int)(*rp) + p) & 0xff);
5295                rp++;
5296             }
5297          }  /* end !UseMMX_paeth */
5298          break;
5299
5300       default:
5301          png_warning(png_ptr, "Ignoring bad row-filter type");
5302          *row=0;
5303          break;
5304    }
5305 }
5306
5307 #endif /* PNG_HAVE_ASSEMBLER_READ_FILTER_ROW */
5308
5309
5310 /*===========================================================================*/
5311 /*                                                                           */
5312 /*                      P N G _ M M X _ S U P P O R T                        */
5313 /*                                                                           */
5314 /*===========================================================================*/
5315
5316 /* GRR NOTES:  (1) the following code assumes 386 or better (pushfl/popfl)
5317  *             (2) all instructions compile with gcc 2.7.2.3 and later
5318  *             (3) the function is moved down here to prevent gcc from
5319  *                  inlining it in multiple places and then barfing be-
5320  *                  cause the ".NOT_SUPPORTED" label is multiply defined
5321  *             [is there a way to signal that a *single* function should
5322  *              not be inlined?  is there a way to modify the label for
5323  *              each inlined instance, e.g., by appending _1, _2, etc.?
5324  *              maybe if don't use leading "." in label name? (nope...sigh)]
5325  */
5326
5327 int PNGAPI
5328 png_mmx_support(void)
5329 {
5330 #if defined(PNG_MMX_CODE_SUPPORTED)
5331     __asm__ __volatile__ (
5332         "pushl %%ebx          \n\t"  // ebx gets clobbered by CPUID instruction
5333         "pushl %%ecx          \n\t"  // so does ecx...
5334         "pushl %%edx          \n\t"  // ...and edx (but ecx & edx safe on Linux)
5335 //      ".byte  0x66          \n\t"  // convert 16-bit pushf to 32-bit pushfd
5336 //      "pushf                \n\t"  // 16-bit pushf
5337         "pushfl               \n\t"  // save Eflag to stack
5338         "popl %%eax           \n\t"  // get Eflag from stack into eax
5339         "movl %%eax, %%ecx    \n\t"  // make another copy of Eflag in ecx
5340         "xorl $0x200000, %%eax \n\t" // toggle ID bit in Eflag (i.e., bit 21)
5341         "pushl %%eax          \n\t"  // save modified Eflag back to stack
5342 //      ".byte  0x66          \n\t"  // convert 16-bit popf to 32-bit popfd
5343 //      "popf                 \n\t"  // 16-bit popf
5344         "popfl                \n\t"  // restore modified value to Eflag reg
5345         "pushfl               \n\t"  // save Eflag to stack
5346         "popl %%eax           \n\t"  // get Eflag from stack
5347         "pushl %%ecx          \n\t"  // save original Eflag to stack
5348         "popfl                \n\t"  // restore original Eflag
5349         "xorl %%ecx, %%eax    \n\t"  // compare new Eflag with original Eflag
5350         "jz 0f                \n\t"  // if same, CPUID instr. is not supported
5351
5352         "xorl %%eax, %%eax    \n\t"  // set eax to zero
5353 //      ".byte  0x0f, 0xa2    \n\t"  // CPUID instruction (two-byte opcode)
5354         "cpuid                \n\t"  // get the CPU identification info
5355         "cmpl $1, %%eax       \n\t"  // make sure eax return non-zero value
5356         "jl 0f                \n\t"  // if eax is zero, MMX is not supported
5357
5358         "xorl %%eax, %%eax    \n\t"  // set eax to zero and...
5359         "incl %%eax           \n\t"  // ...increment eax to 1.  This pair is
5360                                      // faster than the instruction "mov eax, 1"
5361         "cpuid                \n\t"  // get the CPU identification info again
5362         "andl $0x800000, %%edx \n\t" // mask out all bits but MMX bit (23)
5363         "cmpl $0, %%edx       \n\t"  // 0 = MMX not supported
5364         "jz 0f                \n\t"  // non-zero = yes, MMX IS supported
5365
5366         "movl $1, %%eax       \n\t"  // set return value to 1
5367         "jmp  1f              \n\t"  // DONE:  have MMX support
5368
5369     "0:                       \n\t"  // .NOT_SUPPORTED: target label for jump instructions
5370         "movl $0, %%eax       \n\t"  // set return value to 0
5371     "1:                       \n\t"  // .RETURN: target label for jump instructions
5372         "movl %%eax, _mmx_supported \n\t" // save in global static variable, too
5373         "popl %%edx           \n\t"  // restore edx
5374         "popl %%ecx           \n\t"  // restore ecx
5375         "popl %%ebx           \n\t"  // restore ebx
5376
5377 //      "ret                  \n\t"  // DONE:  no MMX support
5378                                      // (fall through to standard C "ret")
5379
5380         :                            // output list (none)
5381
5382         :                            // any variables used on input (none)
5383
5384         : "%eax"                     // clobber list
5385 //      , "%ebx", "%ecx", "%edx"     // GRR:  we handle these manually
5386 //      , "memory"   // if write to a variable gcc thought was in a reg
5387 //      , "cc"       // "condition codes" (flag bits)
5388     );
5389 #else
5390     _mmx_supported = 0;
5391 #endif /* PNG_MMX_CODE_SUPPORTED */
5392
5393     return _mmx_supported;
5394 }
5395
5396
5397 #endif /* PNG_USE_PNGGCCRD */