diff --git a/ChangeLog.md b/ChangeLog.md
index c4f9490..2b89d83 100644
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -61,10 +61,9 @@
 each iMCU row based on which scan generated the pixels in that row, rather than
 always using the block smoothing parameters for the most recent scan.
 
-7. Added SIMD acceleration for progressive Huffman encoding on Arm 64-bit
-(Armv8) platforms.  This speeds up the compression of full-color progressive
-JPEGs by about 30-40% on average (relative to libjpeg-turbo 2.0.x) when using
-modern Armv8 CPUs.
+7. Added SIMD acceleration for progressive Huffman encoding on Arm platforms.
+This speeds up the compression of full-color progressive JPEGs by about 30-40%
+on average (relative to libjpeg-turbo 2.0.x) when using modern Arm CPUs.
 
 8. Added configure-time and run-time auto-detection of Loongson MMI SIMD
 instructions, so that the Loongson MMI SIMD extensions can be included in any
diff --git a/README.md b/README.md
index 924ebd8..01e391e 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
 
 libjpeg-turbo is a JPEG image codec that uses SIMD instructions to accelerate
 baseline JPEG compression and decompression on x86, x86-64, Arm, PowerPC, and
-MIPS systems, as well as progressive JPEG compression on x86, x86-64, and Armv8
+MIPS systems, as well as progressive JPEG compression on x86, x86-64, and Arm
 systems.  On such systems, libjpeg-turbo is generally 2-6x as fast as libjpeg,
 all else being equal.  On other types of systems, libjpeg-turbo can still
 outperform libjpeg by a significant amount, by virtue of its highly-optimized
diff --git a/release/ReadMe.txt b/release/ReadMe.txt
index 64fc294..446ce46 100644
--- a/release/ReadMe.txt
+++ b/release/ReadMe.txt
@@ -1,4 +1,4 @@
-libjpeg-turbo is a JPEG image codec that uses SIMD instructions to accelerate baseline JPEG compression and decompression on x86, x86-64, Arm, PowerPC, and MIPS systems, as well as progressive JPEG compression on x86, x86-64, and Armv8 systems.  On such systems, libjpeg-turbo is generally 2-6x as fast as libjpeg, all else being equal.  On other types of systems, libjpeg-turbo can still outperform libjpeg by a significant amount, by virtue of its highly-optimized Huffman coding routines.  In many cases, the performance of libjpeg-turbo rivals that of proprietary high-speed JPEG codecs.
+libjpeg-turbo is a JPEG image codec that uses SIMD instructions to accelerate baseline JPEG compression and decompression on x86, x86-64, Arm, PowerPC, and MIPS systems, as well as progressive JPEG compression on x86, x86-64, and Arm systems.  On such systems, libjpeg-turbo is generally 2-6x as fast as libjpeg, all else being equal.  On other types of systems, libjpeg-turbo can still outperform libjpeg by a significant amount, by virtue of its highly-optimized Huffman coding routines.  In many cases, the performance of libjpeg-turbo rivals that of proprietary high-speed JPEG codecs.
 
 libjpeg-turbo implements both the traditional libjpeg API as well as the less powerful but more straightforward TurboJPEG API.  libjpeg-turbo also features colorspace extensions that allow it to compress from/decompress to 32-bit and big-endian pixel buffers (RGBX, XBGR, etc.), as well as a full-featured Java interface.
 
diff --git a/release/deb-control.in b/release/deb-control.in
index b59f8f9..72bceec 100644
--- a/release/deb-control.in
+++ b/release/deb-control.in
@@ -11,7 +11,7 @@
  libjpeg-turbo is a JPEG image codec that uses SIMD instructions to accelerate
  baseline JPEG compression and decompression on x86, x86-64, Arm, PowerPC, and
  MIPS systems, as well as progressive JPEG compression on x86, x86-64, and
- Armv8 systems.  On such systems, libjpeg-turbo is generally 2-6x as fast as
+ Arm systems.  On such systems, libjpeg-turbo is generally 2-6x as fast as
  libjpeg, all else being equal.  On other types of systems, libjpeg-turbo can
  still outperform libjpeg by a significant amount, by virtue of its
  highly-optimized Huffman coding routines.  In many cases, the performance of
diff --git a/release/rpm.spec.in b/release/rpm.spec.in
index 0cd1825..05410a6 100644
--- a/release/rpm.spec.in
+++ b/release/rpm.spec.in
@@ -53,7 +53,7 @@
 %description
 libjpeg-turbo is a JPEG image codec that uses SIMD instructions to accelerate
 baseline JPEG compression and decompression on x86, x86-64, Arm, PowerPC, and
-MIPS systems, as well as progressive JPEG compression on x86, x86-64, and Armv8
+MIPS systems, as well as progressive JPEG compression on x86, x86-64, and Arm
 systems.  On such systems, libjpeg-turbo is generally 2-6x as fast as libjpeg,
 all else being equal.  On other types of systems, libjpeg-turbo can still
 outperform libjpeg by a significant amount, by virtue of its highly-optimized
diff --git a/simd/CMakeLists.txt b/simd/CMakeLists.txt
index b9dfdcb..f976417 100644
--- a/simd/CMakeLists.txt
+++ b/simd/CMakeLists.txt
@@ -265,8 +265,8 @@
 
 file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/gastest.S)
 
-set(SIMD_SOURCES arm/jcgray-neon.c arm/jcsample-neon.c arm/jdsample-neon.c
-  arm/jfdctfst-neon.c arm/jquanti-neon.c)
+set(SIMD_SOURCES arm/jcgray-neon.c arm/jcphuff-neon.c arm/jcsample-neon.c
+  arm/jdsample-neon.c arm/jfdctfst-neon.c arm/jquanti-neon.c)
 if(NEON_INTRINSICS)
   set(SIMD_SOURCES ${SIMD_SOURCES} arm/jccolor-neon.c arm/jidctint-neon.c)
 endif()
diff --git a/simd/arm/aarch32/jsimd.c b/simd/arm/aarch32/jsimd.c
index cd90c63..e054a45 100644
--- a/simd/arm/aarch32/jsimd.c
+++ b/simd/arm/aarch32/jsimd.c
@@ -825,6 +825,16 @@
 GLOBAL(int)
 jsimd_can_encode_mcu_AC_first_prepare(void)
 {
+  init_simd();
+
+  if (DCTSIZE != 8)
+    return 0;
+  if (sizeof(JCOEF) != 2)
+    return 0;
+
+  if (simd_support & JSIMD_NEON)
+    return 1;
+
   return 0;
 }
 
@@ -833,11 +843,23 @@
                                   const int *jpeg_natural_order_start, int Sl,
                                   int Al, JCOEF *values, size_t *zerobits)
 {
+  jsimd_encode_mcu_AC_first_prepare_neon(block, jpeg_natural_order_start,
+                                         Sl, Al, values, zerobits);
 }
 
 GLOBAL(int)
 jsimd_can_encode_mcu_AC_refine_prepare(void)
 {
+  init_simd();
+
+  if (DCTSIZE != 8)
+    return 0;
+  if (sizeof(JCOEF) != 2)
+    return 0;
+
+  if (simd_support & JSIMD_NEON)
+    return 1;
+
   return 0;
 }
 
@@ -846,5 +868,7 @@
                                    const int *jpeg_natural_order_start, int Sl,
                                    int Al, JCOEF *absvalues, size_t *bits)
 {
-  return 0;
+  return jsimd_encode_mcu_AC_refine_prepare_neon(block,
+                                                 jpeg_natural_order_start, Sl,
+                                                 Al, absvalues, bits);
 }
diff --git a/simd/arm/aarch64/jsimd_neon.S b/simd/arm/aarch64/jsimd_neon.S
index a99703e..9ded729 100644
--- a/simd/arm/aarch64/jsimd_neon.S
+++ b/simd/arm/aarch64/jsimd_neon.S
@@ -235,22 +235,6 @@
 
 #endif
 
-/* Constants for jsimd_encode_mcu_AC_first_prepare_neon() */
-
-.balign 16
-Ljsimd_encode_mcu_AC_first_prepare_neon_consts:
-    .byte 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, \
-          0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80
-
-/* Constants for jsimd_encode_mcu_AC_refine_prepare_neon() */
-
-.balign 16
-Ljsimd_encode_mcu_AC_refine_prepare_neon_consts:
-    .byte 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, \
-          0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80
-
-.text
-
 
 #define RESPECT_STRICT_ALIGNMENT  1
 
@@ -2754,625 +2738,3 @@
 .purgem checkbuf47
 
 #endif  /* NEON_INTRINSICS */
-
-
-/*****************************************************************************/
-
-/*
- * Macros to load data for jsimd_encode_mcu_AC_first_prepare_neon() and
- * jsimd_encode_mcu_AC_refine_prepare_neon()
- */
-
-.macro LOAD16
-    ldr             T0d, [LUT, #(0 * 4)]
-    ldr             T1d, [LUT, #(8 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[0], [T0]
-    ld1             {Y1.h}[0], [T1]
-
-    ldr             T0d, [LUT, #(1 * 4)]
-    ldr             T1d, [LUT, #(9 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[1], [T0]
-    ld1             {Y1.h}[1], [T1]
-
-    ldr             T0d, [LUT, #(2 * 4)]
-    ldr             T1d, [LUT, #(10 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[2], [T0]
-    ld1             {Y1.h}[2], [T1]
-
-    ldr             T0d, [LUT, #(3 * 4)]
-    ldr             T1d, [LUT, #(11 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[3], [T0]
-    ld1             {Y1.h}[3], [T1]
-
-    ldr             T0d, [LUT, #(4 * 4)]
-    ldr             T1d, [LUT, #(12 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[4], [T0]
-    ld1             {Y1.h}[4], [T1]
-
-    ldr             T0d, [LUT, #(5 * 4)]
-    ldr             T1d, [LUT, #(13 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[5], [T0]
-    ld1             {Y1.h}[5], [T1]
-
-    ldr             T0d, [LUT, #(6 * 4)]
-    ldr             T1d, [LUT, #(14 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[6], [T0]
-    ld1             {Y1.h}[6], [T1]
-
-    ldr             T0d, [LUT, #(7 * 4)]
-    ldr             T1d, [LUT, #(15 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[7], [T0]
-    ld1             {Y1.h}[7], [T1]
-
-    add             LUT, LUT, #(16 * 4)
-.endm
-
-.macro LOAD15
-    eor             Y1.16b, Y1.16b, Y1.16b
-
-    ldr             T0d, [LUT, #(0 * 4)]
-    ldr             T1d, [LUT, #(8 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[0], [T0]
-    ld1             {Y1.h}[0], [T1]
-
-    ldr             T0d, [LUT, #(1 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[1], [T0]
-
-    ldr             T0d, [LUT, #(2 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[2], [T0]
-
-    ldr             T0d, [LUT, #(3 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[3], [T0]
-
-    ldr             T0d, [LUT, #(4 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[4], [T0]
-
-    ldr             T0d, [LUT, #(5 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[5], [T0]
-
-    ldr             T0d, [LUT, #(6 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[6], [T0]
-
-    ldr             T0d, [LUT, #(7 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[7], [T0]
-
-    cmp             LENEND, #2
-    b.lt            1515f
-    ldr             T1d, [LUT, #(9 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y1.h}[1], [T1]
-
-    cmp             LENEND, #3
-    b.lt            1515f
-    ldr             T1d, [LUT, #(10 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y1.h}[2], [T1]
-
-    cmp             LENEND, #4
-    b.lt            1515f
-    ldr             T1d, [LUT, #(11 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y1.h}[3], [T1]
-
-    cmp             LENEND, #5
-    b.lt            1515f
-    ldr             T1d, [LUT, #(12 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y1.h}[4], [T1]
-
-    cmp             LENEND, #6
-    b.lt            1515f
-    ldr             T1d, [LUT, #(13 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y1.h}[5], [T1]
-
-    cmp             LENEND, #7
-    b.lt            1515f
-    ldr             T1d, [LUT, #(14 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y1.h}[6], [T1]
-
-1515:
-.endm
-
-.macro LOAD8
-    ldr             T0d, [LUT, #(0 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[0], [T0]
-
-    ldr             T0d, [LUT, #(1 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[1], [T0]
-
-    ldr             T0d, [LUT, #(2 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[2], [T0]
-
-    ldr             T0d, [LUT, #(3 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[3], [T0]
-
-    ldr             T0d, [LUT, #(4 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[4], [T0]
-
-    ldr             T0d, [LUT, #(5 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[5], [T0]
-
-    ldr             T0d, [LUT, #(6 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[6], [T0]
-
-    ldr             T0d, [LUT, #(7 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[7], [T0]
-.endm
-
-.macro LOAD7
-    eor             Y0.16b, Y0.16b, Y0.16b
-
-    ldr             T0d, [LUT, #(0 * 4)]
-    add             T0, BLOCK, T0, lsl #1
-    ld1             {Y0.h}[0], [T0]
-
-    cmp             LENEND, #2
-    b.lt            77f
-    ldr             T1d, [LUT, #(1 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[1], [T1]
-
-    cmp             LENEND, #3
-    b.lt            77f
-    ldr             T1d, [LUT, #(2 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[2], [T1]
-
-    cmp             LENEND, #4
-    b.lt            77f
-    ldr             T1d, [LUT, #(3 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[3], [T1]
-
-    cmp             LENEND, #5
-    b.lt            77f
-    ldr             T1d, [LUT, #(4 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[4], [T1]
-
-    cmp             LENEND, #6
-    b.lt            77f
-    ldr             T1d, [LUT, #(5 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[5], [T1]
-
-    cmp             LENEND, #7
-    b.lt            77f
-    ldr             T1d, [LUT, #(6 * 4)]
-    add             T1, BLOCK, T1, lsl #1
-    ld1             {Y0.h}[6], [T1]
-
-77:
-.endm
-
-.macro REDUCE0
-    ld1             {v0.8h, v1.8h, v2.8h, v3.8h}, [VALUES], #64
-    ld1             {v4.8h, v5.8h, v6.8h, v7.8h}, [VALUES], #64
-
-    cmeq            v0.8h, v0.8h, #0
-    cmeq            v1.8h, v1.8h, #0
-    cmeq            v2.8h, v2.8h, #0
-    cmeq            v3.8h, v3.8h, #0
-    cmeq            v4.8h, v4.8h, #0
-    cmeq            v5.8h, v5.8h, #0
-    cmeq            v6.8h, v6.8h, #0
-    cmeq            v7.8h, v7.8h, #0
-
-    xtn             v0.8b, v0.8h
-    xtn             v2.8b, v2.8h
-    xtn             v4.8b, v4.8h
-    xtn             v6.8b, v6.8h
-    xtn2            v0.16b, v1.8h
-    xtn2            v2.16b, v3.8h
-    xtn2            v4.16b, v5.8h
-    xtn2            v6.16b, v7.8h
-
-    and             v0.16b, v0.16b, ANDMASK.16b
-    and             v2.16b, v2.16b, ANDMASK.16b
-    and             v4.16b, v4.16b, ANDMASK.16b
-    and             v6.16b, v6.16b, ANDMASK.16b
-    addp            v0.16b, v0.16b, v2.16b
-    addp            v4.16b, v4.16b, v6.16b
-    addp            v0.16b, v0.16b, v4.16b
-    addp            v0.16b, v0.16b, v0.16b
-    umov            T0, v0.D[0]
-    mvn             T0, T0
-    str             T0, [BITS]
-.endm
-
-/*
- * Prepare data for jsimd_encode_mcu_AC_first().
- *
- * GLOBAL(int)
- * jsimd_encode_mcu_AC_first_prepare_neon(const JCOEF *block,
- *                                        const int *jpeg_natural_order_start,
- *                                        int Sl, int Al, JCOEF *values,
- *                                        size_t *zerobits)
- *
- * x0 = const JCOEF *block
- * x1 = const int *jpeg_natural_order_start
- * w2 = int Sl
- * w3 = int Al
- * x4 = JCOEF *values
- * x5 = size_t *zerobits
- *
- */
-
-    ZERO            .req v0
-    Y0              .req v2
-    Y1              .req v3
-    N0              .req v4
-    N1              .req v5
-    AL              .req v6
-    ANDMASK         .req v20
-    K               .req w12
-    LUT             .req x1
-    T0              .req x10
-    T0d             .req w10
-    T1              .req x11
-    T1d             .req w11
-    BLOCK           .req x0
-    VALUES          .req x4
-    XORVALUES       .req x14
-    LEN             .req w2
-    LENEND          .req w9
-    BITS            .req x5
-
-asm_function jsimd_encode_mcu_AC_first_prepare_neon
-    get_symbol_loc  T0, Ljsimd_encode_mcu_AC_first_prepare_neon_consts
-    neg             w3, w3                        /* Al = -Al */
-    eor             ZERO.16b, ZERO.16b, ZERO.16b
-    ld1             {ANDMASK.16b}, [T0]
-    dup             AL.8h, w3
-    add             XORVALUES, VALUES, #(/*DCTSIZE2*/ 64 * 2)
-    and             LENEND, LEN, 7
-    lsr             K, LEN, 4
-    cbz             K, 3f
-1:
-    LOAD16
-    cmlt            N0.8h, Y0.8h, #0
-    cmlt            N1.8h, Y1.8h, #0
-    abs             Y0.8h, Y0.8h
-    abs             Y1.8h, Y1.8h
-    ushl            Y0.8h, Y0.8h, AL.8h
-    ushl            Y1.8h, Y1.8h, AL.8h
-    eor             N0.16b, N0.16b, Y0.16b
-    eor             N1.16b, N1.16b, Y1.16b
-    st1             {Y0.8h, Y1.8h}, [VALUES], #32
-    st1             {N0.8h, N1.8h}, [XORVALUES], #32
-    subs            K, K, #1
-    b.ne            1b
-3:
-    tst             LEN, #8
-    b.eq            3f
-    tst             LEN, #7
-    b.eq            2f
-
-    LOAD15
-    cmlt            N0.8h, Y0.8h, #0
-    cmlt            N1.8h, Y1.8h, #0
-    abs             Y0.8h, Y0.8h
-    abs             Y1.8h, Y1.8h
-    ushl            Y0.8h, Y0.8h, AL.8h
-    ushl            Y1.8h, Y1.8h, AL.8h
-    eor             N0.16b, N0.16b, Y0.16b
-    eor             N1.16b, N1.16b, Y1.16b
-    st1             {Y0.8h, Y1.8h}, [VALUES], #32
-    st1             {N0.8h, N1.8h}, [XORVALUES], #32
-    b               4f
-2:
-    LOAD8
-    cmlt            N0.8h, Y0.8h, #0
-    abs             Y0.8h, Y0.8h
-    ushl            Y0.8h, Y0.8h, AL.8h
-    eor             N0.16b, N0.16b, Y0.16b
-    st1             {Y0.8h}, [VALUES], #16
-    st1             {N0.8h}, [XORVALUES], #16
-    b               4f
-3:
-    cbz             LENEND, 4f
-    LOAD7
-    cmlt            N0.8h, Y0.8h, #0
-    abs             Y0.8h, Y0.8h
-    ushl            Y0.8h, Y0.8h, AL.8h
-    eor             N0.16b, N0.16b, Y0.16b
-    st1             {Y0.8h}, [VALUES], #16
-    st1             {N0.8h}, [XORVALUES], #16
-    /* b               4f */
-    /* fallthrough */
-4:
-    add             K, LEN, #7
-    lsr             K, K, #3
-    subs            K, K, #(/*DCTSIZE2*/ 64 / 8)
-    b.eq            5f
-1:
-    st1             {ZERO.8h}, [VALUES], #16
-    st1             {ZERO.8h}, [XORVALUES], #16
-    adds            K, K, #1
-    b.ne            1b
-5:
-    sub             VALUES, VALUES, #(/*DCTSIZE2*/ 64 * 2)
-
-    REDUCE0
-
-    br              x30
-
-    .unreq          ZERO
-    .unreq          Y0
-    .unreq          Y1
-    .unreq          N0
-    .unreq          N1
-    .unreq          AL
-    .unreq          ANDMASK
-    .unreq          K
-    .unreq          LUT
-    .unreq          T0
-    .unreq          T0d
-    .unreq          T1
-    .unreq          T1d
-    .unreq          BLOCK
-    .unreq          VALUES
-    .unreq          XORVALUES
-    .unreq          LEN
-    .unreq          LENEND
-    .unreq          BITS
-
-/*
- * Prepare data for jsimd_encode_mcu_AC_refine.
- *
- * GLOBAL(int)
- * jsimd_encode_mcu_AC_refine_prepare_neon(const JCOEF *block,
- *                                         const int *jpeg_natural_order_start,
- *                                         int Sl, int Al, JCOEF *absvalues,
- *                                         size_t *bits)
- *
- * x0 = const JCOEF *block
- * x1 = const int *jpeg_natural_order_start
- * w2 = int Sl
- * w3 = int Al
- * x4 = JCOEF *absvalues
- * x5 = size_t *bits
- *
- */
-
-    ZERO            .req v0
-    ONE             .req v1
-    Y0              .req v2
-    Y1              .req v3
-    N0              .req v4
-    N1              .req v5
-    AL              .req v6
-    ANDMASK         .req v20
-    K               .req w12
-    KK              .req w13
-    EOB             .req w14
-    SIGN            .req x15
-    LUT             .req x1
-    T0              .req x10
-    T0d             .req w10
-    T1              .req x11
-    T1d             .req w11
-    BLOCK           .req x0
-    VALUES          .req x4
-    LEN             .req w2
-    LENEND          .req w9
-    BITS            .req x5
-
-asm_function jsimd_encode_mcu_AC_refine_prepare_neon
-    get_symbol_loc  T0, Ljsimd_encode_mcu_AC_refine_prepare_neon_consts
-    neg             w3, w3                        /* Al = -Al */
-    movi            ONE.8h, #1
-    eor             SIGN, SIGN, SIGN
-    eor             ZERO.16b, ZERO.16b, ZERO.16b
-    eor             EOB, EOB, EOB
-    ld1             {ANDMASK.16b}, [T0]
-    eor             KK, KK, KK
-    dup             AL.8h, w3
-    and             LENEND, LEN, 7
-    lsr             K, LEN, 4
-    cbz             K, 3f
-1:
-    LOAD16
-    cmlt            N0.8h, Y0.8h, #0
-    cmlt            N1.8h, Y1.8h, #0
-    abs             Y0.8h, Y0.8h
-    abs             Y1.8h, Y1.8h
-    ushl            Y0.8h, Y0.8h, AL.8h
-    ushl            Y1.8h, Y1.8h, AL.8h
-    st1             {Y0.8h, Y1.8h}, [VALUES], #32
-    xtn             N0.8b, N0.8h
-    xtn             N1.8b, N1.8h
-    cmeq            Y0.8h, Y0.8h, ONE.8h
-    cmeq            Y1.8h, Y1.8h, ONE.8h
-    xtn             Y0.8b, Y0.8h
-    xtn             Y1.8b, Y1.8h
-    and             N0.8b, N0.8b, ANDMASK.8b
-    and             N1.8b, N1.8b, ANDMASK.8b
-    and             Y0.8b, Y0.8b, ANDMASK.8b
-    and             Y1.8b, Y1.8b, ANDMASK.8b
-    addv            B28, N0.8b
-    addv            B29, N1.8b
-    addv            B30, Y0.8b
-    addv            B31, Y1.8b
-    ins             v28.b[1], v29.b[0]
-    ins             v30.b[1], v31.b[0]
-    umov            T0d, v28.h[0]    /* lsignbits.val16u[k>>4] = _mm_movemask_epi8(neg); */
-    umov            T1d, v30.h[0]    /* idx = _mm_movemask_epi8(x1); */
-    lsr             SIGN, SIGN, #16  /* make room for sizebits */
-    orr             SIGN, SIGN, T0, lsl #48
-    cbz             T1d, 2f
-    rbit            T1d, T1d
-    clz             T1d, T1d
-    add             EOB, KK, T1d     /* EOB = k + idx; */
-2:
-    add             KK, KK, #16
-    subs            K, K, #1
-    b.ne            1b
-3:
-    tst             LEN, #8
-    b.eq            3f
-    tst             LEN, #7
-    b.eq            2f
-
-    LOAD15
-    cmlt            N0.8h, Y0.8h, #0
-    cmlt            N1.8h, Y1.8h, #0
-    abs             Y0.8h, Y0.8h
-    abs             Y1.8h, Y1.8h
-    ushl            Y0.8h, Y0.8h, AL.8h
-    ushl            Y1.8h, Y1.8h, AL.8h
-    st1             {Y0.8h, Y1.8h}, [VALUES], #32
-    xtn             N0.8b, N0.8h
-    xtn             N1.8b, N1.8h
-    cmeq            Y0.8h, Y0.8h, ONE.8h
-    cmeq            Y1.8h, Y1.8h, ONE.8h
-    xtn             Y0.8b, Y0.8h
-    xtn             Y1.8b, Y1.8h
-    and             N0.8b, N0.8b, ANDMASK.8b
-    and             N1.8b, N1.8b, ANDMASK.8b
-    and             Y0.8b, Y0.8b, ANDMASK.8b
-    and             Y1.8b, Y1.8b, ANDMASK.8b
-    addv            B28, N0.8b
-    addv            B29, N1.8b
-    addv            B30, Y0.8b
-    addv            B31, Y1.8b
-    ins             v28.b[1], v29.b[0]
-    ins             v30.b[1], v31.b[0]
-    umov            T0d, v28.h[0]    /* lsignbits.val16u[k>>4] = _mm_movemask_epi8(neg); */
-    umov            T1d, v30.h[0]    /* idx = _mm_movemask_epi8(x1); */
-    lsr             SIGN, SIGN, #16  /* make room for sizebits */
-    orr             SIGN, SIGN, T0, lsl #48
-    cbz             T1d, 4f
-    rbit            T1d, T1d
-    clz             T1d, T1d
-    add             EOB, KK, T1d     /* EOB = k + idx; */
-    b               4f
-2:
-    LOAD8
-    cmlt            N0.8h, Y0.8h, #0
-    abs             Y0.8h, Y0.8h
-    ushl            Y0.8h, Y0.8h, AL.8h
-    st1             {Y0.8h}, [VALUES], #16
-    xtn             N0.8b, N0.8h
-    cmeq            Y0.8h, Y0.8h, ONE.8h
-    xtn             Y0.8b, Y0.8h
-    and             N0.8b, N0.8b, ANDMASK.8b
-    and             Y0.8b, Y0.8b, ANDMASK.8b
-    addv            B28, N0.8b
-    addv            B30, Y0.8b
-    umov            T0d, v28.b[0]    /* lsignbits.val16u[k>>4] = _mm_movemask_epi8(neg); */
-    umov            T1d, v30.b[0]    /* idx = _mm_movemask_epi8(x1); */
-    lsr             SIGN, SIGN, #8   /* make room for sizebits */
-    orr             SIGN, SIGN, T0, lsl #56
-    cbz             T1d, 4f
-    rbit            T1d, T1d
-    clz             T1d, T1d
-    add             EOB, KK, T1d     /* EOB = k + idx; */
-    b               4f
-3:
-    cbz             LENEND, 4f
-    LOAD7
-    cmlt            N0.8h, Y0.8h, #0
-    abs             Y0.8h, Y0.8h
-    ushl            Y0.8h, Y0.8h, AL.8h
-    st1             {Y0.8h}, [VALUES], #16
-    xtn             N0.8b, N0.8h
-    cmeq            Y0.8h, Y0.8h, ONE.8h
-    xtn             Y0.8b, Y0.8h
-    and             N0.8b, N0.8b, ANDMASK.8b
-    and             Y0.8b, Y0.8b, ANDMASK.8b
-    addv            B28, N0.8b
-    addv            B30, Y0.8b
-    umov            T0d, v28.b[0]    /* lsignbits.val16u[k>>4] = _mm_movemask_epi8(neg); */
-    umov            T1d, v30.b[0]    /* idx = _mm_movemask_epi8(x1); */
-    lsr             SIGN, SIGN, #8   /* make room for sizebits */
-    orr             SIGN, SIGN, T0, lsl #56
-    cbz             T1d, 4f
-    rbit            T1d, T1d
-    clz             T1d, T1d
-    add             EOB, KK, T1d     /* EOB = k + idx; */
-    /* b               4f */
-    /* fallthrough */
-4:
-    add             K, LEN, #7
-    lsr             K, K, #3
-    subs            K, K, #(/*DCTSIZE2*/ 64 / 8)
-    b.eq            5f
-1:
-    st1             {ZERO.8h}, [VALUES], #16
-    lsr             SIGN, SIGN, #8
-    adds            K, K, #1
-    b.ne            1b
-5:
-    mvn             SIGN, SIGN
-    sub             VALUES, VALUES, #(/*DCTSIZE2*/ 64 * 2)
-    str             SIGN, [BITS, #8]
-
-    REDUCE0
-
-    mov             w0, EOB
-    br              x30
-
-    .unreq          ZERO
-    .unreq          ONE
-    .unreq          Y0
-    .unreq          Y1
-    .unreq          N0
-    .unreq          N1
-    .unreq          AL
-    .unreq          ANDMASK
-    .unreq          K
-    .unreq          KK
-    .unreq          EOB
-    .unreq          SIGN
-    .unreq          LUT
-    .unreq          T0
-    .unreq          T0d
-    .unreq          T1
-    .unreq          T1d
-    .unreq          BLOCK
-    .unreq          VALUES
-    .unreq          LEN
-    .unreq          LENEND
-    .unreq          BITS
-
-.purgem LOAD16
-.purgem LOAD15
-.purgem LOAD8
-.purgem LOAD7
-.purgem REDUCE0
diff --git a/simd/arm/jcphuff-neon.c b/simd/arm/jcphuff-neon.c
new file mode 100644
index 0000000..61f94c2
--- /dev/null
+++ b/simd/arm/jcphuff-neon.c
@@ -0,0 +1,588 @@
+/*
+ * jcphuff-neon.c - prepare data for progressive Huffman encoding (Arm Neon)
+ *
+ * Copyright (C) 2020, Arm Limited.  All Rights Reserved.
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty.  In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ *    claim that you wrote the original software. If you use this software
+ *    in a product, an acknowledgment in the product documentation would be
+ *    appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ *    misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+#define JPEG_INTERNALS
+#include "../../jinclude.h"
+#include "../../jpeglib.h"
+#include "../../jsimd.h"
+#include "../../jdct.h"
+#include "../../jsimddct.h"
+#include "../jsimd.h"
+
+#include <arm_neon.h>
+
+
+/* Data preparation for encode_mcu_AC_first().
+ *
+ * The equivalent scalar C function (encode_mcu_AC_first_prepare()) can be
+ * found in jcphuff.c.
+ */
+
+void jsimd_encode_mcu_AC_first_prepare_neon
+  (const JCOEF *block, const int *jpeg_natural_order_start, int Sl, int Al,
+   JCOEF *values, size_t *zerobits)
+{
+  JCOEF *values_ptr = values;
+  JCOEF *diff_values_ptr = values + DCTSIZE2;
+
+  /* Rows of coefficients to zero (since they haven't been processed) */
+  int i, rows_to_zero = 8;
+
+  for (i = 0; i < Sl / 16; i++) {
+    int16x8_t coefs1 = vld1q_dup_s16(block + jpeg_natural_order_start[0]);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[1], coefs1, 1);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[2], coefs1, 2);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[3], coefs1, 3);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[4], coefs1, 4);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[5], coefs1, 5);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[6], coefs1, 6);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[7], coefs1, 7);
+    int16x8_t coefs2 = vld1q_dup_s16(block + jpeg_natural_order_start[8]);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[9], coefs2, 1);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[10], coefs2, 2);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[11], coefs2, 3);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[12], coefs2, 4);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[13], coefs2, 5);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[14], coefs2, 6);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[15], coefs2, 7);
+
+    /* Isolate sign of coefficients. */
+    int16x8_t sign_coefs1 = vshrq_n_s16(coefs1, 15);
+    int16x8_t sign_coefs2 = vshrq_n_s16(coefs2, 15);
+    /* Compute absolute value of coefficients and apply point transform Al. */
+    int16x8_t abs_coefs1 = vabsq_s16(coefs1);
+    int16x8_t abs_coefs2 = vabsq_s16(coefs2);
+    coefs1 = vshlq_s16(abs_coefs1, vdupq_n_s16(-Al));
+    coefs2 = vshlq_s16(abs_coefs2, vdupq_n_s16(-Al));
+
+    /* Compute diff values. */
+    int16x8_t diff1 = veorq_s16(coefs1, sign_coefs1);
+    int16x8_t diff2 = veorq_s16(coefs2, sign_coefs2);
+
+    /* Store transformed coefficients and diff values. */
+    vst1q_s16(values_ptr, coefs1);
+    vst1q_s16(values_ptr + DCTSIZE, coefs2);
+    vst1q_s16(diff_values_ptr, diff1);
+    vst1q_s16(diff_values_ptr + DCTSIZE, diff2);
+    values_ptr += 16;
+    diff_values_ptr += 16;
+    jpeg_natural_order_start += 16;
+    rows_to_zero -= 2;
+  }
+
+  /* Same operation but for remaining partial vector */
+  int remaining_coefs = Sl % 16;
+  if (remaining_coefs > 8) {
+    int16x8_t coefs1 = vld1q_dup_s16(block + jpeg_natural_order_start[0]);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[1], coefs1, 1);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[2], coefs1, 2);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[3], coefs1, 3);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[4], coefs1, 4);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[5], coefs1, 5);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[6], coefs1, 6);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[7], coefs1, 7);
+    int16x8_t coefs2 = vdupq_n_s16(0);
+    switch (remaining_coefs) {
+    case 15:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[14], coefs2, 6);
+    case 14:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[13], coefs2, 5);
+    case 13:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[12], coefs2, 4);
+    case 12:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[11], coefs2, 3);
+    case 11:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[10], coefs2, 2);
+    case 10:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[9], coefs2, 1);
+    case 9:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[8], coefs2, 0);
+    default:
+      break;
+    }
+
+    /* Isolate sign of coefficients. */
+    int16x8_t sign_coefs1 = vshrq_n_s16(coefs1, 15);
+    int16x8_t sign_coefs2 = vshrq_n_s16(coefs2, 15);
+    /* Compute absolute value of coefficients and apply point transform Al. */
+    int16x8_t abs_coefs1 = vabsq_s16(coefs1);
+    int16x8_t abs_coefs2 = vabsq_s16(coefs2);
+    coefs1 = vshlq_s16(abs_coefs1, vdupq_n_s16(-Al));
+    coefs2 = vshlq_s16(abs_coefs2, vdupq_n_s16(-Al));
+
+    /* Compute diff values. */
+    int16x8_t diff1 = veorq_s16(coefs1, sign_coefs1);
+    int16x8_t diff2 = veorq_s16(coefs2, sign_coefs2);
+
+    /* Store transformed coefficients and diff values. */
+    vst1q_s16(values_ptr, coefs1);
+    vst1q_s16(values_ptr + DCTSIZE, coefs2);
+    vst1q_s16(diff_values_ptr, diff1);
+    vst1q_s16(diff_values_ptr + DCTSIZE, diff2);
+    values_ptr += 16;
+    diff_values_ptr += 16;
+    rows_to_zero -= 2;
+
+  } else if (remaining_coefs > 0) {
+    int16x8_t coefs = vdupq_n_s16(0);
+
+    switch (remaining_coefs) {
+    case 8:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[7], coefs, 7);
+    case 7:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[6], coefs, 6);
+    case 6:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[5], coefs, 5);
+    case 5:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[4], coefs, 4);
+    case 4:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[3], coefs, 3);
+    case 3:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[2], coefs, 2);
+    case 2:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[1], coefs, 1);
+    case 1:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[0], coefs, 0);
+    default:
+      break;
+    }
+
+    /* Isolate sign of coefficients. */
+    int16x8_t sign_coefs = vshrq_n_s16(coefs, 15);
+    /* Compute absolute value of coefficients and apply point transform Al. */
+    int16x8_t abs_coefs = vabsq_s16(coefs);
+    coefs = vshlq_s16(abs_coefs, vdupq_n_s16(-Al));
+
+    /* Compute diff values. */
+    int16x8_t diff = veorq_s16(coefs, sign_coefs);
+
+    /* Store transformed coefficients and diff values. */
+    vst1q_s16(values_ptr, coefs);
+    vst1q_s16(diff_values_ptr, diff);
+    values_ptr += 8;
+    diff_values_ptr += 8;
+    rows_to_zero--;
+  }
+
+  /* Zero remaining memory in the values and diff_values blocks. */
+  for (i = 0; i < rows_to_zero; i++) {
+    vst1q_s16(values_ptr, vdupq_n_s16(0));
+    vst1q_s16(diff_values_ptr, vdupq_n_s16(0));
+    values_ptr += 8;
+    diff_values_ptr += 8;
+  }
+
+  /* Construct zerobits bitmap.  A set bit means that the corresponding
+   * coefficient != 0.
+   */
+  int16x8_t row0 = vld1q_s16(values + 0 * DCTSIZE);
+  int16x8_t row1 = vld1q_s16(values + 1 * DCTSIZE);
+  int16x8_t row2 = vld1q_s16(values + 2 * DCTSIZE);
+  int16x8_t row3 = vld1q_s16(values + 3 * DCTSIZE);
+  int16x8_t row4 = vld1q_s16(values + 4 * DCTSIZE);
+  int16x8_t row5 = vld1q_s16(values + 5 * DCTSIZE);
+  int16x8_t row6 = vld1q_s16(values + 6 * DCTSIZE);
+  int16x8_t row7 = vld1q_s16(values + 7 * DCTSIZE);
+
+  uint8x8_t row0_eq0 = vmovn_u16(vceqq_s16(row0, vdupq_n_s16(0)));
+  uint8x8_t row1_eq0 = vmovn_u16(vceqq_s16(row1, vdupq_n_s16(0)));
+  uint8x8_t row2_eq0 = vmovn_u16(vceqq_s16(row2, vdupq_n_s16(0)));
+  uint8x8_t row3_eq0 = vmovn_u16(vceqq_s16(row3, vdupq_n_s16(0)));
+  uint8x8_t row4_eq0 = vmovn_u16(vceqq_s16(row4, vdupq_n_s16(0)));
+  uint8x8_t row5_eq0 = vmovn_u16(vceqq_s16(row5, vdupq_n_s16(0)));
+  uint8x8_t row6_eq0 = vmovn_u16(vceqq_s16(row6, vdupq_n_s16(0)));
+  uint8x8_t row7_eq0 = vmovn_u16(vceqq_s16(row7, vdupq_n_s16(0)));
+
+  const uint8x8_t bitmap_mask =
+    { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 };
+
+  row0_eq0 = vand_u8(row0_eq0, bitmap_mask);
+  row1_eq0 = vand_u8(row1_eq0, bitmap_mask);
+  row2_eq0 = vand_u8(row2_eq0, bitmap_mask);
+  row3_eq0 = vand_u8(row3_eq0, bitmap_mask);
+  row4_eq0 = vand_u8(row4_eq0, bitmap_mask);
+  row5_eq0 = vand_u8(row5_eq0, bitmap_mask);
+  row6_eq0 = vand_u8(row6_eq0, bitmap_mask);
+  row7_eq0 = vand_u8(row7_eq0, bitmap_mask);
+
+  uint8x8_t bitmap_rows_01 = vpadd_u8(row0_eq0, row1_eq0);
+  uint8x8_t bitmap_rows_23 = vpadd_u8(row2_eq0, row3_eq0);
+  uint8x8_t bitmap_rows_45 = vpadd_u8(row4_eq0, row5_eq0);
+  uint8x8_t bitmap_rows_67 = vpadd_u8(row6_eq0, row7_eq0);
+  uint8x8_t bitmap_rows_0123 = vpadd_u8(bitmap_rows_01, bitmap_rows_23);
+  uint8x8_t bitmap_rows_4567 = vpadd_u8(bitmap_rows_45, bitmap_rows_67);
+  uint8x8_t bitmap_all = vpadd_u8(bitmap_rows_0123, bitmap_rows_4567);
+
+#if defined(__aarch64__)
+  /* Move bitmap to a 64-bit scalar register. */
+  uint64_t bitmap = vget_lane_u64(vreinterpret_u64_u8(bitmap_all), 0);
+  /* Store zerobits bitmap. */
+  *zerobits = ~bitmap;
+#else
+  /* Move bitmap to two 32-bit scalar registers. */
+  uint32_t bitmap0 = vget_lane_u32(vreinterpret_u32_u8(bitmap_all), 0);
+  uint32_t bitmap1 = vget_lane_u32(vreinterpret_u32_u8(bitmap_all), 1);
+  /* Store zerobits bitmap. */
+  zerobits[0] = ~bitmap0;
+  zerobits[1] = ~bitmap1;
+#endif
+}
+
+
+/* Data preparation for encode_mcu_AC_refine().
+ *
+ * The equivalent scalar C function (encode_mcu_AC_refine_prepare()) can be
+ * found in jcphuff.c.
+ */
+
+int jsimd_encode_mcu_AC_refine_prepare_neon
+  (const JCOEF *block, const int *jpeg_natural_order_start, int Sl, int Al,
+   JCOEF *absvalues, size_t *bits)
+{
+  /* Temporary storage buffers for data used to compute the signbits bitmap and
+   * the end-of-block (EOB) position
+   */
+  uint8_t coef_sign_bits[64];
+  uint8_t coef_eq1_bits[64];
+
+  JCOEF *absvalues_ptr = absvalues;
+  uint8_t *coef_sign_bits_ptr = coef_sign_bits;
+  uint8_t *eq1_bits_ptr = coef_eq1_bits;
+
+  /* Rows of coefficients to zero (since they haven't been processed) */
+  int i, rows_to_zero = 8;
+
+  for (i = 0; i < Sl / 16; i++) {
+    int16x8_t coefs1 = vld1q_dup_s16(block + jpeg_natural_order_start[0]);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[1], coefs1, 1);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[2], coefs1, 2);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[3], coefs1, 3);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[4], coefs1, 4);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[5], coefs1, 5);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[6], coefs1, 6);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[7], coefs1, 7);
+    int16x8_t coefs2 = vld1q_dup_s16(block + jpeg_natural_order_start[8]);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[9], coefs2, 1);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[10], coefs2, 2);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[11], coefs2, 3);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[12], coefs2, 4);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[13], coefs2, 5);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[14], coefs2, 6);
+    coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[15], coefs2, 7);
+
+    /* Compute and store data for signbits bitmap. */
+    uint8x8_t sign_coefs1 =
+      vmovn_u16(vreinterpretq_u16_s16(vshrq_n_s16(coefs1, 15)));
+    uint8x8_t sign_coefs2 =
+      vmovn_u16(vreinterpretq_u16_s16(vshrq_n_s16(coefs2, 15)));
+    vst1_u8(coef_sign_bits_ptr, sign_coefs1);
+    vst1_u8(coef_sign_bits_ptr + DCTSIZE, sign_coefs2);
+
+    /* Compute absolute value of coefficients and apply point transform Al. */
+    int16x8_t abs_coefs1 = vabsq_s16(coefs1);
+    int16x8_t abs_coefs2 = vabsq_s16(coefs2);
+    coefs1 = vshlq_s16(abs_coefs1, vdupq_n_s16(-Al));
+    coefs2 = vshlq_s16(abs_coefs2, vdupq_n_s16(-Al));
+    vst1q_s16(absvalues_ptr, coefs1);
+    vst1q_s16(absvalues_ptr + DCTSIZE, coefs2);
+
+    /* Test whether transformed coefficient values == 1 (used to find EOB
+     * position.)
+     */
+    uint8x8_t coefs_eq11 = vmovn_u16(vceqq_s16(coefs1, vdupq_n_s16(1)));
+    uint8x8_t coefs_eq12 = vmovn_u16(vceqq_s16(coefs2, vdupq_n_s16(1)));
+    vst1_u8(eq1_bits_ptr, coefs_eq11);
+    vst1_u8(eq1_bits_ptr + DCTSIZE, coefs_eq12);
+
+    absvalues_ptr += 16;
+    coef_sign_bits_ptr += 16;
+    eq1_bits_ptr += 16;
+    jpeg_natural_order_start += 16;
+    rows_to_zero -= 2;
+  }
+
+  /* Same operation but for remaining partial vector */
+  int remaining_coefs = Sl % 16;
+  if (remaining_coefs > 8) {
+    int16x8_t coefs1 = vld1q_dup_s16(block + jpeg_natural_order_start[0]);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[1], coefs1, 1);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[2], coefs1, 2);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[3], coefs1, 3);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[4], coefs1, 4);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[5], coefs1, 5);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[6], coefs1, 6);
+    coefs1 = vld1q_lane_s16(block + jpeg_natural_order_start[7], coefs1, 7);
+    int16x8_t coefs2 = vdupq_n_s16(0);
+    switch (remaining_coefs) {
+    case 15:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[14], coefs2, 6);
+    case 14:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[13], coefs2, 5);
+    case 13:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[12], coefs2, 4);
+    case 12:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[11], coefs2, 3);
+    case 11:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[10], coefs2, 2);
+    case 10:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[9], coefs2, 1);
+    case 9:
+      coefs2 = vld1q_lane_s16(block + jpeg_natural_order_start[8], coefs2, 0);
+    default:
+      break;
+    }
+
+    /* Compute and store data for signbits bitmap. */
+    uint8x8_t sign_coefs1 =
+      vmovn_u16(vreinterpretq_u16_s16(vshrq_n_s16(coefs1, 15)));
+    uint8x8_t sign_coefs2 =
+      vmovn_u16(vreinterpretq_u16_s16(vshrq_n_s16(coefs2, 15)));
+    vst1_u8(coef_sign_bits_ptr, sign_coefs1);
+    vst1_u8(coef_sign_bits_ptr + DCTSIZE, sign_coefs2);
+
+    /* Compute absolute value of coefficients and apply point transform Al. */
+    int16x8_t abs_coefs1 = vabsq_s16(coefs1);
+    int16x8_t abs_coefs2 = vabsq_s16(coefs2);
+    coefs1 = vshlq_s16(abs_coefs1, vdupq_n_s16(-Al));
+    coefs2 = vshlq_s16(abs_coefs2, vdupq_n_s16(-Al));
+    vst1q_s16(absvalues_ptr, coefs1);
+    vst1q_s16(absvalues_ptr + DCTSIZE, coefs2);
+
+    /* Test whether transformed coefficient values == 1 (used to find EOB
+     * position.)
+     */
+    uint8x8_t coefs_eq11 = vmovn_u16(vceqq_s16(coefs1, vdupq_n_s16(1)));
+    uint8x8_t coefs_eq12 = vmovn_u16(vceqq_s16(coefs2, vdupq_n_s16(1)));
+    vst1_u8(eq1_bits_ptr, coefs_eq11);
+    vst1_u8(eq1_bits_ptr + DCTSIZE, coefs_eq12);
+
+    absvalues_ptr += 16;
+    coef_sign_bits_ptr += 16;
+    eq1_bits_ptr += 16;
+    jpeg_natural_order_start += 16;
+    rows_to_zero -= 2;
+
+  } else if (remaining_coefs > 0) {
+    int16x8_t coefs = vdupq_n_s16(0);
+
+    switch (remaining_coefs) {
+    case 8:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[7], coefs, 7);
+    case 7:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[6], coefs, 6);
+    case 6:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[5], coefs, 5);
+    case 5:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[4], coefs, 4);
+    case 4:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[3], coefs, 3);
+    case 3:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[2], coefs, 2);
+    case 2:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[1], coefs, 1);
+    case 1:
+      coefs = vld1q_lane_s16(block + jpeg_natural_order_start[0], coefs, 0);
+    default:
+      break;
+    }
+
+    /* Compute and store data for signbits bitmap. */
+    uint8x8_t sign_coefs =
+      vmovn_u16(vreinterpretq_u16_s16(vshrq_n_s16(coefs, 15)));
+    vst1_u8(coef_sign_bits_ptr, sign_coefs);
+
+    /* Compute absolute value of coefficients and apply point transform Al. */
+    int16x8_t abs_coefs = vabsq_s16(coefs);
+    coefs = vshlq_s16(abs_coefs, vdupq_n_s16(-Al));
+    vst1q_s16(absvalues_ptr, coefs);
+
+    /* Test whether transformed coefficient values == 1 (used to find EOB
+     * position.)
+     */
+    uint8x8_t coefs_eq1 = vmovn_u16(vceqq_s16(coefs, vdupq_n_s16(1)));
+    vst1_u8(eq1_bits_ptr, coefs_eq1);
+
+    absvalues_ptr += 8;
+    coef_sign_bits_ptr += 8;
+    eq1_bits_ptr += 8;
+    rows_to_zero--;
+  }
+
+  /* Zero remaining memory in blocks. */
+  for (i = 0; i < rows_to_zero; i++) {
+    vst1q_s16(absvalues_ptr, vdupq_n_s16(0));
+    vst1_u8(coef_sign_bits_ptr, vdup_n_u8(0));
+    vst1_u8(eq1_bits_ptr, vdup_n_u8(0));
+    absvalues_ptr += 8;
+    coef_sign_bits_ptr += 8;
+    eq1_bits_ptr += 8;
+  }
+
+  /* Construct zerobits bitmap. */
+  int16x8_t abs_row0 = vld1q_s16(absvalues + 0 * DCTSIZE);
+  int16x8_t abs_row1 = vld1q_s16(absvalues + 1 * DCTSIZE);
+  int16x8_t abs_row2 = vld1q_s16(absvalues + 2 * DCTSIZE);
+  int16x8_t abs_row3 = vld1q_s16(absvalues + 3 * DCTSIZE);
+  int16x8_t abs_row4 = vld1q_s16(absvalues + 4 * DCTSIZE);
+  int16x8_t abs_row5 = vld1q_s16(absvalues + 5 * DCTSIZE);
+  int16x8_t abs_row6 = vld1q_s16(absvalues + 6 * DCTSIZE);
+  int16x8_t abs_row7 = vld1q_s16(absvalues + 7 * DCTSIZE);
+
+  uint8x8_t abs_row0_eq0 = vmovn_u16(vceqq_s16(abs_row0, vdupq_n_s16(0)));
+  uint8x8_t abs_row1_eq0 = vmovn_u16(vceqq_s16(abs_row1, vdupq_n_s16(0)));
+  uint8x8_t abs_row2_eq0 = vmovn_u16(vceqq_s16(abs_row2, vdupq_n_s16(0)));
+  uint8x8_t abs_row3_eq0 = vmovn_u16(vceqq_s16(abs_row3, vdupq_n_s16(0)));
+  uint8x8_t abs_row4_eq0 = vmovn_u16(vceqq_s16(abs_row4, vdupq_n_s16(0)));
+  uint8x8_t abs_row5_eq0 = vmovn_u16(vceqq_s16(abs_row5, vdupq_n_s16(0)));
+  uint8x8_t abs_row6_eq0 = vmovn_u16(vceqq_s16(abs_row6, vdupq_n_s16(0)));
+  uint8x8_t abs_row7_eq0 = vmovn_u16(vceqq_s16(abs_row7, vdupq_n_s16(0)));
+
+  const uint8x8_t bitmap_mask =
+    { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 };
+
+  abs_row0_eq0 = vand_u8(abs_row0_eq0, bitmap_mask);
+  abs_row1_eq0 = vand_u8(abs_row1_eq0, bitmap_mask);
+  abs_row2_eq0 = vand_u8(abs_row2_eq0, bitmap_mask);
+  abs_row3_eq0 = vand_u8(abs_row3_eq0, bitmap_mask);
+  abs_row4_eq0 = vand_u8(abs_row4_eq0, bitmap_mask);
+  abs_row5_eq0 = vand_u8(abs_row5_eq0, bitmap_mask);
+  abs_row6_eq0 = vand_u8(abs_row6_eq0, bitmap_mask);
+  abs_row7_eq0 = vand_u8(abs_row7_eq0, bitmap_mask);
+
+  uint8x8_t bitmap_rows_01 = vpadd_u8(abs_row0_eq0, abs_row1_eq0);
+  uint8x8_t bitmap_rows_23 = vpadd_u8(abs_row2_eq0, abs_row3_eq0);
+  uint8x8_t bitmap_rows_45 = vpadd_u8(abs_row4_eq0, abs_row5_eq0);
+  uint8x8_t bitmap_rows_67 = vpadd_u8(abs_row6_eq0, abs_row7_eq0);
+  uint8x8_t bitmap_rows_0123 = vpadd_u8(bitmap_rows_01, bitmap_rows_23);
+  uint8x8_t bitmap_rows_4567 = vpadd_u8(bitmap_rows_45, bitmap_rows_67);
+  uint8x8_t bitmap_all = vpadd_u8(bitmap_rows_0123, bitmap_rows_4567);
+
+#if defined(__aarch64__)
+  /* Move bitmap to a 64-bit scalar register. */
+  uint64_t bitmap = vget_lane_u64(vreinterpret_u64_u8(bitmap_all), 0);
+  /* Store zerobits bitmap. */
+  bits[0] = ~bitmap;
+#else
+  /* Move bitmap to two 32-bit scalar registers. */
+  uint32_t bitmap0 = vget_lane_u32(vreinterpret_u32_u8(bitmap_all), 0);
+  uint32_t bitmap1 = vget_lane_u32(vreinterpret_u32_u8(bitmap_all), 1);
+  /* Store zerobits bitmap. */
+  bits[0] = ~bitmap0;
+  bits[1] = ~bitmap1;
+#endif
+
+  /* Construct signbits bitmap. */
+  uint8x8_t signbits_row0 = vld1_u8(coef_sign_bits + 0 * DCTSIZE);
+  uint8x8_t signbits_row1 = vld1_u8(coef_sign_bits + 1 * DCTSIZE);
+  uint8x8_t signbits_row2 = vld1_u8(coef_sign_bits + 2 * DCTSIZE);
+  uint8x8_t signbits_row3 = vld1_u8(coef_sign_bits + 3 * DCTSIZE);
+  uint8x8_t signbits_row4 = vld1_u8(coef_sign_bits + 4 * DCTSIZE);
+  uint8x8_t signbits_row5 = vld1_u8(coef_sign_bits + 5 * DCTSIZE);
+  uint8x8_t signbits_row6 = vld1_u8(coef_sign_bits + 6 * DCTSIZE);
+  uint8x8_t signbits_row7 = vld1_u8(coef_sign_bits + 7 * DCTSIZE);
+
+  signbits_row0 = vand_u8(signbits_row0, bitmap_mask);
+  signbits_row1 = vand_u8(signbits_row1, bitmap_mask);
+  signbits_row2 = vand_u8(signbits_row2, bitmap_mask);
+  signbits_row3 = vand_u8(signbits_row3, bitmap_mask);
+  signbits_row4 = vand_u8(signbits_row4, bitmap_mask);
+  signbits_row5 = vand_u8(signbits_row5, bitmap_mask);
+  signbits_row6 = vand_u8(signbits_row6, bitmap_mask);
+  signbits_row7 = vand_u8(signbits_row7, bitmap_mask);
+
+  bitmap_rows_01 = vpadd_u8(signbits_row0, signbits_row1);
+  bitmap_rows_23 = vpadd_u8(signbits_row2, signbits_row3);
+  bitmap_rows_45 = vpadd_u8(signbits_row4, signbits_row5);
+  bitmap_rows_67 = vpadd_u8(signbits_row6, signbits_row7);
+  bitmap_rows_0123 = vpadd_u8(bitmap_rows_01, bitmap_rows_23);
+  bitmap_rows_4567 = vpadd_u8(bitmap_rows_45, bitmap_rows_67);
+  bitmap_all = vpadd_u8(bitmap_rows_0123, bitmap_rows_4567);
+
+#if defined(__aarch64__)
+  /* Move bitmap to a 64-bit scalar register. */
+  bitmap = vget_lane_u64(vreinterpret_u64_u8(bitmap_all), 0);
+  /* Store signbits bitmap. */
+  bits[1] = ~bitmap;
+#else
+  /* Move bitmap to two 32-bit scalar registers. */
+  bitmap0 = vget_lane_u32(vreinterpret_u32_u8(bitmap_all), 0);
+  bitmap1 = vget_lane_u32(vreinterpret_u32_u8(bitmap_all), 1);
+  /* Store signbits bitmap. */
+  bits[2] = ~bitmap0;
+  bits[3] = ~bitmap1;
+#endif
+
+  /* Construct bitmap to find EOB position (the index of the last coefficient
+   * equal to 1.)
+   */
+  uint8x8_t row0_eq1 = vld1_u8(coef_eq1_bits + 0 * DCTSIZE);
+  uint8x8_t row1_eq1 = vld1_u8(coef_eq1_bits + 1 * DCTSIZE);
+  uint8x8_t row2_eq1 = vld1_u8(coef_eq1_bits + 2 * DCTSIZE);
+  uint8x8_t row3_eq1 = vld1_u8(coef_eq1_bits + 3 * DCTSIZE);
+  uint8x8_t row4_eq1 = vld1_u8(coef_eq1_bits + 4 * DCTSIZE);
+  uint8x8_t row5_eq1 = vld1_u8(coef_eq1_bits + 5 * DCTSIZE);
+  uint8x8_t row6_eq1 = vld1_u8(coef_eq1_bits + 6 * DCTSIZE);
+  uint8x8_t row7_eq1 = vld1_u8(coef_eq1_bits + 7 * DCTSIZE);
+
+  row0_eq1 = vand_u8(row0_eq1, bitmap_mask);
+  row1_eq1 = vand_u8(row1_eq1, bitmap_mask);
+  row2_eq1 = vand_u8(row2_eq1, bitmap_mask);
+  row3_eq1 = vand_u8(row3_eq1, bitmap_mask);
+  row4_eq1 = vand_u8(row4_eq1, bitmap_mask);
+  row5_eq1 = vand_u8(row5_eq1, bitmap_mask);
+  row6_eq1 = vand_u8(row6_eq1, bitmap_mask);
+  row7_eq1 = vand_u8(row7_eq1, bitmap_mask);
+
+  bitmap_rows_01 = vpadd_u8(row0_eq1, row1_eq1);
+  bitmap_rows_23 = vpadd_u8(row2_eq1, row3_eq1);
+  bitmap_rows_45 = vpadd_u8(row4_eq1, row5_eq1);
+  bitmap_rows_67 = vpadd_u8(row6_eq1, row7_eq1);
+  bitmap_rows_0123 = vpadd_u8(bitmap_rows_01, bitmap_rows_23);
+  bitmap_rows_4567 = vpadd_u8(bitmap_rows_45, bitmap_rows_67);
+  bitmap_all = vpadd_u8(bitmap_rows_0123, bitmap_rows_4567);
+
+#if defined(__aarch64__)
+  /* Move bitmap to a 64-bit scalar register. */
+  bitmap = vget_lane_u64(vreinterpret_u64_u8(bitmap_all), 0);
+
+  /* Return EOB position. */
+  if (bitmap == 0) {
+    /* EOB position is defined to be 0 if all coefficients != 1. */
+    return 0;
+  } else {
+    return 63 - __builtin_clzl(bitmap);
+  }
+#else
+  /* Move bitmap to two 32-bit scalar registers. */
+  bitmap0 = vget_lane_u32(vreinterpret_u32_u8(bitmap_all), 0);
+  bitmap1 = vget_lane_u32(vreinterpret_u32_u8(bitmap_all), 1);
+
+  /* Return EOB position. */
+  if (bitmap0 == 0 && bitmap1 == 0) {
+    return 0;
+  } else if (bitmap1 != 0) {
+    return 63 - __builtin_clz(bitmap1);
+  } else {
+    return 31 - __builtin_clz(bitmap0);
+  }
+#endif
+}