455 lines
10 KiB
ArmAsm
455 lines
10 KiB
ArmAsm
/*
|
|
* Copyright (c) 2012-2018 Apple Inc. All rights reserved.
|
|
*
|
|
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
|
|
*
|
|
* This file contains Original Code and/or Modifications of Original Code
|
|
* as defined in and that are subject to the Apple Public Source License
|
|
* Version 2.0 (the 'License'). You may not use this file except in
|
|
* compliance with the License. The rights granted to you under the License
|
|
* may not be used to create, or enable the creation or redistribution of,
|
|
* unlawful or unlicensed copies of an Apple operating system, or to
|
|
* circumvent, violate, or enable the circumvention or violation of, any
|
|
* terms of an Apple operating system software license agreement.
|
|
*
|
|
* Please obtain a copy of the License at
|
|
* http://www.opensource.apple.com/apsl/ and read it before using this file.
|
|
*
|
|
* The Original Code and all software distributed under the License are
|
|
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
|
|
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
|
|
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
|
|
* Please see the License for the specific language governing rights and
|
|
* limitations under the License.
|
|
*
|
|
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
|
|
*/
|
|
|
|
/*
|
|
* This assembly was previously cloned from ../arm/cpu_in_cksum.s (__arm__)
|
|
* with __arm64__ tagged ARM64_TODO . This code revision is optimized based
|
|
* on the 64-bit part in netinet/cpu_in_cksum.c
|
|
*
|
|
* cclee - CoreOS - Vector & Numerics. 06/20/2012.
|
|
*/
|
|
|
|
#ifdef KERNEL
|
|
#define CKSUM_ERR _kprintf
|
|
#else
|
|
#ifndef LIBSYSCALL_INTERFACE
|
|
#error "LIBSYSCALL_INTERFACE not defined"
|
|
#endif /* !LIBSYSCALL_INTERFACE */
|
|
#define CKSUM_ERR _fprintf_stderr
|
|
#endif /* !KERNEL */
|
|
|
|
/*
|
|
* XXX: adi@apple.com:
|
|
*
|
|
* Ugly, but we have little choice, since relying on genassym and <assym.s>
|
|
* is not possible unless this code lives in osfmk. Note also that this
|
|
* routine expects "mbuf-like" argument, and it does not expect the mbuf to be
|
|
* authentic; it only cares about 3 fields.
|
|
*/
|
|
#if defined(__LP64__)
|
|
#define M_NEXT 0
|
|
#define M_DATA 16 // 8-byte address, would be aligned to 8-byte boundary
|
|
#define M_LEN 24
|
|
#else
|
|
#define M_NEXT 0
|
|
#define M_DATA 8
|
|
#define M_LEN 12
|
|
#endif
|
|
|
|
.globl _os_cpu_in_cksum_mbuf
|
|
.text
|
|
.align 4
|
|
_os_cpu_in_cksum_mbuf:
|
|
|
|
|
|
/*
|
|
* 64-bit version.
|
|
*
|
|
* This function returns the partial 16-bit checksum accumulated in
|
|
* a 32-bit variable (withouth 1's complement); caller is responsible
|
|
* for folding the 32-bit sum into 16-bit and performinng the 1's
|
|
* complement if applicable
|
|
*/
|
|
|
|
/*
|
|
* uint32_t
|
|
* os_cpu_in_cksum_mbuf(struct mbuf *m, int len, int off, uint32_t initial_sum)
|
|
* {
|
|
* int mlen;
|
|
* uint64_t sum, partial;
|
|
* unsigned int final_acc;
|
|
* uint8_t *data;
|
|
* boolean_t needs_swap, started_on_odd;
|
|
*
|
|
* VERIFY(len >= 0);
|
|
* VERIFY(off >= 0);
|
|
*
|
|
* needs_swap = FALSE;
|
|
* started_on_odd = FALSE;
|
|
* sum = initial_sum;
|
|
*/
|
|
|
|
#define m x0
|
|
#define len x1
|
|
#define off x2
|
|
#define sum x3
|
|
#define needs_swap x4
|
|
#define started_on_odd x5
|
|
#define mlen x6
|
|
#define Wmlen w6
|
|
#define t x7
|
|
#define data x8
|
|
#if defined(__LP64__)
|
|
#define ptr_m x0
|
|
#define ptr_data x8
|
|
#else
|
|
#define ptr_m w0
|
|
#define ptr_data w8
|
|
#endif
|
|
|
|
|
|
mov needs_swap, #0 // needs_swap = FALSE;
|
|
mov started_on_odd, #0 // started_on_odd = FALSE;
|
|
mov w3, w3 // clear higher half
|
|
|
|
|
|
/*
|
|
* for (;;) {
|
|
* if (PREDICT_FALSE(m == NULL)) {
|
|
* CKSUM_ERR("%s: out of data\n", __func__);
|
|
* return (-1);
|
|
* }
|
|
* mlen = m->m_len;
|
|
* if (mlen > off) {
|
|
* mlen -= off;
|
|
* data = mtod(m, uint8_t *) + off;
|
|
* goto post_initial_offset;
|
|
* }
|
|
* off -= mlen;
|
|
* if (len == 0)
|
|
* break;
|
|
* m = m->m_next;
|
|
* }
|
|
*/
|
|
|
|
0:
|
|
cbz m, Lin_cksum_whoops // if (m == NULL) return -1;
|
|
ldr Wmlen, [m, #M_LEN] // mlen = m->m_len;
|
|
cmp mlen, off
|
|
b.le 1f
|
|
ldr ptr_data, [m, #M_DATA] // mtod(m, uint8_t *)
|
|
sub mlen, mlen, off // mlen -= off;
|
|
add data, data, off // data = mtod(m, uint8_t *) + off;
|
|
b L_post_initial_offset
|
|
1:
|
|
sub off, off, mlen
|
|
cbnz len, 2f
|
|
mov x0, x3
|
|
ret lr
|
|
2:
|
|
ldr ptr_m, [m, #M_NEXT]
|
|
b 0b
|
|
|
|
L_loop: // for (; len > 0; m = m->m_next) {
|
|
/*
|
|
* if (PREDICT_FALSE(m == NULL)) {
|
|
* CKSUM_ERR("%s: out of data\n", __func__);
|
|
* return (-1);
|
|
* }
|
|
* mlen = m->m_len;
|
|
* data = mtod(m, uint8_t *);
|
|
*/
|
|
cbz m, Lin_cksum_whoops // if (m == NULL) return -1;
|
|
ldr Wmlen, [m, #M_LEN] // mlen = m->m_len;
|
|
ldr ptr_data, [m, #M_DATA] // mtod(m, uint8_t *)
|
|
|
|
L_post_initial_offset:
|
|
/*
|
|
* if (mlen == 0) continue;
|
|
* if (mlen > len) mlen = len;
|
|
* len -= mlen;
|
|
*/
|
|
|
|
cbz mlen, L_continue
|
|
cmp mlen, len
|
|
csel mlen, mlen, len, le
|
|
sub len, len, mlen
|
|
|
|
/*
|
|
* partial = 0;
|
|
* if ((uintptr_t)data & 1) {
|
|
* started_on_odd = !started_on_odd;
|
|
* partial = *data << 8;
|
|
* ++data;
|
|
* --mlen;
|
|
* }
|
|
* needs_swap = started_on_odd;
|
|
*/
|
|
|
|
tst data, #1
|
|
mov x7, #0
|
|
mov x10, #0
|
|
b.eq 1f
|
|
ldrb w7, [data], #1
|
|
eor started_on_odd, started_on_odd, #1
|
|
sub mlen, mlen, #1
|
|
lsl w7, w7, #8
|
|
1:
|
|
|
|
|
|
/*
|
|
* if ((uintptr_t)data & 2) {
|
|
* if (mlen < 2)
|
|
* goto trailing_bytes;
|
|
* partial += *(uint16_t *)(void *)data;
|
|
* data += 2;
|
|
* mlen -= 2;
|
|
* }
|
|
*/
|
|
tst data, #2
|
|
mov needs_swap, started_on_odd
|
|
b.eq 1f
|
|
cmp mlen, #2
|
|
b.lt L_trailing_bytes
|
|
ldrh w9, [data], #2
|
|
sub mlen, mlen, #2
|
|
add w7, w7, w9
|
|
1:
|
|
|
|
/*
|
|
* if ((uintptr_t)data & 4) {
|
|
* if (mlen < 4)
|
|
* goto L2_bytes;
|
|
* partial += *(uint32_t *)(void *)data;
|
|
* data += 4;
|
|
* mlen -= 4;
|
|
* }
|
|
*/
|
|
// align on 8-bytes boundary if applicable
|
|
tst data, #4
|
|
b.eq 1f
|
|
cmp mlen, #4
|
|
b.lt L2_bytes
|
|
ldr w9, [data], #4
|
|
sub mlen, mlen, #4
|
|
adds w7, w7, w9
|
|
adc x7, x7, x10 // assumes x10 still is #0 as set above
|
|
1:
|
|
|
|
/*
|
|
* while (mlen >= 64) {
|
|
* __builtin_prefetch(data + 32);
|
|
* __builtin_prefetch(data + 64);
|
|
* partial += *(uint32_t *)(void *)data;
|
|
* partial += *(uint32_t *)(void *)(data + 4);
|
|
* partial += *(uint32_t *)(void *)(data + 8);
|
|
* partial += *(uint32_t *)(void *)(data + 12);
|
|
* partial += *(uint32_t *)(void *)(data + 16);
|
|
* partial += *(uint32_t *)(void *)(data + 20);
|
|
* partial += *(uint32_t *)(void *)(data + 24);
|
|
* partial += *(uint32_t *)(void *)(data + 28);
|
|
* partial += *(uint32_t *)(void *)(data + 32);
|
|
* partial += *(uint32_t *)(void *)(data + 36);
|
|
* partial += *(uint32_t *)(void *)(data + 40);
|
|
* partial += *(uint32_t *)(void *)(data + 44);
|
|
* partial += *(uint32_t *)(void *)(data + 48);
|
|
* partial += *(uint32_t *)(void *)(data + 52);
|
|
* partial += *(uint32_t *)(void *)(data + 56);
|
|
* partial += *(uint32_t *)(void *)(data + 60);
|
|
* data += 64;
|
|
* mlen -= 64;
|
|
* // if (PREDICT_FALSE(partial & (3ULL << 62))) {
|
|
* // if (needs_swap)
|
|
* // partial = (partial << 8) +
|
|
* // (partial >> 56);
|
|
* // sum += (partial >> 32);
|
|
* // sum += (partial & 0xffffffff);
|
|
* // partial = 0;
|
|
* // }
|
|
* }
|
|
*/
|
|
|
|
// pre-decrement mlen by 64, and if < 64 bytes, try 32 bytes next
|
|
subs mlen, mlen, #64
|
|
b.lt L32_bytes
|
|
|
|
// save used vector registers
|
|
sub sp, sp, #8*16
|
|
mov x11, sp
|
|
st1.4s {v0, v1, v2, v3}, [x11], #4*16
|
|
st1.4s {v4, v5, v6, v7}, [x11], #4*16
|
|
|
|
// spread partial into 8 8-byte registers in v0-v3
|
|
fmov s3, w7
|
|
eor.16b v0, v0, v0
|
|
eor.16b v1, v1, v1
|
|
eor.16b v2, v2, v2
|
|
|
|
// load the 1st 64 bytes (16 32-bit words)
|
|
ld1.4s {v4,v5,v6,v7},[data],#64
|
|
|
|
// branch to finish off if mlen<64
|
|
subs mlen, mlen, #64
|
|
b.lt L64_finishup
|
|
|
|
/*
|
|
* loop for loading and accumulating 16 32-bit words into
|
|
* 8 8-byte accumulators per iteration.
|
|
*/
|
|
L64_loop:
|
|
subs mlen, mlen, #64 // mlen -= 64
|
|
|
|
uadalp.2d v0, v4
|
|
ld1.4s {v4},[data], #16
|
|
|
|
uadalp.2d v1, v5
|
|
ld1.4s {v5},[data], #16
|
|
|
|
uadalp.2d v2, v6
|
|
ld1.4s {v6},[data], #16
|
|
|
|
uadalp.2d v3, v7
|
|
ld1.4s {v7},[data], #16
|
|
|
|
b.ge L64_loop
|
|
|
|
L64_finishup:
|
|
uadalp.2d v0, v4
|
|
uadalp.2d v1, v5
|
|
uadalp.2d v2, v6
|
|
uadalp.2d v3, v7
|
|
|
|
add.2d v0, v0, v1
|
|
add.2d v2, v2, v3
|
|
addp.2d d0, v0
|
|
addp.2d d2, v2
|
|
add.2d v0, v0, v2
|
|
fmov x7, d0 // partial in x7 now
|
|
|
|
// restore used vector registers
|
|
ld1.4s {v0, v1, v2, v3}, [sp], #4*16
|
|
ld1.4s {v4, v5, v6, v7}, [sp], #4*16
|
|
|
|
L32_bytes:
|
|
tst mlen, #32
|
|
b.eq L16_bytes
|
|
ldp x9, x10, [data], #16
|
|
ldp x11, x12, [data], #16
|
|
adds x7, x7, x9
|
|
mov x9, #0
|
|
adcs x7, x7, x10
|
|
adcs x7, x7, x11
|
|
adcs x7, x7, x12
|
|
adc x7, x7, x9
|
|
|
|
L16_bytes:
|
|
tst mlen, #16
|
|
b.eq L8_bytes
|
|
ldp x9, x10, [data], #16
|
|
adds x7, x7, x9
|
|
mov x9, #0
|
|
adcs x7, x7, x10
|
|
adc x7, x7, x9
|
|
|
|
L8_bytes:
|
|
tst mlen, #8
|
|
mov x10, #0
|
|
b.eq L4_bytes
|
|
ldr x9,[data],#8
|
|
adds x7, x7, x9
|
|
adc x7, x7, x10
|
|
|
|
L4_bytes:
|
|
tst mlen, #4
|
|
b.eq L2_bytes
|
|
ldr w9,[data],#4
|
|
adds x7, x7, x9
|
|
adc x7, x7, x10
|
|
|
|
L2_bytes:
|
|
tst mlen, #2
|
|
b.eq L_trailing_bytes
|
|
ldrh w9,[data],#2
|
|
adds x7, x7, x9
|
|
adc x7, x7, x10
|
|
|
|
L_trailing_bytes:
|
|
tst mlen, #1
|
|
b.eq L0_bytes
|
|
ldrb w9,[data],#1
|
|
adds x7, x7, x9
|
|
adc x7, x7, x10
|
|
eor started_on_odd, started_on_odd, #1
|
|
|
|
L0_bytes:
|
|
/*
|
|
* if (needs_swap)
|
|
* partial = (partial << 8) + (partial >> 56);
|
|
*/
|
|
cbz needs_swap, 1f
|
|
ror x7, x7, #56
|
|
1:
|
|
/*
|
|
* sum += (partial >> 32) + (partial & 0xffffffff);
|
|
* sum = (sum >> 32) + (sum & 0xffffffff);
|
|
* }
|
|
*/
|
|
|
|
add x3, x3, x7, lsr #32
|
|
mov w7, w7
|
|
add x3, x3, x7
|
|
mov w7, w3
|
|
add x3, x7, x3, lsr #32
|
|
|
|
L_continue:
|
|
cmp len, #0
|
|
ldr ptr_m, [m, #M_NEXT] // m = m->m_next
|
|
b.gt L_loop
|
|
|
|
/*
|
|
* final_acc = (sum >> 48) + ((sum >> 32) & 0xffff) +
|
|
* ((sum >> 16) & 0xffff) + (sum & 0xffff);
|
|
* final_acc = (final_acc >> 16) + (final_acc & 0xffff);
|
|
* final_acc = (final_acc >> 16) + (final_acc & 0xffff);
|
|
* return (final_acc & 0xffff);
|
|
* }
|
|
*/
|
|
|
|
mov w4, #0x00ffff
|
|
and x0, x4, x3, lsr #48
|
|
and x1, x4, x3, lsr #32
|
|
and x2, x4, x3, lsr #16
|
|
and x3, x4, x3
|
|
add w0, w0, w1
|
|
add w2, w2, w3
|
|
add w0, w0, w2
|
|
and w1, w4, w0, lsr #16
|
|
and w0, w4, w0
|
|
add w0, w0, w1
|
|
and w1, w4, w0, lsr #16
|
|
and w0, w4, w0
|
|
add w0, w0, w1
|
|
/*
|
|
* If we were to 1's complement it (XOR with 0xffff):
|
|
*
|
|
* eor w0, w0, w4
|
|
*/
|
|
and w0, w0, w4
|
|
|
|
ret lr
|
|
|
|
Lin_cksum_whoops:
|
|
adrp x0, Lin_cksum_whoops_str@page
|
|
add x0, x0, Lin_cksum_whoops_str@pageoff
|
|
bl #CKSUM_ERR
|
|
mov x0, #-1
|
|
ret lr
|
|
|
|
Lin_cksum_whoops_str:
|
|
.asciz "os_cpu_in_cksum_mbuf: out of data\n"
|
|
.align 5
|