1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2013 ARM Ltd.
* Copyright (C) 2013 Linaro.
*
* This code is based on glibc cortex strings work originally authored by Linaro
* be found @
*
* http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
* files/head:/src/aarch64/
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
/*
* calculate the length of a string
*
* Parameters:
* x0 - const string pointer
* Returns:
* x0 - the return length of specific string
*/
/* Arguments and results. */
srcin .req x0
len .req x0
/* Locals and temporaries. */
src .req x1
data1 .req x2
data2 .req x3
data2a .req x4
has_nul1 .req x5
has_nul2 .req x6
tmp1 .req x7
tmp2 .req x8
tmp3 .req x9
tmp4 .req x10
zeroones .req x11
pos .req x12
#define REP8_01 0x0101010101010101
#define REP8_7f 0x7f7f7f7f7f7f7f7f
#define REP8_80 0x8080808080808080
SYM_FUNC_START_WEAK_PI(strlen)
mov zeroones, #REP8_01
bic src, srcin, #15
ands tmp1, srcin, #15
b.ne .Lmisaligned
/*
* NUL detection works on the principle that (X - 1) & (~X) & 0x80
* (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
* can be done in parallel across the entire word.
*/
/*
* The inner loop deals with two Dwords at a time. This has a
* slightly higher start-up cost, but we should win quite quickly,
* especially on cores with a high number of issue slots per
* cycle, as we get much better parallelism out of the operations.
*/
.Lloop:
ldp data1, data2, [src], #16
.Lrealigned:
sub tmp1, data1, zeroones
orr tmp2, data1, #REP8_7f
sub tmp3, data2, zeroones
orr tmp4, data2, #REP8_7f
bic has_nul1, tmp1, tmp2
bics has_nul2, tmp3, tmp4
ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
b.eq .Lloop
sub len, src, srcin
cbz has_nul1, .Lnul_in_data2
CPU_BE( mov data2, data1 ) /*prepare data to re-calculate the syndrome*/
sub len, len, #8
mov has_nul2, has_nul1
.Lnul_in_data2:
/*
* For big-endian, carry propagation (if the final byte in the
* string is 0x01) means we cannot use has_nul directly. The
* easiest way to get the correct byte is to byte-swap the data
* and calculate the syndrome a second time.
*/
CPU_BE( rev data2, data2 )
CPU_BE( sub tmp1, data2, zeroones )
CPU_BE( orr tmp2, data2, #REP8_7f )
CPU_BE( bic has_nul2, tmp1, tmp2 )
sub len, len, #8
rev has_nul2, has_nul2
clz pos, has_nul2
add len, len, pos, lsr #3 /* Bits to bytes. */
ret
.Lmisaligned:
cmp tmp1, #8
neg tmp1, tmp1
ldp data1, data2, [src], #16
lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */
mov tmp2, #~0
/* Big-endian. Early bytes are at MSB. */
CPU_BE( lsl tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */
/* Little-endian. Early bytes are at LSB. */
CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */
orr data1, data1, tmp2
orr data2a, data2, tmp2
csinv data1, data1, xzr, le
csel data2, data2, data2a, le
b .Lrealigned
SYM_FUNC_END_PI(strlen)
EXPORT_SYMBOL_NOKASAN(strlen)
|
