FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
memcpy_avx2.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*-
16  * BSD LICENSE
17  *
18  * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
19  * All rights reserved.
20  *
21  * Redistribution and use in source and binary forms, with or without
22  * modification, are permitted provided that the following conditions
23  * are met:
24  *
25  * * Redistributions of source code must retain the above copyright
26  * notice, this list of conditions and the following disclaimer.
27  * * Redistributions in binary form must reproduce the above copyright
28  * notice, this list of conditions and the following disclaimer in
29  * the documentation and/or other materials provided with the
30  * distribution.
31  * * Neither the name of Intel Corporation nor the names of its
32  * contributors may be used to endorse or promote products derived
33  * from this software without specific prior written permission.
34  *
35  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46  */
47 
48 #ifndef included_clib_memcpy_avx2_h
49 #define included_clib_memcpy_avx2_h
50 
51 #include <stdint.h>
52 #include <x86intrin.h>
53 #include <vppinfra/warnings.h>
54 
55 /* *INDENT-OFF* */
56 WARN_OFF (stringop-overflow)
57 /* *INDENT-ON* */
58 
59 static inline void
60 clib_mov16 (u8 * dst, const u8 * src)
61 {
62  __m128i xmm0;
63 
64  xmm0 = _mm_loadu_si128 ((const __m128i *) src);
65  _mm_storeu_si128 ((__m128i *) dst, xmm0);
66 }
67 
68 static inline void
69 clib_mov32 (u8 * dst, const u8 * src)
70 {
71  __m256i ymm0;
72 
73  ymm0 = _mm256_loadu_si256 ((const __m256i *) src);
74  _mm256_storeu_si256 ((__m256i *) dst, ymm0);
75 }
76 
77 static inline void
78 clib_mov64 (u8 * dst, const u8 * src)
79 {
80  clib_mov32 ((u8 *) dst + 0 * 32, (const u8 *) src + 0 * 32);
81  clib_mov32 ((u8 *) dst + 1 * 32, (const u8 *) src + 1 * 32);
82 }
83 
84 static inline void
85 clib_mov128 (u8 * dst, const u8 * src)
86 {
87  clib_mov64 ((u8 *) dst + 0 * 64, (const u8 *) src + 0 * 64);
88  clib_mov64 ((u8 *) dst + 1 * 64, (const u8 *) src + 1 * 64);
89 }
90 
91 static inline void
92 clib_mov128blocks (u8 * dst, const u8 * src, size_t n)
93 {
94  __m256i ymm0, ymm1, ymm2, ymm3;
95 
96  while (n >= 128)
97  {
98  ymm0 =
99  _mm256_loadu_si256 ((const __m256i *) ((const u8 *) src + 0 * 32));
100  n -= 128;
101  ymm1 =
102  _mm256_loadu_si256 ((const __m256i *) ((const u8 *) src + 1 * 32));
103  ymm2 =
104  _mm256_loadu_si256 ((const __m256i *) ((const u8 *) src + 2 * 32));
105  ymm3 =
106  _mm256_loadu_si256 ((const __m256i *) ((const u8 *) src + 3 * 32));
107  src = (const u8 *) src + 128;
108  _mm256_storeu_si256 ((__m256i *) ((u8 *) dst + 0 * 32), ymm0);
109  _mm256_storeu_si256 ((__m256i *) ((u8 *) dst + 1 * 32), ymm1);
110  _mm256_storeu_si256 ((__m256i *) ((u8 *) dst + 2 * 32), ymm2);
111  _mm256_storeu_si256 ((__m256i *) ((u8 *) dst + 3 * 32), ymm3);
112  dst = (u8 *) dst + 128;
113  }
114 }
115 
116 static inline void *
117 clib_memcpy_fast_avx2 (void *dst, const void *src, size_t n)
118 {
119  uword dstu = (uword) dst;
120  uword srcu = (uword) src;
121  void *ret = dst;
122  size_t dstofss;
123  size_t bits;
124 
125  /**
126  * Copy less than 16 bytes
127  */
128  if (n < 16)
129  {
130  if (n & 0x01)
131  {
132  *(u8 *) dstu = *(const u8 *) srcu;
133  srcu = (uword) ((const u8 *) srcu + 1);
134  dstu = (uword) ((u8 *) dstu + 1);
135  }
136  if (n & 0x02)
137  {
138  *(u16 *) dstu = *(const u16 *) srcu;
139  srcu = (uword) ((const u16 *) srcu + 1);
140  dstu = (uword) ((u16 *) dstu + 1);
141  }
142  if (n & 0x04)
143  {
144  *(u32 *) dstu = *(const u32 *) srcu;
145  srcu = (uword) ((const u32 *) srcu + 1);
146  dstu = (uword) ((u32 *) dstu + 1);
147  }
148  if (n & 0x08)
149  {
150  *(u64 *) dstu = *(const u64 *) srcu;
151  }
152  return ret;
153  }
154 
155  /**
156  * Fast way when copy size doesn't exceed 512 bytes
157  */
158  if (n <= 32)
159  {
160  clib_mov16 ((u8 *) dst, (const u8 *) src);
161  clib_mov16 ((u8 *) dst - 16 + n, (const u8 *) src - 16 + n);
162  return ret;
163  }
164  if (n <= 48)
165  {
166  clib_mov16 ((u8 *) dst, (const u8 *) src);
167  clib_mov16 ((u8 *) dst + 16, (const u8 *) src + 16);
168  clib_mov16 ((u8 *) dst - 16 + n, (const u8 *) src - 16 + n);
169  return ret;
170  }
171  if (n <= 64)
172  {
173  clib_mov32 ((u8 *) dst, (const u8 *) src);
174  clib_mov32 ((u8 *) dst - 32 + n, (const u8 *) src - 32 + n);
175  return ret;
176  }
177  if (n <= 256)
178  {
179  if (n >= 128)
180  {
181  n -= 128;
182  clib_mov128 ((u8 *) dst, (const u8 *) src);
183  src = (const u8 *) src + 128;
184  dst = (u8 *) dst + 128;
185  }
186  COPY_BLOCK_128_BACK31:
187  if (n >= 64)
188  {
189  n -= 64;
190  clib_mov64 ((u8 *) dst, (const u8 *) src);
191  src = (const u8 *) src + 64;
192  dst = (u8 *) dst + 64;
193  }
194  if (n > 32)
195  {
196  clib_mov32 ((u8 *) dst, (const u8 *) src);
197  clib_mov32 ((u8 *) dst - 32 + n, (const u8 *) src - 32 + n);
198  return ret;
199  }
200  if (n > 0)
201  {
202  clib_mov32 ((u8 *) dst - 32 + n, (const u8 *) src - 32 + n);
203  }
204  return ret;
205  }
206 
207  /**
208  * Make store aligned when copy size exceeds 256 bytes
209  */
210  dstofss = (uword) dst & 0x1F;
211  if (dstofss > 0)
212  {
213  dstofss = 32 - dstofss;
214  n -= dstofss;
215  clib_mov32 ((u8 *) dst, (const u8 *) src);
216  src = (const u8 *) src + dstofss;
217  dst = (u8 *) dst + dstofss;
218  }
219 
220  /**
221  * Copy 128-byte blocks.
222  */
223  clib_mov128blocks ((u8 *) dst, (const u8 *) src, n);
224  bits = n;
225  n = n & 127;
226  bits -= n;
227  src = (const u8 *) src + bits;
228  dst = (u8 *) dst + bits;
229 
230  /**
231  * Copy whatever left
232  */
233  goto COPY_BLOCK_128_BACK31;
234 }
235 
236 /* *INDENT-OFF* */
237 WARN_ON (stringop-overflow)
238 /* *INDENT-ON* */
239 
240 #endif /* included_clib_memcpy_avx2_h */
241 
242 
243 /*
244  * fd.io coding-style-patch-verification: ON
245  *
246  * Local Variables:
247  * eval: (c-set-style "gnu")
248  * End:
249  */
u16
unsigned short u16
Definition: types.h:57
clib_mov16
static void clib_mov16(u8 *dst, const u8 *src)
Definition: memcpy_avx2.h:60
uword
u64 uword
Definition: types.h:112
clib_mov128
static void clib_mov128(u8 *dst, const u8 *src)
Definition: memcpy_avx2.h:85
src
vl_api_address_t src
Definition: gre.api:54
clib_mov32
static void clib_mov32(u8 *dst, const u8 *src)
Definition: memcpy_avx2.h:69
WARN_ON
#define WARN_ON(x)
Definition: warnings.h:81
u64
unsigned long u64
Definition: types.h:89
WARN_OFF
#define WARN_OFF(x)
Definition: warnings.h:80
u32
unsigned int u32
Definition: types.h:88
dst
vl_api_ip4_address_t dst
Definition: pnat.api:41
clib_mov64
static void clib_mov64(u8 *dst, const u8 *src)
Definition: memcpy_avx2.h:78
u8
unsigned char u8
Definition: types.h:56
clib_mov128blocks
static void clib_mov128blocks(u8 *dst, const u8 *src, size_t n)
Definition: memcpy_avx2.h:92
warnings.h
clib_memcpy_fast_avx2
static void * clib_memcpy_fast_avx2(void *dst, const void *src, size_t n)
Definition: memcpy_avx2.h:117