FD.io VPP  v21.06-3-gbb25fbf28
Vector Packet Processing
memcpy_avx512.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*-
16  * BSD LICENSE
17  *
18  * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
19  * All rights reserved.
20  *
21  * Redistribution and use in source and binary forms, with or without
22  * modification, are permitted provided that the following conditions
23  * are met:
24  *
25  * * Redistributions of source code must retain the above copyright
26  * notice, this list of conditions and the following disclaimer.
27  * * Redistributions in binary form must reproduce the above copyright
28  * notice, this list of conditions and the following disclaimer in
29  * the documentation and/or other materials provided with the
30  * distribution.
31  * * Neither the name of Intel Corporation nor the names of its
32  * contributors may be used to endorse or promote products derived
33  * from this software without specific prior written permission.
34  *
35  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46  */
47 
48 #ifndef included_clib_memcpy_avx512_h
49 #define included_clib_memcpy_avx512_h
50 
51 #include <stdint.h>
52 #include <x86intrin.h>
53 #include <vppinfra/warnings.h>
54 
55 /* *INDENT-OFF* */
56 WARN_OFF (stringop-overflow)
57 /* *INDENT-ON* */
58 
59 static inline void
60 clib_mov16 (u8 * dst, const u8 * src)
61 {
62  __m128i xmm0;
63 
64  xmm0 = _mm_loadu_si128 ((const __m128i *) src);
65  _mm_storeu_si128 ((__m128i *) dst, xmm0);
66 }
67 
68 static inline void
69 clib_mov32 (u8 * dst, const u8 * src)
70 {
71  __m256i ymm0;
72 
73  ymm0 = _mm256_loadu_si256 ((const __m256i *) src);
74  _mm256_storeu_si256 ((__m256i *) dst, ymm0);
75 }
76 
77 static inline void
78 clib_mov64 (u8 * dst, const u8 * src)
79 {
80  __m512i zmm0;
81 
82  zmm0 = _mm512_loadu_si512 ((const void *) src);
83  _mm512_storeu_si512 ((void *) dst, zmm0);
84 }
85 
86 static inline void
87 clib_mov128 (u8 * dst, const u8 * src)
88 {
89  clib_mov64 (dst + 0 * 64, src + 0 * 64);
90  clib_mov64 (dst + 1 * 64, src + 1 * 64);
91 }
92 
93 static inline void
94 clib_mov256 (u8 * dst, const u8 * src)
95 {
96  clib_mov128 (dst + 0 * 128, src + 0 * 128);
97  clib_mov128 (dst + 1 * 128, src + 1 * 128);
98 }
99 
100 static inline void
101 clib_mov128blocks (u8 * dst, const u8 * src, size_t n)
102 {
103  __m512i zmm0, zmm1;
104 
105  while (n >= 128)
106  {
107  zmm0 = _mm512_loadu_si512 ((const void *) (src + 0 * 64));
108  n -= 128;
109  zmm1 = _mm512_loadu_si512 ((const void *) (src + 1 * 64));
110  src = src + 128;
111  _mm512_storeu_si512 ((void *) (dst + 0 * 64), zmm0);
112  _mm512_storeu_si512 ((void *) (dst + 1 * 64), zmm1);
113  dst = dst + 128;
114  }
115 }
116 
117 static inline void
118 clib_mov512blocks (u8 * dst, const u8 * src, size_t n)
119 {
120  __m512i zmm0, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7;
121 
122  while (n >= 512)
123  {
124  zmm0 = _mm512_loadu_si512 ((const void *) (src + 0 * 64));
125  n -= 512;
126  zmm1 = _mm512_loadu_si512 ((const void *) (src + 1 * 64));
127  zmm2 = _mm512_loadu_si512 ((const void *) (src + 2 * 64));
128  zmm3 = _mm512_loadu_si512 ((const void *) (src + 3 * 64));
129  zmm4 = _mm512_loadu_si512 ((const void *) (src + 4 * 64));
130  zmm5 = _mm512_loadu_si512 ((const void *) (src + 5 * 64));
131  zmm6 = _mm512_loadu_si512 ((const void *) (src + 6 * 64));
132  zmm7 = _mm512_loadu_si512 ((const void *) (src + 7 * 64));
133  src = src + 512;
134  _mm512_storeu_si512 ((void *) (dst + 0 * 64), zmm0);
135  _mm512_storeu_si512 ((void *) (dst + 1 * 64), zmm1);
136  _mm512_storeu_si512 ((void *) (dst + 2 * 64), zmm2);
137  _mm512_storeu_si512 ((void *) (dst + 3 * 64), zmm3);
138  _mm512_storeu_si512 ((void *) (dst + 4 * 64), zmm4);
139  _mm512_storeu_si512 ((void *) (dst + 5 * 64), zmm5);
140  _mm512_storeu_si512 ((void *) (dst + 6 * 64), zmm6);
141  _mm512_storeu_si512 ((void *) (dst + 7 * 64), zmm7);
142  dst = dst + 512;
143  }
144 }
145 
146 static inline void *
147 clib_memcpy_fast_avx512 (void *dst, const void *src, size_t n)
148 {
149  uword dstu = (uword) dst;
150  uword srcu = (uword) src;
151  void *ret = dst;
152  size_t dstofss;
153  size_t bits;
154 
155  /**
156  * Copy less than 16 bytes
157  */
158  if (n < 16)
159  {
160  if (n & 0x01)
161  {
162  *(u8 *) dstu = *(const u8 *) srcu;
163  srcu = (uword) ((const u8 *) srcu + 1);
164  dstu = (uword) ((u8 *) dstu + 1);
165  }
166  if (n & 0x02)
167  {
168  *(u16 *) dstu = *(const u16 *) srcu;
169  srcu = (uword) ((const u16 *) srcu + 1);
170  dstu = (uword) ((u16 *) dstu + 1);
171  }
172  if (n & 0x04)
173  {
174  *(u32 *) dstu = *(const u32 *) srcu;
175  srcu = (uword) ((const u32 *) srcu + 1);
176  dstu = (uword) ((u32 *) dstu + 1);
177  }
178  if (n & 0x08)
179  *(u64 *) dstu = *(const u64 *) srcu;
180  return ret;
181  }
182 
183  /**
184  * Fast way when copy size doesn't exceed 512 bytes
185  */
186  if (n <= 32)
187  {
188  clib_mov16 ((u8 *) dst, (const u8 *) src);
189  clib_mov16 ((u8 *) dst - 16 + n, (const u8 *) src - 16 + n);
190  return ret;
191  }
192  if (n <= 64)
193  {
194  clib_mov32 ((u8 *) dst, (const u8 *) src);
195  clib_mov32 ((u8 *) dst - 32 + n, (const u8 *) src - 32 + n);
196  return ret;
197  }
198  if (n <= 512)
199  {
200  if (n >= 256)
201  {
202  n -= 256;
203  clib_mov256 ((u8 *) dst, (const u8 *) src);
204  src = (const u8 *) src + 256;
205  dst = (u8 *) dst + 256;
206  }
207  if (n >= 128)
208  {
209  n -= 128;
210  clib_mov128 ((u8 *) dst, (const u8 *) src);
211  src = (const u8 *) src + 128;
212  dst = (u8 *) dst + 128;
213  }
214  COPY_BLOCK_128_BACK63:
215  if (n > 64)
216  {
217  clib_mov64 ((u8 *) dst, (const u8 *) src);
218  clib_mov64 ((u8 *) dst - 64 + n, (const u8 *) src - 64 + n);
219  return ret;
220  }
221  if (n > 0)
222  clib_mov64 ((u8 *) dst - 64 + n, (const u8 *) src - 64 + n);
223  return ret;
224  }
225 
226  /**
227  * Make store aligned when copy size exceeds 512 bytes
228  */
229  dstofss = (uword) dst & 0x3F;
230  if (dstofss > 0)
231  {
232  dstofss = 64 - dstofss;
233  n -= dstofss;
234  clib_mov64 ((u8 *) dst, (const u8 *) src);
235  src = (const u8 *) src + dstofss;
236  dst = (u8 *) dst + dstofss;
237  }
238 
239  /**
240  * Copy 512-byte blocks.
241  * Use copy block function for better instruction order control,
242  * which is important when load is unaligned.
243  */
244  clib_mov512blocks ((u8 *) dst, (const u8 *) src, n);
245  bits = n;
246  n = n & 511;
247  bits -= n;
248  src = (const u8 *) src + bits;
249  dst = (u8 *) dst + bits;
250 
251  /**
252  * Copy 128-byte blocks.
253  * Use copy block function for better instruction order control,
254  * which is important when load is unaligned.
255  */
256  if (n >= 128)
257  {
258  clib_mov128blocks ((u8 *) dst, (const u8 *) src, n);
259  bits = n;
260  n = n & 127;
261  bits -= n;
262  src = (const u8 *) src + bits;
263  dst = (u8 *) dst + bits;
264  }
265 
266  /**
267  * Copy whatever left
268  */
269  goto COPY_BLOCK_128_BACK63;
270 }
271 
272 /* *INDENT-OFF* */
273 WARN_ON (stringop-overflow)
274 /* *INDENT-ON* */
275 
276 #endif /* included_clib_memcpy_avx512_h */
277 
278 
279 /*
280  * fd.io coding-style-patch-verification: ON
281  *
282  * Local Variables:
283  * eval: (c-set-style "gnu")
284  * End:
285  */
clib_mov128
static void clib_mov128(u8 *dst, const u8 *src)
Definition: memcpy_avx512.h:87
u16
unsigned short u16
Definition: types.h:57
uword
u64 uword
Definition: types.h:112
src
vl_api_address_t src
Definition: gre.api:54
WARN_ON
#define WARN_ON(x)
Definition: warnings.h:81
clib_mov32
static void clib_mov32(u8 *dst, const u8 *src)
Definition: memcpy_avx512.h:69
clib_mov256
static void clib_mov256(u8 *dst, const u8 *src)
Definition: memcpy_avx512.h:94
u64
unsigned long u64
Definition: types.h:89
WARN_OFF
#define WARN_OFF(x)
Definition: warnings.h:80
u32
unsigned int u32
Definition: types.h:88
dst
vl_api_ip4_address_t dst
Definition: pnat.api:41
clib_mov512blocks
static void clib_mov512blocks(u8 *dst, const u8 *src, size_t n)
Definition: memcpy_avx512.h:118
u8
unsigned char u8
Definition: types.h:56
warnings.h
clib_mov128blocks
static void clib_mov128blocks(u8 *dst, const u8 *src, size_t n)
Definition: memcpy_avx512.h:101
clib_memcpy_fast_avx512
static void * clib_memcpy_fast_avx512(void *dst, const void *src, size_t n)
Definition: memcpy_avx512.h:147
clib_mov64
static void clib_mov64(u8 *dst, const u8 *src)
Definition: memcpy_avx512.h:78
clib_mov16
static void clib_mov16(u8 *dst, const u8 *src)
Definition: memcpy_avx512.h:60