FD.io VPP  v18.10-34-gcce845e
Vector Packet Processing
serialize.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  Copyright (c) 2005 Eliot Dresselhaus
17 
18  Permission is hereby granted, free of charge, to any person obtaining
19  a copy of this software and associated documentation files (the
20  "Software"), to deal in the Software without restriction, including
21  without limitation the rights to use, copy, modify, merge, publish,
22  distribute, sublicense, and/or sell copies of the Software, and to
23  permit persons to whom the Software is furnished to do so, subject to
24  the following conditions:
25 
26  The above copyright notice and this permission notice shall be
27  included in all copies or substantial portions of the Software.
28 
29  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
33  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
34  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
35  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
36 */
37 
38 /* Turn data structures into byte streams for saving or transport. */
39 
40 #include <vppinfra/heap.h>
41 #include <vppinfra/pool.h>
42 #include <vppinfra/serialize.h>
43 
44 void
45 serialize_64 (serialize_main_t * m, va_list * va)
46 {
47  u64 x = va_arg (*va, u64);
48  u32 lo, hi;
49  lo = x;
50  hi = x >> 32;
51  serialize_integer (m, lo, sizeof (lo));
52  serialize_integer (m, hi, sizeof (hi));
53 }
54 
55 void
56 serialize_32 (serialize_main_t * m, va_list * va)
57 {
58  u32 x = va_arg (*va, u32);
59  serialize_integer (m, x, sizeof (x));
60 }
61 
62 void
63 serialize_16 (serialize_main_t * m, va_list * va)
64 {
65  u32 x = va_arg (*va, u32);
66  serialize_integer (m, x, sizeof (u16));
67 }
68 
69 void
70 serialize_8 (serialize_main_t * m, va_list * va)
71 {
72  u32 x = va_arg (*va, u32);
73  serialize_integer (m, x, sizeof (u8));
74 }
75 
76 void
77 unserialize_64 (serialize_main_t * m, va_list * va)
78 {
79  u64 *x = va_arg (*va, u64 *);
80  u32 lo, hi;
81  unserialize_integer (m, &lo, sizeof (lo));
82  unserialize_integer (m, &hi, sizeof (hi));
83  *x = ((u64) hi << 32) | (u64) lo;
84 }
85 
86 void
87 unserialize_32 (serialize_main_t * m, va_list * va)
88 {
89  u32 *x = va_arg (*va, u32 *);
90  unserialize_integer (m, x, sizeof (x[0]));
91 }
92 
93 void
94 unserialize_16 (serialize_main_t * m, va_list * va)
95 {
96  u16 *x = va_arg (*va, u16 *);
97  u32 t;
98  unserialize_integer (m, &t, sizeof (x[0]));
99  x[0] = t;
100 }
101 
102 void
103 unserialize_8 (serialize_main_t * m, va_list * va)
104 {
105  u8 *x = va_arg (*va, u8 *);
106  u32 t;
107  unserialize_integer (m, &t, sizeof (x[0]));
108  x[0] = t;
109 }
110 
111 void
112 serialize_f64 (serialize_main_t * m, va_list * va)
113 {
114  f64 x = va_arg (*va, f64);
115  union
116  {
117  f64 f;
118  u64 i;
119  } y;
120  y.f = x;
121  serialize (m, serialize_64, y.i);
122 }
123 
124 void
125 serialize_f32 (serialize_main_t * m, va_list * va)
126 {
127  f32 x = va_arg (*va, f64);
128  union
129  {
130  f32 f;
131  u32 i;
132  } y;
133  y.f = x;
134  serialize_integer (m, y.i, sizeof (y.i));
135 }
136 
137 void
139 {
140  f64 *x = va_arg (*va, f64 *);
141  union
142  {
143  f64 f;
144  u64 i;
145  } y;
146  unserialize (m, unserialize_64, &y.i);
147  *x = y.f;
148 }
149 
150 void
152 {
153  f32 *x = va_arg (*va, f32 *);
154  union
155  {
156  f32 f;
157  u32 i;
158  } y;
159  unserialize_integer (m, &y.i, sizeof (y.i));
160  *x = y.f;
161 }
162 
163 void
165 {
166  u32 len = s ? strlen (s) : 0;
167  void *p;
168 
170  if (len > 0)
171  {
172  p = serialize_get (m, len);
173  clib_memcpy (p, s, len);
174  }
175 }
176 
177 void
179 {
180  char *p, *r = 0;
181  u32 len;
182 
184 
185  /*
186  * Given broken enough data, we could get len = 0xFFFFFFFF.
187  * Add one, it overflows, we call vec_new (char, 0), then
188  * memcpy until we bus error.
189  */
190  if (len > 0 && len != 0xFFFFFFFF)
191  {
192  r = vec_new (char, len + 1);
193  p = unserialize_get (m, len);
194  clib_memcpy (r, p, len);
195 
196  /* Null terminate. */
197  r[len] = 0;
198  }
199  *s = r;
200 }
201 
202 /* vec_serialize/vec_unserialize helper functions for basic vector types. */
203 void
205 {
206  u8 *s = va_arg (*va, u8 *);
207  u32 n = va_arg (*va, u32);
208  u8 *p = serialize_get (m, n * sizeof (u8));
209  clib_memcpy (p, s, n * sizeof (u8));
210 }
211 
212 void
214 {
215  u8 *s = va_arg (*va, u8 *);
216  u32 n = va_arg (*va, u32);
217  u8 *p = unserialize_get (m, n);
218  clib_memcpy (s, p, n);
219 }
220 
221 #define _(n_bits) \
222  void serialize_vec_##n_bits (serialize_main_t * m, va_list * va) \
223  { \
224  u##n_bits * s = va_arg (*va, u##n_bits *); \
225  u32 n = va_arg (*va, u32); \
226  u##n_bits * p = serialize_get (m, n * sizeof (s[0])); \
227  \
228  while (n >= 4) \
229  { \
230  p[0] = clib_host_to_net_u##n_bits (s[0]); \
231  p[1] = clib_host_to_net_u##n_bits (s[1]); \
232  p[2] = clib_host_to_net_u##n_bits (s[2]); \
233  p[3] = clib_host_to_net_u##n_bits (s[3]); \
234  s += 4; \
235  p += 4; \
236  n -= 4; \
237  } \
238  \
239  while (n >= 1) \
240  { \
241  p[0] = clib_host_to_net_u##n_bits (s[0]); \
242  s += 1; \
243  p += 1; \
244  n -= 1; \
245  } \
246  } \
247  \
248  void unserialize_vec_##n_bits (serialize_main_t * m, va_list * va) \
249  { \
250  u##n_bits * s = va_arg (*va, u##n_bits *); \
251  u32 n = va_arg (*va, u32); \
252  u##n_bits * p = unserialize_get (m, n * sizeof (s[0])); \
253  \
254  while (n >= 4) \
255  { \
256  s[0] = clib_net_to_host_mem_u##n_bits (&p[0]); \
257  s[1] = clib_net_to_host_mem_u##n_bits (&p[1]); \
258  s[2] = clib_net_to_host_mem_u##n_bits (&p[2]); \
259  s[3] = clib_net_to_host_mem_u##n_bits (&p[3]); \
260  s += 4; \
261  p += 4; \
262  n -= 4; \
263  } \
264  \
265  while (n >= 1) \
266  { \
267  s[0] = clib_net_to_host_mem_u##n_bits (&p[0]); \
268  s += 1; \
269  p += 1; \
270  n -= 1; \
271  } \
272  }
273 
274 _(16);
275 _(32);
276 _(64);
277 
278 #undef _
279 
280 #define SERIALIZE_VECTOR_CHUNK_SIZE 64
281 
282 void
284 {
285  void *vec = va_arg (*va, void *);
286  u32 elt_bytes = va_arg (*va, u32);
287  serialize_function_t *f = va_arg (*va, serialize_function_t *);
288  u32 l = vec_len (vec);
289  void *p = vec;
290 
291  serialize_integer (m, l, sizeof (l));
292 
293  /* Serialize vector in chunks for cache locality. */
294  while (l != 0)
295  {
297  serialize (m, f, p, n);
298  l -= n;
299  p += SERIALIZE_VECTOR_CHUNK_SIZE * elt_bytes;
300  }
301 }
302 
303 void *
305  u32 elt_bytes,
306  u32 header_bytes,
307  u32 align, u32 max_length, serialize_function_t * f)
308 {
309  void *v, *p;
310  u32 l;
311 
312  unserialize_integer (m, &l, sizeof (l));
313  if (l > max_length)
314  serialize_error (&m->header,
315  clib_error_create ("bad vector length %d", l));
316  p = v = _vec_resize ((void *) 0, l, (uword) l * elt_bytes, header_bytes,
317  /* align */ align);
318 
319  while (l != 0)
320  {
322  unserialize (m, f, p, n);
323  l -= n;
324  p += SERIALIZE_VECTOR_CHUNK_SIZE * elt_bytes;
325  }
326  return v;
327 }
328 
329 void
331 {
332  void **vec = va_arg (*va, void **);
333  u32 elt_bytes = va_arg (*va, u32);
334  serialize_function_t *f = va_arg (*va, serialize_function_t *);
335  u32 align = va_arg (*va, u32);
336 
337  *vec = unserialize_vector_ha (m, elt_bytes,
338  /* header_bytes */ 0,
339  /* align */ align,
340  /* max_length */ ~0,
341  f);
342 }
343 
344 void
346 {
347  void **vec = va_arg (*va, void **);
348  u32 elt_bytes = va_arg (*va, u32);
349  serialize_function_t *f = va_arg (*va, serialize_function_t *);
350 
351  *vec = unserialize_vector_ha (m, elt_bytes,
352  /* header_bytes */ 0,
353  /* align */ 0,
354  /* max_length */ ~0,
355  f);
356 }
357 
358 void
360 {
361  u32 l, i, n_u32s;
362 
363  l = vec_len (b);
364  n_u32s = l * sizeof (b[0]) / sizeof (u32);
365  serialize_integer (m, n_u32s, sizeof (n_u32s));
366 
367  /* Send 32 bit words, low-order word first on 64 bit. */
368  for (i = 0; i < l; i++)
369  {
370  serialize_integer (m, b[i], sizeof (u32));
371  if (BITS (uword) == 64)
372  serialize_integer (m, (u64) b[i] >> (u64) 32, sizeof (u32));
373  }
374 }
375 
376 uword *
378 {
379  uword *b = 0;
380  u32 i, n_u32s;
381 
382  unserialize_integer (m, &n_u32s, sizeof (n_u32s));
383  if (n_u32s == 0)
384  return b;
385 
386  i = (n_u32s * sizeof (u32) + sizeof (b[0]) - 1) / sizeof (b[0]);
387  vec_resize (b, i);
388  for (i = 0; i < n_u32s; i++)
389  {
390  u32 data;
391  unserialize_integer (m, &data, sizeof (u32));
392 
393  /* Low-word is first on 64 bit. */
394  if (BITS (uword) == 64)
395  {
396  if ((i % 2) == 0)
397  b[i / 2] |= (u64) data << (u64) 0;
398  else
399  b[i / 2] |= (u64) data << (u64) 32;
400  }
401  else
402  {
403  b[i] = data;
404  }
405  }
406 
407  return b;
408 }
409 
410 void
411 serialize_pool (serialize_main_t * m, va_list * va)
412 {
413  void *pool = va_arg (*va, void *);
414  u32 elt_bytes = va_arg (*va, u32);
415  serialize_function_t *f = va_arg (*va, serialize_function_t *);
416  u32 l, lo, hi;
417  pool_header_t *p;
418 
419  l = vec_len (pool);
420  serialize_integer (m, l, sizeof (u32));
421  if (l == 0)
422  return;
423  p = pool_header (pool);
424 
425  /* No need to send free bitmap. Need to send index vector
426  to guarantee that unserialized pool will be identical. */
428 
429  pool_foreach_region (lo, hi, pool,
430  serialize (m, f, pool + lo * elt_bytes, hi - lo));
431 }
432 
433 static void *
435  u32 elt_bytes, u32 align, serialize_function_t * f)
436 {
437  void *v;
438  u32 i, l, lo, hi;
439  pool_header_t *p;
440 
441  unserialize_integer (m, &l, sizeof (l));
442  if (l == 0)
443  {
444  return 0;
445  }
446 
447  v = _vec_resize ((void *) 0, l, (uword) l * elt_bytes, sizeof (p[0]),
448  align);
449  p = pool_header (v);
450 
452 
453  /* Construct free bitmap. */
454  p->free_bitmap = 0;
455  for (i = 0; i < vec_len (p->free_indices); i++)
456  p->free_bitmap = clib_bitmap_ori (p->free_bitmap, p->free_indices[i]);
457 
458  pool_foreach_region (lo, hi, v,
459  unserialize (m, f, v + lo * elt_bytes, hi - lo));
460 
461  return v;
462 }
463 
464 void
466 {
467  void **result = va_arg (*va, void **);
468  u32 elt_bytes = va_arg (*va, u32);
469  serialize_function_t *f = va_arg (*va, serialize_function_t *);
470  *result = unserialize_pool_helper (m, elt_bytes, /* align */ 0, f);
471 }
472 
473 void
475 {
476  void **result = va_arg (*va, void **);
477  u32 elt_bytes = va_arg (*va, u32);
478  u32 align = va_arg (*va, u32);
479  serialize_function_t *f = va_arg (*va, serialize_function_t *);
480  *result = unserialize_pool_helper (m, elt_bytes, align, f);
481 }
482 
483 static void
485 {
486  heap_elt_t *e = va_arg (*va, heap_elt_t *);
487  u32 i, n = va_arg (*va, u32);
488  for (i = 0; i < n; i++)
489  {
490  serialize_integer (m, e[i].offset, sizeof (e[i].offset));
491  serialize_integer (m, e[i].next, sizeof (e[i].next));
492  serialize_integer (m, e[i].prev, sizeof (e[i].prev));
493  }
494 }
495 
496 static void
498 {
499  heap_elt_t *e = va_arg (*va, heap_elt_t *);
500  u32 i, n = va_arg (*va, u32);
501  for (i = 0; i < n; i++)
502  {
503  unserialize_integer (m, &e[i].offset, sizeof (e[i].offset));
504  unserialize_integer (m, &e[i].next, sizeof (e[i].next));
505  unserialize_integer (m, &e[i].prev, sizeof (e[i].prev));
506  }
507 }
508 
509 void
510 serialize_heap (serialize_main_t * m, va_list * va)
511 {
512  void *heap = va_arg (*va, void *);
513  serialize_function_t *f = va_arg (*va, serialize_function_t *);
514  u32 i, l;
515  heap_header_t *h;
516 
517  l = vec_len (heap);
518  serialize_integer (m, l, sizeof (u32));
519  if (l == 0)
520  return;
521 
522  h = heap_header (heap);
523 
524 #define foreach_serialize_heap_header_integer \
525  _ (head) _ (tail) _ (used_count) _ (max_len) _ (flags) _ (elt_bytes)
526 
527 #define _(f) serialize_integer (m, h->f, sizeof (h->f));
529 #undef _
530 
531  serialize_integer (m, vec_len (h->free_lists), sizeof (u32));
532  for (i = 0; i < vec_len (h->free_lists); i++)
534 
538 
539  /* Serialize data in heap. */
540  {
541  heap_elt_t *e, *end;
542  e = h->elts + h->head;
543  end = h->elts + h->tail;
544  while (1)
545  {
546  if (!heap_is_free (e))
547  {
548  void *v = heap + heap_offset (e) * h->elt_bytes;
549  u32 n = heap_elt_size (heap, e);
550  serialize (m, f, v, n);
551  }
552  if (e == end)
553  break;
554  e = heap_next (e);
555  }
556  }
557 }
558 
559 void
561 {
562  void **result = va_arg (*va, void **);
563  serialize_function_t *f = va_arg (*va, serialize_function_t *);
564  u32 i, vl, fl;
566  void *heap;
567 
568  unserialize_integer (m, &vl, sizeof (u32));
569  if (vl == 0)
570  {
571  *result = 0;
572  return;
573  }
574 
575  memset (&h, 0, sizeof (h));
576 #define _(f) unserialize_integer (m, &h.f, sizeof (h.f));
578 #undef _
579 
580  unserialize_integer (m, &fl, sizeof (u32));
581  vec_resize (h.free_lists, fl);
582 
583  for (i = 0; i < vec_len (h.free_lists); i++)
585 
589 
590  /* Re-construct used elt bitmap. */
591  if (CLIB_DEBUG > 0)
592  {
593  heap_elt_t *e;
594  vec_foreach (e, h.elts)
595  {
596  if (!heap_is_free (e))
597  h.used_elt_bitmap = clib_bitmap_ori (h.used_elt_bitmap, e - h.elts);
598  }
599  }
600 
601  heap = *result = _heap_new (vl, h.elt_bytes);
602  heap_header (heap)[0] = h;
603 
604  /* Unserialize data in heap. */
605  {
606  heap_elt_t *e, *end;
607  e = h.elts + h.head;
608  end = h.elts + h.tail;
609  while (1)
610  {
611  if (!heap_is_free (e))
612  {
613  void *v = heap + heap_offset (e) * h.elt_bytes;
614  u32 n = heap_elt_size (heap, e);
615  unserialize (m, f, v, n);
616  }
617  if (e == end)
618  break;
619  e = heap_next (e);
620  }
621  }
622 }
623 
624 void
625 serialize_magic (serialize_main_t * m, void *magic, u32 magic_bytes)
626 {
627  void *p;
628  serialize_integer (m, magic_bytes, sizeof (magic_bytes));
629  p = serialize_get (m, magic_bytes);
630  clib_memcpy (p, magic, magic_bytes);
631 }
632 
633 void
634 unserialize_check_magic (serialize_main_t * m, void *magic, u32 magic_bytes)
635 {
636  u32 l;
637  void *d;
638 
639  unserialize_integer (m, &l, sizeof (l));
640  if (l != magic_bytes)
641  {
642  bad:
643  serialize_error_return (m, "bad magic number");
644  }
645  d = serialize_get (m, magic_bytes);
646  if (memcmp (magic, d, magic_bytes))
647  goto bad;
648 }
649 
650 clib_error_t *
651 va_serialize (serialize_main_t * sm, va_list * va)
652 {
654  serialize_function_t *f = va_arg (*va, serialize_function_t *);
655  clib_error_t *error = 0;
656 
657  m->recursion_level += 1;
658  if (m->recursion_level == 1)
659  {
660  uword r = clib_setjmp (&m->error_longjmp, 0);
661  error = uword_to_pointer (r, clib_error_t *);
662  }
663 
664  if (!error)
665  f (sm, va);
666 
667  m->recursion_level -= 1;
668  return error;
669 }
670 
671 clib_error_t *
673 {
674  clib_error_t *error;
675  va_list va;
676 
677  va_start (va, m);
678  error = va_serialize (m, &va);
679  va_end (va);
680  return error;
681 }
682 
683 clib_error_t *
685 {
686  clib_error_t *error;
687  va_list va;
688 
689  va_start (va, m);
690  error = va_serialize (m, &va);
691  va_end (va);
692  return error;
693 }
694 
695 static void *
697  serialize_stream_t * s,
698  uword n_bytes_to_write, uword flags)
699 {
700  uword cur_bi, n_left_b, n_left_o;
701 
703  cur_bi = s->current_buffer_index;
704  n_left_b = s->n_buffer_bytes - cur_bi;
705  n_left_o = vec_len (s->overflow_buffer);
706 
707  /* Prepend overflow buffer if present. */
708  do
709  {
710  if (n_left_o > 0 && n_left_b > 0)
711  {
712  uword n = clib_min (n_left_b, n_left_o);
713  clib_memcpy (s->buffer + cur_bi, s->overflow_buffer, n);
714  cur_bi += n;
715  n_left_b -= n;
716  n_left_o -= n;
717  if (n_left_o == 0)
718  _vec_len (s->overflow_buffer) = 0;
719  else
720  vec_delete (s->overflow_buffer, n, 0);
721  }
722 
723  /* Call data function when buffer is complete. Data function should
724  dispatch with current buffer and give us a new one to write more
725  data into. */
726  if (n_left_b == 0)
727  {
728  s->current_buffer_index = cur_bi;
729  m->data_function (m, s);
730  cur_bi = s->current_buffer_index;
731  n_left_b = s->n_buffer_bytes - cur_bi;
732  }
733  }
734  while (n_left_o > 0);
735 
736  if (n_left_o > 0 || n_left_b < n_bytes_to_write)
737  {
738  u8 *r;
739  vec_add2 (s->overflow_buffer, r, n_bytes_to_write);
740  return r;
741  }
742  else
743  {
744  s->current_buffer_index = cur_bi + n_bytes_to_write;
745  return s->buffer + cur_bi;
746  }
747 }
748 
749 static void *
751  serialize_stream_t * s,
752  uword n_bytes_to_read, uword flags)
753 {
754  uword cur_bi, cur_oi, n_left_b, n_left_o, n_left_to_read;
755 
757 
758  cur_bi = s->current_buffer_index;
759  cur_oi = s->current_overflow_index;
760 
761  n_left_b = s->n_buffer_bytes - cur_bi;
762  n_left_o = vec_len (s->overflow_buffer) - cur_oi;
763 
764  /* Read from overflow? */
765  if (n_left_o >= n_bytes_to_read)
766  {
767  s->current_overflow_index = cur_oi + n_bytes_to_read;
768  return vec_elt_at_index (s->overflow_buffer, cur_oi);
769  }
770 
771  /* Reset overflow buffer. */
772  if (n_left_o == 0 && s->overflow_buffer)
773  {
774  s->current_overflow_index = 0;
775  _vec_len (s->overflow_buffer) = 0;
776  }
777 
778  n_left_to_read = n_bytes_to_read;
779  while (n_left_to_read > 0)
780  {
781  uword n;
782 
783  /* If we don't have enough data between overflow and normal buffer
784  call read function. */
785  if (n_left_o + n_left_b < n_bytes_to_read)
786  {
787  /* Save any left over buffer in overflow vector. */
788  if (n_left_b > 0)
789  {
790  vec_add (s->overflow_buffer, s->buffer + cur_bi, n_left_b);
791  n_left_o += n_left_b;
792  n_left_to_read -= n_left_b;
793  /* Advance buffer to end --- even if
794  SERIALIZE_FLAG_NO_ADVANCE_CURRENT_BUFFER_INDEX is set. */
795  cur_bi = s->n_buffer_bytes;
796  n_left_b = 0;
797  }
798 
799  if (m->data_function)
800  {
801  m->data_function (m, s);
802  cur_bi = s->current_buffer_index;
803  n_left_b = s->n_buffer_bytes - cur_bi;
804  }
805  }
806 
807  /* For first time through loop return if we have enough data
808  in normal buffer and overflow vector is empty. */
809  if (n_left_o == 0
810  && n_left_to_read == n_bytes_to_read && n_left_b >= n_left_to_read)
811  {
812  s->current_buffer_index = cur_bi + n_bytes_to_read;
813  return s->buffer + cur_bi;
814  }
815 
817  {
818  /* This can happen for a peek at end of file.
819  Pad overflow buffer with 0s. */
820  vec_resize (s->overflow_buffer, n_left_to_read);
821  n_left_o += n_left_to_read;
822  n_left_to_read = 0;
823  }
824  else
825  {
826  /* Copy from buffer to overflow vector. */
827  n = clib_min (n_left_to_read, n_left_b);
828  vec_add (s->overflow_buffer, s->buffer + cur_bi, n);
829  cur_bi += n;
830  n_left_b -= n;
831  n_left_o += n;
832  n_left_to_read -= n;
833  }
834  }
835 
836  s->current_buffer_index = cur_bi;
837  s->current_overflow_index = cur_oi + n_bytes_to_read;
838  return vec_elt_at_index (s->overflow_buffer, cur_oi);
839 }
840 
841 void *
843  serialize_stream_t * s,
844  uword n_bytes, uword flags)
845 {
847  serialize_write_not_inline) (m, s, n_bytes, flags));
848 }
849 
850 static void
853 {
855  return;
856 
857  if (flags & SERIALIZE_FLAG_IS_WRITE)
858  /* "Write" 0 bytes to flush overflow vector. */
859  serialize_write_not_inline (m, s, /* n bytes */ 0, flags);
860 
862 
863  /* Call it one last time to flush buffer and close. */
864  m->data_function (m, s);
865 
867 }
868 
869 void
871 {
874 }
875 
876 void
878 {
880 }
881 
882 void
883 serialize_open_data (serialize_main_t * m, u8 * data, uword n_data_bytes)
884 {
885  memset (m, 0, sizeof (m[0]));
886  m->stream.buffer = data;
887  m->stream.n_buffer_bytes = n_data_bytes;
888 }
889 
890 void
891 unserialize_open_data (serialize_main_t * m, u8 * data, uword n_data_bytes)
892 {
893  serialize_open_data (m, data, n_data_bytes);
894 }
895 
896 static void
898 {
900  {
901  /* Double buffer size. */
902  uword l = vec_len (s->buffer);
903  vec_resize (s->buffer, l > 0 ? l : 64);
904  s->n_buffer_bytes = vec_len (s->buffer);
905  }
906 }
907 
908 void
910 {
911  memset (m, 0, sizeof (m[0]));
913  m->stream.buffer = vector;
915  m->stream.n_buffer_bytes = vec_len (vector);
916 }
917 
918 void *
920 {
921  serialize_stream_t *s = &m->stream;
922  void *result;
923 
924  serialize_close (m); /* frees overflow buffer */
925 
926  if (s->buffer)
927  _vec_len (s->buffer) = s->current_buffer_index;
928  result = s->buffer;
929  memset (m, 0, sizeof (m[0]));
930  return result;
931 }
932 
933 void
935  void *data, uword data_stride, uword n_data)
936 {
937  u8 *d = data;
938  u8 *p;
939  uword n_left = n_data;
940 
941  while (n_left >= 4)
942  {
943  p = serialize_get (m, 4 * sizeof (d[0]));
944  p[0] = d[0 * data_stride];
945  p[1] = d[1 * data_stride];
946  p[2] = d[2 * data_stride];
947  p[3] = d[3 * data_stride];
948  n_left -= 4;
949  d += 4 * data_stride;
950  }
951 
952  if (n_left > 0)
953  {
954  p = serialize_get (m, n_left * sizeof (p[0]));
955  while (n_left > 0)
956  {
957  p[0] = d[0];
958  p += 1;
959  d += 1 * data_stride;
960  n_left -= 1;
961  }
962  }
963 }
964 
965 void
967  void *data, uword data_stride, uword n_data)
968 {
969  void *d = data;
970  u16 *p;
971  uword n_left = n_data;
972 
973  while (n_left >= 4)
974  {
975  p = serialize_get (m, 4 * sizeof (p[0]));
976  clib_mem_unaligned (p + 0, u16) =
977  clib_host_to_net_mem_u16 (d + 0 * data_stride);
978  clib_mem_unaligned (p + 1, u16) =
979  clib_host_to_net_mem_u16 (d + 1 * data_stride);
980  clib_mem_unaligned (p + 2, u16) =
981  clib_host_to_net_mem_u16 (d + 2 * data_stride);
982  clib_mem_unaligned (p + 3, u16) =
983  clib_host_to_net_mem_u16 (d + 3 * data_stride);
984  n_left -= 4;
985  d += 4 * data_stride;
986  }
987 
988  if (n_left > 0)
989  {
990  p = serialize_get (m, n_left * sizeof (p[0]));
991  while (n_left > 0)
992  {
993  clib_mem_unaligned (p + 0, u16) =
994  clib_host_to_net_mem_u16 (d + 0 * data_stride);
995  p += 1;
996  d += 1 * data_stride;
997  n_left -= 1;
998  }
999  }
1000 }
1001 
1002 void
1004  void *data, uword data_stride, uword n_data)
1005 {
1006  void *d = data;
1007  u32 *p;
1008  uword n_left = n_data;
1009 
1010  while (n_left >= 4)
1011  {
1012  p = serialize_get (m, 4 * sizeof (p[0]));
1013  clib_mem_unaligned (p + 0, u32) =
1014  clib_host_to_net_mem_u32 (d + 0 * data_stride);
1015  clib_mem_unaligned (p + 1, u32) =
1016  clib_host_to_net_mem_u32 (d + 1 * data_stride);
1017  clib_mem_unaligned (p + 2, u32) =
1018  clib_host_to_net_mem_u32 (d + 2 * data_stride);
1019  clib_mem_unaligned (p + 3, u32) =
1020  clib_host_to_net_mem_u32 (d + 3 * data_stride);
1021  n_left -= 4;
1022  d += 4 * data_stride;
1023  }
1024 
1025  if (n_left > 0)
1026  {
1027  p = serialize_get (m, n_left * sizeof (p[0]));
1028  while (n_left > 0)
1029  {
1030  clib_mem_unaligned (p + 0, u32) =
1031  clib_host_to_net_mem_u32 (d + 0 * data_stride);
1032  p += 1;
1033  d += 1 * data_stride;
1034  n_left -= 1;
1035  }
1036  }
1037 }
1038 
1039 void
1041  void *data, uword data_stride, uword n_data)
1042 {
1043  u8 *d = data;
1044  u8 *p;
1045  uword n_left = n_data;
1046 
1047  while (n_left >= 4)
1048  {
1049  p = unserialize_get (m, 4 * sizeof (d[0]));
1050  d[0 * data_stride] = p[0];
1051  d[1 * data_stride] = p[1];
1052  d[2 * data_stride] = p[2];
1053  d[3 * data_stride] = p[3];
1054  n_left -= 4;
1055  d += 4 * data_stride;
1056  }
1057 
1058  if (n_left > 0)
1059  {
1060  p = unserialize_get (m, n_left * sizeof (p[0]));
1061  while (n_left > 0)
1062  {
1063  d[0] = p[0];
1064  p += 1;
1065  d += 1 * data_stride;
1066  n_left -= 1;
1067  }
1068  }
1069 }
1070 
1071 void
1073  void *data, uword data_stride, uword n_data)
1074 {
1075  void *d = data;
1076  u16 *p;
1077  uword n_left = n_data;
1078 
1079  while (n_left >= 4)
1080  {
1081  p = unserialize_get (m, 4 * sizeof (p[0]));
1082  clib_mem_unaligned (d + 0 * data_stride, u16) =
1083  clib_net_to_host_mem_u16 (p + 0);
1084  clib_mem_unaligned (d + 1 * data_stride, u16) =
1085  clib_net_to_host_mem_u16 (p + 1);
1086  clib_mem_unaligned (d + 2 * data_stride, u16) =
1087  clib_net_to_host_mem_u16 (p + 2);
1088  clib_mem_unaligned (d + 3 * data_stride, u16) =
1089  clib_net_to_host_mem_u16 (p + 3);
1090  n_left -= 4;
1091  d += 4 * data_stride;
1092  }
1093 
1094  if (n_left > 0)
1095  {
1096  p = unserialize_get (m, n_left * sizeof (p[0]));
1097  while (n_left > 0)
1098  {
1099  clib_mem_unaligned (d + 0 * data_stride, u16) =
1100  clib_net_to_host_mem_u16 (p + 0);
1101  p += 1;
1102  d += 1 * data_stride;
1103  n_left -= 1;
1104  }
1105  }
1106 }
1107 
1108 void
1110  void *data, uword data_stride, uword n_data)
1111 {
1112  void *d = data;
1113  u32 *p;
1114  uword n_left = n_data;
1115 
1116  while (n_left >= 4)
1117  {
1118  p = unserialize_get (m, 4 * sizeof (p[0]));
1119  clib_mem_unaligned (d + 0 * data_stride, u32) =
1120  clib_net_to_host_mem_u32 (p + 0);
1121  clib_mem_unaligned (d + 1 * data_stride, u32) =
1122  clib_net_to_host_mem_u32 (p + 1);
1123  clib_mem_unaligned (d + 2 * data_stride, u32) =
1124  clib_net_to_host_mem_u32 (p + 2);
1125  clib_mem_unaligned (d + 3 * data_stride, u32) =
1126  clib_net_to_host_mem_u32 (p + 3);
1127  n_left -= 4;
1128  d += 4 * data_stride;
1129  }
1130 
1131  if (n_left > 0)
1132  {
1133  p = unserialize_get (m, n_left * sizeof (p[0]));
1134  while (n_left > 0)
1135  {
1136  clib_mem_unaligned (d + 0 * data_stride, u32) =
1137  clib_net_to_host_mem_u32 (p + 0);
1138  p += 1;
1139  d += 1 * data_stride;
1140  n_left -= 1;
1141  }
1142  }
1143 }
1144 
1145 #ifdef CLIB_UNIX
1146 
1147 #include <unistd.h>
1148 #include <fcntl.h>
1149 
1150 static void
1152 {
1153  int fd, n;
1154 
1155  fd = s->data_function_opaque;
1156  n = write (fd, s->buffer, s->current_buffer_index);
1157  if (n < 0)
1158  {
1159  if (!unix_error_is_fatal (errno))
1160  n = 0;
1161  else
1162  serialize_error (m, clib_error_return_unix (0, "write"));
1163  }
1164  if (n == s->current_buffer_index)
1165  _vec_len (s->buffer) = 0;
1166  else
1167  vec_delete (s->buffer, n, 0);
1169 }
1170 
1171 static void
1173 {
1174  int fd, n;
1175 
1176  fd = s->data_function_opaque;
1177  n = read (fd, s->buffer, vec_len (s->buffer));
1178  if (n < 0)
1179  {
1180  if (!unix_error_is_fatal (errno))
1181  n = 0;
1182  else
1183  serialize_error (m, clib_error_return_unix (0, "read"));
1184  }
1185  else if (n == 0)
1187  s->current_buffer_index = 0;
1188  s->n_buffer_bytes = n;
1189 }
1190 
1191 static void
1193  uword is_read)
1194 {
1195  memset (m, 0, sizeof (m[0]));
1196  vec_resize (m->stream.buffer, 4096);
1197 
1198  if (!is_read)
1199  {
1201  _vec_len (m->stream.buffer) = 0;
1202  }
1203 
1205  m->stream.data_function_opaque = fd;
1206 }
1207 
1208 void
1210 {
1211  serialize_open_clib_file_descriptor_helper (m, fd, /* is_read */ 0);
1212 }
1213 
1214 void
1216 {
1217  serialize_open_clib_file_descriptor_helper (m, fd, /* is_read */ 1);
1218 }
1219 
1220 static clib_error_t *
1222  uword is_read)
1223 {
1224  int fd, mode;
1225 
1226  mode = is_read ? O_RDONLY : O_RDWR | O_CREAT | O_TRUNC;
1227  fd = open (file, mode, 0666);
1228  if (fd < 0)
1229  return clib_error_return_unix (0, "open `%s'", file);
1230 
1232  return 0;
1233 }
1234 
1235 clib_error_t *
1237 {
1238  return serialize_open_clib_file_helper (m, file, /* is_read */ 0);
1239 }
1240 
1241 clib_error_t *
1243 {
1244  return serialize_open_clib_file_helper (m, file, /* is_read */ 1);
1245 }
1246 
1247 #endif /* CLIB_UNIX */
1248 
1249 /*
1250  * fd.io coding-style-patch-verification: ON
1251  *
1252  * Local Variables:
1253  * eval: (c-set-style "gnu")
1254  * End:
1255  */
vmrglw vmrglh hi
u32 vl(void *p)
Definition: threads.c:31
static void serialize_vector_write(serialize_main_header_t *m, serialize_stream_t *s)
Definition: serialize.c:897
#define clib_min(x, y)
Definition: clib.h:291
void serialize_f32(serialize_main_t *m, va_list *va)
Definition: serialize.c:125
void unserialize_check_magic(serialize_main_t *m, void *magic, u32 magic_bytes)
Definition: serialize.c:634
u32 current_overflow_index
Definition: serialize.h:69
static u64 unserialize_likely_small_unsigned_integer(serialize_main_t *m)
Definition: serialize.h:254
#define serialize_error_return(m, args...)
Definition: serialize.h:112
clib_error_t * serialize_open_clib_file(serialize_main_t *m, char *file)
Definition: serialize.c:1236
void serialize_bitmap(serialize_main_t *m, uword *b)
Definition: serialize.c:359
void serialize_multiple_4(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:1003
#define vec_serialize(m, v, f)
Definition: serialize.h:371
unsigned long u64
Definition: types.h:89
Fixed length block allocator.
serialize_main_header_t header
Definition: serialize.h:146
static void serialize_error(serialize_main_header_t *m, clib_error_t *error)
Definition: serialize.h:107
#define vec_unserialize(m, v, f)
Definition: serialize.h:374
u32 * free_indices
Vector of free indices.
Definition: pool.h:58
u32 tail
Definition: heap.h:144
static void serialize_stream_set_end_of_stream(serialize_stream_t *s)
Definition: serialize.h:81
void unserialize_32(serialize_main_t *m, va_list *va)
Definition: serialize.c:87
void unserialize_multiple_2(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:1072
void serialize_f64(serialize_main_t *m, va_list *va)
Definition: serialize.c:112
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
u32 head
Definition: heap.h:144
void serialize_open_clib_file_descriptor(serialize_main_t *m, int fd)
Definition: serialize.c:1209
uword clib_setjmp(clib_longjmp_t *save, uword return_value_not_taken)
unsigned char u8
Definition: types.h:56
double f64
Definition: types.h:142
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:600
u32 * free_elts
Definition: heap.h:133
#define SERIALIZE_VECTOR_CHUNK_SIZE
Definition: serialize.c:280
#define foreach_serialize_heap_header_integer
memset(h->entries, 0, sizeof(h->entries[0])*entries)
void * unserialize_vector_ha(serialize_main_t *m, u32 elt_bytes, u32 header_bytes, u32 align, u32 max_length, serialize_function_t *f)
Definition: serialize.c:304
void unserialize_open_data(serialize_main_t *m, u8 *data, uword n_data_bytes)
Definition: serialize.c:891
#define vec_new(T, N)
Create new vector of given type and length (unspecified alignment, no header).
Definition: vec.h:309
void unserialize_heap(serialize_main_t *m, va_list *va)
Definition: serialize.c:560
static uword serialize_stream_is_end_of_stream(serialize_stream_t *s)
Definition: serialize.h:87
static void * serialize_get(serialize_main_t *m, uword n_bytes)
Definition: serialize.h:178
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
void unserialize_aligned_vector(serialize_main_t *m, va_list *va)
Definition: serialize.c:330
static uword heap_is_free(heap_elt_t *e)
Definition: heap.h:85
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:240
unsigned int u32
Definition: types.h:88
#define clib_error_create(args...)
Definition: error.h:96
static void * serialize_read_not_inline(serialize_main_header_t *m, serialize_stream_t *s, uword n_bytes_to_read, uword flags)
Definition: serialize.c:750
uword * used_elt_bitmap
Definition: heap.h:141
static heap_elt_t * heap_next(heap_elt_t *e)
Definition: heap.h:97
void serialize_16(serialize_main_t *m, va_list *va)
Definition: serialize.c:63
#define fl(x, y)
#define SERIALIZE_FLAG_IS_READ
Definition: serialize.h:119
void unserialize_vec_8(serialize_main_t *m, va_list *va)
Definition: serialize.c:213
serialize_function_t serialize_vec_32
Definition: serialize.h:364
static pool_header_t * pool_header(void *v)
Get pool header from user pool pointer.
Definition: pool.h:77
void serialize_open_data(serialize_main_t *m, u8 *data, uword n_data_bytes)
Definition: serialize.c:883
static void serialize_likely_small_unsigned_integer(serialize_main_t *m, u64 x)
Definition: serialize.h:218
void serialize_pool(serialize_main_t *m, va_list *va)
Definition: serialize.c:411
lo
void unserialize_multiple_4(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:1109
#define v
Definition: acl.c:496
unsigned short u16
Definition: types.h:57
#define clib_error_return_unix(e, args...)
Definition: error.h:102
heap_elt_t * elts
Definition: heap.h:126
void unserialize_8(serialize_main_t *m, va_list *va)
Definition: serialize.c:103
void serialize_8(serialize_main_t *m, va_list *va)
Definition: serialize.c:70
static void serialize_open_clib_file_descriptor_helper(serialize_main_t *m, int fd, uword is_read)
Definition: serialize.c:1192
static void * unserialize_get(serialize_main_t *m, uword n_bytes)
Definition: serialize.h:171
static void serialize_read_write_close(serialize_main_header_t *m, serialize_stream_t *s, uword flags)
Definition: serialize.c:851
void unserialize_cstring(serialize_main_t *m, char **s)
Definition: serialize.c:178
void * serialize_read_write_not_inline(serialize_main_header_t *m, serialize_stream_t *s, uword n_bytes, uword flags)
Definition: serialize.c:842
u32 flags
Definition: vhost_user.h:115
u32 elt_bytes
Definition: heap.h:149
serialize_stream_t stream
Definition: serialize.h:147
static void clib_file_read(serialize_main_header_t *m, serialize_stream_t *s)
Definition: serialize.c:1172
clib_error_t * serialize(serialize_main_t *m,...)
Definition: serialize.c:672
u32 current_buffer_index
Definition: serialize.h:62
void serialize_open_vector(serialize_main_t *m, u8 *vector)
Definition: serialize.c:909
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
clib_error_t * unserialize_open_clib_file(serialize_main_t *m, char *file)
Definition: serialize.c:1242
#define clib_memcpy(a, b, c)
Definition: string.h:75
static uword heap_offset(heap_elt_t *e)
Definition: heap.h:91
serialize_data_function_t * data_function
Definition: serialize.h:97
static void unserialize_integer(serialize_main_t *m, void *x, u32 n_bytes)
Definition: serialize.h:201
static void serialize_integer(serialize_main_t *m, u64 x, u32 n_bytes)
Definition: serialize.h:185
void unserialize_close(serialize_main_t *m)
Definition: serialize.c:877
static void serialize_vec_heap_elt(serialize_main_t *m, va_list *va)
Definition: serialize.c:484
uword data_function_opaque
Definition: serialize.h:74
u32 * small_free_elt_free_index
Definition: heap.h:130
void serialize_multiple_2(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:966
void serialize_magic(serialize_main_t *m, void *magic, u32 magic_bytes)
Definition: serialize.c:625
#define uword_to_pointer(u, type)
Definition: types.h:136
void unserialize_16(serialize_main_t *m, va_list *va)
Definition: serialize.c:94
void serialize_vec_8(serialize_main_t *m, va_list *va)
Definition: serialize.c:204
#define ASSERT(truth)
void unserialize_aligned_pool(serialize_main_t *m, va_list *va)
Definition: serialize.c:474
void unserialize_multiple_1(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:1040
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:786
static word unix_error_is_fatal(word error)
Definition: error.h:118
void serialize_close(serialize_main_t *m)
Definition: serialize.c:870
#define pool_foreach_region(LO, HI, POOL, BODY)
Optimized iteration through pool.
Definition: pool.h:369
clib_error_t * unserialize(serialize_main_t *m,...)
Definition: serialize.c:684
#define clib_mem_unaligned(pointer, type)
Definition: types.h:155
static uword heap_elt_size(void *v, heap_elt_t *e)
Definition: heap.h:109
static heap_header_t * heap_header(void *v)
Definition: heap.h:161
float f32
Definition: types.h:143
void serialize_cstring(serialize_main_t *m, char *s)
Definition: serialize.c:164
static void unserialize_vec_heap_elt(serialize_main_t *m, va_list *va)
Definition: serialize.c:497
uword * free_bitmap
Bitmap of indices of free objects.
Definition: pool.h:55
void serialize_32(serialize_main_t *m, va_list *va)
Definition: serialize.c:56
template key/value backing page structure
Definition: bihash_doc.h:44
static void * serialize_write_not_inline(serialize_main_header_t *m, serialize_stream_t *s, uword n_bytes_to_write, uword flags)
Definition: serialize.c:696
void unserialize_f64(serialize_main_t *m, va_list *va)
Definition: serialize.c:138
void unserialize_vector(serialize_main_t *m, va_list *va)
Definition: serialize.c:345
void unserialize_f32(serialize_main_t *m, va_list *va)
Definition: serialize.c:151
uword * unserialize_bitmap(serialize_main_t *m)
Definition: serialize.c:377
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void( serialize_function_t)(serialize_main_t *m, va_list *va)
Definition: serialize.h:168
void serialize_heap(serialize_main_t *m, va_list *va)
Definition: serialize.c:510
void serialize_vector(serialize_main_t *m, va_list *va)
Definition: serialize.c:283
clib_error_t * va_serialize(serialize_main_t *sm, va_list *va)
Definition: serialize.c:651
u64 uword
Definition: types.h:112
void unserialize_pool(serialize_main_t *m, va_list *va)
Definition: serialize.c:465
static void * unserialize_pool_helper(serialize_main_t *m, u32 elt_bytes, u32 align, serialize_function_t *f)
Definition: serialize.c:434
static void clib_file_write(serialize_main_header_t *m, serialize_stream_t *s)
Definition: serialize.c:1151
#define SERIALIZE_FLAG_IS_WRITE
Definition: serialize.h:120
static clib_error_t * serialize_open_clib_file_helper(serialize_main_t *m, char *file, uword is_read)
Definition: serialize.c:1221
#define vec_foreach(var, vec)
Vector iterator.
void unserialize_open_clib_file_descriptor(serialize_main_t *m, int fd)
Definition: serialize.c:1215
void * serialize_close_vector(serialize_main_t *m)
Definition: serialize.c:919
void unserialize_64(serialize_main_t *m, va_list *va)
Definition: serialize.c:77
#define BITS(x)
Definition: clib.h:60
void serialize_64(serialize_main_t *m, va_list *va)
Definition: serialize.c:45
void serialize_multiple_1(serialize_main_t *m, void *data, uword data_stride, uword n_data)
Definition: serialize.c:934
clib_longjmp_t error_longjmp
Definition: serialize.h:103
serialize_function_t unserialize_vec_32
Definition: serialize.h:364
u32 ** free_lists
Definition: heap.h:136