EPICS Base  7.0.6.1
 All Classes Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
valgrind.h
1 /* -*- c -*-
2  ----------------------------------------------------------------
3 
4  Notice that the following BSD-style license applies to this one
5  file (valgrind.h) only. The rest of Valgrind is licensed under the
6  terms of the GNU General Public License, version 2, unless
7  otherwise indicated. See the COPYING file in the source
8  distribution for details.
9 
10  ----------------------------------------------------------------
11 
12  This file is part of Valgrind, a dynamic binary instrumentation
13  framework.
14 
15  Copyright (C) 2000-2013 Julian Seward. All rights reserved.
16 
17  Redistribution and use in source and binary forms, with or without
18  modification, are permitted provided that the following conditions
19  are met:
20 
21  1. Redistributions of source code must retain the above copyright
22  notice, this list of conditions and the following disclaimer.
23 
24  2. The origin of this software must not be misrepresented; you must
25  not claim that you wrote the original software. If you use this
26  software in a product, an acknowledgment in the product
27  documentation would be appreciated but is not required.
28 
29  3. Altered source versions must be plainly marked as such, and must
30  not be misrepresented as being the original software.
31 
32  4. The name of the author may not be used to endorse or promote
33  products derived from this software without specific prior written
34  permission.
35 
36  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
37  OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
38  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
39  ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
40  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
41  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
42  GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
43  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
44  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
45  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
46  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47 
48  ----------------------------------------------------------------
49 
50  Notice that the above BSD-style license applies to this one file
51  (valgrind.h) only. The entire rest of Valgrind is licensed under
52  the terms of the GNU General Public License, version 2. See the
53  COPYING file in the source distribution for details.
54 
55  ----------------------------------------------------------------
56 */
57 
58 
59 /* This file is for inclusion into client (your!) code.
60 
61  You can use these macros to manipulate and query Valgrind's
62  execution inside your own programs.
63 
64  The resulting executables will still run without Valgrind, just a
65  little bit more slowly than they otherwise would, but otherwise
66  unchanged. When not running on valgrind, each client request
67  consumes very few (eg. 7) instructions, so the resulting performance
68  loss is negligible unless you plan to execute client requests
69  millions of times per second. Nevertheless, if that is still a
70  problem, you can compile with the NVALGRIND symbol defined (gcc
71  -DNVALGRIND) so that client requests are not even compiled in. */
72 
73 #ifndef __VALGRIND_H
74 #define __VALGRIND_H
75 
76 
77 /* ------------------------------------------------------------------ */
78 /* VERSION NUMBER OF VALGRIND */
79 /* ------------------------------------------------------------------ */
80 
81 /* Specify Valgrind's version number, so that user code can
82  conditionally compile based on our version number. Note that these
83  were introduced at version 3.6 and so do not exist in version 3.5
84  or earlier. The recommended way to use them to check for "version
85  X.Y or later" is (eg)
86 
87 #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__) \
88  && (__VALGRIND_MAJOR__ > 3 \
89  || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6))
90 */
91 #define __VALGRIND_MAJOR__ 3
92 #define __VALGRIND_MINOR__ 10
93 
94 
95 #include <stdarg.h>
96 
97 /* Nb: this file might be included in a file compiled with -ansi. So
98  we can't use C++ style "//" comments nor the "asm" keyword (instead
99  use "__asm__"). */
100 
101 /* Derive some tags indicating what the target platform is. Note
102  that in this file we're using the compiler's CPP symbols for
103  identifying architectures, which are different to the ones we use
104  within the rest of Valgrind. Note, __powerpc__ is active for both
105  32 and 64-bit PPC, whereas __powerpc64__ is only active for the
106  latter (on Linux, that is).
107 
108  Misc note: how to find out what's predefined in gcc by default:
109  gcc -Wp,-dM somefile.c
110 */
111 #undef PLAT_x86_darwin
112 #undef PLAT_amd64_darwin
113 #undef PLAT_x86_win32
114 #undef PLAT_amd64_win64
115 #undef PLAT_x86_linux
116 #undef PLAT_amd64_linux
117 #undef PLAT_ppc32_linux
118 #undef PLAT_ppc64be_linux
119 #undef PLAT_ppc64le_linux
120 #undef PLAT_arm_linux
121 #undef PLAT_arm64_linux
122 #undef PLAT_s390x_linux
123 #undef PLAT_mips32_linux
124 #undef PLAT_mips64_linux
125 
126 
127 #if defined(__APPLE__) && defined(__i386__)
128 # define PLAT_x86_darwin 1
129 #elif defined(__APPLE__) && defined(__x86_64__)
130 # define PLAT_amd64_darwin 1
131 #elif (defined(__MINGW32__) && !defined(__MINGW64__)) \
132  || defined(__CYGWIN32__) \
133  || (defined(_WIN32) && defined(_M_IX86) && defined(__GNUC__))
134 # define PLAT_x86_win32 1
135 #elif defined(__MINGW64__) \
136  || (defined(_WIN64) && defined(_M_X64) && defined(__GNUC__))
137 # define PLAT_amd64_win64 1
138 #elif defined(__linux__) && defined(__i386__)
139 # define PLAT_x86_linux 1
140 #elif defined(__linux__) && defined(__x86_64__)
141 # define PLAT_amd64_linux 1
142 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__)
143 # define PLAT_ppc32_linux 1
144 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF != 2
145 /* Big Endian uses ELF version 1 */
146 # define PLAT_ppc64be_linux 1
147 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF == 2
148 /* Little Endian uses ELF version 2 */
149 # define PLAT_ppc64le_linux 1
150 #elif defined(__linux__) && defined(__arm__) && !defined(__aarch64__)
151 # define PLAT_arm_linux 1
152 #elif defined(__linux__) && defined(__aarch64__) && !defined(__arm__)
153 # define PLAT_arm64_linux 1
154 #elif defined(__linux__) && defined(__s390__) && defined(__s390x__)
155 # define PLAT_s390x_linux 1
156 #elif defined(__linux__) && defined(__mips__) && (__mips==64)
157 # define PLAT_mips64_linux 1
158 #elif defined(__linux__) && defined(__mips__) && (__mips!=64)
159 # define PLAT_mips32_linux 1
160 #else
161 /* If we're not compiling for our target platform, don't generate
162  any inline asms. */
163 # if !defined(NVALGRIND)
164 # define NVALGRIND 1
165 # endif
166 #endif
167 
168 
169 /* ------------------------------------------------------------------ */
170 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS. There is nothing */
171 /* in here of use to end-users -- skip to the next section. */
172 /* ------------------------------------------------------------------ */
173 
174 /*
175  * VALGRIND_DO_CLIENT_REQUEST(): a statement that invokes a Valgrind client
176  * request. Accepts both pointers and integers as arguments.
177  *
178  * VALGRIND_DO_CLIENT_REQUEST_STMT(): a statement that invokes a Valgrind
179  * client request that does not return a value.
180 
181  * VALGRIND_DO_CLIENT_REQUEST_EXPR(): a C expression that invokes a Valgrind
182  * client request and whose value equals the client request result. Accepts
183  * both pointers and integers as arguments. Note that such calls are not
184  * necessarily pure functions -- they may have side effects.
185  */
186 
187 #define VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default, \
188  _zzq_request, _zzq_arg1, _zzq_arg2, \
189  _zzq_arg3, _zzq_arg4, _zzq_arg5) \
190  do { (_zzq_rlval) = VALGRIND_DO_CLIENT_REQUEST_EXPR((_zzq_default), \
191  (_zzq_request), (_zzq_arg1), (_zzq_arg2), \
192  (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
193 
194 #define VALGRIND_DO_CLIENT_REQUEST_STMT(_zzq_request, _zzq_arg1, \
195  _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
196  do { (void) VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \
197  (_zzq_request), (_zzq_arg1), (_zzq_arg2), \
198  (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
199 
200 #if defined(NVALGRIND)
201 
202 /* Define NVALGRIND to completely remove the Valgrind magic sequence
203  from the compiled code (analogous to NDEBUG's effects on
204  assert()) */
205 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
206  _zzq_default, _zzq_request, \
207  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
208  (_zzq_default)
209 
210 #else /* ! NVALGRIND */
211 
212 /* The following defines the magic code sequences which the JITter
213  spots and handles magically. Don't look too closely at them as
214  they will rot your brain.
215 
216  The assembly code sequences for all architectures is in this one
217  file. This is because this file must be stand-alone, and we don't
218  want to have multiple files.
219 
220  For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
221  value gets put in the return slot, so that everything works when
222  this is executed not under Valgrind. Args are passed in a memory
223  block, and so there's no intrinsic limit to the number that could
224  be passed, but it's currently five.
225 
226  The macro args are:
227  _zzq_rlval result lvalue
228  _zzq_default default value (result returned when running on real CPU)
229  _zzq_request request code
230  _zzq_arg1..5 request params
231 
232  The other two macros are used to support function wrapping, and are
233  a lot simpler. VALGRIND_GET_NR_CONTEXT returns the value of the
234  guest's NRADDR pseudo-register and whatever other information is
235  needed to safely run the call original from the wrapper: on
236  ppc64-linux, the R2 value at the divert point is also needed. This
237  information is abstracted into a user-visible type, OrigFn.
238 
239  VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
240  guest, but guarantees that the branch instruction will not be
241  redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
242  branch-and-link-to-r11. VALGRIND_CALL_NOREDIR is just text, not a
243  complete inline asm, since it needs to be combined with more magic
244  inline asm stuff to be useful.
245 */
246 
247 /* ------------------------- x86-{linux,darwin} ---------------- */
248 
249 #if defined(PLAT_x86_linux) || defined(PLAT_x86_darwin) \
250  || (defined(PLAT_x86_win32) && defined(__GNUC__))
251 
252 typedef
253  struct {
254  unsigned int nraddr; /* where's the code? */
255  }
256  OrigFn;
257 
258 #define __SPECIAL_INSTRUCTION_PREAMBLE \
259  "roll $3, %%edi ; roll $13, %%edi\n\t" \
260  "roll $29, %%edi ; roll $19, %%edi\n\t"
261 
262 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
263  _zzq_default, _zzq_request, \
264  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
265  __extension__ \
266  ({volatile unsigned int _zzq_args[6]; \
267  volatile unsigned int _zzq_result; \
268  _zzq_args[0] = (unsigned int)(_zzq_request); \
269  _zzq_args[1] = (unsigned int)(_zzq_arg1); \
270  _zzq_args[2] = (unsigned int)(_zzq_arg2); \
271  _zzq_args[3] = (unsigned int)(_zzq_arg3); \
272  _zzq_args[4] = (unsigned int)(_zzq_arg4); \
273  _zzq_args[5] = (unsigned int)(_zzq_arg5); \
274  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
275  /* %EDX = client_request ( %EAX ) */ \
276  "xchgl %%ebx,%%ebx" \
277  : "=d" (_zzq_result) \
278  : "a" (&_zzq_args[0]), "0" (_zzq_default) \
279  : "cc", "memory" \
280  ); \
281  _zzq_result; \
282  })
283 
284 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
285  { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
286  volatile unsigned int __addr; \
287  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
288  /* %EAX = guest_NRADDR */ \
289  "xchgl %%ecx,%%ecx" \
290  : "=a" (__addr) \
291  : \
292  : "cc", "memory" \
293  ); \
294  _zzq_orig->nraddr = __addr; \
295  }
296 
297 #define VALGRIND_CALL_NOREDIR_EAX \
298  __SPECIAL_INSTRUCTION_PREAMBLE \
299  /* call-noredir *%EAX */ \
300  "xchgl %%edx,%%edx\n\t"
301 
302 #define VALGRIND_VEX_INJECT_IR() \
303  do { \
304  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
305  "xchgl %%edi,%%edi\n\t" \
306  : : : "cc", "memory" \
307  ); \
308  } while (0)
309 
310 #endif /* PLAT_x86_linux || PLAT_x86_darwin || (PLAT_x86_win32 && __GNUC__) */
311 
312 /* ------------------------- x86-Win32 ------------------------- */
313 
314 #if defined(PLAT_x86_win32) && !defined(__GNUC__)
315 
316 typedef
317  struct {
318  unsigned int nraddr; /* where's the code? */
319  }
320  OrigFn;
321 
322 #if defined(_MSC_VER)
323 
324 #define __SPECIAL_INSTRUCTION_PREAMBLE \
325  __asm rol edi, 3 __asm rol edi, 13 \
326  __asm rol edi, 29 __asm rol edi, 19
327 
328 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
329  _zzq_default, _zzq_request, \
330  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
331  valgrind_do_client_request_expr((uintptr_t)(_zzq_default), \
332  (uintptr_t)(_zzq_request), (uintptr_t)(_zzq_arg1), \
333  (uintptr_t)(_zzq_arg2), (uintptr_t)(_zzq_arg3), \
334  (uintptr_t)(_zzq_arg4), (uintptr_t)(_zzq_arg5))
335 
336 static __inline uintptr_t
337 valgrind_do_client_request_expr(uintptr_t _zzq_default, uintptr_t _zzq_request,
338  uintptr_t _zzq_arg1, uintptr_t _zzq_arg2,
339  uintptr_t _zzq_arg3, uintptr_t _zzq_arg4,
340  uintptr_t _zzq_arg5)
341 {
342  volatile uintptr_t _zzq_args[6];
343  volatile unsigned int _zzq_result;
344  _zzq_args[0] = (uintptr_t)(_zzq_request);
345  _zzq_args[1] = (uintptr_t)(_zzq_arg1);
346  _zzq_args[2] = (uintptr_t)(_zzq_arg2);
347  _zzq_args[3] = (uintptr_t)(_zzq_arg3);
348  _zzq_args[4] = (uintptr_t)(_zzq_arg4);
349  _zzq_args[5] = (uintptr_t)(_zzq_arg5);
350  __asm { __asm lea eax, _zzq_args __asm mov edx, _zzq_default
351  __SPECIAL_INSTRUCTION_PREAMBLE
352  /* %EDX = client_request ( %EAX ) */
353  __asm xchg ebx,ebx
354  __asm mov _zzq_result, edx
355  }
356  return _zzq_result;
357 }
358 
359 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
360  { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
361  volatile unsigned int __addr; \
362  __asm { __SPECIAL_INSTRUCTION_PREAMBLE \
363  /* %EAX = guest_NRADDR */ \
364  __asm xchg ecx,ecx \
365  __asm mov __addr, eax \
366  } \
367  _zzq_orig->nraddr = __addr; \
368  }
369 
370 #define VALGRIND_CALL_NOREDIR_EAX ERROR
371 
372 #define VALGRIND_VEX_INJECT_IR() \
373  do { \
374  __asm { __SPECIAL_INSTRUCTION_PREAMBLE \
375  __asm xchg edi,edi \
376  } \
377  } while (0)
378 
379 #else
380 #error Unsupported compiler.
381 #endif
382 
383 #endif /* PLAT_x86_win32 */
384 
385 /* ------------------------ amd64-{linux,darwin} --------------- */
386 
387 #if defined(PLAT_amd64_linux) || defined(PLAT_amd64_darwin) \
388  || (defined(PLAT_amd64_win64) && defined(__GNUC__))
389 
390 typedef
391  struct {
392  unsigned long long int nraddr; /* where's the code? */
393  }
394  OrigFn;
395 
396 #define __SPECIAL_INSTRUCTION_PREAMBLE \
397  "rolq $3, %%rdi ; rolq $13, %%rdi\n\t" \
398  "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
399 
400 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
401  _zzq_default, _zzq_request, \
402  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
403  __extension__ \
404  ({ volatile unsigned long long int _zzq_args[6]; \
405  volatile unsigned long long int _zzq_result; \
406  _zzq_args[0] = (unsigned long long int)(_zzq_request); \
407  _zzq_args[1] = (unsigned long long int)(_zzq_arg1); \
408  _zzq_args[2] = (unsigned long long int)(_zzq_arg2); \
409  _zzq_args[3] = (unsigned long long int)(_zzq_arg3); \
410  _zzq_args[4] = (unsigned long long int)(_zzq_arg4); \
411  _zzq_args[5] = (unsigned long long int)(_zzq_arg5); \
412  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
413  /* %RDX = client_request ( %RAX ) */ \
414  "xchgq %%rbx,%%rbx" \
415  : "=d" (_zzq_result) \
416  : "a" (&_zzq_args[0]), "0" (_zzq_default) \
417  : "cc", "memory" \
418  ); \
419  _zzq_result; \
420  })
421 
422 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
423  { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
424  volatile unsigned long long int __addr; \
425  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
426  /* %RAX = guest_NRADDR */ \
427  "xchgq %%rcx,%%rcx" \
428  : "=a" (__addr) \
429  : \
430  : "cc", "memory" \
431  ); \
432  _zzq_orig->nraddr = __addr; \
433  }
434 
435 #define VALGRIND_CALL_NOREDIR_RAX \
436  __SPECIAL_INSTRUCTION_PREAMBLE \
437  /* call-noredir *%RAX */ \
438  "xchgq %%rdx,%%rdx\n\t"
439 
440 #define VALGRIND_VEX_INJECT_IR() \
441  do { \
442  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
443  "xchgq %%rdi,%%rdi\n\t" \
444  : : : "cc", "memory" \
445  ); \
446  } while (0)
447 
448 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
449 
450 /* ------------------------- amd64-Win64 ------------------------- */
451 
452 #if defined(PLAT_amd64_win64) && !defined(__GNUC__)
453 
454 #error Unsupported compiler.
455 
456 #endif /* PLAT_amd64_win64 */
457 
458 /* ------------------------ ppc32-linux ------------------------ */
459 
460 #if defined(PLAT_ppc32_linux)
461 
462 typedef
463  struct {
464  unsigned int nraddr; /* where's the code? */
465  }
466  OrigFn;
467 
468 #define __SPECIAL_INSTRUCTION_PREAMBLE \
469  "rlwinm 0,0,3,0,31 ; rlwinm 0,0,13,0,31\n\t" \
470  "rlwinm 0,0,29,0,31 ; rlwinm 0,0,19,0,31\n\t"
471 
472 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
473  _zzq_default, _zzq_request, \
474  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
475  \
476  __extension__ \
477  ({ unsigned int _zzq_args[6]; \
478  unsigned int _zzq_result; \
479  unsigned int* _zzq_ptr; \
480  _zzq_args[0] = (unsigned int)(_zzq_request); \
481  _zzq_args[1] = (unsigned int)(_zzq_arg1); \
482  _zzq_args[2] = (unsigned int)(_zzq_arg2); \
483  _zzq_args[3] = (unsigned int)(_zzq_arg3); \
484  _zzq_args[4] = (unsigned int)(_zzq_arg4); \
485  _zzq_args[5] = (unsigned int)(_zzq_arg5); \
486  _zzq_ptr = _zzq_args; \
487  __asm__ volatile("mr 3,%1\n\t" /*default*/ \
488  "mr 4,%2\n\t" /*ptr*/ \
489  __SPECIAL_INSTRUCTION_PREAMBLE \
490  /* %R3 = client_request ( %R4 ) */ \
491  "or 1,1,1\n\t" \
492  "mr %0,3" /*result*/ \
493  : "=b" (_zzq_result) \
494  : "b" (_zzq_default), "b" (_zzq_ptr) \
495  : "cc", "memory", "r3", "r4"); \
496  _zzq_result; \
497  })
498 
499 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
500  { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
501  unsigned int __addr; \
502  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
503  /* %R3 = guest_NRADDR */ \
504  "or 2,2,2\n\t" \
505  "mr %0,3" \
506  : "=b" (__addr) \
507  : \
508  : "cc", "memory", "r3" \
509  ); \
510  _zzq_orig->nraddr = __addr; \
511  }
512 
513 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
514  __SPECIAL_INSTRUCTION_PREAMBLE \
515  /* branch-and-link-to-noredir *%R11 */ \
516  "or 3,3,3\n\t"
517 
518 #define VALGRIND_VEX_INJECT_IR() \
519  do { \
520  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
521  "or 5,5,5\n\t" \
522  ); \
523  } while (0)
524 
525 #endif /* PLAT_ppc32_linux */
526 
527 /* ------------------------ ppc64-linux ------------------------ */
528 
529 #if defined(PLAT_ppc64be_linux)
530 
531 typedef
532  struct {
533  unsigned long long int nraddr; /* where's the code? */
534  unsigned long long int r2; /* what tocptr do we need? */
535  }
536  OrigFn;
537 
538 #define __SPECIAL_INSTRUCTION_PREAMBLE \
539  "rotldi 0,0,3 ; rotldi 0,0,13\n\t" \
540  "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
541 
542 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
543  _zzq_default, _zzq_request, \
544  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
545  \
546  __extension__ \
547  ({ unsigned long long int _zzq_args[6]; \
548  unsigned long long int _zzq_result; \
549  unsigned long long int* _zzq_ptr; \
550  _zzq_args[0] = (unsigned long long int)(_zzq_request); \
551  _zzq_args[1] = (unsigned long long int)(_zzq_arg1); \
552  _zzq_args[2] = (unsigned long long int)(_zzq_arg2); \
553  _zzq_args[3] = (unsigned long long int)(_zzq_arg3); \
554  _zzq_args[4] = (unsigned long long int)(_zzq_arg4); \
555  _zzq_args[5] = (unsigned long long int)(_zzq_arg5); \
556  _zzq_ptr = _zzq_args; \
557  __asm__ volatile("mr 3,%1\n\t" /*default*/ \
558  "mr 4,%2\n\t" /*ptr*/ \
559  __SPECIAL_INSTRUCTION_PREAMBLE \
560  /* %R3 = client_request ( %R4 ) */ \
561  "or 1,1,1\n\t" \
562  "mr %0,3" /*result*/ \
563  : "=b" (_zzq_result) \
564  : "b" (_zzq_default), "b" (_zzq_ptr) \
565  : "cc", "memory", "r3", "r4"); \
566  _zzq_result; \
567  })
568 
569 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
570  { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
571  unsigned long long int __addr; \
572  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
573  /* %R3 = guest_NRADDR */ \
574  "or 2,2,2\n\t" \
575  "mr %0,3" \
576  : "=b" (__addr) \
577  : \
578  : "cc", "memory", "r3" \
579  ); \
580  _zzq_orig->nraddr = __addr; \
581  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
582  /* %R3 = guest_NRADDR_GPR2 */ \
583  "or 4,4,4\n\t" \
584  "mr %0,3" \
585  : "=b" (__addr) \
586  : \
587  : "cc", "memory", "r3" \
588  ); \
589  _zzq_orig->r2 = __addr; \
590  }
591 
592 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
593  __SPECIAL_INSTRUCTION_PREAMBLE \
594  /* branch-and-link-to-noredir *%R11 */ \
595  "or 3,3,3\n\t"
596 
597 #define VALGRIND_VEX_INJECT_IR() \
598  do { \
599  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
600  "or 5,5,5\n\t" \
601  ); \
602  } while (0)
603 
604 #endif /* PLAT_ppc64be_linux */
605 
606 #if defined(PLAT_ppc64le_linux)
607 
608 typedef
609  struct {
610  unsigned long long int nraddr; /* where's the code? */
611  unsigned long long int r2; /* what tocptr do we need? */
612  }
613  OrigFn;
614 
615 #define __SPECIAL_INSTRUCTION_PREAMBLE \
616  "rotldi 0,0,3 ; rotldi 0,0,13\n\t" \
617  "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
618 
619 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
620  _zzq_default, _zzq_request, \
621  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
622  \
623  __extension__ \
624  ({ unsigned long long int _zzq_args[6]; \
625  unsigned long long int _zzq_result; \
626  unsigned long long int* _zzq_ptr; \
627  _zzq_args[0] = (unsigned long long int)(_zzq_request); \
628  _zzq_args[1] = (unsigned long long int)(_zzq_arg1); \
629  _zzq_args[2] = (unsigned long long int)(_zzq_arg2); \
630  _zzq_args[3] = (unsigned long long int)(_zzq_arg3); \
631  _zzq_args[4] = (unsigned long long int)(_zzq_arg4); \
632  _zzq_args[5] = (unsigned long long int)(_zzq_arg5); \
633  _zzq_ptr = _zzq_args; \
634  __asm__ volatile("mr 3,%1\n\t" /*default*/ \
635  "mr 4,%2\n\t" /*ptr*/ \
636  __SPECIAL_INSTRUCTION_PREAMBLE \
637  /* %R3 = client_request ( %R4 ) */ \
638  "or 1,1,1\n\t" \
639  "mr %0,3" /*result*/ \
640  : "=b" (_zzq_result) \
641  : "b" (_zzq_default), "b" (_zzq_ptr) \
642  : "cc", "memory", "r3", "r4"); \
643  _zzq_result; \
644  })
645 
646 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
647  { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
648  unsigned long long int __addr; \
649  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
650  /* %R3 = guest_NRADDR */ \
651  "or 2,2,2\n\t" \
652  "mr %0,3" \
653  : "=b" (__addr) \
654  : \
655  : "cc", "memory", "r3" \
656  ); \
657  _zzq_orig->nraddr = __addr; \
658  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
659  /* %R3 = guest_NRADDR_GPR2 */ \
660  "or 4,4,4\n\t" \
661  "mr %0,3" \
662  : "=b" (__addr) \
663  : \
664  : "cc", "memory", "r3" \
665  ); \
666  _zzq_orig->r2 = __addr; \
667  }
668 
669 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
670  __SPECIAL_INSTRUCTION_PREAMBLE \
671  /* branch-and-link-to-noredir *%R12 */ \
672  "or 3,3,3\n\t"
673 
674 #define VALGRIND_VEX_INJECT_IR() \
675  do { \
676  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
677  "or 5,5,5\n\t" \
678  ); \
679  } while (0)
680 
681 #endif /* PLAT_ppc64le_linux */
682 
683 /* ------------------------- arm-linux ------------------------- */
684 
685 #if defined(PLAT_arm_linux)
686 
687 typedef
688  struct {
689  unsigned int nraddr; /* where's the code? */
690  }
691  OrigFn;
692 
693 #define __SPECIAL_INSTRUCTION_PREAMBLE \
694  "mov r12, r12, ror #3 ; mov r12, r12, ror #13 \n\t" \
695  "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t"
696 
697 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
698  _zzq_default, _zzq_request, \
699  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
700  \
701  __extension__ \
702  ({volatile unsigned int _zzq_args[6]; \
703  volatile unsigned int _zzq_result; \
704  _zzq_args[0] = (unsigned int)(_zzq_request); \
705  _zzq_args[1] = (unsigned int)(_zzq_arg1); \
706  _zzq_args[2] = (unsigned int)(_zzq_arg2); \
707  _zzq_args[3] = (unsigned int)(_zzq_arg3); \
708  _zzq_args[4] = (unsigned int)(_zzq_arg4); \
709  _zzq_args[5] = (unsigned int)(_zzq_arg5); \
710  __asm__ volatile("mov r3, %1\n\t" /*default*/ \
711  "mov r4, %2\n\t" /*ptr*/ \
712  __SPECIAL_INSTRUCTION_PREAMBLE \
713  /* R3 = client_request ( R4 ) */ \
714  "orr r10, r10, r10\n\t" \
715  "mov %0, r3" /*result*/ \
716  : "=r" (_zzq_result) \
717  : "r" (_zzq_default), "r" (&_zzq_args[0]) \
718  : "cc","memory", "r3", "r4"); \
719  _zzq_result; \
720  })
721 
722 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
723  { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
724  unsigned int __addr; \
725  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
726  /* R3 = guest_NRADDR */ \
727  "orr r11, r11, r11\n\t" \
728  "mov %0, r3" \
729  : "=r" (__addr) \
730  : \
731  : "cc", "memory", "r3" \
732  ); \
733  _zzq_orig->nraddr = __addr; \
734  }
735 
736 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
737  __SPECIAL_INSTRUCTION_PREAMBLE \
738  /* branch-and-link-to-noredir *%R4 */ \
739  "orr r12, r12, r12\n\t"
740 
741 #define VALGRIND_VEX_INJECT_IR() \
742  do { \
743  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
744  "orr r9, r9, r9\n\t" \
745  : : : "cc", "memory" \
746  ); \
747  } while (0)
748 
749 #endif /* PLAT_arm_linux */
750 
751 /* ------------------------ arm64-linux ------------------------- */
752 
753 #if defined(PLAT_arm64_linux)
754 
755 typedef
756  struct {
757  unsigned long long int nraddr; /* where's the code? */
758  }
759  OrigFn;
760 
761 #define __SPECIAL_INSTRUCTION_PREAMBLE \
762  "ror x12, x12, #3 ; ror x12, x12, #13 \n\t" \
763  "ror x12, x12, #51 ; ror x12, x12, #61 \n\t"
764 
765 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
766  _zzq_default, _zzq_request, \
767  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
768  \
769  __extension__ \
770  ({volatile unsigned long long int _zzq_args[6]; \
771  volatile unsigned long long int _zzq_result; \
772  _zzq_args[0] = (unsigned long long int)(_zzq_request); \
773  _zzq_args[1] = (unsigned long long int)(_zzq_arg1); \
774  _zzq_args[2] = (unsigned long long int)(_zzq_arg2); \
775  _zzq_args[3] = (unsigned long long int)(_zzq_arg3); \
776  _zzq_args[4] = (unsigned long long int)(_zzq_arg4); \
777  _zzq_args[5] = (unsigned long long int)(_zzq_arg5); \
778  __asm__ volatile("mov x3, %1\n\t" /*default*/ \
779  "mov x4, %2\n\t" /*ptr*/ \
780  __SPECIAL_INSTRUCTION_PREAMBLE \
781  /* X3 = client_request ( X4 ) */ \
782  "orr x10, x10, x10\n\t" \
783  "mov %0, x3" /*result*/ \
784  : "=r" (_zzq_result) \
785  : "r" (_zzq_default), "r" (&_zzq_args[0]) \
786  : "cc","memory", "x3", "x4"); \
787  _zzq_result; \
788  })
789 
790 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
791  { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
792  unsigned long long int __addr; \
793  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
794  /* X3 = guest_NRADDR */ \
795  "orr x11, x11, x11\n\t" \
796  "mov %0, x3" \
797  : "=r" (__addr) \
798  : \
799  : "cc", "memory", "x3" \
800  ); \
801  _zzq_orig->nraddr = __addr; \
802  }
803 
804 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
805  __SPECIAL_INSTRUCTION_PREAMBLE \
806  /* branch-and-link-to-noredir X8 */ \
807  "orr x12, x12, x12\n\t"
808 
809 #define VALGRIND_VEX_INJECT_IR() \
810  do { \
811  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
812  "orr x9, x9, x9\n\t" \
813  : : : "cc", "memory" \
814  ); \
815  } while (0)
816 
817 #endif /* PLAT_arm64_linux */
818 
819 /* ------------------------ s390x-linux ------------------------ */
820 
821 #if defined(PLAT_s390x_linux)
822 
823 typedef
824  struct {
825  unsigned long long int nraddr; /* where's the code? */
826  }
827  OrigFn;
828 
829 /* __SPECIAL_INSTRUCTION_PREAMBLE will be used to identify Valgrind specific
830  * code. This detection is implemented in platform specific toIR.c
831  * (e.g. VEX/priv/guest_s390_decoder.c).
832  */
833 #define __SPECIAL_INSTRUCTION_PREAMBLE \
834  "lr 15,15\n\t" \
835  "lr 1,1\n\t" \
836  "lr 2,2\n\t" \
837  "lr 3,3\n\t"
838 
839 #define __CLIENT_REQUEST_CODE "lr 2,2\n\t"
840 #define __GET_NR_CONTEXT_CODE "lr 3,3\n\t"
841 #define __CALL_NO_REDIR_CODE "lr 4,4\n\t"
842 #define __VEX_INJECT_IR_CODE "lr 5,5\n\t"
843 
844 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
845  _zzq_default, _zzq_request, \
846  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
847  __extension__ \
848  ({volatile unsigned long long int _zzq_args[6]; \
849  volatile unsigned long long int _zzq_result; \
850  _zzq_args[0] = (unsigned long long int)(_zzq_request); \
851  _zzq_args[1] = (unsigned long long int)(_zzq_arg1); \
852  _zzq_args[2] = (unsigned long long int)(_zzq_arg2); \
853  _zzq_args[3] = (unsigned long long int)(_zzq_arg3); \
854  _zzq_args[4] = (unsigned long long int)(_zzq_arg4); \
855  _zzq_args[5] = (unsigned long long int)(_zzq_arg5); \
856  __asm__ volatile(/* r2 = args */ \
857  "lgr 2,%1\n\t" \
858  /* r3 = default */ \
859  "lgr 3,%2\n\t" \
860  __SPECIAL_INSTRUCTION_PREAMBLE \
861  __CLIENT_REQUEST_CODE \
862  /* results = r3 */ \
863  "lgr %0, 3\n\t" \
864  : "=d" (_zzq_result) \
865  : "a" (&_zzq_args[0]), "0" (_zzq_default) \
866  : "cc", "2", "3", "memory" \
867  ); \
868  _zzq_result; \
869  })
870 
871 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
872  { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
873  volatile unsigned long long int __addr; \
874  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
875  __GET_NR_CONTEXT_CODE \
876  "lgr %0, 3\n\t" \
877  : "=a" (__addr) \
878  : \
879  : "cc", "3", "memory" \
880  ); \
881  _zzq_orig->nraddr = __addr; \
882  }
883 
884 #define VALGRIND_CALL_NOREDIR_R1 \
885  __SPECIAL_INSTRUCTION_PREAMBLE \
886  __CALL_NO_REDIR_CODE
887 
888 #define VALGRIND_VEX_INJECT_IR() \
889  do { \
890  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
891  __VEX_INJECT_IR_CODE); \
892  } while (0)
893 
894 #endif /* PLAT_s390x_linux */
895 
896 /* ------------------------- mips32-linux ---------------- */
897 
898 #if defined(PLAT_mips32_linux)
899 
900 typedef
901  struct {
902  unsigned int nraddr; /* where's the code? */
903  }
904  OrigFn;
905 
906 /* .word 0x342
907  * .word 0x742
908  * .word 0xC2
909  * .word 0x4C2*/
910 #define __SPECIAL_INSTRUCTION_PREAMBLE \
911  "srl $0, $0, 13\n\t" \
912  "srl $0, $0, 29\n\t" \
913  "srl $0, $0, 3\n\t" \
914  "srl $0, $0, 19\n\t"
915 
916 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
917  _zzq_default, _zzq_request, \
918  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
919  __extension__ \
920  ({ volatile unsigned int _zzq_args[6]; \
921  volatile unsigned int _zzq_result; \
922  _zzq_args[0] = (unsigned int)(_zzq_request); \
923  _zzq_args[1] = (unsigned int)(_zzq_arg1); \
924  _zzq_args[2] = (unsigned int)(_zzq_arg2); \
925  _zzq_args[3] = (unsigned int)(_zzq_arg3); \
926  _zzq_args[4] = (unsigned int)(_zzq_arg4); \
927  _zzq_args[5] = (unsigned int)(_zzq_arg5); \
928  __asm__ volatile("move $11, %1\n\t" /*default*/ \
929  "move $12, %2\n\t" /*ptr*/ \
930  __SPECIAL_INSTRUCTION_PREAMBLE \
931  /* T3 = client_request ( T4 ) */ \
932  "or $13, $13, $13\n\t" \
933  "move %0, $11\n\t" /*result*/ \
934  : "=r" (_zzq_result) \
935  : "r" (_zzq_default), "r" (&_zzq_args[0]) \
936  : "$11", "$12"); \
937  _zzq_result; \
938  })
939 
940 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
941  { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
942  volatile unsigned int __addr; \
943  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
944  /* %t9 = guest_NRADDR */ \
945  "or $14, $14, $14\n\t" \
946  "move %0, $11" /*result*/ \
947  : "=r" (__addr) \
948  : \
949  : "$11" \
950  ); \
951  _zzq_orig->nraddr = __addr; \
952  }
953 
954 #define VALGRIND_CALL_NOREDIR_T9 \
955  __SPECIAL_INSTRUCTION_PREAMBLE \
956  /* call-noredir *%t9 */ \
957  "or $15, $15, $15\n\t"
958 
959 #define VALGRIND_VEX_INJECT_IR() \
960  do { \
961  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
962  "or $11, $11, $11\n\t" \
963  ); \
964  } while (0)
965 
966 
967 #endif /* PLAT_mips32_linux */
968 
969 /* ------------------------- mips64-linux ---------------- */
970 
971 #if defined(PLAT_mips64_linux)
972 
973 typedef
974  struct {
975  unsigned long long nraddr; /* where's the code? */
976  }
977  OrigFn;
978 
979 /* dsll $0,$0, 3
980  * dsll $0,$0, 13
981  * dsll $0,$0, 29
982  * dsll $0,$0, 19*/
983 #define __SPECIAL_INSTRUCTION_PREAMBLE \
984  "dsll $0,$0, 3 ; dsll $0,$0,13\n\t" \
985  "dsll $0,$0,29 ; dsll $0,$0,19\n\t"
986 
987 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \
988  _zzq_default, _zzq_request, \
989  _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \
990  __extension__ \
991  ({ volatile unsigned long long int _zzq_args[6]; \
992  volatile unsigned long long int _zzq_result; \
993  _zzq_args[0] = (unsigned long long int)(_zzq_request); \
994  _zzq_args[1] = (unsigned long long int)(_zzq_arg1); \
995  _zzq_args[2] = (unsigned long long int)(_zzq_arg2); \
996  _zzq_args[3] = (unsigned long long int)(_zzq_arg3); \
997  _zzq_args[4] = (unsigned long long int)(_zzq_arg4); \
998  _zzq_args[5] = (unsigned long long int)(_zzq_arg5); \
999  __asm__ volatile("move $11, %1\n\t" /*default*/ \
1000  "move $12, %2\n\t" /*ptr*/ \
1001  __SPECIAL_INSTRUCTION_PREAMBLE \
1002  /* $11 = client_request ( $12 ) */ \
1003  "or $13, $13, $13\n\t" \
1004  "move %0, $11\n\t" /*result*/ \
1005  : "=r" (_zzq_result) \
1006  : "r" (_zzq_default), "r" (&_zzq_args[0]) \
1007  : "$11", "$12"); \
1008  _zzq_result; \
1009  })
1010 
1011 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \
1012  { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \
1013  volatile unsigned long long int __addr; \
1014  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
1015  /* $11 = guest_NRADDR */ \
1016  "or $14, $14, $14\n\t" \
1017  "move %0, $11" /*result*/ \
1018  : "=r" (__addr) \
1019  : \
1020  : "$11"); \
1021  _zzq_orig->nraddr = __addr; \
1022  }
1023 
1024 #define VALGRIND_CALL_NOREDIR_T9 \
1025  __SPECIAL_INSTRUCTION_PREAMBLE \
1026  /* call-noredir $25 */ \
1027  "or $15, $15, $15\n\t"
1028 
1029 #define VALGRIND_VEX_INJECT_IR() \
1030  do { \
1031  __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
1032  "or $11, $11, $11\n\t" \
1033  ); \
1034  } while (0)
1035 
1036 #endif /* PLAT_mips64_linux */
1037 
1038 /* Insert assembly code for other platforms here... */
1039 
1040 #endif /* NVALGRIND */
1041 
1042 
1043 /* ------------------------------------------------------------------ */
1044 /* PLATFORM SPECIFICS for FUNCTION WRAPPING. This is all very */
1045 /* ugly. It's the least-worst tradeoff I can think of. */
1046 /* ------------------------------------------------------------------ */
1047 
1048 /* This section defines magic (a.k.a appalling-hack) macros for doing
1049  guaranteed-no-redirection macros, so as to get from function
1050  wrappers to the functions they are wrapping. The whole point is to
1051  construct standard call sequences, but to do the call itself with a
1052  special no-redirect call pseudo-instruction that the JIT
1053  understands and handles specially. This section is long and
1054  repetitious, and I can't see a way to make it shorter.
1055 
1056  The naming scheme is as follows:
1057 
1058  CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
1059 
1060  'W' stands for "word" and 'v' for "void". Hence there are
1061  different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
1062  and for each, the possibility of returning a word-typed result, or
1063  no result.
1064 */
1065 
1066 /* Use these to write the name of your wrapper. NOTE: duplicates
1067  VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. NOTE also: inserts
1068  the default behavior equivalence class tag "0000" into the name.
1069  See pub_tool_redir.h for details -- normally you don't need to
1070  think about this, though. */
1071 
1072 /* Use an extra level of macroisation so as to ensure the soname/fnname
1073  args are fully macro-expanded before pasting them together. */
1074 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
1075 
1076 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname) \
1077  VG_CONCAT4(_vgw00000ZU_,soname,_,fnname)
1078 
1079 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname) \
1080  VG_CONCAT4(_vgw00000ZZ_,soname,_,fnname)
1081 
1082 /* Use this macro from within a wrapper function to collect the
1083  context (address and possibly other info) of the original function.
1084  Once you have that you can then use it in one of the CALL_FN_
1085  macros. The type of the argument _lval is OrigFn. */
1086 #define VALGRIND_GET_ORIG_FN(_lval) VALGRIND_GET_NR_CONTEXT(_lval)
1087 
1088 /* Also provide end-user facilities for function replacement, rather
1089  than wrapping. A replacement function differs from a wrapper in
1090  that it has no way to get hold of the original function being
1091  called, and hence no way to call onwards to it. In a replacement
1092  function, VALGRIND_GET_ORIG_FN always returns zero. */
1093 
1094 #define I_REPLACE_SONAME_FNNAME_ZU(soname,fnname) \
1095  VG_CONCAT4(_vgr00000ZU_,soname,_,fnname)
1096 
1097 #define I_REPLACE_SONAME_FNNAME_ZZ(soname,fnname) \
1098  VG_CONCAT4(_vgr00000ZZ_,soname,_,fnname)
1099 
1100 /* Derivatives of the main macros below, for calling functions
1101  returning void. */
1102 
1103 #define CALL_FN_v_v(fnptr) \
1104  do { volatile unsigned long _junk; \
1105  CALL_FN_W_v(_junk,fnptr); } while (0)
1106 
1107 #define CALL_FN_v_W(fnptr, arg1) \
1108  do { volatile unsigned long _junk; \
1109  CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
1110 
1111 #define CALL_FN_v_WW(fnptr, arg1,arg2) \
1112  do { volatile unsigned long _junk; \
1113  CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
1114 
1115 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3) \
1116  do { volatile unsigned long _junk; \
1117  CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
1118 
1119 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4) \
1120  do { volatile unsigned long _junk; \
1121  CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
1122 
1123 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5) \
1124  do { volatile unsigned long _junk; \
1125  CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
1126 
1127 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6) \
1128  do { volatile unsigned long _junk; \
1129  CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
1130 
1131 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7) \
1132  do { volatile unsigned long _junk; \
1133  CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
1134 
1135 /* ------------------------- x86-{linux,darwin} ---------------- */
1136 
1137 #if defined(PLAT_x86_linux) || defined(PLAT_x86_darwin)
1138 
1139 /* These regs are trashed by the hidden call. No need to mention eax
1140  as gcc can already see that, plus causes gcc to bomb. */
1141 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
1142 
1143 /* Macros to save and align the stack before making a function
1144  call and restore it afterwards as gcc may not keep the stack
1145  pointer aligned if it doesn't realize calls are being made
1146  to other functions. */
1147 
1148 #define VALGRIND_ALIGN_STACK \
1149  "movl %%esp,%%edi\n\t" \
1150  "andl $0xfffffff0,%%esp\n\t"
1151 #define VALGRIND_RESTORE_STACK \
1152  "movl %%edi,%%esp\n\t"
1153 
1154 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
1155  long) == 4. */
1156 
1157 #define CALL_FN_W_v(lval, orig) \
1158  do { \
1159  volatile OrigFn _orig = (orig); \
1160  volatile unsigned long _argvec[1]; \
1161  volatile unsigned long _res; \
1162  _argvec[0] = (unsigned long)_orig.nraddr; \
1163  __asm__ volatile( \
1164  VALGRIND_ALIGN_STACK \
1165  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1166  VALGRIND_CALL_NOREDIR_EAX \
1167  VALGRIND_RESTORE_STACK \
1168  : /*out*/ "=a" (_res) \
1169  : /*in*/ "a" (&_argvec[0]) \
1170  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1171  ); \
1172  lval = (__typeof__(lval)) _res; \
1173  } while (0)
1174 
1175 #define CALL_FN_W_W(lval, orig, arg1) \
1176  do { \
1177  volatile OrigFn _orig = (orig); \
1178  volatile unsigned long _argvec[2]; \
1179  volatile unsigned long _res; \
1180  _argvec[0] = (unsigned long)_orig.nraddr; \
1181  _argvec[1] = (unsigned long)(arg1); \
1182  __asm__ volatile( \
1183  VALGRIND_ALIGN_STACK \
1184  "subl $12, %%esp\n\t" \
1185  "pushl 4(%%eax)\n\t" \
1186  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1187  VALGRIND_CALL_NOREDIR_EAX \
1188  VALGRIND_RESTORE_STACK \
1189  : /*out*/ "=a" (_res) \
1190  : /*in*/ "a" (&_argvec[0]) \
1191  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1192  ); \
1193  lval = (__typeof__(lval)) _res; \
1194  } while (0)
1195 
1196 #define CALL_FN_W_WW(lval, orig, arg1,arg2) \
1197  do { \
1198  volatile OrigFn _orig = (orig); \
1199  volatile unsigned long _argvec[3]; \
1200  volatile unsigned long _res; \
1201  _argvec[0] = (unsigned long)_orig.nraddr; \
1202  _argvec[1] = (unsigned long)(arg1); \
1203  _argvec[2] = (unsigned long)(arg2); \
1204  __asm__ volatile( \
1205  VALGRIND_ALIGN_STACK \
1206  "subl $8, %%esp\n\t" \
1207  "pushl 8(%%eax)\n\t" \
1208  "pushl 4(%%eax)\n\t" \
1209  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1210  VALGRIND_CALL_NOREDIR_EAX \
1211  VALGRIND_RESTORE_STACK \
1212  : /*out*/ "=a" (_res) \
1213  : /*in*/ "a" (&_argvec[0]) \
1214  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1215  ); \
1216  lval = (__typeof__(lval)) _res; \
1217  } while (0)
1218 
1219 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
1220  do { \
1221  volatile OrigFn _orig = (orig); \
1222  volatile unsigned long _argvec[4]; \
1223  volatile unsigned long _res; \
1224  _argvec[0] = (unsigned long)_orig.nraddr; \
1225  _argvec[1] = (unsigned long)(arg1); \
1226  _argvec[2] = (unsigned long)(arg2); \
1227  _argvec[3] = (unsigned long)(arg3); \
1228  __asm__ volatile( \
1229  VALGRIND_ALIGN_STACK \
1230  "subl $4, %%esp\n\t" \
1231  "pushl 12(%%eax)\n\t" \
1232  "pushl 8(%%eax)\n\t" \
1233  "pushl 4(%%eax)\n\t" \
1234  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1235  VALGRIND_CALL_NOREDIR_EAX \
1236  VALGRIND_RESTORE_STACK \
1237  : /*out*/ "=a" (_res) \
1238  : /*in*/ "a" (&_argvec[0]) \
1239  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1240  ); \
1241  lval = (__typeof__(lval)) _res; \
1242  } while (0)
1243 
1244 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
1245  do { \
1246  volatile OrigFn _orig = (orig); \
1247  volatile unsigned long _argvec[5]; \
1248  volatile unsigned long _res; \
1249  _argvec[0] = (unsigned long)_orig.nraddr; \
1250  _argvec[1] = (unsigned long)(arg1); \
1251  _argvec[2] = (unsigned long)(arg2); \
1252  _argvec[3] = (unsigned long)(arg3); \
1253  _argvec[4] = (unsigned long)(arg4); \
1254  __asm__ volatile( \
1255  VALGRIND_ALIGN_STACK \
1256  "pushl 16(%%eax)\n\t" \
1257  "pushl 12(%%eax)\n\t" \
1258  "pushl 8(%%eax)\n\t" \
1259  "pushl 4(%%eax)\n\t" \
1260  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1261  VALGRIND_CALL_NOREDIR_EAX \
1262  VALGRIND_RESTORE_STACK \
1263  : /*out*/ "=a" (_res) \
1264  : /*in*/ "a" (&_argvec[0]) \
1265  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1266  ); \
1267  lval = (__typeof__(lval)) _res; \
1268  } while (0)
1269 
1270 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
1271  do { \
1272  volatile OrigFn _orig = (orig); \
1273  volatile unsigned long _argvec[6]; \
1274  volatile unsigned long _res; \
1275  _argvec[0] = (unsigned long)_orig.nraddr; \
1276  _argvec[1] = (unsigned long)(arg1); \
1277  _argvec[2] = (unsigned long)(arg2); \
1278  _argvec[3] = (unsigned long)(arg3); \
1279  _argvec[4] = (unsigned long)(arg4); \
1280  _argvec[5] = (unsigned long)(arg5); \
1281  __asm__ volatile( \
1282  VALGRIND_ALIGN_STACK \
1283  "subl $12, %%esp\n\t" \
1284  "pushl 20(%%eax)\n\t" \
1285  "pushl 16(%%eax)\n\t" \
1286  "pushl 12(%%eax)\n\t" \
1287  "pushl 8(%%eax)\n\t" \
1288  "pushl 4(%%eax)\n\t" \
1289  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1290  VALGRIND_CALL_NOREDIR_EAX \
1291  VALGRIND_RESTORE_STACK \
1292  : /*out*/ "=a" (_res) \
1293  : /*in*/ "a" (&_argvec[0]) \
1294  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1295  ); \
1296  lval = (__typeof__(lval)) _res; \
1297  } while (0)
1298 
1299 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
1300  do { \
1301  volatile OrigFn _orig = (orig); \
1302  volatile unsigned long _argvec[7]; \
1303  volatile unsigned long _res; \
1304  _argvec[0] = (unsigned long)_orig.nraddr; \
1305  _argvec[1] = (unsigned long)(arg1); \
1306  _argvec[2] = (unsigned long)(arg2); \
1307  _argvec[3] = (unsigned long)(arg3); \
1308  _argvec[4] = (unsigned long)(arg4); \
1309  _argvec[5] = (unsigned long)(arg5); \
1310  _argvec[6] = (unsigned long)(arg6); \
1311  __asm__ volatile( \
1312  VALGRIND_ALIGN_STACK \
1313  "subl $8, %%esp\n\t" \
1314  "pushl 24(%%eax)\n\t" \
1315  "pushl 20(%%eax)\n\t" \
1316  "pushl 16(%%eax)\n\t" \
1317  "pushl 12(%%eax)\n\t" \
1318  "pushl 8(%%eax)\n\t" \
1319  "pushl 4(%%eax)\n\t" \
1320  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1321  VALGRIND_CALL_NOREDIR_EAX \
1322  VALGRIND_RESTORE_STACK \
1323  : /*out*/ "=a" (_res) \
1324  : /*in*/ "a" (&_argvec[0]) \
1325  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1326  ); \
1327  lval = (__typeof__(lval)) _res; \
1328  } while (0)
1329 
1330 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1331  arg7) \
1332  do { \
1333  volatile OrigFn _orig = (orig); \
1334  volatile unsigned long _argvec[8]; \
1335  volatile unsigned long _res; \
1336  _argvec[0] = (unsigned long)_orig.nraddr; \
1337  _argvec[1] = (unsigned long)(arg1); \
1338  _argvec[2] = (unsigned long)(arg2); \
1339  _argvec[3] = (unsigned long)(arg3); \
1340  _argvec[4] = (unsigned long)(arg4); \
1341  _argvec[5] = (unsigned long)(arg5); \
1342  _argvec[6] = (unsigned long)(arg6); \
1343  _argvec[7] = (unsigned long)(arg7); \
1344  __asm__ volatile( \
1345  VALGRIND_ALIGN_STACK \
1346  "subl $4, %%esp\n\t" \
1347  "pushl 28(%%eax)\n\t" \
1348  "pushl 24(%%eax)\n\t" \
1349  "pushl 20(%%eax)\n\t" \
1350  "pushl 16(%%eax)\n\t" \
1351  "pushl 12(%%eax)\n\t" \
1352  "pushl 8(%%eax)\n\t" \
1353  "pushl 4(%%eax)\n\t" \
1354  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1355  VALGRIND_CALL_NOREDIR_EAX \
1356  VALGRIND_RESTORE_STACK \
1357  : /*out*/ "=a" (_res) \
1358  : /*in*/ "a" (&_argvec[0]) \
1359  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1360  ); \
1361  lval = (__typeof__(lval)) _res; \
1362  } while (0)
1363 
1364 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1365  arg7,arg8) \
1366  do { \
1367  volatile OrigFn _orig = (orig); \
1368  volatile unsigned long _argvec[9]; \
1369  volatile unsigned long _res; \
1370  _argvec[0] = (unsigned long)_orig.nraddr; \
1371  _argvec[1] = (unsigned long)(arg1); \
1372  _argvec[2] = (unsigned long)(arg2); \
1373  _argvec[3] = (unsigned long)(arg3); \
1374  _argvec[4] = (unsigned long)(arg4); \
1375  _argvec[5] = (unsigned long)(arg5); \
1376  _argvec[6] = (unsigned long)(arg6); \
1377  _argvec[7] = (unsigned long)(arg7); \
1378  _argvec[8] = (unsigned long)(arg8); \
1379  __asm__ volatile( \
1380  VALGRIND_ALIGN_STACK \
1381  "pushl 32(%%eax)\n\t" \
1382  "pushl 28(%%eax)\n\t" \
1383  "pushl 24(%%eax)\n\t" \
1384  "pushl 20(%%eax)\n\t" \
1385  "pushl 16(%%eax)\n\t" \
1386  "pushl 12(%%eax)\n\t" \
1387  "pushl 8(%%eax)\n\t" \
1388  "pushl 4(%%eax)\n\t" \
1389  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1390  VALGRIND_CALL_NOREDIR_EAX \
1391  VALGRIND_RESTORE_STACK \
1392  : /*out*/ "=a" (_res) \
1393  : /*in*/ "a" (&_argvec[0]) \
1394  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1395  ); \
1396  lval = (__typeof__(lval)) _res; \
1397  } while (0)
1398 
1399 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1400  arg7,arg8,arg9) \
1401  do { \
1402  volatile OrigFn _orig = (orig); \
1403  volatile unsigned long _argvec[10]; \
1404  volatile unsigned long _res; \
1405  _argvec[0] = (unsigned long)_orig.nraddr; \
1406  _argvec[1] = (unsigned long)(arg1); \
1407  _argvec[2] = (unsigned long)(arg2); \
1408  _argvec[3] = (unsigned long)(arg3); \
1409  _argvec[4] = (unsigned long)(arg4); \
1410  _argvec[5] = (unsigned long)(arg5); \
1411  _argvec[6] = (unsigned long)(arg6); \
1412  _argvec[7] = (unsigned long)(arg7); \
1413  _argvec[8] = (unsigned long)(arg8); \
1414  _argvec[9] = (unsigned long)(arg9); \
1415  __asm__ volatile( \
1416  VALGRIND_ALIGN_STACK \
1417  "subl $12, %%esp\n\t" \
1418  "pushl 36(%%eax)\n\t" \
1419  "pushl 32(%%eax)\n\t" \
1420  "pushl 28(%%eax)\n\t" \
1421  "pushl 24(%%eax)\n\t" \
1422  "pushl 20(%%eax)\n\t" \
1423  "pushl 16(%%eax)\n\t" \
1424  "pushl 12(%%eax)\n\t" \
1425  "pushl 8(%%eax)\n\t" \
1426  "pushl 4(%%eax)\n\t" \
1427  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1428  VALGRIND_CALL_NOREDIR_EAX \
1429  VALGRIND_RESTORE_STACK \
1430  : /*out*/ "=a" (_res) \
1431  : /*in*/ "a" (&_argvec[0]) \
1432  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1433  ); \
1434  lval = (__typeof__(lval)) _res; \
1435  } while (0)
1436 
1437 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1438  arg7,arg8,arg9,arg10) \
1439  do { \
1440  volatile OrigFn _orig = (orig); \
1441  volatile unsigned long _argvec[11]; \
1442  volatile unsigned long _res; \
1443  _argvec[0] = (unsigned long)_orig.nraddr; \
1444  _argvec[1] = (unsigned long)(arg1); \
1445  _argvec[2] = (unsigned long)(arg2); \
1446  _argvec[3] = (unsigned long)(arg3); \
1447  _argvec[4] = (unsigned long)(arg4); \
1448  _argvec[5] = (unsigned long)(arg5); \
1449  _argvec[6] = (unsigned long)(arg6); \
1450  _argvec[7] = (unsigned long)(arg7); \
1451  _argvec[8] = (unsigned long)(arg8); \
1452  _argvec[9] = (unsigned long)(arg9); \
1453  _argvec[10] = (unsigned long)(arg10); \
1454  __asm__ volatile( \
1455  VALGRIND_ALIGN_STACK \
1456  "subl $8, %%esp\n\t" \
1457  "pushl 40(%%eax)\n\t" \
1458  "pushl 36(%%eax)\n\t" \
1459  "pushl 32(%%eax)\n\t" \
1460  "pushl 28(%%eax)\n\t" \
1461  "pushl 24(%%eax)\n\t" \
1462  "pushl 20(%%eax)\n\t" \
1463  "pushl 16(%%eax)\n\t" \
1464  "pushl 12(%%eax)\n\t" \
1465  "pushl 8(%%eax)\n\t" \
1466  "pushl 4(%%eax)\n\t" \
1467  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1468  VALGRIND_CALL_NOREDIR_EAX \
1469  VALGRIND_RESTORE_STACK \
1470  : /*out*/ "=a" (_res) \
1471  : /*in*/ "a" (&_argvec[0]) \
1472  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1473  ); \
1474  lval = (__typeof__(lval)) _res; \
1475  } while (0)
1476 
1477 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \
1478  arg6,arg7,arg8,arg9,arg10, \
1479  arg11) \
1480  do { \
1481  volatile OrigFn _orig = (orig); \
1482  volatile unsigned long _argvec[12]; \
1483  volatile unsigned long _res; \
1484  _argvec[0] = (unsigned long)_orig.nraddr; \
1485  _argvec[1] = (unsigned long)(arg1); \
1486  _argvec[2] = (unsigned long)(arg2); \
1487  _argvec[3] = (unsigned long)(arg3); \
1488  _argvec[4] = (unsigned long)(arg4); \
1489  _argvec[5] = (unsigned long)(arg5); \
1490  _argvec[6] = (unsigned long)(arg6); \
1491  _argvec[7] = (unsigned long)(arg7); \
1492  _argvec[8] = (unsigned long)(arg8); \
1493  _argvec[9] = (unsigned long)(arg9); \
1494  _argvec[10] = (unsigned long)(arg10); \
1495  _argvec[11] = (unsigned long)(arg11); \
1496  __asm__ volatile( \
1497  VALGRIND_ALIGN_STACK \
1498  "subl $4, %%esp\n\t" \
1499  "pushl 44(%%eax)\n\t" \
1500  "pushl 40(%%eax)\n\t" \
1501  "pushl 36(%%eax)\n\t" \
1502  "pushl 32(%%eax)\n\t" \
1503  "pushl 28(%%eax)\n\t" \
1504  "pushl 24(%%eax)\n\t" \
1505  "pushl 20(%%eax)\n\t" \
1506  "pushl 16(%%eax)\n\t" \
1507  "pushl 12(%%eax)\n\t" \
1508  "pushl 8(%%eax)\n\t" \
1509  "pushl 4(%%eax)\n\t" \
1510  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1511  VALGRIND_CALL_NOREDIR_EAX \
1512  VALGRIND_RESTORE_STACK \
1513  : /*out*/ "=a" (_res) \
1514  : /*in*/ "a" (&_argvec[0]) \
1515  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1516  ); \
1517  lval = (__typeof__(lval)) _res; \
1518  } while (0)
1519 
1520 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \
1521  arg6,arg7,arg8,arg9,arg10, \
1522  arg11,arg12) \
1523  do { \
1524  volatile OrigFn _orig = (orig); \
1525  volatile unsigned long _argvec[13]; \
1526  volatile unsigned long _res; \
1527  _argvec[0] = (unsigned long)_orig.nraddr; \
1528  _argvec[1] = (unsigned long)(arg1); \
1529  _argvec[2] = (unsigned long)(arg2); \
1530  _argvec[3] = (unsigned long)(arg3); \
1531  _argvec[4] = (unsigned long)(arg4); \
1532  _argvec[5] = (unsigned long)(arg5); \
1533  _argvec[6] = (unsigned long)(arg6); \
1534  _argvec[7] = (unsigned long)(arg7); \
1535  _argvec[8] = (unsigned long)(arg8); \
1536  _argvec[9] = (unsigned long)(arg9); \
1537  _argvec[10] = (unsigned long)(arg10); \
1538  _argvec[11] = (unsigned long)(arg11); \
1539  _argvec[12] = (unsigned long)(arg12); \
1540  __asm__ volatile( \
1541  VALGRIND_ALIGN_STACK \
1542  "pushl 48(%%eax)\n\t" \
1543  "pushl 44(%%eax)\n\t" \
1544  "pushl 40(%%eax)\n\t" \
1545  "pushl 36(%%eax)\n\t" \
1546  "pushl 32(%%eax)\n\t" \
1547  "pushl 28(%%eax)\n\t" \
1548  "pushl 24(%%eax)\n\t" \
1549  "pushl 20(%%eax)\n\t" \
1550  "pushl 16(%%eax)\n\t" \
1551  "pushl 12(%%eax)\n\t" \
1552  "pushl 8(%%eax)\n\t" \
1553  "pushl 4(%%eax)\n\t" \
1554  "movl (%%eax), %%eax\n\t" /* target->%eax */ \
1555  VALGRIND_CALL_NOREDIR_EAX \
1556  VALGRIND_RESTORE_STACK \
1557  : /*out*/ "=a" (_res) \
1558  : /*in*/ "a" (&_argvec[0]) \
1559  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \
1560  ); \
1561  lval = (__typeof__(lval)) _res; \
1562  } while (0)
1563 
1564 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
1565 
1566 /* ------------------------ amd64-{linux,darwin} --------------- */
1567 
1568 #if defined(PLAT_amd64_linux) || defined(PLAT_amd64_darwin)
1569 
1570 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1571 
1572 /* These regs are trashed by the hidden call. */
1573 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi", \
1574  "rdi", "r8", "r9", "r10", "r11"
1575 
1576 /* This is all pretty complex. It's so as to make stack unwinding
1577  work reliably. See bug 243270. The basic problem is the sub and
1578  add of 128 of %rsp in all of the following macros. If gcc believes
1579  the CFA is in %rsp, then unwinding may fail, because what's at the
1580  CFA is not what gcc "expected" when it constructs the CFIs for the
1581  places where the macros are instantiated.
1582 
1583  But we can't just add a CFI annotation to increase the CFA offset
1584  by 128, to match the sub of 128 from %rsp, because we don't know
1585  whether gcc has chosen %rsp as the CFA at that point, or whether it
1586  has chosen some other register (eg, %rbp). In the latter case,
1587  adding a CFI annotation to change the CFA offset is simply wrong.
1588 
1589  So the solution is to get hold of the CFA using
1590  __builtin_dwarf_cfa(), put it in a known register, and add a
1591  CFI annotation to say what the register is. We choose %rbp for
1592  this (perhaps perversely), because:
1593 
1594  (1) %rbp is already subject to unwinding. If a new register was
1595  chosen then the unwinder would have to unwind it in all stack
1596  traces, which is expensive, and
1597 
1598  (2) %rbp is already subject to precise exception updates in the
1599  JIT. If a new register was chosen, we'd have to have precise
1600  exceptions for it too, which reduces performance of the
1601  generated code.
1602 
1603  However .. one extra complication. We can't just whack the result
1604  of __builtin_dwarf_cfa() into %rbp and then add %rbp to the
1605  list of trashed registers at the end of the inline assembly
1606  fragments; gcc won't allow %rbp to appear in that list. Hence
1607  instead we need to stash %rbp in %r15 for the duration of the asm,
1608  and say that %r15 is trashed instead. gcc seems happy to go with
1609  that.
1610 
1611  Oh .. and this all needs to be conditionalised so that it is
1612  unchanged from before this commit, when compiled with older gccs
1613  that don't support __builtin_dwarf_cfa. Furthermore, since
1614  this header file is freestanding, it has to be independent of
1615  config.h, and so the following conditionalisation cannot depend on
1616  configure time checks.
1617 
1618  Although it's not clear from
1619  'defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)',
1620  this expression excludes Darwin.
1621  .cfi directives in Darwin assembly appear to be completely
1622  different and I haven't investigated how they work.
1623 
1624  For even more entertainment value, note we have to use the
1625  completely undocumented __builtin_dwarf_cfa(), which appears to
1626  really compute the CFA, whereas __builtin_frame_address(0) claims
1627  to but actually doesn't. See
1628  https://bugs.kde.org/show_bug.cgi?id=243270#c47
1629 */
1630 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
1631 # define __FRAME_POINTER \
1632  ,"r"(__builtin_dwarf_cfa())
1633 # define VALGRIND_CFI_PROLOGUE \
1634  "movq %%rbp, %%r15\n\t" \
1635  "movq %2, %%rbp\n\t" \
1636  ".cfi_remember_state\n\t" \
1637  ".cfi_def_cfa rbp, 0\n\t"
1638 # define VALGRIND_CFI_EPILOGUE \
1639  "movq %%r15, %%rbp\n\t" \
1640  ".cfi_restore_state\n\t"
1641 #else
1642 # define __FRAME_POINTER
1643 # define VALGRIND_CFI_PROLOGUE
1644 # define VALGRIND_CFI_EPILOGUE
1645 #endif
1646 
1647 /* Macros to save and align the stack before making a function
1648  call and restore it afterwards as gcc may not keep the stack
1649  pointer aligned if it doesn't realize calls are being made
1650  to other functions. */
1651 
1652 #define VALGRIND_ALIGN_STACK \
1653  "movq %%rsp,%%r14\n\t" \
1654  "andq $0xfffffffffffffff0,%%rsp\n\t"
1655 #define VALGRIND_RESTORE_STACK \
1656  "movq %%r14,%%rsp\n\t"
1657 
1658 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1659  long) == 8. */
1660 
1661 /* NB 9 Sept 07. There is a nasty kludge here in all these CALL_FN_
1662  macros. In order not to trash the stack redzone, we need to drop
1663  %rsp by 128 before the hidden call, and restore afterwards. The
1664  nastiness is that it is only by luck that the stack still appears
1665  to be unwindable during the hidden call - since then the behavior
1666  of any routine using this macro does not match what the CFI data
1667  says. Sigh.
1668 
1669  Why is this important? Imagine that a wrapper has a stack
1670  allocated local, and passes to the hidden call, a pointer to it.
1671  Because gcc does not know about the hidden call, it may allocate
1672  that local in the redzone. Unfortunately the hidden call may then
1673  trash it before it comes to use it. So we must step clear of the
1674  redzone, for the duration of the hidden call, to make it safe.
1675 
1676  Probably the same problem afflicts the other redzone-style ABIs too
1677  (ppc64-linux); but for those, the stack is
1678  self describing (none of this CFI nonsense) so at least messing
1679  with the stack pointer doesn't give a danger of non-unwindable
1680  stack. */
1681 
1682 #define CALL_FN_W_v(lval, orig) \
1683  do { \
1684  volatile OrigFn _orig = (orig); \
1685  volatile unsigned long _argvec[1]; \
1686  volatile unsigned long _res; \
1687  _argvec[0] = (unsigned long)_orig.nraddr; \
1688  __asm__ volatile( \
1689  VALGRIND_CFI_PROLOGUE \
1690  VALGRIND_ALIGN_STACK \
1691  "subq $128,%%rsp\n\t" \
1692  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
1693  VALGRIND_CALL_NOREDIR_RAX \
1694  VALGRIND_RESTORE_STACK \
1695  VALGRIND_CFI_EPILOGUE \
1696  : /*out*/ "=a" (_res) \
1697  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
1698  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1699  ); \
1700  lval = (__typeof__(lval)) _res; \
1701  } while (0)
1702 
1703 #define CALL_FN_W_W(lval, orig, arg1) \
1704  do { \
1705  volatile OrigFn _orig = (orig); \
1706  volatile unsigned long _argvec[2]; \
1707  volatile unsigned long _res; \
1708  _argvec[0] = (unsigned long)_orig.nraddr; \
1709  _argvec[1] = (unsigned long)(arg1); \
1710  __asm__ volatile( \
1711  VALGRIND_CFI_PROLOGUE \
1712  VALGRIND_ALIGN_STACK \
1713  "subq $128,%%rsp\n\t" \
1714  "movq 8(%%rax), %%rdi\n\t" \
1715  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
1716  VALGRIND_CALL_NOREDIR_RAX \
1717  VALGRIND_RESTORE_STACK \
1718  VALGRIND_CFI_EPILOGUE \
1719  : /*out*/ "=a" (_res) \
1720  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
1721  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1722  ); \
1723  lval = (__typeof__(lval)) _res; \
1724  } while (0)
1725 
1726 #define CALL_FN_W_WW(lval, orig, arg1,arg2) \
1727  do { \
1728  volatile OrigFn _orig = (orig); \
1729  volatile unsigned long _argvec[3]; \
1730  volatile unsigned long _res; \
1731  _argvec[0] = (unsigned long)_orig.nraddr; \
1732  _argvec[1] = (unsigned long)(arg1); \
1733  _argvec[2] = (unsigned long)(arg2); \
1734  __asm__ volatile( \
1735  VALGRIND_CFI_PROLOGUE \
1736  VALGRIND_ALIGN_STACK \
1737  "subq $128,%%rsp\n\t" \
1738  "movq 16(%%rax), %%rsi\n\t" \
1739  "movq 8(%%rax), %%rdi\n\t" \
1740  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
1741  VALGRIND_CALL_NOREDIR_RAX \
1742  VALGRIND_RESTORE_STACK \
1743  VALGRIND_CFI_EPILOGUE \
1744  : /*out*/ "=a" (_res) \
1745  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
1746  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1747  ); \
1748  lval = (__typeof__(lval)) _res; \
1749  } while (0)
1750 
1751 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
1752  do { \
1753  volatile OrigFn _orig = (orig); \
1754  volatile unsigned long _argvec[4]; \
1755  volatile unsigned long _res; \
1756  _argvec[0] = (unsigned long)_orig.nraddr; \
1757  _argvec[1] = (unsigned long)(arg1); \
1758  _argvec[2] = (unsigned long)(arg2); \
1759  _argvec[3] = (unsigned long)(arg3); \
1760  __asm__ volatile( \
1761  VALGRIND_CFI_PROLOGUE \
1762  VALGRIND_ALIGN_STACK \
1763  "subq $128,%%rsp\n\t" \
1764  "movq 24(%%rax), %%rdx\n\t" \
1765  "movq 16(%%rax), %%rsi\n\t" \
1766  "movq 8(%%rax), %%rdi\n\t" \
1767  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
1768  VALGRIND_CALL_NOREDIR_RAX \
1769  VALGRIND_RESTORE_STACK \
1770  VALGRIND_CFI_EPILOGUE \
1771  : /*out*/ "=a" (_res) \
1772  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
1773  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1774  ); \
1775  lval = (__typeof__(lval)) _res; \
1776  } while (0)
1777 
1778 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
1779  do { \
1780  volatile OrigFn _orig = (orig); \
1781  volatile unsigned long _argvec[5]; \
1782  volatile unsigned long _res; \
1783  _argvec[0] = (unsigned long)_orig.nraddr; \
1784  _argvec[1] = (unsigned long)(arg1); \
1785  _argvec[2] = (unsigned long)(arg2); \
1786  _argvec[3] = (unsigned long)(arg3); \
1787  _argvec[4] = (unsigned long)(arg4); \
1788  __asm__ volatile( \
1789  VALGRIND_CFI_PROLOGUE \
1790  VALGRIND_ALIGN_STACK \
1791  "subq $128,%%rsp\n\t" \
1792  "movq 32(%%rax), %%rcx\n\t" \
1793  "movq 24(%%rax), %%rdx\n\t" \
1794  "movq 16(%%rax), %%rsi\n\t" \
1795  "movq 8(%%rax), %%rdi\n\t" \
1796  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
1797  VALGRIND_CALL_NOREDIR_RAX \
1798  VALGRIND_RESTORE_STACK \
1799  VALGRIND_CFI_EPILOGUE \
1800  : /*out*/ "=a" (_res) \
1801  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
1802  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1803  ); \
1804  lval = (__typeof__(lval)) _res; \
1805  } while (0)
1806 
1807 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
1808  do { \
1809  volatile OrigFn _orig = (orig); \
1810  volatile unsigned long _argvec[6]; \
1811  volatile unsigned long _res; \
1812  _argvec[0] = (unsigned long)_orig.nraddr; \
1813  _argvec[1] = (unsigned long)(arg1); \
1814  _argvec[2] = (unsigned long)(arg2); \
1815  _argvec[3] = (unsigned long)(arg3); \
1816  _argvec[4] = (unsigned long)(arg4); \
1817  _argvec[5] = (unsigned long)(arg5); \
1818  __asm__ volatile( \
1819  VALGRIND_CFI_PROLOGUE \
1820  VALGRIND_ALIGN_STACK \
1821  "subq $128,%%rsp\n\t" \
1822  "movq 40(%%rax), %%r8\n\t" \
1823  "movq 32(%%rax), %%rcx\n\t" \
1824  "movq 24(%%rax), %%rdx\n\t" \
1825  "movq 16(%%rax), %%rsi\n\t" \
1826  "movq 8(%%rax), %%rdi\n\t" \
1827  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
1828  VALGRIND_CALL_NOREDIR_RAX \
1829  VALGRIND_RESTORE_STACK \
1830  VALGRIND_CFI_EPILOGUE \
1831  : /*out*/ "=a" (_res) \
1832  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
1833  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1834  ); \
1835  lval = (__typeof__(lval)) _res; \
1836  } while (0)
1837 
1838 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
1839  do { \
1840  volatile OrigFn _orig = (orig); \
1841  volatile unsigned long _argvec[7]; \
1842  volatile unsigned long _res; \
1843  _argvec[0] = (unsigned long)_orig.nraddr; \
1844  _argvec[1] = (unsigned long)(arg1); \
1845  _argvec[2] = (unsigned long)(arg2); \
1846  _argvec[3] = (unsigned long)(arg3); \
1847  _argvec[4] = (unsigned long)(arg4); \
1848  _argvec[5] = (unsigned long)(arg5); \
1849  _argvec[6] = (unsigned long)(arg6); \
1850  __asm__ volatile( \
1851  VALGRIND_CFI_PROLOGUE \
1852  VALGRIND_ALIGN_STACK \
1853  "subq $128,%%rsp\n\t" \
1854  "movq 48(%%rax), %%r9\n\t" \
1855  "movq 40(%%rax), %%r8\n\t" \
1856  "movq 32(%%rax), %%rcx\n\t" \
1857  "movq 24(%%rax), %%rdx\n\t" \
1858  "movq 16(%%rax), %%rsi\n\t" \
1859  "movq 8(%%rax), %%rdi\n\t" \
1860  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
1861  VALGRIND_CALL_NOREDIR_RAX \
1862  VALGRIND_RESTORE_STACK \
1863  VALGRIND_CFI_EPILOGUE \
1864  : /*out*/ "=a" (_res) \
1865  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
1866  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1867  ); \
1868  lval = (__typeof__(lval)) _res; \
1869  } while (0)
1870 
1871 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1872  arg7) \
1873  do { \
1874  volatile OrigFn _orig = (orig); \
1875  volatile unsigned long _argvec[8]; \
1876  volatile unsigned long _res; \
1877  _argvec[0] = (unsigned long)_orig.nraddr; \
1878  _argvec[1] = (unsigned long)(arg1); \
1879  _argvec[2] = (unsigned long)(arg2); \
1880  _argvec[3] = (unsigned long)(arg3); \
1881  _argvec[4] = (unsigned long)(arg4); \
1882  _argvec[5] = (unsigned long)(arg5); \
1883  _argvec[6] = (unsigned long)(arg6); \
1884  _argvec[7] = (unsigned long)(arg7); \
1885  __asm__ volatile( \
1886  VALGRIND_CFI_PROLOGUE \
1887  VALGRIND_ALIGN_STACK \
1888  "subq $136,%%rsp\n\t" \
1889  "pushq 56(%%rax)\n\t" \
1890  "movq 48(%%rax), %%r9\n\t" \
1891  "movq 40(%%rax), %%r8\n\t" \
1892  "movq 32(%%rax), %%rcx\n\t" \
1893  "movq 24(%%rax), %%rdx\n\t" \
1894  "movq 16(%%rax), %%rsi\n\t" \
1895  "movq 8(%%rax), %%rdi\n\t" \
1896  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
1897  VALGRIND_CALL_NOREDIR_RAX \
1898  VALGRIND_RESTORE_STACK \
1899  VALGRIND_CFI_EPILOGUE \
1900  : /*out*/ "=a" (_res) \
1901  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
1902  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1903  ); \
1904  lval = (__typeof__(lval)) _res; \
1905  } while (0)
1906 
1907 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1908  arg7,arg8) \
1909  do { \
1910  volatile OrigFn _orig = (orig); \
1911  volatile unsigned long _argvec[9]; \
1912  volatile unsigned long _res; \
1913  _argvec[0] = (unsigned long)_orig.nraddr; \
1914  _argvec[1] = (unsigned long)(arg1); \
1915  _argvec[2] = (unsigned long)(arg2); \
1916  _argvec[3] = (unsigned long)(arg3); \
1917  _argvec[4] = (unsigned long)(arg4); \
1918  _argvec[5] = (unsigned long)(arg5); \
1919  _argvec[6] = (unsigned long)(arg6); \
1920  _argvec[7] = (unsigned long)(arg7); \
1921  _argvec[8] = (unsigned long)(arg8); \
1922  __asm__ volatile( \
1923  VALGRIND_CFI_PROLOGUE \
1924  VALGRIND_ALIGN_STACK \
1925  "subq $128,%%rsp\n\t" \
1926  "pushq 64(%%rax)\n\t" \
1927  "pushq 56(%%rax)\n\t" \
1928  "movq 48(%%rax), %%r9\n\t" \
1929  "movq 40(%%rax), %%r8\n\t" \
1930  "movq 32(%%rax), %%rcx\n\t" \
1931  "movq 24(%%rax), %%rdx\n\t" \
1932  "movq 16(%%rax), %%rsi\n\t" \
1933  "movq 8(%%rax), %%rdi\n\t" \
1934  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
1935  VALGRIND_CALL_NOREDIR_RAX \
1936  VALGRIND_RESTORE_STACK \
1937  VALGRIND_CFI_EPILOGUE \
1938  : /*out*/ "=a" (_res) \
1939  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
1940  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1941  ); \
1942  lval = (__typeof__(lval)) _res; \
1943  } while (0)
1944 
1945 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1946  arg7,arg8,arg9) \
1947  do { \
1948  volatile OrigFn _orig = (orig); \
1949  volatile unsigned long _argvec[10]; \
1950  volatile unsigned long _res; \
1951  _argvec[0] = (unsigned long)_orig.nraddr; \
1952  _argvec[1] = (unsigned long)(arg1); \
1953  _argvec[2] = (unsigned long)(arg2); \
1954  _argvec[3] = (unsigned long)(arg3); \
1955  _argvec[4] = (unsigned long)(arg4); \
1956  _argvec[5] = (unsigned long)(arg5); \
1957  _argvec[6] = (unsigned long)(arg6); \
1958  _argvec[7] = (unsigned long)(arg7); \
1959  _argvec[8] = (unsigned long)(arg8); \
1960  _argvec[9] = (unsigned long)(arg9); \
1961  __asm__ volatile( \
1962  VALGRIND_CFI_PROLOGUE \
1963  VALGRIND_ALIGN_STACK \
1964  "subq $136,%%rsp\n\t" \
1965  "pushq 72(%%rax)\n\t" \
1966  "pushq 64(%%rax)\n\t" \
1967  "pushq 56(%%rax)\n\t" \
1968  "movq 48(%%rax), %%r9\n\t" \
1969  "movq 40(%%rax), %%r8\n\t" \
1970  "movq 32(%%rax), %%rcx\n\t" \
1971  "movq 24(%%rax), %%rdx\n\t" \
1972  "movq 16(%%rax), %%rsi\n\t" \
1973  "movq 8(%%rax), %%rdi\n\t" \
1974  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
1975  VALGRIND_CALL_NOREDIR_RAX \
1976  VALGRIND_RESTORE_STACK \
1977  VALGRIND_CFI_EPILOGUE \
1978  : /*out*/ "=a" (_res) \
1979  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
1980  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1981  ); \
1982  lval = (__typeof__(lval)) _res; \
1983  } while (0)
1984 
1985 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
1986  arg7,arg8,arg9,arg10) \
1987  do { \
1988  volatile OrigFn _orig = (orig); \
1989  volatile unsigned long _argvec[11]; \
1990  volatile unsigned long _res; \
1991  _argvec[0] = (unsigned long)_orig.nraddr; \
1992  _argvec[1] = (unsigned long)(arg1); \
1993  _argvec[2] = (unsigned long)(arg2); \
1994  _argvec[3] = (unsigned long)(arg3); \
1995  _argvec[4] = (unsigned long)(arg4); \
1996  _argvec[5] = (unsigned long)(arg5); \
1997  _argvec[6] = (unsigned long)(arg6); \
1998  _argvec[7] = (unsigned long)(arg7); \
1999  _argvec[8] = (unsigned long)(arg8); \
2000  _argvec[9] = (unsigned long)(arg9); \
2001  _argvec[10] = (unsigned long)(arg10); \
2002  __asm__ volatile( \
2003  VALGRIND_CFI_PROLOGUE \
2004  VALGRIND_ALIGN_STACK \
2005  "subq $128,%%rsp\n\t" \
2006  "pushq 80(%%rax)\n\t" \
2007  "pushq 72(%%rax)\n\t" \
2008  "pushq 64(%%rax)\n\t" \
2009  "pushq 56(%%rax)\n\t" \
2010  "movq 48(%%rax), %%r9\n\t" \
2011  "movq 40(%%rax), %%r8\n\t" \
2012  "movq 32(%%rax), %%rcx\n\t" \
2013  "movq 24(%%rax), %%rdx\n\t" \
2014  "movq 16(%%rax), %%rsi\n\t" \
2015  "movq 8(%%rax), %%rdi\n\t" \
2016  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
2017  VALGRIND_CALL_NOREDIR_RAX \
2018  VALGRIND_RESTORE_STACK \
2019  VALGRIND_CFI_EPILOGUE \
2020  : /*out*/ "=a" (_res) \
2021  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
2022  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2023  ); \
2024  lval = (__typeof__(lval)) _res; \
2025  } while (0)
2026 
2027 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2028  arg7,arg8,arg9,arg10,arg11) \
2029  do { \
2030  volatile OrigFn _orig = (orig); \
2031  volatile unsigned long _argvec[12]; \
2032  volatile unsigned long _res; \
2033  _argvec[0] = (unsigned long)_orig.nraddr; \
2034  _argvec[1] = (unsigned long)(arg1); \
2035  _argvec[2] = (unsigned long)(arg2); \
2036  _argvec[3] = (unsigned long)(arg3); \
2037  _argvec[4] = (unsigned long)(arg4); \
2038  _argvec[5] = (unsigned long)(arg5); \
2039  _argvec[6] = (unsigned long)(arg6); \
2040  _argvec[7] = (unsigned long)(arg7); \
2041  _argvec[8] = (unsigned long)(arg8); \
2042  _argvec[9] = (unsigned long)(arg9); \
2043  _argvec[10] = (unsigned long)(arg10); \
2044  _argvec[11] = (unsigned long)(arg11); \
2045  __asm__ volatile( \
2046  VALGRIND_CFI_PROLOGUE \
2047  VALGRIND_ALIGN_STACK \
2048  "subq $136,%%rsp\n\t" \
2049  "pushq 88(%%rax)\n\t" \
2050  "pushq 80(%%rax)\n\t" \
2051  "pushq 72(%%rax)\n\t" \
2052  "pushq 64(%%rax)\n\t" \
2053  "pushq 56(%%rax)\n\t" \
2054  "movq 48(%%rax), %%r9\n\t" \
2055  "movq 40(%%rax), %%r8\n\t" \
2056  "movq 32(%%rax), %%rcx\n\t" \
2057  "movq 24(%%rax), %%rdx\n\t" \
2058  "movq 16(%%rax), %%rsi\n\t" \
2059  "movq 8(%%rax), %%rdi\n\t" \
2060  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
2061  VALGRIND_CALL_NOREDIR_RAX \
2062  VALGRIND_RESTORE_STACK \
2063  VALGRIND_CFI_EPILOGUE \
2064  : /*out*/ "=a" (_res) \
2065  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
2066  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2067  ); \
2068  lval = (__typeof__(lval)) _res; \
2069  } while (0)
2070 
2071 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2072  arg7,arg8,arg9,arg10,arg11,arg12) \
2073  do { \
2074  volatile OrigFn _orig = (orig); \
2075  volatile unsigned long _argvec[13]; \
2076  volatile unsigned long _res; \
2077  _argvec[0] = (unsigned long)_orig.nraddr; \
2078  _argvec[1] = (unsigned long)(arg1); \
2079  _argvec[2] = (unsigned long)(arg2); \
2080  _argvec[3] = (unsigned long)(arg3); \
2081  _argvec[4] = (unsigned long)(arg4); \
2082  _argvec[5] = (unsigned long)(arg5); \
2083  _argvec[6] = (unsigned long)(arg6); \
2084  _argvec[7] = (unsigned long)(arg7); \
2085  _argvec[8] = (unsigned long)(arg8); \
2086  _argvec[9] = (unsigned long)(arg9); \
2087  _argvec[10] = (unsigned long)(arg10); \
2088  _argvec[11] = (unsigned long)(arg11); \
2089  _argvec[12] = (unsigned long)(arg12); \
2090  __asm__ volatile( \
2091  VALGRIND_CFI_PROLOGUE \
2092  VALGRIND_ALIGN_STACK \
2093  "subq $128,%%rsp\n\t" \
2094  "pushq 96(%%rax)\n\t" \
2095  "pushq 88(%%rax)\n\t" \
2096  "pushq 80(%%rax)\n\t" \
2097  "pushq 72(%%rax)\n\t" \
2098  "pushq 64(%%rax)\n\t" \
2099  "pushq 56(%%rax)\n\t" \
2100  "movq 48(%%rax), %%r9\n\t" \
2101  "movq 40(%%rax), %%r8\n\t" \
2102  "movq 32(%%rax), %%rcx\n\t" \
2103  "movq 24(%%rax), %%rdx\n\t" \
2104  "movq 16(%%rax), %%rsi\n\t" \
2105  "movq 8(%%rax), %%rdi\n\t" \
2106  "movq (%%rax), %%rax\n\t" /* target->%rax */ \
2107  VALGRIND_CALL_NOREDIR_RAX \
2108  VALGRIND_RESTORE_STACK \
2109  VALGRIND_CFI_EPILOGUE \
2110  : /*out*/ "=a" (_res) \
2111  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
2112  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2113  ); \
2114  lval = (__typeof__(lval)) _res; \
2115  } while (0)
2116 
2117 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
2118 
2119 /* ------------------------ ppc32-linux ------------------------ */
2120 
2121 #if defined(PLAT_ppc32_linux)
2122 
2123 /* This is useful for finding out about the on-stack stuff:
2124 
2125  extern int f9 ( int,int,int,int,int,int,int,int,int );
2126  extern int f10 ( int,int,int,int,int,int,int,int,int,int );
2127  extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
2128  extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
2129 
2130  int g9 ( void ) {
2131  return f9(11,22,33,44,55,66,77,88,99);
2132  }
2133  int g10 ( void ) {
2134  return f10(11,22,33,44,55,66,77,88,99,110);
2135  }
2136  int g11 ( void ) {
2137  return f11(11,22,33,44,55,66,77,88,99,110,121);
2138  }
2139  int g12 ( void ) {
2140  return f12(11,22,33,44,55,66,77,88,99,110,121,132);
2141  }
2142 */
2143 
2144 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2145 
2146 /* These regs are trashed by the hidden call. */
2147 #define __CALLER_SAVED_REGS \
2148  "lr", "ctr", "xer", \
2149  "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \
2150  "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \
2151  "r11", "r12", "r13"
2152 
2153 /* Macros to save and align the stack before making a function
2154  call and restore it afterwards as gcc may not keep the stack
2155  pointer aligned if it doesn't realize calls are being made
2156  to other functions. */
2157 
2158 #define VALGRIND_ALIGN_STACK \
2159  "mr 28,1\n\t" \
2160  "rlwinm 1,1,0,0,27\n\t"
2161 #define VALGRIND_RESTORE_STACK \
2162  "mr 1,28\n\t"
2163 
2164 /* These CALL_FN_ macros assume that on ppc32-linux,
2165  sizeof(unsigned long) == 4. */
2166 
2167 #define CALL_FN_W_v(lval, orig) \
2168  do { \
2169  volatile OrigFn _orig = (orig); \
2170  volatile unsigned long _argvec[1]; \
2171  volatile unsigned long _res; \
2172  _argvec[0] = (unsigned long)_orig.nraddr; \
2173  __asm__ volatile( \
2174  VALGRIND_ALIGN_STACK \
2175  "mr 11,%1\n\t" \
2176  "lwz 11,0(11)\n\t" /* target->r11 */ \
2177  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2178  VALGRIND_RESTORE_STACK \
2179  "mr %0,3" \
2180  : /*out*/ "=r" (_res) \
2181  : /*in*/ "r" (&_argvec[0]) \
2182  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2183  ); \
2184  lval = (__typeof__(lval)) _res; \
2185  } while (0)
2186 
2187 #define CALL_FN_W_W(lval, orig, arg1) \
2188  do { \
2189  volatile OrigFn _orig = (orig); \
2190  volatile unsigned long _argvec[2]; \
2191  volatile unsigned long _res; \
2192  _argvec[0] = (unsigned long)_orig.nraddr; \
2193  _argvec[1] = (unsigned long)arg1; \
2194  __asm__ volatile( \
2195  VALGRIND_ALIGN_STACK \
2196  "mr 11,%1\n\t" \
2197  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2198  "lwz 11,0(11)\n\t" /* target->r11 */ \
2199  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2200  VALGRIND_RESTORE_STACK \
2201  "mr %0,3" \
2202  : /*out*/ "=r" (_res) \
2203  : /*in*/ "r" (&_argvec[0]) \
2204  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2205  ); \
2206  lval = (__typeof__(lval)) _res; \
2207  } while (0)
2208 
2209 #define CALL_FN_W_WW(lval, orig, arg1,arg2) \
2210  do { \
2211  volatile OrigFn _orig = (orig); \
2212  volatile unsigned long _argvec[3]; \
2213  volatile unsigned long _res; \
2214  _argvec[0] = (unsigned long)_orig.nraddr; \
2215  _argvec[1] = (unsigned long)arg1; \
2216  _argvec[2] = (unsigned long)arg2; \
2217  __asm__ volatile( \
2218  VALGRIND_ALIGN_STACK \
2219  "mr 11,%1\n\t" \
2220  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2221  "lwz 4,8(11)\n\t" \
2222  "lwz 11,0(11)\n\t" /* target->r11 */ \
2223  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2224  VALGRIND_RESTORE_STACK \
2225  "mr %0,3" \
2226  : /*out*/ "=r" (_res) \
2227  : /*in*/ "r" (&_argvec[0]) \
2228  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2229  ); \
2230  lval = (__typeof__(lval)) _res; \
2231  } while (0)
2232 
2233 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
2234  do { \
2235  volatile OrigFn _orig = (orig); \
2236  volatile unsigned long _argvec[4]; \
2237  volatile unsigned long _res; \
2238  _argvec[0] = (unsigned long)_orig.nraddr; \
2239  _argvec[1] = (unsigned long)arg1; \
2240  _argvec[2] = (unsigned long)arg2; \
2241  _argvec[3] = (unsigned long)arg3; \
2242  __asm__ volatile( \
2243  VALGRIND_ALIGN_STACK \
2244  "mr 11,%1\n\t" \
2245  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2246  "lwz 4,8(11)\n\t" \
2247  "lwz 5,12(11)\n\t" \
2248  "lwz 11,0(11)\n\t" /* target->r11 */ \
2249  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2250  VALGRIND_RESTORE_STACK \
2251  "mr %0,3" \
2252  : /*out*/ "=r" (_res) \
2253  : /*in*/ "r" (&_argvec[0]) \
2254  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2255  ); \
2256  lval = (__typeof__(lval)) _res; \
2257  } while (0)
2258 
2259 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
2260  do { \
2261  volatile OrigFn _orig = (orig); \
2262  volatile unsigned long _argvec[5]; \
2263  volatile unsigned long _res; \
2264  _argvec[0] = (unsigned long)_orig.nraddr; \
2265  _argvec[1] = (unsigned long)arg1; \
2266  _argvec[2] = (unsigned long)arg2; \
2267  _argvec[3] = (unsigned long)arg3; \
2268  _argvec[4] = (unsigned long)arg4; \
2269  __asm__ volatile( \
2270  VALGRIND_ALIGN_STACK \
2271  "mr 11,%1\n\t" \
2272  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2273  "lwz 4,8(11)\n\t" \
2274  "lwz 5,12(11)\n\t" \
2275  "lwz 6,16(11)\n\t" /* arg4->r6 */ \
2276  "lwz 11,0(11)\n\t" /* target->r11 */ \
2277  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2278  VALGRIND_RESTORE_STACK \
2279  "mr %0,3" \
2280  : /*out*/ "=r" (_res) \
2281  : /*in*/ "r" (&_argvec[0]) \
2282  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2283  ); \
2284  lval = (__typeof__(lval)) _res; \
2285  } while (0)
2286 
2287 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
2288  do { \
2289  volatile OrigFn _orig = (orig); \
2290  volatile unsigned long _argvec[6]; \
2291  volatile unsigned long _res; \
2292  _argvec[0] = (unsigned long)_orig.nraddr; \
2293  _argvec[1] = (unsigned long)arg1; \
2294  _argvec[2] = (unsigned long)arg2; \
2295  _argvec[3] = (unsigned long)arg3; \
2296  _argvec[4] = (unsigned long)arg4; \
2297  _argvec[5] = (unsigned long)arg5; \
2298  __asm__ volatile( \
2299  VALGRIND_ALIGN_STACK \
2300  "mr 11,%1\n\t" \
2301  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2302  "lwz 4,8(11)\n\t" \
2303  "lwz 5,12(11)\n\t" \
2304  "lwz 6,16(11)\n\t" /* arg4->r6 */ \
2305  "lwz 7,20(11)\n\t" \
2306  "lwz 11,0(11)\n\t" /* target->r11 */ \
2307  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2308  VALGRIND_RESTORE_STACK \
2309  "mr %0,3" \
2310  : /*out*/ "=r" (_res) \
2311  : /*in*/ "r" (&_argvec[0]) \
2312  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2313  ); \
2314  lval = (__typeof__(lval)) _res; \
2315  } while (0)
2316 
2317 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
2318  do { \
2319  volatile OrigFn _orig = (orig); \
2320  volatile unsigned long _argvec[7]; \
2321  volatile unsigned long _res; \
2322  _argvec[0] = (unsigned long)_orig.nraddr; \
2323  _argvec[1] = (unsigned long)arg1; \
2324  _argvec[2] = (unsigned long)arg2; \
2325  _argvec[3] = (unsigned long)arg3; \
2326  _argvec[4] = (unsigned long)arg4; \
2327  _argvec[5] = (unsigned long)arg5; \
2328  _argvec[6] = (unsigned long)arg6; \
2329  __asm__ volatile( \
2330  VALGRIND_ALIGN_STACK \
2331  "mr 11,%1\n\t" \
2332  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2333  "lwz 4,8(11)\n\t" \
2334  "lwz 5,12(11)\n\t" \
2335  "lwz 6,16(11)\n\t" /* arg4->r6 */ \
2336  "lwz 7,20(11)\n\t" \
2337  "lwz 8,24(11)\n\t" \
2338  "lwz 11,0(11)\n\t" /* target->r11 */ \
2339  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2340  VALGRIND_RESTORE_STACK \
2341  "mr %0,3" \
2342  : /*out*/ "=r" (_res) \
2343  : /*in*/ "r" (&_argvec[0]) \
2344  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2345  ); \
2346  lval = (__typeof__(lval)) _res; \
2347  } while (0)
2348 
2349 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2350  arg7) \
2351  do { \
2352  volatile OrigFn _orig = (orig); \
2353  volatile unsigned long _argvec[8]; \
2354  volatile unsigned long _res; \
2355  _argvec[0] = (unsigned long)_orig.nraddr; \
2356  _argvec[1] = (unsigned long)arg1; \
2357  _argvec[2] = (unsigned long)arg2; \
2358  _argvec[3] = (unsigned long)arg3; \
2359  _argvec[4] = (unsigned long)arg4; \
2360  _argvec[5] = (unsigned long)arg5; \
2361  _argvec[6] = (unsigned long)arg6; \
2362  _argvec[7] = (unsigned long)arg7; \
2363  __asm__ volatile( \
2364  VALGRIND_ALIGN_STACK \
2365  "mr 11,%1\n\t" \
2366  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2367  "lwz 4,8(11)\n\t" \
2368  "lwz 5,12(11)\n\t" \
2369  "lwz 6,16(11)\n\t" /* arg4->r6 */ \
2370  "lwz 7,20(11)\n\t" \
2371  "lwz 8,24(11)\n\t" \
2372  "lwz 9,28(11)\n\t" \
2373  "lwz 11,0(11)\n\t" /* target->r11 */ \
2374  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2375  VALGRIND_RESTORE_STACK \
2376  "mr %0,3" \
2377  : /*out*/ "=r" (_res) \
2378  : /*in*/ "r" (&_argvec[0]) \
2379  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2380  ); \
2381  lval = (__typeof__(lval)) _res; \
2382  } while (0)
2383 
2384 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2385  arg7,arg8) \
2386  do { \
2387  volatile OrigFn _orig = (orig); \
2388  volatile unsigned long _argvec[9]; \
2389  volatile unsigned long _res; \
2390  _argvec[0] = (unsigned long)_orig.nraddr; \
2391  _argvec[1] = (unsigned long)arg1; \
2392  _argvec[2] = (unsigned long)arg2; \
2393  _argvec[3] = (unsigned long)arg3; \
2394  _argvec[4] = (unsigned long)arg4; \
2395  _argvec[5] = (unsigned long)arg5; \
2396  _argvec[6] = (unsigned long)arg6; \
2397  _argvec[7] = (unsigned long)arg7; \
2398  _argvec[8] = (unsigned long)arg8; \
2399  __asm__ volatile( \
2400  VALGRIND_ALIGN_STACK \
2401  "mr 11,%1\n\t" \
2402  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2403  "lwz 4,8(11)\n\t" \
2404  "lwz 5,12(11)\n\t" \
2405  "lwz 6,16(11)\n\t" /* arg4->r6 */ \
2406  "lwz 7,20(11)\n\t" \
2407  "lwz 8,24(11)\n\t" \
2408  "lwz 9,28(11)\n\t" \
2409  "lwz 10,32(11)\n\t" /* arg8->r10 */ \
2410  "lwz 11,0(11)\n\t" /* target->r11 */ \
2411  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2412  VALGRIND_RESTORE_STACK \
2413  "mr %0,3" \
2414  : /*out*/ "=r" (_res) \
2415  : /*in*/ "r" (&_argvec[0]) \
2416  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2417  ); \
2418  lval = (__typeof__(lval)) _res; \
2419  } while (0)
2420 
2421 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2422  arg7,arg8,arg9) \
2423  do { \
2424  volatile OrigFn _orig = (orig); \
2425  volatile unsigned long _argvec[10]; \
2426  volatile unsigned long _res; \
2427  _argvec[0] = (unsigned long)_orig.nraddr; \
2428  _argvec[1] = (unsigned long)arg1; \
2429  _argvec[2] = (unsigned long)arg2; \
2430  _argvec[3] = (unsigned long)arg3; \
2431  _argvec[4] = (unsigned long)arg4; \
2432  _argvec[5] = (unsigned long)arg5; \
2433  _argvec[6] = (unsigned long)arg6; \
2434  _argvec[7] = (unsigned long)arg7; \
2435  _argvec[8] = (unsigned long)arg8; \
2436  _argvec[9] = (unsigned long)arg9; \
2437  __asm__ volatile( \
2438  VALGRIND_ALIGN_STACK \
2439  "mr 11,%1\n\t" \
2440  "addi 1,1,-16\n\t" \
2441  /* arg9 */ \
2442  "lwz 3,36(11)\n\t" \
2443  "stw 3,8(1)\n\t" \
2444  /* args1-8 */ \
2445  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2446  "lwz 4,8(11)\n\t" \
2447  "lwz 5,12(11)\n\t" \
2448  "lwz 6,16(11)\n\t" /* arg4->r6 */ \
2449  "lwz 7,20(11)\n\t" \
2450  "lwz 8,24(11)\n\t" \
2451  "lwz 9,28(11)\n\t" \
2452  "lwz 10,32(11)\n\t" /* arg8->r10 */ \
2453  "lwz 11,0(11)\n\t" /* target->r11 */ \
2454  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2455  VALGRIND_RESTORE_STACK \
2456  "mr %0,3" \
2457  : /*out*/ "=r" (_res) \
2458  : /*in*/ "r" (&_argvec[0]) \
2459  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2460  ); \
2461  lval = (__typeof__(lval)) _res; \
2462  } while (0)
2463 
2464 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2465  arg7,arg8,arg9,arg10) \
2466  do { \
2467  volatile OrigFn _orig = (orig); \
2468  volatile unsigned long _argvec[11]; \
2469  volatile unsigned long _res; \
2470  _argvec[0] = (unsigned long)_orig.nraddr; \
2471  _argvec[1] = (unsigned long)arg1; \
2472  _argvec[2] = (unsigned long)arg2; \
2473  _argvec[3] = (unsigned long)arg3; \
2474  _argvec[4] = (unsigned long)arg4; \
2475  _argvec[5] = (unsigned long)arg5; \
2476  _argvec[6] = (unsigned long)arg6; \
2477  _argvec[7] = (unsigned long)arg7; \
2478  _argvec[8] = (unsigned long)arg8; \
2479  _argvec[9] = (unsigned long)arg9; \
2480  _argvec[10] = (unsigned long)arg10; \
2481  __asm__ volatile( \
2482  VALGRIND_ALIGN_STACK \
2483  "mr 11,%1\n\t" \
2484  "addi 1,1,-16\n\t" \
2485  /* arg10 */ \
2486  "lwz 3,40(11)\n\t" \
2487  "stw 3,12(1)\n\t" \
2488  /* arg9 */ \
2489  "lwz 3,36(11)\n\t" \
2490  "stw 3,8(1)\n\t" \
2491  /* args1-8 */ \
2492  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2493  "lwz 4,8(11)\n\t" \
2494  "lwz 5,12(11)\n\t" \
2495  "lwz 6,16(11)\n\t" /* arg4->r6 */ \
2496  "lwz 7,20(11)\n\t" \
2497  "lwz 8,24(11)\n\t" \
2498  "lwz 9,28(11)\n\t" \
2499  "lwz 10,32(11)\n\t" /* arg8->r10 */ \
2500  "lwz 11,0(11)\n\t" /* target->r11 */ \
2501  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2502  VALGRIND_RESTORE_STACK \
2503  "mr %0,3" \
2504  : /*out*/ "=r" (_res) \
2505  : /*in*/ "r" (&_argvec[0]) \
2506  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2507  ); \
2508  lval = (__typeof__(lval)) _res; \
2509  } while (0)
2510 
2511 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2512  arg7,arg8,arg9,arg10,arg11) \
2513  do { \
2514  volatile OrigFn _orig = (orig); \
2515  volatile unsigned long _argvec[12]; \
2516  volatile unsigned long _res; \
2517  _argvec[0] = (unsigned long)_orig.nraddr; \
2518  _argvec[1] = (unsigned long)arg1; \
2519  _argvec[2] = (unsigned long)arg2; \
2520  _argvec[3] = (unsigned long)arg3; \
2521  _argvec[4] = (unsigned long)arg4; \
2522  _argvec[5] = (unsigned long)arg5; \
2523  _argvec[6] = (unsigned long)arg6; \
2524  _argvec[7] = (unsigned long)arg7; \
2525  _argvec[8] = (unsigned long)arg8; \
2526  _argvec[9] = (unsigned long)arg9; \
2527  _argvec[10] = (unsigned long)arg10; \
2528  _argvec[11] = (unsigned long)arg11; \
2529  __asm__ volatile( \
2530  VALGRIND_ALIGN_STACK \
2531  "mr 11,%1\n\t" \
2532  "addi 1,1,-32\n\t" \
2533  /* arg11 */ \
2534  "lwz 3,44(11)\n\t" \
2535  "stw 3,16(1)\n\t" \
2536  /* arg10 */ \
2537  "lwz 3,40(11)\n\t" \
2538  "stw 3,12(1)\n\t" \
2539  /* arg9 */ \
2540  "lwz 3,36(11)\n\t" \
2541  "stw 3,8(1)\n\t" \
2542  /* args1-8 */ \
2543  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2544  "lwz 4,8(11)\n\t" \
2545  "lwz 5,12(11)\n\t" \
2546  "lwz 6,16(11)\n\t" /* arg4->r6 */ \
2547  "lwz 7,20(11)\n\t" \
2548  "lwz 8,24(11)\n\t" \
2549  "lwz 9,28(11)\n\t" \
2550  "lwz 10,32(11)\n\t" /* arg8->r10 */ \
2551  "lwz 11,0(11)\n\t" /* target->r11 */ \
2552  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2553  VALGRIND_RESTORE_STACK \
2554  "mr %0,3" \
2555  : /*out*/ "=r" (_res) \
2556  : /*in*/ "r" (&_argvec[0]) \
2557  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2558  ); \
2559  lval = (__typeof__(lval)) _res; \
2560  } while (0)
2561 
2562 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2563  arg7,arg8,arg9,arg10,arg11,arg12) \
2564  do { \
2565  volatile OrigFn _orig = (orig); \
2566  volatile unsigned long _argvec[13]; \
2567  volatile unsigned long _res; \
2568  _argvec[0] = (unsigned long)_orig.nraddr; \
2569  _argvec[1] = (unsigned long)arg1; \
2570  _argvec[2] = (unsigned long)arg2; \
2571  _argvec[3] = (unsigned long)arg3; \
2572  _argvec[4] = (unsigned long)arg4; \
2573  _argvec[5] = (unsigned long)arg5; \
2574  _argvec[6] = (unsigned long)arg6; \
2575  _argvec[7] = (unsigned long)arg7; \
2576  _argvec[8] = (unsigned long)arg8; \
2577  _argvec[9] = (unsigned long)arg9; \
2578  _argvec[10] = (unsigned long)arg10; \
2579  _argvec[11] = (unsigned long)arg11; \
2580  _argvec[12] = (unsigned long)arg12; \
2581  __asm__ volatile( \
2582  VALGRIND_ALIGN_STACK \
2583  "mr 11,%1\n\t" \
2584  "addi 1,1,-32\n\t" \
2585  /* arg12 */ \
2586  "lwz 3,48(11)\n\t" \
2587  "stw 3,20(1)\n\t" \
2588  /* arg11 */ \
2589  "lwz 3,44(11)\n\t" \
2590  "stw 3,16(1)\n\t" \
2591  /* arg10 */ \
2592  "lwz 3,40(11)\n\t" \
2593  "stw 3,12(1)\n\t" \
2594  /* arg9 */ \
2595  "lwz 3,36(11)\n\t" \
2596  "stw 3,8(1)\n\t" \
2597  /* args1-8 */ \
2598  "lwz 3,4(11)\n\t" /* arg1->r3 */ \
2599  "lwz 4,8(11)\n\t" \
2600  "lwz 5,12(11)\n\t" \
2601  "lwz 6,16(11)\n\t" /* arg4->r6 */ \
2602  "lwz 7,20(11)\n\t" \
2603  "lwz 8,24(11)\n\t" \
2604  "lwz 9,28(11)\n\t" \
2605  "lwz 10,32(11)\n\t" /* arg8->r10 */ \
2606  "lwz 11,0(11)\n\t" /* target->r11 */ \
2607  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2608  VALGRIND_RESTORE_STACK \
2609  "mr %0,3" \
2610  : /*out*/ "=r" (_res) \
2611  : /*in*/ "r" (&_argvec[0]) \
2612  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2613  ); \
2614  lval = (__typeof__(lval)) _res; \
2615  } while (0)
2616 
2617 #endif /* PLAT_ppc32_linux */
2618 
2619 /* ------------------------ ppc64-linux ------------------------ */
2620 
2621 #if defined(PLAT_ppc64be_linux)
2622 
2623 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2624 
2625 /* These regs are trashed by the hidden call. */
2626 #define __CALLER_SAVED_REGS \
2627  "lr", "ctr", "xer", \
2628  "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \
2629  "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \
2630  "r11", "r12", "r13"
2631 
2632 /* Macros to save and align the stack before making a function
2633  call and restore it afterwards as gcc may not keep the stack
2634  pointer aligned if it doesn't realize calls are being made
2635  to other functions. */
2636 
2637 #define VALGRIND_ALIGN_STACK \
2638  "mr 28,1\n\t" \
2639  "rldicr 1,1,0,59\n\t"
2640 #define VALGRIND_RESTORE_STACK \
2641  "mr 1,28\n\t"
2642 
2643 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
2644  long) == 8. */
2645 
2646 #define CALL_FN_W_v(lval, orig) \
2647  do { \
2648  volatile OrigFn _orig = (orig); \
2649  volatile unsigned long _argvec[3+0]; \
2650  volatile unsigned long _res; \
2651  /* _argvec[0] holds current r2 across the call */ \
2652  _argvec[1] = (unsigned long)_orig.r2; \
2653  _argvec[2] = (unsigned long)_orig.nraddr; \
2654  __asm__ volatile( \
2655  VALGRIND_ALIGN_STACK \
2656  "mr 11,%1\n\t" \
2657  "std 2,-16(11)\n\t" /* save tocptr */ \
2658  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2659  "ld 11, 0(11)\n\t" /* target->r11 */ \
2660  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2661  "mr 11,%1\n\t" \
2662  "mr %0,3\n\t" \
2663  "ld 2,-16(11)\n\t" /* restore tocptr */ \
2664  VALGRIND_RESTORE_STACK \
2665  : /*out*/ "=r" (_res) \
2666  : /*in*/ "r" (&_argvec[2]) \
2667  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2668  ); \
2669  lval = (__typeof__(lval)) _res; \
2670  } while (0)
2671 
2672 #define CALL_FN_W_W(lval, orig, arg1) \
2673  do { \
2674  volatile OrigFn _orig = (orig); \
2675  volatile unsigned long _argvec[3+1]; \
2676  volatile unsigned long _res; \
2677  /* _argvec[0] holds current r2 across the call */ \
2678  _argvec[1] = (unsigned long)_orig.r2; \
2679  _argvec[2] = (unsigned long)_orig.nraddr; \
2680  _argvec[2+1] = (unsigned long)arg1; \
2681  __asm__ volatile( \
2682  VALGRIND_ALIGN_STACK \
2683  "mr 11,%1\n\t" \
2684  "std 2,-16(11)\n\t" /* save tocptr */ \
2685  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2686  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
2687  "ld 11, 0(11)\n\t" /* target->r11 */ \
2688  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2689  "mr 11,%1\n\t" \
2690  "mr %0,3\n\t" \
2691  "ld 2,-16(11)\n\t" /* restore tocptr */ \
2692  VALGRIND_RESTORE_STACK \
2693  : /*out*/ "=r" (_res) \
2694  : /*in*/ "r" (&_argvec[2]) \
2695  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2696  ); \
2697  lval = (__typeof__(lval)) _res; \
2698  } while (0)
2699 
2700 #define CALL_FN_W_WW(lval, orig, arg1,arg2) \
2701  do { \
2702  volatile OrigFn _orig = (orig); \
2703  volatile unsigned long _argvec[3+2]; \
2704  volatile unsigned long _res; \
2705  /* _argvec[0] holds current r2 across the call */ \
2706  _argvec[1] = (unsigned long)_orig.r2; \
2707  _argvec[2] = (unsigned long)_orig.nraddr; \
2708  _argvec[2+1] = (unsigned long)arg1; \
2709  _argvec[2+2] = (unsigned long)arg2; \
2710  __asm__ volatile( \
2711  VALGRIND_ALIGN_STACK \
2712  "mr 11,%1\n\t" \
2713  "std 2,-16(11)\n\t" /* save tocptr */ \
2714  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2715  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
2716  "ld 4, 16(11)\n\t" /* arg2->r4 */ \
2717  "ld 11, 0(11)\n\t" /* target->r11 */ \
2718  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2719  "mr 11,%1\n\t" \
2720  "mr %0,3\n\t" \
2721  "ld 2,-16(11)\n\t" /* restore tocptr */ \
2722  VALGRIND_RESTORE_STACK \
2723  : /*out*/ "=r" (_res) \
2724  : /*in*/ "r" (&_argvec[2]) \
2725  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2726  ); \
2727  lval = (__typeof__(lval)) _res; \
2728  } while (0)
2729 
2730 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
2731  do { \
2732  volatile OrigFn _orig = (orig); \
2733  volatile unsigned long _argvec[3+3]; \
2734  volatile unsigned long _res; \
2735  /* _argvec[0] holds current r2 across the call */ \
2736  _argvec[1] = (unsigned long)_orig.r2; \
2737  _argvec[2] = (unsigned long)_orig.nraddr; \
2738  _argvec[2+1] = (unsigned long)arg1; \
2739  _argvec[2+2] = (unsigned long)arg2; \
2740  _argvec[2+3] = (unsigned long)arg3; \
2741  __asm__ volatile( \
2742  VALGRIND_ALIGN_STACK \
2743  "mr 11,%1\n\t" \
2744  "std 2,-16(11)\n\t" /* save tocptr */ \
2745  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2746  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
2747  "ld 4, 16(11)\n\t" /* arg2->r4 */ \
2748  "ld 5, 24(11)\n\t" /* arg3->r5 */ \
2749  "ld 11, 0(11)\n\t" /* target->r11 */ \
2750  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2751  "mr 11,%1\n\t" \
2752  "mr %0,3\n\t" \
2753  "ld 2,-16(11)\n\t" /* restore tocptr */ \
2754  VALGRIND_RESTORE_STACK \
2755  : /*out*/ "=r" (_res) \
2756  : /*in*/ "r" (&_argvec[2]) \
2757  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2758  ); \
2759  lval = (__typeof__(lval)) _res; \
2760  } while (0)
2761 
2762 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
2763  do { \
2764  volatile OrigFn _orig = (orig); \
2765  volatile unsigned long _argvec[3+4]; \
2766  volatile unsigned long _res; \
2767  /* _argvec[0] holds current r2 across the call */ \
2768  _argvec[1] = (unsigned long)_orig.r2; \
2769  _argvec[2] = (unsigned long)_orig.nraddr; \
2770  _argvec[2+1] = (unsigned long)arg1; \
2771  _argvec[2+2] = (unsigned long)arg2; \
2772  _argvec[2+3] = (unsigned long)arg3; \
2773  _argvec[2+4] = (unsigned long)arg4; \
2774  __asm__ volatile( \
2775  VALGRIND_ALIGN_STACK \
2776  "mr 11,%1\n\t" \
2777  "std 2,-16(11)\n\t" /* save tocptr */ \
2778  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2779  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
2780  "ld 4, 16(11)\n\t" /* arg2->r4 */ \
2781  "ld 5, 24(11)\n\t" /* arg3->r5 */ \
2782  "ld 6, 32(11)\n\t" /* arg4->r6 */ \
2783  "ld 11, 0(11)\n\t" /* target->r11 */ \
2784  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2785  "mr 11,%1\n\t" \
2786  "mr %0,3\n\t" \
2787  "ld 2,-16(11)\n\t" /* restore tocptr */ \
2788  VALGRIND_RESTORE_STACK \
2789  : /*out*/ "=r" (_res) \
2790  : /*in*/ "r" (&_argvec[2]) \
2791  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2792  ); \
2793  lval = (__typeof__(lval)) _res; \
2794  } while (0)
2795 
2796 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
2797  do { \
2798  volatile OrigFn _orig = (orig); \
2799  volatile unsigned long _argvec[3+5]; \
2800  volatile unsigned long _res; \
2801  /* _argvec[0] holds current r2 across the call */ \
2802  _argvec[1] = (unsigned long)_orig.r2; \
2803  _argvec[2] = (unsigned long)_orig.nraddr; \
2804  _argvec[2+1] = (unsigned long)arg1; \
2805  _argvec[2+2] = (unsigned long)arg2; \
2806  _argvec[2+3] = (unsigned long)arg3; \
2807  _argvec[2+4] = (unsigned long)arg4; \
2808  _argvec[2+5] = (unsigned long)arg5; \
2809  __asm__ volatile( \
2810  VALGRIND_ALIGN_STACK \
2811  "mr 11,%1\n\t" \
2812  "std 2,-16(11)\n\t" /* save tocptr */ \
2813  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2814  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
2815  "ld 4, 16(11)\n\t" /* arg2->r4 */ \
2816  "ld 5, 24(11)\n\t" /* arg3->r5 */ \
2817  "ld 6, 32(11)\n\t" /* arg4->r6 */ \
2818  "ld 7, 40(11)\n\t" /* arg5->r7 */ \
2819  "ld 11, 0(11)\n\t" /* target->r11 */ \
2820  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2821  "mr 11,%1\n\t" \
2822  "mr %0,3\n\t" \
2823  "ld 2,-16(11)\n\t" /* restore tocptr */ \
2824  VALGRIND_RESTORE_STACK \
2825  : /*out*/ "=r" (_res) \
2826  : /*in*/ "r" (&_argvec[2]) \
2827  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2828  ); \
2829  lval = (__typeof__(lval)) _res; \
2830  } while (0)
2831 
2832 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
2833  do { \
2834  volatile OrigFn _orig = (orig); \
2835  volatile unsigned long _argvec[3+6]; \
2836  volatile unsigned long _res; \
2837  /* _argvec[0] holds current r2 across the call */ \
2838  _argvec[1] = (unsigned long)_orig.r2; \
2839  _argvec[2] = (unsigned long)_orig.nraddr; \
2840  _argvec[2+1] = (unsigned long)arg1; \
2841  _argvec[2+2] = (unsigned long)arg2; \
2842  _argvec[2+3] = (unsigned long)arg3; \
2843  _argvec[2+4] = (unsigned long)arg4; \
2844  _argvec[2+5] = (unsigned long)arg5; \
2845  _argvec[2+6] = (unsigned long)arg6; \
2846  __asm__ volatile( \
2847  VALGRIND_ALIGN_STACK \
2848  "mr 11,%1\n\t" \
2849  "std 2,-16(11)\n\t" /* save tocptr */ \
2850  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2851  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
2852  "ld 4, 16(11)\n\t" /* arg2->r4 */ \
2853  "ld 5, 24(11)\n\t" /* arg3->r5 */ \
2854  "ld 6, 32(11)\n\t" /* arg4->r6 */ \
2855  "ld 7, 40(11)\n\t" /* arg5->r7 */ \
2856  "ld 8, 48(11)\n\t" /* arg6->r8 */ \
2857  "ld 11, 0(11)\n\t" /* target->r11 */ \
2858  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2859  "mr 11,%1\n\t" \
2860  "mr %0,3\n\t" \
2861  "ld 2,-16(11)\n\t" /* restore tocptr */ \
2862  VALGRIND_RESTORE_STACK \
2863  : /*out*/ "=r" (_res) \
2864  : /*in*/ "r" (&_argvec[2]) \
2865  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2866  ); \
2867  lval = (__typeof__(lval)) _res; \
2868  } while (0)
2869 
2870 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2871  arg7) \
2872  do { \
2873  volatile OrigFn _orig = (orig); \
2874  volatile unsigned long _argvec[3+7]; \
2875  volatile unsigned long _res; \
2876  /* _argvec[0] holds current r2 across the call */ \
2877  _argvec[1] = (unsigned long)_orig.r2; \
2878  _argvec[2] = (unsigned long)_orig.nraddr; \
2879  _argvec[2+1] = (unsigned long)arg1; \
2880  _argvec[2+2] = (unsigned long)arg2; \
2881  _argvec[2+3] = (unsigned long)arg3; \
2882  _argvec[2+4] = (unsigned long)arg4; \
2883  _argvec[2+5] = (unsigned long)arg5; \
2884  _argvec[2+6] = (unsigned long)arg6; \
2885  _argvec[2+7] = (unsigned long)arg7; \
2886  __asm__ volatile( \
2887  VALGRIND_ALIGN_STACK \
2888  "mr 11,%1\n\t" \
2889  "std 2,-16(11)\n\t" /* save tocptr */ \
2890  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2891  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
2892  "ld 4, 16(11)\n\t" /* arg2->r4 */ \
2893  "ld 5, 24(11)\n\t" /* arg3->r5 */ \
2894  "ld 6, 32(11)\n\t" /* arg4->r6 */ \
2895  "ld 7, 40(11)\n\t" /* arg5->r7 */ \
2896  "ld 8, 48(11)\n\t" /* arg6->r8 */ \
2897  "ld 9, 56(11)\n\t" /* arg7->r9 */ \
2898  "ld 11, 0(11)\n\t" /* target->r11 */ \
2899  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2900  "mr 11,%1\n\t" \
2901  "mr %0,3\n\t" \
2902  "ld 2,-16(11)\n\t" /* restore tocptr */ \
2903  VALGRIND_RESTORE_STACK \
2904  : /*out*/ "=r" (_res) \
2905  : /*in*/ "r" (&_argvec[2]) \
2906  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2907  ); \
2908  lval = (__typeof__(lval)) _res; \
2909  } while (0)
2910 
2911 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2912  arg7,arg8) \
2913  do { \
2914  volatile OrigFn _orig = (orig); \
2915  volatile unsigned long _argvec[3+8]; \
2916  volatile unsigned long _res; \
2917  /* _argvec[0] holds current r2 across the call */ \
2918  _argvec[1] = (unsigned long)_orig.r2; \
2919  _argvec[2] = (unsigned long)_orig.nraddr; \
2920  _argvec[2+1] = (unsigned long)arg1; \
2921  _argvec[2+2] = (unsigned long)arg2; \
2922  _argvec[2+3] = (unsigned long)arg3; \
2923  _argvec[2+4] = (unsigned long)arg4; \
2924  _argvec[2+5] = (unsigned long)arg5; \
2925  _argvec[2+6] = (unsigned long)arg6; \
2926  _argvec[2+7] = (unsigned long)arg7; \
2927  _argvec[2+8] = (unsigned long)arg8; \
2928  __asm__ volatile( \
2929  VALGRIND_ALIGN_STACK \
2930  "mr 11,%1\n\t" \
2931  "std 2,-16(11)\n\t" /* save tocptr */ \
2932  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2933  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
2934  "ld 4, 16(11)\n\t" /* arg2->r4 */ \
2935  "ld 5, 24(11)\n\t" /* arg3->r5 */ \
2936  "ld 6, 32(11)\n\t" /* arg4->r6 */ \
2937  "ld 7, 40(11)\n\t" /* arg5->r7 */ \
2938  "ld 8, 48(11)\n\t" /* arg6->r8 */ \
2939  "ld 9, 56(11)\n\t" /* arg7->r9 */ \
2940  "ld 10, 64(11)\n\t" /* arg8->r10 */ \
2941  "ld 11, 0(11)\n\t" /* target->r11 */ \
2942  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2943  "mr 11,%1\n\t" \
2944  "mr %0,3\n\t" \
2945  "ld 2,-16(11)\n\t" /* restore tocptr */ \
2946  VALGRIND_RESTORE_STACK \
2947  : /*out*/ "=r" (_res) \
2948  : /*in*/ "r" (&_argvec[2]) \
2949  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2950  ); \
2951  lval = (__typeof__(lval)) _res; \
2952  } while (0)
2953 
2954 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
2955  arg7,arg8,arg9) \
2956  do { \
2957  volatile OrigFn _orig = (orig); \
2958  volatile unsigned long _argvec[3+9]; \
2959  volatile unsigned long _res; \
2960  /* _argvec[0] holds current r2 across the call */ \
2961  _argvec[1] = (unsigned long)_orig.r2; \
2962  _argvec[2] = (unsigned long)_orig.nraddr; \
2963  _argvec[2+1] = (unsigned long)arg1; \
2964  _argvec[2+2] = (unsigned long)arg2; \
2965  _argvec[2+3] = (unsigned long)arg3; \
2966  _argvec[2+4] = (unsigned long)arg4; \
2967  _argvec[2+5] = (unsigned long)arg5; \
2968  _argvec[2+6] = (unsigned long)arg6; \
2969  _argvec[2+7] = (unsigned long)arg7; \
2970  _argvec[2+8] = (unsigned long)arg8; \
2971  _argvec[2+9] = (unsigned long)arg9; \
2972  __asm__ volatile( \
2973  VALGRIND_ALIGN_STACK \
2974  "mr 11,%1\n\t" \
2975  "std 2,-16(11)\n\t" /* save tocptr */ \
2976  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
2977  "addi 1,1,-128\n\t" /* expand stack frame */ \
2978  /* arg9 */ \
2979  "ld 3,72(11)\n\t" \
2980  "std 3,112(1)\n\t" \
2981  /* args1-8 */ \
2982  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
2983  "ld 4, 16(11)\n\t" /* arg2->r4 */ \
2984  "ld 5, 24(11)\n\t" /* arg3->r5 */ \
2985  "ld 6, 32(11)\n\t" /* arg4->r6 */ \
2986  "ld 7, 40(11)\n\t" /* arg5->r7 */ \
2987  "ld 8, 48(11)\n\t" /* arg6->r8 */ \
2988  "ld 9, 56(11)\n\t" /* arg7->r9 */ \
2989  "ld 10, 64(11)\n\t" /* arg8->r10 */ \
2990  "ld 11, 0(11)\n\t" /* target->r11 */ \
2991  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
2992  "mr 11,%1\n\t" \
2993  "mr %0,3\n\t" \
2994  "ld 2,-16(11)\n\t" /* restore tocptr */ \
2995  VALGRIND_RESTORE_STACK \
2996  : /*out*/ "=r" (_res) \
2997  : /*in*/ "r" (&_argvec[2]) \
2998  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
2999  ); \
3000  lval = (__typeof__(lval)) _res; \
3001  } while (0)
3002 
3003 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3004  arg7,arg8,arg9,arg10) \
3005  do { \
3006  volatile OrigFn _orig = (orig); \
3007  volatile unsigned long _argvec[3+10]; \
3008  volatile unsigned long _res; \
3009  /* _argvec[0] holds current r2 across the call */ \
3010  _argvec[1] = (unsigned long)_orig.r2; \
3011  _argvec[2] = (unsigned long)_orig.nraddr; \
3012  _argvec[2+1] = (unsigned long)arg1; \
3013  _argvec[2+2] = (unsigned long)arg2; \
3014  _argvec[2+3] = (unsigned long)arg3; \
3015  _argvec[2+4] = (unsigned long)arg4; \
3016  _argvec[2+5] = (unsigned long)arg5; \
3017  _argvec[2+6] = (unsigned long)arg6; \
3018  _argvec[2+7] = (unsigned long)arg7; \
3019  _argvec[2+8] = (unsigned long)arg8; \
3020  _argvec[2+9] = (unsigned long)arg9; \
3021  _argvec[2+10] = (unsigned long)arg10; \
3022  __asm__ volatile( \
3023  VALGRIND_ALIGN_STACK \
3024  "mr 11,%1\n\t" \
3025  "std 2,-16(11)\n\t" /* save tocptr */ \
3026  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3027  "addi 1,1,-128\n\t" /* expand stack frame */ \
3028  /* arg10 */ \
3029  "ld 3,80(11)\n\t" \
3030  "std 3,120(1)\n\t" \
3031  /* arg9 */ \
3032  "ld 3,72(11)\n\t" \
3033  "std 3,112(1)\n\t" \
3034  /* args1-8 */ \
3035  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
3036  "ld 4, 16(11)\n\t" /* arg2->r4 */ \
3037  "ld 5, 24(11)\n\t" /* arg3->r5 */ \
3038  "ld 6, 32(11)\n\t" /* arg4->r6 */ \
3039  "ld 7, 40(11)\n\t" /* arg5->r7 */ \
3040  "ld 8, 48(11)\n\t" /* arg6->r8 */ \
3041  "ld 9, 56(11)\n\t" /* arg7->r9 */ \
3042  "ld 10, 64(11)\n\t" /* arg8->r10 */ \
3043  "ld 11, 0(11)\n\t" /* target->r11 */ \
3044  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3045  "mr 11,%1\n\t" \
3046  "mr %0,3\n\t" \
3047  "ld 2,-16(11)\n\t" /* restore tocptr */ \
3048  VALGRIND_RESTORE_STACK \
3049  : /*out*/ "=r" (_res) \
3050  : /*in*/ "r" (&_argvec[2]) \
3051  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3052  ); \
3053  lval = (__typeof__(lval)) _res; \
3054  } while (0)
3055 
3056 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3057  arg7,arg8,arg9,arg10,arg11) \
3058  do { \
3059  volatile OrigFn _orig = (orig); \
3060  volatile unsigned long _argvec[3+11]; \
3061  volatile unsigned long _res; \
3062  /* _argvec[0] holds current r2 across the call */ \
3063  _argvec[1] = (unsigned long)_orig.r2; \
3064  _argvec[2] = (unsigned long)_orig.nraddr; \
3065  _argvec[2+1] = (unsigned long)arg1; \
3066  _argvec[2+2] = (unsigned long)arg2; \
3067  _argvec[2+3] = (unsigned long)arg3; \
3068  _argvec[2+4] = (unsigned long)arg4; \
3069  _argvec[2+5] = (unsigned long)arg5; \
3070  _argvec[2+6] = (unsigned long)arg6; \
3071  _argvec[2+7] = (unsigned long)arg7; \
3072  _argvec[2+8] = (unsigned long)arg8; \
3073  _argvec[2+9] = (unsigned long)arg9; \
3074  _argvec[2+10] = (unsigned long)arg10; \
3075  _argvec[2+11] = (unsigned long)arg11; \
3076  __asm__ volatile( \
3077  VALGRIND_ALIGN_STACK \
3078  "mr 11,%1\n\t" \
3079  "std 2,-16(11)\n\t" /* save tocptr */ \
3080  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3081  "addi 1,1,-144\n\t" /* expand stack frame */ \
3082  /* arg11 */ \
3083  "ld 3,88(11)\n\t" \
3084  "std 3,128(1)\n\t" \
3085  /* arg10 */ \
3086  "ld 3,80(11)\n\t" \
3087  "std 3,120(1)\n\t" \
3088  /* arg9 */ \
3089  "ld 3,72(11)\n\t" \
3090  "std 3,112(1)\n\t" \
3091  /* args1-8 */ \
3092  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
3093  "ld 4, 16(11)\n\t" /* arg2->r4 */ \
3094  "ld 5, 24(11)\n\t" /* arg3->r5 */ \
3095  "ld 6, 32(11)\n\t" /* arg4->r6 */ \
3096  "ld 7, 40(11)\n\t" /* arg5->r7 */ \
3097  "ld 8, 48(11)\n\t" /* arg6->r8 */ \
3098  "ld 9, 56(11)\n\t" /* arg7->r9 */ \
3099  "ld 10, 64(11)\n\t" /* arg8->r10 */ \
3100  "ld 11, 0(11)\n\t" /* target->r11 */ \
3101  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3102  "mr 11,%1\n\t" \
3103  "mr %0,3\n\t" \
3104  "ld 2,-16(11)\n\t" /* restore tocptr */ \
3105  VALGRIND_RESTORE_STACK \
3106  : /*out*/ "=r" (_res) \
3107  : /*in*/ "r" (&_argvec[2]) \
3108  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3109  ); \
3110  lval = (__typeof__(lval)) _res; \
3111  } while (0)
3112 
3113 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3114  arg7,arg8,arg9,arg10,arg11,arg12) \
3115  do { \
3116  volatile OrigFn _orig = (orig); \
3117  volatile unsigned long _argvec[3+12]; \
3118  volatile unsigned long _res; \
3119  /* _argvec[0] holds current r2 across the call */ \
3120  _argvec[1] = (unsigned long)_orig.r2; \
3121  _argvec[2] = (unsigned long)_orig.nraddr; \
3122  _argvec[2+1] = (unsigned long)arg1; \
3123  _argvec[2+2] = (unsigned long)arg2; \
3124  _argvec[2+3] = (unsigned long)arg3; \
3125  _argvec[2+4] = (unsigned long)arg4; \
3126  _argvec[2+5] = (unsigned long)arg5; \
3127  _argvec[2+6] = (unsigned long)arg6; \
3128  _argvec[2+7] = (unsigned long)arg7; \
3129  _argvec[2+8] = (unsigned long)arg8; \
3130  _argvec[2+9] = (unsigned long)arg9; \
3131  _argvec[2+10] = (unsigned long)arg10; \
3132  _argvec[2+11] = (unsigned long)arg11; \
3133  _argvec[2+12] = (unsigned long)arg12; \
3134  __asm__ volatile( \
3135  VALGRIND_ALIGN_STACK \
3136  "mr 11,%1\n\t" \
3137  "std 2,-16(11)\n\t" /* save tocptr */ \
3138  "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \
3139  "addi 1,1,-144\n\t" /* expand stack frame */ \
3140  /* arg12 */ \
3141  "ld 3,96(11)\n\t" \
3142  "std 3,136(1)\n\t" \
3143  /* arg11 */ \
3144  "ld 3,88(11)\n\t" \
3145  "std 3,128(1)\n\t" \
3146  /* arg10 */ \
3147  "ld 3,80(11)\n\t" \
3148  "std 3,120(1)\n\t" \
3149  /* arg9 */ \
3150  "ld 3,72(11)\n\t" \
3151  "std 3,112(1)\n\t" \
3152  /* args1-8 */ \
3153  "ld 3, 8(11)\n\t" /* arg1->r3 */ \
3154  "ld 4, 16(11)\n\t" /* arg2->r4 */ \
3155  "ld 5, 24(11)\n\t" /* arg3->r5 */ \
3156  "ld 6, 32(11)\n\t" /* arg4->r6 */ \
3157  "ld 7, 40(11)\n\t" /* arg5->r7 */ \
3158  "ld 8, 48(11)\n\t" /* arg6->r8 */ \
3159  "ld 9, 56(11)\n\t" /* arg7->r9 */ \
3160  "ld 10, 64(11)\n\t" /* arg8->r10 */ \
3161  "ld 11, 0(11)\n\t" /* target->r11 */ \
3162  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \
3163  "mr 11,%1\n\t" \
3164  "mr %0,3\n\t" \
3165  "ld 2,-16(11)\n\t" /* restore tocptr */ \
3166  VALGRIND_RESTORE_STACK \
3167  : /*out*/ "=r" (_res) \
3168  : /*in*/ "r" (&_argvec[2]) \
3169  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3170  ); \
3171  lval = (__typeof__(lval)) _res; \
3172  } while (0)
3173 
3174 #endif /* PLAT_ppc64be_linux */
3175 
3176 /* ------------------------- ppc64le-linux ----------------------- */
3177 #if defined(PLAT_ppc64le_linux)
3178 
3179 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3180 
3181 /* These regs are trashed by the hidden call. */
3182 #define __CALLER_SAVED_REGS \
3183  "lr", "ctr", "xer", \
3184  "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \
3185  "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \
3186  "r11", "r12", "r13"
3187 
3188 /* Macros to save and align the stack before making a function
3189  call and restore it afterwards as gcc may not keep the stack
3190  pointer aligned if it doesn't realize calls are being made
3191  to other functions. */
3192 
3193 #define VALGRIND_ALIGN_STACK \
3194  "mr 28,1\n\t" \
3195  "rldicr 1,1,0,59\n\t"
3196 #define VALGRIND_RESTORE_STACK \
3197  "mr 1,28\n\t"
3198 
3199 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
3200  long) == 8. */
3201 
3202 #define CALL_FN_W_v(lval, orig) \
3203  do { \
3204  volatile OrigFn _orig = (orig); \
3205  volatile unsigned long _argvec[3+0]; \
3206  volatile unsigned long _res; \
3207  /* _argvec[0] holds current r2 across the call */ \
3208  _argvec[1] = (unsigned long)_orig.r2; \
3209  _argvec[2] = (unsigned long)_orig.nraddr; \
3210  __asm__ volatile( \
3211  VALGRIND_ALIGN_STACK \
3212  "mr 12,%1\n\t" \
3213  "std 2,-16(12)\n\t" /* save tocptr */ \
3214  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3215  "ld 12, 0(12)\n\t" /* target->r12 */ \
3216  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3217  "mr 12,%1\n\t" \
3218  "mr %0,3\n\t" \
3219  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3220  VALGRIND_RESTORE_STACK \
3221  : /*out*/ "=r" (_res) \
3222  : /*in*/ "r" (&_argvec[2]) \
3223  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3224  ); \
3225  lval = (__typeof__(lval)) _res; \
3226  } while (0)
3227 
3228 #define CALL_FN_W_W(lval, orig, arg1) \
3229  do { \
3230  volatile OrigFn _orig = (orig); \
3231  volatile unsigned long _argvec[3+1]; \
3232  volatile unsigned long _res; \
3233  /* _argvec[0] holds current r2 across the call */ \
3234  _argvec[1] = (unsigned long)_orig.r2; \
3235  _argvec[2] = (unsigned long)_orig.nraddr; \
3236  _argvec[2+1] = (unsigned long)arg1; \
3237  __asm__ volatile( \
3238  VALGRIND_ALIGN_STACK \
3239  "mr 12,%1\n\t" \
3240  "std 2,-16(12)\n\t" /* save tocptr */ \
3241  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3242  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3243  "ld 12, 0(12)\n\t" /* target->r12 */ \
3244  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3245  "mr 12,%1\n\t" \
3246  "mr %0,3\n\t" \
3247  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3248  VALGRIND_RESTORE_STACK \
3249  : /*out*/ "=r" (_res) \
3250  : /*in*/ "r" (&_argvec[2]) \
3251  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3252  ); \
3253  lval = (__typeof__(lval)) _res; \
3254  } while (0)
3255 
3256 #define CALL_FN_W_WW(lval, orig, arg1,arg2) \
3257  do { \
3258  volatile OrigFn _orig = (orig); \
3259  volatile unsigned long _argvec[3+2]; \
3260  volatile unsigned long _res; \
3261  /* _argvec[0] holds current r2 across the call */ \
3262  _argvec[1] = (unsigned long)_orig.r2; \
3263  _argvec[2] = (unsigned long)_orig.nraddr; \
3264  _argvec[2+1] = (unsigned long)arg1; \
3265  _argvec[2+2] = (unsigned long)arg2; \
3266  __asm__ volatile( \
3267  VALGRIND_ALIGN_STACK \
3268  "mr 12,%1\n\t" \
3269  "std 2,-16(12)\n\t" /* save tocptr */ \
3270  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3271  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3272  "ld 4, 16(12)\n\t" /* arg2->r4 */ \
3273  "ld 12, 0(12)\n\t" /* target->r12 */ \
3274  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3275  "mr 12,%1\n\t" \
3276  "mr %0,3\n\t" \
3277  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3278  VALGRIND_RESTORE_STACK \
3279  : /*out*/ "=r" (_res) \
3280  : /*in*/ "r" (&_argvec[2]) \
3281  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3282  ); \
3283  lval = (__typeof__(lval)) _res; \
3284  } while (0)
3285 
3286 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
3287  do { \
3288  volatile OrigFn _orig = (orig); \
3289  volatile unsigned long _argvec[3+3]; \
3290  volatile unsigned long _res; \
3291  /* _argvec[0] holds current r2 across the call */ \
3292  _argvec[1] = (unsigned long)_orig.r2; \
3293  _argvec[2] = (unsigned long)_orig.nraddr; \
3294  _argvec[2+1] = (unsigned long)arg1; \
3295  _argvec[2+2] = (unsigned long)arg2; \
3296  _argvec[2+3] = (unsigned long)arg3; \
3297  __asm__ volatile( \
3298  VALGRIND_ALIGN_STACK \
3299  "mr 12,%1\n\t" \
3300  "std 2,-16(12)\n\t" /* save tocptr */ \
3301  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3302  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3303  "ld 4, 16(12)\n\t" /* arg2->r4 */ \
3304  "ld 5, 24(12)\n\t" /* arg3->r5 */ \
3305  "ld 12, 0(12)\n\t" /* target->r12 */ \
3306  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3307  "mr 12,%1\n\t" \
3308  "mr %0,3\n\t" \
3309  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3310  VALGRIND_RESTORE_STACK \
3311  : /*out*/ "=r" (_res) \
3312  : /*in*/ "r" (&_argvec[2]) \
3313  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3314  ); \
3315  lval = (__typeof__(lval)) _res; \
3316  } while (0)
3317 
3318 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
3319  do { \
3320  volatile OrigFn _orig = (orig); \
3321  volatile unsigned long _argvec[3+4]; \
3322  volatile unsigned long _res; \
3323  /* _argvec[0] holds current r2 across the call */ \
3324  _argvec[1] = (unsigned long)_orig.r2; \
3325  _argvec[2] = (unsigned long)_orig.nraddr; \
3326  _argvec[2+1] = (unsigned long)arg1; \
3327  _argvec[2+2] = (unsigned long)arg2; \
3328  _argvec[2+3] = (unsigned long)arg3; \
3329  _argvec[2+4] = (unsigned long)arg4; \
3330  __asm__ volatile( \
3331  VALGRIND_ALIGN_STACK \
3332  "mr 12,%1\n\t" \
3333  "std 2,-16(12)\n\t" /* save tocptr */ \
3334  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3335  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3336  "ld 4, 16(12)\n\t" /* arg2->r4 */ \
3337  "ld 5, 24(12)\n\t" /* arg3->r5 */ \
3338  "ld 6, 32(12)\n\t" /* arg4->r6 */ \
3339  "ld 12, 0(12)\n\t" /* target->r12 */ \
3340  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3341  "mr 12,%1\n\t" \
3342  "mr %0,3\n\t" \
3343  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3344  VALGRIND_RESTORE_STACK \
3345  : /*out*/ "=r" (_res) \
3346  : /*in*/ "r" (&_argvec[2]) \
3347  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3348  ); \
3349  lval = (__typeof__(lval)) _res; \
3350  } while (0)
3351 
3352 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
3353  do { \
3354  volatile OrigFn _orig = (orig); \
3355  volatile unsigned long _argvec[3+5]; \
3356  volatile unsigned long _res; \
3357  /* _argvec[0] holds current r2 across the call */ \
3358  _argvec[1] = (unsigned long)_orig.r2; \
3359  _argvec[2] = (unsigned long)_orig.nraddr; \
3360  _argvec[2+1] = (unsigned long)arg1; \
3361  _argvec[2+2] = (unsigned long)arg2; \
3362  _argvec[2+3] = (unsigned long)arg3; \
3363  _argvec[2+4] = (unsigned long)arg4; \
3364  _argvec[2+5] = (unsigned long)arg5; \
3365  __asm__ volatile( \
3366  VALGRIND_ALIGN_STACK \
3367  "mr 12,%1\n\t" \
3368  "std 2,-16(12)\n\t" /* save tocptr */ \
3369  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3370  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3371  "ld 4, 16(12)\n\t" /* arg2->r4 */ \
3372  "ld 5, 24(12)\n\t" /* arg3->r5 */ \
3373  "ld 6, 32(12)\n\t" /* arg4->r6 */ \
3374  "ld 7, 40(12)\n\t" /* arg5->r7 */ \
3375  "ld 12, 0(12)\n\t" /* target->r12 */ \
3376  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3377  "mr 12,%1\n\t" \
3378  "mr %0,3\n\t" \
3379  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3380  VALGRIND_RESTORE_STACK \
3381  : /*out*/ "=r" (_res) \
3382  : /*in*/ "r" (&_argvec[2]) \
3383  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3384  ); \
3385  lval = (__typeof__(lval)) _res; \
3386  } while (0)
3387 
3388 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
3389  do { \
3390  volatile OrigFn _orig = (orig); \
3391  volatile unsigned long _argvec[3+6]; \
3392  volatile unsigned long _res; \
3393  /* _argvec[0] holds current r2 across the call */ \
3394  _argvec[1] = (unsigned long)_orig.r2; \
3395  _argvec[2] = (unsigned long)_orig.nraddr; \
3396  _argvec[2+1] = (unsigned long)arg1; \
3397  _argvec[2+2] = (unsigned long)arg2; \
3398  _argvec[2+3] = (unsigned long)arg3; \
3399  _argvec[2+4] = (unsigned long)arg4; \
3400  _argvec[2+5] = (unsigned long)arg5; \
3401  _argvec[2+6] = (unsigned long)arg6; \
3402  __asm__ volatile( \
3403  VALGRIND_ALIGN_STACK \
3404  "mr 12,%1\n\t" \
3405  "std 2,-16(12)\n\t" /* save tocptr */ \
3406  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3407  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3408  "ld 4, 16(12)\n\t" /* arg2->r4 */ \
3409  "ld 5, 24(12)\n\t" /* arg3->r5 */ \
3410  "ld 6, 32(12)\n\t" /* arg4->r6 */ \
3411  "ld 7, 40(12)\n\t" /* arg5->r7 */ \
3412  "ld 8, 48(12)\n\t" /* arg6->r8 */ \
3413  "ld 12, 0(12)\n\t" /* target->r12 */ \
3414  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3415  "mr 12,%1\n\t" \
3416  "mr %0,3\n\t" \
3417  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3418  VALGRIND_RESTORE_STACK \
3419  : /*out*/ "=r" (_res) \
3420  : /*in*/ "r" (&_argvec[2]) \
3421  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3422  ); \
3423  lval = (__typeof__(lval)) _res; \
3424  } while (0)
3425 
3426 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3427  arg7) \
3428  do { \
3429  volatile OrigFn _orig = (orig); \
3430  volatile unsigned long _argvec[3+7]; \
3431  volatile unsigned long _res; \
3432  /* _argvec[0] holds current r2 across the call */ \
3433  _argvec[1] = (unsigned long)_orig.r2; \
3434  _argvec[2] = (unsigned long)_orig.nraddr; \
3435  _argvec[2+1] = (unsigned long)arg1; \
3436  _argvec[2+2] = (unsigned long)arg2; \
3437  _argvec[2+3] = (unsigned long)arg3; \
3438  _argvec[2+4] = (unsigned long)arg4; \
3439  _argvec[2+5] = (unsigned long)arg5; \
3440  _argvec[2+6] = (unsigned long)arg6; \
3441  _argvec[2+7] = (unsigned long)arg7; \
3442  __asm__ volatile( \
3443  VALGRIND_ALIGN_STACK \
3444  "mr 12,%1\n\t" \
3445  "std 2,-16(12)\n\t" /* save tocptr */ \
3446  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3447  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3448  "ld 4, 16(12)\n\t" /* arg2->r4 */ \
3449  "ld 5, 24(12)\n\t" /* arg3->r5 */ \
3450  "ld 6, 32(12)\n\t" /* arg4->r6 */ \
3451  "ld 7, 40(12)\n\t" /* arg5->r7 */ \
3452  "ld 8, 48(12)\n\t" /* arg6->r8 */ \
3453  "ld 9, 56(12)\n\t" /* arg7->r9 */ \
3454  "ld 12, 0(12)\n\t" /* target->r12 */ \
3455  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3456  "mr 12,%1\n\t" \
3457  "mr %0,3\n\t" \
3458  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3459  VALGRIND_RESTORE_STACK \
3460  : /*out*/ "=r" (_res) \
3461  : /*in*/ "r" (&_argvec[2]) \
3462  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3463  ); \
3464  lval = (__typeof__(lval)) _res; \
3465  } while (0)
3466 
3467 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3468  arg7,arg8) \
3469  do { \
3470  volatile OrigFn _orig = (orig); \
3471  volatile unsigned long _argvec[3+8]; \
3472  volatile unsigned long _res; \
3473  /* _argvec[0] holds current r2 across the call */ \
3474  _argvec[1] = (unsigned long)_orig.r2; \
3475  _argvec[2] = (unsigned long)_orig.nraddr; \
3476  _argvec[2+1] = (unsigned long)arg1; \
3477  _argvec[2+2] = (unsigned long)arg2; \
3478  _argvec[2+3] = (unsigned long)arg3; \
3479  _argvec[2+4] = (unsigned long)arg4; \
3480  _argvec[2+5] = (unsigned long)arg5; \
3481  _argvec[2+6] = (unsigned long)arg6; \
3482  _argvec[2+7] = (unsigned long)arg7; \
3483  _argvec[2+8] = (unsigned long)arg8; \
3484  __asm__ volatile( \
3485  VALGRIND_ALIGN_STACK \
3486  "mr 12,%1\n\t" \
3487  "std 2,-16(12)\n\t" /* save tocptr */ \
3488  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3489  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3490  "ld 4, 16(12)\n\t" /* arg2->r4 */ \
3491  "ld 5, 24(12)\n\t" /* arg3->r5 */ \
3492  "ld 6, 32(12)\n\t" /* arg4->r6 */ \
3493  "ld 7, 40(12)\n\t" /* arg5->r7 */ \
3494  "ld 8, 48(12)\n\t" /* arg6->r8 */ \
3495  "ld 9, 56(12)\n\t" /* arg7->r9 */ \
3496  "ld 10, 64(12)\n\t" /* arg8->r10 */ \
3497  "ld 12, 0(12)\n\t" /* target->r12 */ \
3498  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3499  "mr 12,%1\n\t" \
3500  "mr %0,3\n\t" \
3501  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3502  VALGRIND_RESTORE_STACK \
3503  : /*out*/ "=r" (_res) \
3504  : /*in*/ "r" (&_argvec[2]) \
3505  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3506  ); \
3507  lval = (__typeof__(lval)) _res; \
3508  } while (0)
3509 
3510 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3511  arg7,arg8,arg9) \
3512  do { \
3513  volatile OrigFn _orig = (orig); \
3514  volatile unsigned long _argvec[3+9]; \
3515  volatile unsigned long _res; \
3516  /* _argvec[0] holds current r2 across the call */ \
3517  _argvec[1] = (unsigned long)_orig.r2; \
3518  _argvec[2] = (unsigned long)_orig.nraddr; \
3519  _argvec[2+1] = (unsigned long)arg1; \
3520  _argvec[2+2] = (unsigned long)arg2; \
3521  _argvec[2+3] = (unsigned long)arg3; \
3522  _argvec[2+4] = (unsigned long)arg4; \
3523  _argvec[2+5] = (unsigned long)arg5; \
3524  _argvec[2+6] = (unsigned long)arg6; \
3525  _argvec[2+7] = (unsigned long)arg7; \
3526  _argvec[2+8] = (unsigned long)arg8; \
3527  _argvec[2+9] = (unsigned long)arg9; \
3528  __asm__ volatile( \
3529  VALGRIND_ALIGN_STACK \
3530  "mr 12,%1\n\t" \
3531  "std 2,-16(12)\n\t" /* save tocptr */ \
3532  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3533  "addi 1,1,-128\n\t" /* expand stack frame */ \
3534  /* arg9 */ \
3535  "ld 3,72(12)\n\t" \
3536  "std 3,96(1)\n\t" \
3537  /* args1-8 */ \
3538  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3539  "ld 4, 16(12)\n\t" /* arg2->r4 */ \
3540  "ld 5, 24(12)\n\t" /* arg3->r5 */ \
3541  "ld 6, 32(12)\n\t" /* arg4->r6 */ \
3542  "ld 7, 40(12)\n\t" /* arg5->r7 */ \
3543  "ld 8, 48(12)\n\t" /* arg6->r8 */ \
3544  "ld 9, 56(12)\n\t" /* arg7->r9 */ \
3545  "ld 10, 64(12)\n\t" /* arg8->r10 */ \
3546  "ld 12, 0(12)\n\t" /* target->r12 */ \
3547  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3548  "mr 12,%1\n\t" \
3549  "mr %0,3\n\t" \
3550  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3551  VALGRIND_RESTORE_STACK \
3552  : /*out*/ "=r" (_res) \
3553  : /*in*/ "r" (&_argvec[2]) \
3554  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3555  ); \
3556  lval = (__typeof__(lval)) _res; \
3557  } while (0)
3558 
3559 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3560  arg7,arg8,arg9,arg10) \
3561  do { \
3562  volatile OrigFn _orig = (orig); \
3563  volatile unsigned long _argvec[3+10]; \
3564  volatile unsigned long _res; \
3565  /* _argvec[0] holds current r2 across the call */ \
3566  _argvec[1] = (unsigned long)_orig.r2; \
3567  _argvec[2] = (unsigned long)_orig.nraddr; \
3568  _argvec[2+1] = (unsigned long)arg1; \
3569  _argvec[2+2] = (unsigned long)arg2; \
3570  _argvec[2+3] = (unsigned long)arg3; \
3571  _argvec[2+4] = (unsigned long)arg4; \
3572  _argvec[2+5] = (unsigned long)arg5; \
3573  _argvec[2+6] = (unsigned long)arg6; \
3574  _argvec[2+7] = (unsigned long)arg7; \
3575  _argvec[2+8] = (unsigned long)arg8; \
3576  _argvec[2+9] = (unsigned long)arg9; \
3577  _argvec[2+10] = (unsigned long)arg10; \
3578  __asm__ volatile( \
3579  VALGRIND_ALIGN_STACK \
3580  "mr 12,%1\n\t" \
3581  "std 2,-16(12)\n\t" /* save tocptr */ \
3582  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3583  "addi 1,1,-128\n\t" /* expand stack frame */ \
3584  /* arg10 */ \
3585  "ld 3,80(12)\n\t" \
3586  "std 3,104(1)\n\t" \
3587  /* arg9 */ \
3588  "ld 3,72(12)\n\t" \
3589  "std 3,96(1)\n\t" \
3590  /* args1-8 */ \
3591  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3592  "ld 4, 16(12)\n\t" /* arg2->r4 */ \
3593  "ld 5, 24(12)\n\t" /* arg3->r5 */ \
3594  "ld 6, 32(12)\n\t" /* arg4->r6 */ \
3595  "ld 7, 40(12)\n\t" /* arg5->r7 */ \
3596  "ld 8, 48(12)\n\t" /* arg6->r8 */ \
3597  "ld 9, 56(12)\n\t" /* arg7->r9 */ \
3598  "ld 10, 64(12)\n\t" /* arg8->r10 */ \
3599  "ld 12, 0(12)\n\t" /* target->r12 */ \
3600  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3601  "mr 12,%1\n\t" \
3602  "mr %0,3\n\t" \
3603  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3604  VALGRIND_RESTORE_STACK \
3605  : /*out*/ "=r" (_res) \
3606  : /*in*/ "r" (&_argvec[2]) \
3607  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3608  ); \
3609  lval = (__typeof__(lval)) _res; \
3610  } while (0)
3611 
3612 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3613  arg7,arg8,arg9,arg10,arg11) \
3614  do { \
3615  volatile OrigFn _orig = (orig); \
3616  volatile unsigned long _argvec[3+11]; \
3617  volatile unsigned long _res; \
3618  /* _argvec[0] holds current r2 across the call */ \
3619  _argvec[1] = (unsigned long)_orig.r2; \
3620  _argvec[2] = (unsigned long)_orig.nraddr; \
3621  _argvec[2+1] = (unsigned long)arg1; \
3622  _argvec[2+2] = (unsigned long)arg2; \
3623  _argvec[2+3] = (unsigned long)arg3; \
3624  _argvec[2+4] = (unsigned long)arg4; \
3625  _argvec[2+5] = (unsigned long)arg5; \
3626  _argvec[2+6] = (unsigned long)arg6; \
3627  _argvec[2+7] = (unsigned long)arg7; \
3628  _argvec[2+8] = (unsigned long)arg8; \
3629  _argvec[2+9] = (unsigned long)arg9; \
3630  _argvec[2+10] = (unsigned long)arg10; \
3631  _argvec[2+11] = (unsigned long)arg11; \
3632  __asm__ volatile( \
3633  VALGRIND_ALIGN_STACK \
3634  "mr 12,%1\n\t" \
3635  "std 2,-16(12)\n\t" /* save tocptr */ \
3636  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3637  "addi 1,1,-144\n\t" /* expand stack frame */ \
3638  /* arg11 */ \
3639  "ld 3,88(12)\n\t" \
3640  "std 3,112(1)\n\t" \
3641  /* arg10 */ \
3642  "ld 3,80(12)\n\t" \
3643  "std 3,104(1)\n\t" \
3644  /* arg9 */ \
3645  "ld 3,72(12)\n\t" \
3646  "std 3,96(1)\n\t" \
3647  /* args1-8 */ \
3648  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3649  "ld 4, 16(12)\n\t" /* arg2->r4 */ \
3650  "ld 5, 24(12)\n\t" /* arg3->r5 */ \
3651  "ld 6, 32(12)\n\t" /* arg4->r6 */ \
3652  "ld 7, 40(12)\n\t" /* arg5->r7 */ \
3653  "ld 8, 48(12)\n\t" /* arg6->r8 */ \
3654  "ld 9, 56(12)\n\t" /* arg7->r9 */ \
3655  "ld 10, 64(12)\n\t" /* arg8->r10 */ \
3656  "ld 12, 0(12)\n\t" /* target->r12 */ \
3657  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3658  "mr 12,%1\n\t" \
3659  "mr %0,3\n\t" \
3660  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3661  VALGRIND_RESTORE_STACK \
3662  : /*out*/ "=r" (_res) \
3663  : /*in*/ "r" (&_argvec[2]) \
3664  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3665  ); \
3666  lval = (__typeof__(lval)) _res; \
3667  } while (0)
3668 
3669 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3670  arg7,arg8,arg9,arg10,arg11,arg12) \
3671  do { \
3672  volatile OrigFn _orig = (orig); \
3673  volatile unsigned long _argvec[3+12]; \
3674  volatile unsigned long _res; \
3675  /* _argvec[0] holds current r2 across the call */ \
3676  _argvec[1] = (unsigned long)_orig.r2; \
3677  _argvec[2] = (unsigned long)_orig.nraddr; \
3678  _argvec[2+1] = (unsigned long)arg1; \
3679  _argvec[2+2] = (unsigned long)arg2; \
3680  _argvec[2+3] = (unsigned long)arg3; \
3681  _argvec[2+4] = (unsigned long)arg4; \
3682  _argvec[2+5] = (unsigned long)arg5; \
3683  _argvec[2+6] = (unsigned long)arg6; \
3684  _argvec[2+7] = (unsigned long)arg7; \
3685  _argvec[2+8] = (unsigned long)arg8; \
3686  _argvec[2+9] = (unsigned long)arg9; \
3687  _argvec[2+10] = (unsigned long)arg10; \
3688  _argvec[2+11] = (unsigned long)arg11; \
3689  _argvec[2+12] = (unsigned long)arg12; \
3690  __asm__ volatile( \
3691  VALGRIND_ALIGN_STACK \
3692  "mr 12,%1\n\t" \
3693  "std 2,-16(12)\n\t" /* save tocptr */ \
3694  "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \
3695  "addi 1,1,-144\n\t" /* expand stack frame */ \
3696  /* arg12 */ \
3697  "ld 3,96(12)\n\t" \
3698  "std 3,120(1)\n\t" \
3699  /* arg11 */ \
3700  "ld 3,88(12)\n\t" \
3701  "std 3,112(1)\n\t" \
3702  /* arg10 */ \
3703  "ld 3,80(12)\n\t" \
3704  "std 3,104(1)\n\t" \
3705  /* arg9 */ \
3706  "ld 3,72(12)\n\t" \
3707  "std 3,96(1)\n\t" \
3708  /* args1-8 */ \
3709  "ld 3, 8(12)\n\t" /* arg1->r3 */ \
3710  "ld 4, 16(12)\n\t" /* arg2->r4 */ \
3711  "ld 5, 24(12)\n\t" /* arg3->r5 */ \
3712  "ld 6, 32(12)\n\t" /* arg4->r6 */ \
3713  "ld 7, 40(12)\n\t" /* arg5->r7 */ \
3714  "ld 8, 48(12)\n\t" /* arg6->r8 */ \
3715  "ld 9, 56(12)\n\t" /* arg7->r9 */ \
3716  "ld 10, 64(12)\n\t" /* arg8->r10 */ \
3717  "ld 12, 0(12)\n\t" /* target->r12 */ \
3718  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \
3719  "mr 12,%1\n\t" \
3720  "mr %0,3\n\t" \
3721  "ld 2,-16(12)\n\t" /* restore tocptr */ \
3722  VALGRIND_RESTORE_STACK \
3723  : /*out*/ "=r" (_res) \
3724  : /*in*/ "r" (&_argvec[2]) \
3725  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \
3726  ); \
3727  lval = (__typeof__(lval)) _res; \
3728  } while (0)
3729 
3730 #endif /* PLAT_ppc64le_linux */
3731 
3732 /* ------------------------- arm-linux ------------------------- */
3733 
3734 #if defined(PLAT_arm_linux)
3735 
3736 /* These regs are trashed by the hidden call. */
3737 #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4","r14"
3738 
3739 /* Macros to save and align the stack before making a function
3740  call and restore it afterwards as gcc may not keep the stack
3741  pointer aligned if it doesn't realize calls are being made
3742  to other functions. */
3743 
3744 /* This is a bit tricky. We store the original stack pointer in r10
3745  as it is callee-saves. gcc doesn't allow the use of r11 for some
3746  reason. Also, we can't directly "bic" the stack pointer in thumb
3747  mode since r13 isn't an allowed register number in that context.
3748  So use r4 as a temporary, since that is about to get trashed
3749  anyway, just after each use of this macro. Side effect is we need
3750  to be very careful about any future changes, since
3751  VALGRIND_ALIGN_STACK simply assumes r4 is usable. */
3752 #define VALGRIND_ALIGN_STACK \
3753  "mov r10, sp\n\t" \
3754  "mov r4, sp\n\t" \
3755  "bic r4, r4, #7\n\t" \
3756  "mov sp, r4\n\t"
3757 #define VALGRIND_RESTORE_STACK \
3758  "mov sp, r10\n\t"
3759 
3760 /* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned
3761  long) == 4. */
3762 
3763 #define CALL_FN_W_v(lval, orig) \
3764  do { \
3765  volatile OrigFn _orig = (orig); \
3766  volatile unsigned long _argvec[1]; \
3767  volatile unsigned long _res; \
3768  _argvec[0] = (unsigned long)_orig.nraddr; \
3769  __asm__ volatile( \
3770  VALGRIND_ALIGN_STACK \
3771  "ldr r4, [%1] \n\t" /* target->r4 */ \
3772  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
3773  VALGRIND_RESTORE_STACK \
3774  "mov %0, r0\n" \
3775  : /*out*/ "=r" (_res) \
3776  : /*in*/ "0" (&_argvec[0]) \
3777  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
3778  ); \
3779  lval = (__typeof__(lval)) _res; \
3780  } while (0)
3781 
3782 #define CALL_FN_W_W(lval, orig, arg1) \
3783  do { \
3784  volatile OrigFn _orig = (orig); \
3785  volatile unsigned long _argvec[2]; \
3786  volatile unsigned long _res; \
3787  _argvec[0] = (unsigned long)_orig.nraddr; \
3788  _argvec[1] = (unsigned long)(arg1); \
3789  __asm__ volatile( \
3790  VALGRIND_ALIGN_STACK \
3791  "ldr r0, [%1, #4] \n\t" \
3792  "ldr r4, [%1] \n\t" /* target->r4 */ \
3793  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
3794  VALGRIND_RESTORE_STACK \
3795  "mov %0, r0\n" \
3796  : /*out*/ "=r" (_res) \
3797  : /*in*/ "0" (&_argvec[0]) \
3798  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
3799  ); \
3800  lval = (__typeof__(lval)) _res; \
3801  } while (0)
3802 
3803 #define CALL_FN_W_WW(lval, orig, arg1,arg2) \
3804  do { \
3805  volatile OrigFn _orig = (orig); \
3806  volatile unsigned long _argvec[3]; \
3807  volatile unsigned long _res; \
3808  _argvec[0] = (unsigned long)_orig.nraddr; \
3809  _argvec[1] = (unsigned long)(arg1); \
3810  _argvec[2] = (unsigned long)(arg2); \
3811  __asm__ volatile( \
3812  VALGRIND_ALIGN_STACK \
3813  "ldr r0, [%1, #4] \n\t" \
3814  "ldr r1, [%1, #8] \n\t" \
3815  "ldr r4, [%1] \n\t" /* target->r4 */ \
3816  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
3817  VALGRIND_RESTORE_STACK \
3818  "mov %0, r0\n" \
3819  : /*out*/ "=r" (_res) \
3820  : /*in*/ "0" (&_argvec[0]) \
3821  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
3822  ); \
3823  lval = (__typeof__(lval)) _res; \
3824  } while (0)
3825 
3826 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
3827  do { \
3828  volatile OrigFn _orig = (orig); \
3829  volatile unsigned long _argvec[4]; \
3830  volatile unsigned long _res; \
3831  _argvec[0] = (unsigned long)_orig.nraddr; \
3832  _argvec[1] = (unsigned long)(arg1); \
3833  _argvec[2] = (unsigned long)(arg2); \
3834  _argvec[3] = (unsigned long)(arg3); \
3835  __asm__ volatile( \
3836  VALGRIND_ALIGN_STACK \
3837  "ldr r0, [%1, #4] \n\t" \
3838  "ldr r1, [%1, #8] \n\t" \
3839  "ldr r2, [%1, #12] \n\t" \
3840  "ldr r4, [%1] \n\t" /* target->r4 */ \
3841  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
3842  VALGRIND_RESTORE_STACK \
3843  "mov %0, r0\n" \
3844  : /*out*/ "=r" (_res) \
3845  : /*in*/ "0" (&_argvec[0]) \
3846  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
3847  ); \
3848  lval = (__typeof__(lval)) _res; \
3849  } while (0)
3850 
3851 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
3852  do { \
3853  volatile OrigFn _orig = (orig); \
3854  volatile unsigned long _argvec[5]; \
3855  volatile unsigned long _res; \
3856  _argvec[0] = (unsigned long)_orig.nraddr; \
3857  _argvec[1] = (unsigned long)(arg1); \
3858  _argvec[2] = (unsigned long)(arg2); \
3859  _argvec[3] = (unsigned long)(arg3); \
3860  _argvec[4] = (unsigned long)(arg4); \
3861  __asm__ volatile( \
3862  VALGRIND_ALIGN_STACK \
3863  "ldr r0, [%1, #4] \n\t" \
3864  "ldr r1, [%1, #8] \n\t" \
3865  "ldr r2, [%1, #12] \n\t" \
3866  "ldr r3, [%1, #16] \n\t" \
3867  "ldr r4, [%1] \n\t" /* target->r4 */ \
3868  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
3869  VALGRIND_RESTORE_STACK \
3870  "mov %0, r0" \
3871  : /*out*/ "=r" (_res) \
3872  : /*in*/ "0" (&_argvec[0]) \
3873  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
3874  ); \
3875  lval = (__typeof__(lval)) _res; \
3876  } while (0)
3877 
3878 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
3879  do { \
3880  volatile OrigFn _orig = (orig); \
3881  volatile unsigned long _argvec[6]; \
3882  volatile unsigned long _res; \
3883  _argvec[0] = (unsigned long)_orig.nraddr; \
3884  _argvec[1] = (unsigned long)(arg1); \
3885  _argvec[2] = (unsigned long)(arg2); \
3886  _argvec[3] = (unsigned long)(arg3); \
3887  _argvec[4] = (unsigned long)(arg4); \
3888  _argvec[5] = (unsigned long)(arg5); \
3889  __asm__ volatile( \
3890  VALGRIND_ALIGN_STACK \
3891  "sub sp, sp, #4 \n\t" \
3892  "ldr r0, [%1, #20] \n\t" \
3893  "push {r0} \n\t" \
3894  "ldr r0, [%1, #4] \n\t" \
3895  "ldr r1, [%1, #8] \n\t" \
3896  "ldr r2, [%1, #12] \n\t" \
3897  "ldr r3, [%1, #16] \n\t" \
3898  "ldr r4, [%1] \n\t" /* target->r4 */ \
3899  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
3900  VALGRIND_RESTORE_STACK \
3901  "mov %0, r0" \
3902  : /*out*/ "=r" (_res) \
3903  : /*in*/ "0" (&_argvec[0]) \
3904  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
3905  ); \
3906  lval = (__typeof__(lval)) _res; \
3907  } while (0)
3908 
3909 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
3910  do { \
3911  volatile OrigFn _orig = (orig); \
3912  volatile unsigned long _argvec[7]; \
3913  volatile unsigned long _res; \
3914  _argvec[0] = (unsigned long)_orig.nraddr; \
3915  _argvec[1] = (unsigned long)(arg1); \
3916  _argvec[2] = (unsigned long)(arg2); \
3917  _argvec[3] = (unsigned long)(arg3); \
3918  _argvec[4] = (unsigned long)(arg4); \
3919  _argvec[5] = (unsigned long)(arg5); \
3920  _argvec[6] = (unsigned long)(arg6); \
3921  __asm__ volatile( \
3922  VALGRIND_ALIGN_STACK \
3923  "ldr r0, [%1, #20] \n\t" \
3924  "ldr r1, [%1, #24] \n\t" \
3925  "push {r0, r1} \n\t" \
3926  "ldr r0, [%1, #4] \n\t" \
3927  "ldr r1, [%1, #8] \n\t" \
3928  "ldr r2, [%1, #12] \n\t" \
3929  "ldr r3, [%1, #16] \n\t" \
3930  "ldr r4, [%1] \n\t" /* target->r4 */ \
3931  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
3932  VALGRIND_RESTORE_STACK \
3933  "mov %0, r0" \
3934  : /*out*/ "=r" (_res) \
3935  : /*in*/ "0" (&_argvec[0]) \
3936  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
3937  ); \
3938  lval = (__typeof__(lval)) _res; \
3939  } while (0)
3940 
3941 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3942  arg7) \
3943  do { \
3944  volatile OrigFn _orig = (orig); \
3945  volatile unsigned long _argvec[8]; \
3946  volatile unsigned long _res; \
3947  _argvec[0] = (unsigned long)_orig.nraddr; \
3948  _argvec[1] = (unsigned long)(arg1); \
3949  _argvec[2] = (unsigned long)(arg2); \
3950  _argvec[3] = (unsigned long)(arg3); \
3951  _argvec[4] = (unsigned long)(arg4); \
3952  _argvec[5] = (unsigned long)(arg5); \
3953  _argvec[6] = (unsigned long)(arg6); \
3954  _argvec[7] = (unsigned long)(arg7); \
3955  __asm__ volatile( \
3956  VALGRIND_ALIGN_STACK \
3957  "sub sp, sp, #4 \n\t" \
3958  "ldr r0, [%1, #20] \n\t" \
3959  "ldr r1, [%1, #24] \n\t" \
3960  "ldr r2, [%1, #28] \n\t" \
3961  "push {r0, r1, r2} \n\t" \
3962  "ldr r0, [%1, #4] \n\t" \
3963  "ldr r1, [%1, #8] \n\t" \
3964  "ldr r2, [%1, #12] \n\t" \
3965  "ldr r3, [%1, #16] \n\t" \
3966  "ldr r4, [%1] \n\t" /* target->r4 */ \
3967  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
3968  VALGRIND_RESTORE_STACK \
3969  "mov %0, r0" \
3970  : /*out*/ "=r" (_res) \
3971  : /*in*/ "0" (&_argvec[0]) \
3972  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
3973  ); \
3974  lval = (__typeof__(lval)) _res; \
3975  } while (0)
3976 
3977 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
3978  arg7,arg8) \
3979  do { \
3980  volatile OrigFn _orig = (orig); \
3981  volatile unsigned long _argvec[9]; \
3982  volatile unsigned long _res; \
3983  _argvec[0] = (unsigned long)_orig.nraddr; \
3984  _argvec[1] = (unsigned long)(arg1); \
3985  _argvec[2] = (unsigned long)(arg2); \
3986  _argvec[3] = (unsigned long)(arg3); \
3987  _argvec[4] = (unsigned long)(arg4); \
3988  _argvec[5] = (unsigned long)(arg5); \
3989  _argvec[6] = (unsigned long)(arg6); \
3990  _argvec[7] = (unsigned long)(arg7); \
3991  _argvec[8] = (unsigned long)(arg8); \
3992  __asm__ volatile( \
3993  VALGRIND_ALIGN_STACK \
3994  "ldr r0, [%1, #20] \n\t" \
3995  "ldr r1, [%1, #24] \n\t" \
3996  "ldr r2, [%1, #28] \n\t" \
3997  "ldr r3, [%1, #32] \n\t" \
3998  "push {r0, r1, r2, r3} \n\t" \
3999  "ldr r0, [%1, #4] \n\t" \
4000  "ldr r1, [%1, #8] \n\t" \
4001  "ldr r2, [%1, #12] \n\t" \
4002  "ldr r3, [%1, #16] \n\t" \
4003  "ldr r4, [%1] \n\t" /* target->r4 */ \
4004  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
4005  VALGRIND_RESTORE_STACK \
4006  "mov %0, r0" \
4007  : /*out*/ "=r" (_res) \
4008  : /*in*/ "0" (&_argvec[0]) \
4009  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
4010  ); \
4011  lval = (__typeof__(lval)) _res; \
4012  } while (0)
4013 
4014 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
4015  arg7,arg8,arg9) \
4016  do { \
4017  volatile OrigFn _orig = (orig); \
4018  volatile unsigned long _argvec[10]; \
4019  volatile unsigned long _res; \
4020  _argvec[0] = (unsigned long)_orig.nraddr; \
4021  _argvec[1] = (unsigned long)(arg1); \
4022  _argvec[2] = (unsigned long)(arg2); \
4023  _argvec[3] = (unsigned long)(arg3); \
4024  _argvec[4] = (unsigned long)(arg4); \
4025  _argvec[5] = (unsigned long)(arg5); \
4026  _argvec[6] = (unsigned long)(arg6); \
4027  _argvec[7] = (unsigned long)(arg7); \
4028  _argvec[8] = (unsigned long)(arg8); \
4029  _argvec[9] = (unsigned long)(arg9); \
4030  __asm__ volatile( \
4031  VALGRIND_ALIGN_STACK \
4032  "sub sp, sp, #4 \n\t" \
4033  "ldr r0, [%1, #20] \n\t" \
4034  "ldr r1, [%1, #24] \n\t" \
4035  "ldr r2, [%1, #28] \n\t" \
4036  "ldr r3, [%1, #32] \n\t" \
4037  "ldr r4, [%1, #36] \n\t" \
4038  "push {r0, r1, r2, r3, r4} \n\t" \
4039  "ldr r0, [%1, #4] \n\t" \
4040  "ldr r1, [%1, #8] \n\t" \
4041  "ldr r2, [%1, #12] \n\t" \
4042  "ldr r3, [%1, #16] \n\t" \
4043  "ldr r4, [%1] \n\t" /* target->r4 */ \
4044  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
4045  VALGRIND_RESTORE_STACK \
4046  "mov %0, r0" \
4047  : /*out*/ "=r" (_res) \
4048  : /*in*/ "0" (&_argvec[0]) \
4049  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
4050  ); \
4051  lval = (__typeof__(lval)) _res; \
4052  } while (0)
4053 
4054 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
4055  arg7,arg8,arg9,arg10) \
4056  do { \
4057  volatile OrigFn _orig = (orig); \
4058  volatile unsigned long _argvec[11]; \
4059  volatile unsigned long _res; \
4060  _argvec[0] = (unsigned long)_orig.nraddr; \
4061  _argvec[1] = (unsigned long)(arg1); \
4062  _argvec[2] = (unsigned long)(arg2); \
4063  _argvec[3] = (unsigned long)(arg3); \
4064  _argvec[4] = (unsigned long)(arg4); \
4065  _argvec[5] = (unsigned long)(arg5); \
4066  _argvec[6] = (unsigned long)(arg6); \
4067  _argvec[7] = (unsigned long)(arg7); \
4068  _argvec[8] = (unsigned long)(arg8); \
4069  _argvec[9] = (unsigned long)(arg9); \
4070  _argvec[10] = (unsigned long)(arg10); \
4071  __asm__ volatile( \
4072  VALGRIND_ALIGN_STACK \
4073  "ldr r0, [%1, #40] \n\t" \
4074  "push {r0} \n\t" \
4075  "ldr r0, [%1, #20] \n\t" \
4076  "ldr r1, [%1, #24] \n\t" \
4077  "ldr r2, [%1, #28] \n\t" \
4078  "ldr r3, [%1, #32] \n\t" \
4079  "ldr r4, [%1, #36] \n\t" \
4080  "push {r0, r1, r2, r3, r4} \n\t" \
4081  "ldr r0, [%1, #4] \n\t" \
4082  "ldr r1, [%1, #8] \n\t" \
4083  "ldr r2, [%1, #12] \n\t" \
4084  "ldr r3, [%1, #16] \n\t" \
4085  "ldr r4, [%1] \n\t" /* target->r4 */ \
4086  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
4087  VALGRIND_RESTORE_STACK \
4088  "mov %0, r0" \
4089  : /*out*/ "=r" (_res) \
4090  : /*in*/ "0" (&_argvec[0]) \
4091  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
4092  ); \
4093  lval = (__typeof__(lval)) _res; \
4094  } while (0)
4095 
4096 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \
4097  arg6,arg7,arg8,arg9,arg10, \
4098  arg11) \
4099  do { \
4100  volatile OrigFn _orig = (orig); \
4101  volatile unsigned long _argvec[12]; \
4102  volatile unsigned long _res; \
4103  _argvec[0] = (unsigned long)_orig.nraddr; \
4104  _argvec[1] = (unsigned long)(arg1); \
4105  _argvec[2] = (unsigned long)(arg2); \
4106  _argvec[3] = (unsigned long)(arg3); \
4107  _argvec[4] = (unsigned long)(arg4); \
4108  _argvec[5] = (unsigned long)(arg5); \
4109  _argvec[6] = (unsigned long)(arg6); \
4110  _argvec[7] = (unsigned long)(arg7); \
4111  _argvec[8] = (unsigned long)(arg8); \
4112  _argvec[9] = (unsigned long)(arg9); \
4113  _argvec[10] = (unsigned long)(arg10); \
4114  _argvec[11] = (unsigned long)(arg11); \
4115  __asm__ volatile( \
4116  VALGRIND_ALIGN_STACK \
4117  "sub sp, sp, #4 \n\t" \
4118  "ldr r0, [%1, #40] \n\t" \
4119  "ldr r1, [%1, #44] \n\t" \
4120  "push {r0, r1} \n\t" \
4121  "ldr r0, [%1, #20] \n\t" \
4122  "ldr r1, [%1, #24] \n\t" \
4123  "ldr r2, [%1, #28] \n\t" \
4124  "ldr r3, [%1, #32] \n\t" \
4125  "ldr r4, [%1, #36] \n\t" \
4126  "push {r0, r1, r2, r3, r4} \n\t" \
4127  "ldr r0, [%1, #4] \n\t" \
4128  "ldr r1, [%1, #8] \n\t" \
4129  "ldr r2, [%1, #12] \n\t" \
4130  "ldr r3, [%1, #16] \n\t" \
4131  "ldr r4, [%1] \n\t" /* target->r4 */ \
4132  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
4133  VALGRIND_RESTORE_STACK \
4134  "mov %0, r0" \
4135  : /*out*/ "=r" (_res) \
4136  : /*in*/ "0" (&_argvec[0]) \
4137  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
4138  ); \
4139  lval = (__typeof__(lval)) _res; \
4140  } while (0)
4141 
4142 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \
4143  arg6,arg7,arg8,arg9,arg10, \
4144  arg11,arg12) \
4145  do { \
4146  volatile OrigFn _orig = (orig); \
4147  volatile unsigned long _argvec[13]; \
4148  volatile unsigned long _res; \
4149  _argvec[0] = (unsigned long)_orig.nraddr; \
4150  _argvec[1] = (unsigned long)(arg1); \
4151  _argvec[2] = (unsigned long)(arg2); \
4152  _argvec[3] = (unsigned long)(arg3); \
4153  _argvec[4] = (unsigned long)(arg4); \
4154  _argvec[5] = (unsigned long)(arg5); \
4155  _argvec[6] = (unsigned long)(arg6); \
4156  _argvec[7] = (unsigned long)(arg7); \
4157  _argvec[8] = (unsigned long)(arg8); \
4158  _argvec[9] = (unsigned long)(arg9); \
4159  _argvec[10] = (unsigned long)(arg10); \
4160  _argvec[11] = (unsigned long)(arg11); \
4161  _argvec[12] = (unsigned long)(arg12); \
4162  __asm__ volatile( \
4163  VALGRIND_ALIGN_STACK \
4164  "ldr r0, [%1, #40] \n\t" \
4165  "ldr r1, [%1, #44] \n\t" \
4166  "ldr r2, [%1, #48] \n\t" \
4167  "push {r0, r1, r2} \n\t" \
4168  "ldr r0, [%1, #20] \n\t" \
4169  "ldr r1, [%1, #24] \n\t" \
4170  "ldr r2, [%1, #28] \n\t" \
4171  "ldr r3, [%1, #32] \n\t" \
4172  "ldr r4, [%1, #36] \n\t" \
4173  "push {r0, r1, r2, r3, r4} \n\t" \
4174  "ldr r0, [%1, #4] \n\t" \
4175  "ldr r1, [%1, #8] \n\t" \
4176  "ldr r2, [%1, #12] \n\t" \
4177  "ldr r3, [%1, #16] \n\t" \
4178  "ldr r4, [%1] \n\t" /* target->r4 */ \
4179  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \
4180  VALGRIND_RESTORE_STACK \
4181  "mov %0, r0" \
4182  : /*out*/ "=r" (_res) \
4183  : /*in*/ "0" (&_argvec[0]) \
4184  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \
4185  ); \
4186  lval = (__typeof__(lval)) _res; \
4187  } while (0)
4188 
4189 #endif /* PLAT_arm_linux */
4190 
4191 /* ------------------------ arm64-linux ------------------------ */
4192 
4193 #if defined(PLAT_arm64_linux)
4194 
4195 /* These regs are trashed by the hidden call. */
4196 #define __CALLER_SAVED_REGS \
4197  "x0", "x1", "x2", "x3","x4", "x5", "x6", "x7", "x8", "x9", \
4198  "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", \
4199  "x18", "x19", "x20", "x30", \
4200  "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", \
4201  "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", \
4202  "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", \
4203  "v26", "v27", "v28", "v29", "v30", "v31"
4204 
4205 /* x21 is callee-saved, so we can use it to save and restore SP around
4206  the hidden call. */
4207 #define VALGRIND_ALIGN_STACK \
4208  "mov x21, sp\n\t" \
4209  "bic sp, x21, #15\n\t"
4210 #define VALGRIND_RESTORE_STACK \
4211  "mov sp, x21\n\t"
4212 
4213 /* These CALL_FN_ macros assume that on arm64-linux,
4214  sizeof(unsigned long) == 8. */
4215 
4216 #define CALL_FN_W_v(lval, orig) \
4217  do { \
4218  volatile OrigFn _orig = (orig); \
4219  volatile unsigned long _argvec[1]; \
4220  volatile unsigned long _res; \
4221  _argvec[0] = (unsigned long)_orig.nraddr; \
4222  __asm__ volatile( \
4223  VALGRIND_ALIGN_STACK \
4224  "ldr x8, [%1] \n\t" /* target->x8 */ \
4225  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4226  VALGRIND_RESTORE_STACK \
4227  "mov %0, x0\n" \
4228  : /*out*/ "=r" (_res) \
4229  : /*in*/ "0" (&_argvec[0]) \
4230  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4231  ); \
4232  lval = (__typeof__(lval)) _res; \
4233  } while (0)
4234 
4235 #define CALL_FN_W_W(lval, orig, arg1) \
4236  do { \
4237  volatile OrigFn _orig = (orig); \
4238  volatile unsigned long _argvec[2]; \
4239  volatile unsigned long _res; \
4240  _argvec[0] = (unsigned long)_orig.nraddr; \
4241  _argvec[1] = (unsigned long)(arg1); \
4242  __asm__ volatile( \
4243  VALGRIND_ALIGN_STACK \
4244  "ldr x0, [%1, #8] \n\t" \
4245  "ldr x8, [%1] \n\t" /* target->x8 */ \
4246  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4247  VALGRIND_RESTORE_STACK \
4248  "mov %0, x0\n" \
4249  : /*out*/ "=r" (_res) \
4250  : /*in*/ "0" (&_argvec[0]) \
4251  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4252  ); \
4253  lval = (__typeof__(lval)) _res; \
4254  } while (0)
4255 
4256 #define CALL_FN_W_WW(lval, orig, arg1,arg2) \
4257  do { \
4258  volatile OrigFn _orig = (orig); \
4259  volatile unsigned long _argvec[3]; \
4260  volatile unsigned long _res; \
4261  _argvec[0] = (unsigned long)_orig.nraddr; \
4262  _argvec[1] = (unsigned long)(arg1); \
4263  _argvec[2] = (unsigned long)(arg2); \
4264  __asm__ volatile( \
4265  VALGRIND_ALIGN_STACK \
4266  "ldr x0, [%1, #8] \n\t" \
4267  "ldr x1, [%1, #16] \n\t" \
4268  "ldr x8, [%1] \n\t" /* target->x8 */ \
4269  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4270  VALGRIND_RESTORE_STACK \
4271  "mov %0, x0\n" \
4272  : /*out*/ "=r" (_res) \
4273  : /*in*/ "0" (&_argvec[0]) \
4274  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4275  ); \
4276  lval = (__typeof__(lval)) _res; \
4277  } while (0)
4278 
4279 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
4280  do { \
4281  volatile OrigFn _orig = (orig); \
4282  volatile unsigned long _argvec[4]; \
4283  volatile unsigned long _res; \
4284  _argvec[0] = (unsigned long)_orig.nraddr; \
4285  _argvec[1] = (unsigned long)(arg1); \
4286  _argvec[2] = (unsigned long)(arg2); \
4287  _argvec[3] = (unsigned long)(arg3); \
4288  __asm__ volatile( \
4289  VALGRIND_ALIGN_STACK \
4290  "ldr x0, [%1, #8] \n\t" \
4291  "ldr x1, [%1, #16] \n\t" \
4292  "ldr x2, [%1, #24] \n\t" \
4293  "ldr x8, [%1] \n\t" /* target->x8 */ \
4294  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4295  VALGRIND_RESTORE_STACK \
4296  "mov %0, x0\n" \
4297  : /*out*/ "=r" (_res) \
4298  : /*in*/ "0" (&_argvec[0]) \
4299  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4300  ); \
4301  lval = (__typeof__(lval)) _res; \
4302  } while (0)
4303 
4304 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
4305  do { \
4306  volatile OrigFn _orig = (orig); \
4307  volatile unsigned long _argvec[5]; \
4308  volatile unsigned long _res; \
4309  _argvec[0] = (unsigned long)_orig.nraddr; \
4310  _argvec[1] = (unsigned long)(arg1); \
4311  _argvec[2] = (unsigned long)(arg2); \
4312  _argvec[3] = (unsigned long)(arg3); \
4313  _argvec[4] = (unsigned long)(arg4); \
4314  __asm__ volatile( \
4315  VALGRIND_ALIGN_STACK \
4316  "ldr x0, [%1, #8] \n\t" \
4317  "ldr x1, [%1, #16] \n\t" \
4318  "ldr x2, [%1, #24] \n\t" \
4319  "ldr x3, [%1, #32] \n\t" \
4320  "ldr x8, [%1] \n\t" /* target->x8 */ \
4321  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4322  VALGRIND_RESTORE_STACK \
4323  "mov %0, x0" \
4324  : /*out*/ "=r" (_res) \
4325  : /*in*/ "0" (&_argvec[0]) \
4326  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4327  ); \
4328  lval = (__typeof__(lval)) _res; \
4329  } while (0)
4330 
4331 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
4332  do { \
4333  volatile OrigFn _orig = (orig); \
4334  volatile unsigned long _argvec[6]; \
4335  volatile unsigned long _res; \
4336  _argvec[0] = (unsigned long)_orig.nraddr; \
4337  _argvec[1] = (unsigned long)(arg1); \
4338  _argvec[2] = (unsigned long)(arg2); \
4339  _argvec[3] = (unsigned long)(arg3); \
4340  _argvec[4] = (unsigned long)(arg4); \
4341  _argvec[5] = (unsigned long)(arg5); \
4342  __asm__ volatile( \
4343  VALGRIND_ALIGN_STACK \
4344  "ldr x0, [%1, #8] \n\t" \
4345  "ldr x1, [%1, #16] \n\t" \
4346  "ldr x2, [%1, #24] \n\t" \
4347  "ldr x3, [%1, #32] \n\t" \
4348  "ldr x4, [%1, #40] \n\t" \
4349  "ldr x8, [%1] \n\t" /* target->x8 */ \
4350  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4351  VALGRIND_RESTORE_STACK \
4352  "mov %0, x0" \
4353  : /*out*/ "=r" (_res) \
4354  : /*in*/ "0" (&_argvec[0]) \
4355  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4356  ); \
4357  lval = (__typeof__(lval)) _res; \
4358  } while (0)
4359 
4360 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
4361  do { \
4362  volatile OrigFn _orig = (orig); \
4363  volatile unsigned long _argvec[7]; \
4364  volatile unsigned long _res; \
4365  _argvec[0] = (unsigned long)_orig.nraddr; \
4366  _argvec[1] = (unsigned long)(arg1); \
4367  _argvec[2] = (unsigned long)(arg2); \
4368  _argvec[3] = (unsigned long)(arg3); \
4369  _argvec[4] = (unsigned long)(arg4); \
4370  _argvec[5] = (unsigned long)(arg5); \
4371  _argvec[6] = (unsigned long)(arg6); \
4372  __asm__ volatile( \
4373  VALGRIND_ALIGN_STACK \
4374  "ldr x0, [%1, #8] \n\t" \
4375  "ldr x1, [%1, #16] \n\t" \
4376  "ldr x2, [%1, #24] \n\t" \
4377  "ldr x3, [%1, #32] \n\t" \
4378  "ldr x4, [%1, #40] \n\t" \
4379  "ldr x5, [%1, #48] \n\t" \
4380  "ldr x8, [%1] \n\t" /* target->x8 */ \
4381  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4382  VALGRIND_RESTORE_STACK \
4383  "mov %0, x0" \
4384  : /*out*/ "=r" (_res) \
4385  : /*in*/ "0" (&_argvec[0]) \
4386  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4387  ); \
4388  lval = (__typeof__(lval)) _res; \
4389  } while (0)
4390 
4391 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
4392  arg7) \
4393  do { \
4394  volatile OrigFn _orig = (orig); \
4395  volatile unsigned long _argvec[8]; \
4396  volatile unsigned long _res; \
4397  _argvec[0] = (unsigned long)_orig.nraddr; \
4398  _argvec[1] = (unsigned long)(arg1); \
4399  _argvec[2] = (unsigned long)(arg2); \
4400  _argvec[3] = (unsigned long)(arg3); \
4401  _argvec[4] = (unsigned long)(arg4); \
4402  _argvec[5] = (unsigned long)(arg5); \
4403  _argvec[6] = (unsigned long)(arg6); \
4404  _argvec[7] = (unsigned long)(arg7); \
4405  __asm__ volatile( \
4406  VALGRIND_ALIGN_STACK \
4407  "ldr x0, [%1, #8] \n\t" \
4408  "ldr x1, [%1, #16] \n\t" \
4409  "ldr x2, [%1, #24] \n\t" \
4410  "ldr x3, [%1, #32] \n\t" \
4411  "ldr x4, [%1, #40] \n\t" \
4412  "ldr x5, [%1, #48] \n\t" \
4413  "ldr x6, [%1, #56] \n\t" \
4414  "ldr x8, [%1] \n\t" /* target->x8 */ \
4415  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4416  VALGRIND_RESTORE_STACK \
4417  "mov %0, x0" \
4418  : /*out*/ "=r" (_res) \
4419  : /*in*/ "0" (&_argvec[0]) \
4420  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4421  ); \
4422  lval = (__typeof__(lval)) _res; \
4423  } while (0)
4424 
4425 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
4426  arg7,arg8) \
4427  do { \
4428  volatile OrigFn _orig = (orig); \
4429  volatile unsigned long _argvec[9]; \
4430  volatile unsigned long _res; \
4431  _argvec[0] = (unsigned long)_orig.nraddr; \
4432  _argvec[1] = (unsigned long)(arg1); \
4433  _argvec[2] = (unsigned long)(arg2); \
4434  _argvec[3] = (unsigned long)(arg3); \
4435  _argvec[4] = (unsigned long)(arg4); \
4436  _argvec[5] = (unsigned long)(arg5); \
4437  _argvec[6] = (unsigned long)(arg6); \
4438  _argvec[7] = (unsigned long)(arg7); \
4439  _argvec[8] = (unsigned long)(arg8); \
4440  __asm__ volatile( \
4441  VALGRIND_ALIGN_STACK \
4442  "ldr x0, [%1, #8] \n\t" \
4443  "ldr x1, [%1, #16] \n\t" \
4444  "ldr x2, [%1, #24] \n\t" \
4445  "ldr x3, [%1, #32] \n\t" \
4446  "ldr x4, [%1, #40] \n\t" \
4447  "ldr x5, [%1, #48] \n\t" \
4448  "ldr x6, [%1, #56] \n\t" \
4449  "ldr x7, [%1, #64] \n\t" \
4450  "ldr x8, [%1] \n\t" /* target->x8 */ \
4451  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4452  VALGRIND_RESTORE_STACK \
4453  "mov %0, x0" \
4454  : /*out*/ "=r" (_res) \
4455  : /*in*/ "0" (&_argvec[0]) \
4456  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4457  ); \
4458  lval = (__typeof__(lval)) _res; \
4459  } while (0)
4460 
4461 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
4462  arg7,arg8,arg9) \
4463  do { \
4464  volatile OrigFn _orig = (orig); \
4465  volatile unsigned long _argvec[10]; \
4466  volatile unsigned long _res; \
4467  _argvec[0] = (unsigned long)_orig.nraddr; \
4468  _argvec[1] = (unsigned long)(arg1); \
4469  _argvec[2] = (unsigned long)(arg2); \
4470  _argvec[3] = (unsigned long)(arg3); \
4471  _argvec[4] = (unsigned long)(arg4); \
4472  _argvec[5] = (unsigned long)(arg5); \
4473  _argvec[6] = (unsigned long)(arg6); \
4474  _argvec[7] = (unsigned long)(arg7); \
4475  _argvec[8] = (unsigned long)(arg8); \
4476  _argvec[9] = (unsigned long)(arg9); \
4477  __asm__ volatile( \
4478  VALGRIND_ALIGN_STACK \
4479  "sub sp, sp, #0x20 \n\t" \
4480  "ldr x0, [%1, #8] \n\t" \
4481  "ldr x1, [%1, #16] \n\t" \
4482  "ldr x2, [%1, #24] \n\t" \
4483  "ldr x3, [%1, #32] \n\t" \
4484  "ldr x4, [%1, #40] \n\t" \
4485  "ldr x5, [%1, #48] \n\t" \
4486  "ldr x6, [%1, #56] \n\t" \
4487  "ldr x7, [%1, #64] \n\t" \
4488  "ldr x8, [%1, #72] \n\t" \
4489  "str x8, [sp, #0] \n\t" \
4490  "ldr x8, [%1] \n\t" /* target->x8 */ \
4491  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4492  VALGRIND_RESTORE_STACK \
4493  "mov %0, x0" \
4494  : /*out*/ "=r" (_res) \
4495  : /*in*/ "0" (&_argvec[0]) \
4496  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4497  ); \
4498  lval = (__typeof__(lval)) _res; \
4499  } while (0)
4500 
4501 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
4502  arg7,arg8,arg9,arg10) \
4503  do { \
4504  volatile OrigFn _orig = (orig); \
4505  volatile unsigned long _argvec[11]; \
4506  volatile unsigned long _res; \
4507  _argvec[0] = (unsigned long)_orig.nraddr; \
4508  _argvec[1] = (unsigned long)(arg1); \
4509  _argvec[2] = (unsigned long)(arg2); \
4510  _argvec[3] = (unsigned long)(arg3); \
4511  _argvec[4] = (unsigned long)(arg4); \
4512  _argvec[5] = (unsigned long)(arg5); \
4513  _argvec[6] = (unsigned long)(arg6); \
4514  _argvec[7] = (unsigned long)(arg7); \
4515  _argvec[8] = (unsigned long)(arg8); \
4516  _argvec[9] = (unsigned long)(arg9); \
4517  _argvec[10] = (unsigned long)(arg10); \
4518  __asm__ volatile( \
4519  VALGRIND_ALIGN_STACK \
4520  "sub sp, sp, #0x20 \n\t" \
4521  "ldr x0, [%1, #8] \n\t" \
4522  "ldr x1, [%1, #16] \n\t" \
4523  "ldr x2, [%1, #24] \n\t" \
4524  "ldr x3, [%1, #32] \n\t" \
4525  "ldr x4, [%1, #40] \n\t" \
4526  "ldr x5, [%1, #48] \n\t" \
4527  "ldr x6, [%1, #56] \n\t" \
4528  "ldr x7, [%1, #64] \n\t" \
4529  "ldr x8, [%1, #72] \n\t" \
4530  "str x8, [sp, #0] \n\t" \
4531  "ldr x8, [%1, #80] \n\t" \
4532  "str x8, [sp, #8] \n\t" \
4533  "ldr x8, [%1] \n\t" /* target->x8 */ \
4534  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4535  VALGRIND_RESTORE_STACK \
4536  "mov %0, x0" \
4537  : /*out*/ "=r" (_res) \
4538  : /*in*/ "0" (&_argvec[0]) \
4539  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4540  ); \
4541  lval = (__typeof__(lval)) _res; \
4542  } while (0)
4543 
4544 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
4545  arg7,arg8,arg9,arg10,arg11) \
4546  do { \
4547  volatile OrigFn _orig = (orig); \
4548  volatile unsigned long _argvec[12]; \
4549  volatile unsigned long _res; \
4550  _argvec[0] = (unsigned long)_orig.nraddr; \
4551  _argvec[1] = (unsigned long)(arg1); \
4552  _argvec[2] = (unsigned long)(arg2); \
4553  _argvec[3] = (unsigned long)(arg3); \
4554  _argvec[4] = (unsigned long)(arg4); \
4555  _argvec[5] = (unsigned long)(arg5); \
4556  _argvec[6] = (unsigned long)(arg6); \
4557  _argvec[7] = (unsigned long)(arg7); \
4558  _argvec[8] = (unsigned long)(arg8); \
4559  _argvec[9] = (unsigned long)(arg9); \
4560  _argvec[10] = (unsigned long)(arg10); \
4561  _argvec[11] = (unsigned long)(arg11); \
4562  __asm__ volatile( \
4563  VALGRIND_ALIGN_STACK \
4564  "sub sp, sp, #0x30 \n\t" \
4565  "ldr x0, [%1, #8] \n\t" \
4566  "ldr x1, [%1, #16] \n\t" \
4567  "ldr x2, [%1, #24] \n\t" \
4568  "ldr x3, [%1, #32] \n\t" \
4569  "ldr x4, [%1, #40] \n\t" \
4570  "ldr x5, [%1, #48] \n\t" \
4571  "ldr x6, [%1, #56] \n\t" \
4572  "ldr x7, [%1, #64] \n\t" \
4573  "ldr x8, [%1, #72] \n\t" \
4574  "str x8, [sp, #0] \n\t" \
4575  "ldr x8, [%1, #80] \n\t" \
4576  "str x8, [sp, #8] \n\t" \
4577  "ldr x8, [%1, #88] \n\t" \
4578  "str x8, [sp, #16] \n\t" \
4579  "ldr x8, [%1] \n\t" /* target->x8 */ \
4580  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4581  VALGRIND_RESTORE_STACK \
4582  "mov %0, x0" \
4583  : /*out*/ "=r" (_res) \
4584  : /*in*/ "0" (&_argvec[0]) \
4585  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4586  ); \
4587  lval = (__typeof__(lval)) _res; \
4588  } while (0)
4589 
4590 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
4591  arg7,arg8,arg9,arg10,arg11, \
4592  arg12) \
4593  do { \
4594  volatile OrigFn _orig = (orig); \
4595  volatile unsigned long _argvec[13]; \
4596  volatile unsigned long _res; \
4597  _argvec[0] = (unsigned long)_orig.nraddr; \
4598  _argvec[1] = (unsigned long)(arg1); \
4599  _argvec[2] = (unsigned long)(arg2); \
4600  _argvec[3] = (unsigned long)(arg3); \
4601  _argvec[4] = (unsigned long)(arg4); \
4602  _argvec[5] = (unsigned long)(arg5); \
4603  _argvec[6] = (unsigned long)(arg6); \
4604  _argvec[7] = (unsigned long)(arg7); \
4605  _argvec[8] = (unsigned long)(arg8); \
4606  _argvec[9] = (unsigned long)(arg9); \
4607  _argvec[10] = (unsigned long)(arg10); \
4608  _argvec[11] = (unsigned long)(arg11); \
4609  _argvec[12] = (unsigned long)(arg12); \
4610  __asm__ volatile( \
4611  VALGRIND_ALIGN_STACK \
4612  "sub sp, sp, #0x30 \n\t" \
4613  "ldr x0, [%1, #8] \n\t" \
4614  "ldr x1, [%1, #16] \n\t" \
4615  "ldr x2, [%1, #24] \n\t" \
4616  "ldr x3, [%1, #32] \n\t" \
4617  "ldr x4, [%1, #40] \n\t" \
4618  "ldr x5, [%1, #48] \n\t" \
4619  "ldr x6, [%1, #56] \n\t" \
4620  "ldr x7, [%1, #64] \n\t" \
4621  "ldr x8, [%1, #72] \n\t" \
4622  "str x8, [sp, #0] \n\t" \
4623  "ldr x8, [%1, #80] \n\t" \
4624  "str x8, [sp, #8] \n\t" \
4625  "ldr x8, [%1, #88] \n\t" \
4626  "str x8, [sp, #16] \n\t" \
4627  "ldr x8, [%1, #96] \n\t" \
4628  "str x8, [sp, #24] \n\t" \
4629  "ldr x8, [%1] \n\t" /* target->x8 */ \
4630  VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \
4631  VALGRIND_RESTORE_STACK \
4632  "mov %0, x0" \
4633  : /*out*/ "=r" (_res) \
4634  : /*in*/ "0" (&_argvec[0]) \
4635  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \
4636  ); \
4637  lval = (__typeof__(lval)) _res; \
4638  } while (0)
4639 
4640 #endif /* PLAT_arm64_linux */
4641 
4642 /* ------------------------- s390x-linux ------------------------- */
4643 
4644 #if defined(PLAT_s390x_linux)
4645 
4646 /* Similar workaround as amd64 (see above), but we use r11 as frame
4647  pointer and save the old r11 in r7. r11 might be used for
4648  argvec, therefore we copy argvec in r1 since r1 is clobbered
4649  after the call anyway. */
4650 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
4651 # define __FRAME_POINTER \
4652  ,"d"(__builtin_dwarf_cfa())
4653 # define VALGRIND_CFI_PROLOGUE \
4654  ".cfi_remember_state\n\t" \
4655  "lgr 1,%1\n\t" /* copy the argvec pointer in r1 */ \
4656  "lgr 7,11\n\t" \
4657  "lgr 11,%2\n\t" \
4658  ".cfi_def_cfa r11, 0\n\t"
4659 # define VALGRIND_CFI_EPILOGUE \
4660  "lgr 11, 7\n\t" \
4661  ".cfi_restore_state\n\t"
4662 #else
4663 # define __FRAME_POINTER
4664 # define VALGRIND_CFI_PROLOGUE \
4665  "lgr 1,%1\n\t"
4666 # define VALGRIND_CFI_EPILOGUE
4667 #endif
4668 
4669 /* Nb: On s390 the stack pointer is properly aligned *at all times*
4670  according to the s390 GCC maintainer. (The ABI specification is not
4671  precise in this regard.) Therefore, VALGRIND_ALIGN_STACK and
4672  VALGRIND_RESTORE_STACK are not defined here. */
4673 
4674 /* These regs are trashed by the hidden call. Note that we overwrite
4675  r14 in s390_irgen_noredir (VEX/priv/guest_s390_irgen.c) to give the
4676  function a proper return address. All others are ABI defined call
4677  clobbers. */
4678 #define __CALLER_SAVED_REGS "0","1","2","3","4","5","14", \
4679  "f0","f1","f2","f3","f4","f5","f6","f7"
4680 
4681 /* Nb: Although r11 is modified in the asm snippets below (inside
4682  VALGRIND_CFI_PROLOGUE) it is not listed in the clobber section, for
4683  two reasons:
4684  (1) r11 is restored in VALGRIND_CFI_EPILOGUE, so effectively it is not
4685  modified
4686  (2) GCC will complain that r11 cannot appear inside a clobber section,
4687  when compiled with -O -fno-omit-frame-pointer
4688  */
4689 
4690 #define CALL_FN_W_v(lval, orig) \
4691  do { \
4692  volatile OrigFn _orig = (orig); \
4693  volatile unsigned long _argvec[1]; \
4694  volatile unsigned long _res; \
4695  _argvec[0] = (unsigned long)_orig.nraddr; \
4696  __asm__ volatile( \
4697  VALGRIND_CFI_PROLOGUE \
4698  "aghi 15,-160\n\t" \
4699  "lg 1, 0(1)\n\t" /* target->r1 */ \
4700  VALGRIND_CALL_NOREDIR_R1 \
4701  "lgr %0, 2\n\t" \
4702  "aghi 15,160\n\t" \
4703  VALGRIND_CFI_EPILOGUE \
4704  : /*out*/ "=d" (_res) \
4705  : /*in*/ "d" (&_argvec[0]) __FRAME_POINTER \
4706  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \
4707  ); \
4708  lval = (__typeof__(lval)) _res; \
4709  } while (0)
4710 
4711 /* The call abi has the arguments in r2-r6 and stack */
4712 #define CALL_FN_W_W(lval, orig, arg1) \
4713  do { \
4714  volatile OrigFn _orig = (orig); \
4715  volatile unsigned long _argvec[2]; \
4716  volatile unsigned long _res; \
4717  _argvec[0] = (unsigned long)_orig.nraddr; \
4718  _argvec[1] = (unsigned long)arg1; \
4719  __asm__ volatile( \
4720  VALGRIND_CFI_PROLOGUE \
4721  "aghi 15,-160\n\t" \
4722  "lg 2, 8(1)\n\t" \
4723  "lg 1, 0(1)\n\t" \
4724  VALGRIND_CALL_NOREDIR_R1 \
4725  "lgr %0, 2\n\t" \
4726  "aghi 15,160\n\t" \
4727  VALGRIND_CFI_EPILOGUE \
4728  : /*out*/ "=d" (_res) \
4729  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
4730  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \
4731  ); \
4732  lval = (__typeof__(lval)) _res; \
4733  } while (0)
4734 
4735 #define CALL_FN_W_WW(lval, orig, arg1, arg2) \
4736  do { \
4737  volatile OrigFn _orig = (orig); \
4738  volatile unsigned long _argvec[3]; \
4739  volatile unsigned long _res; \
4740  _argvec[0] = (unsigned long)_orig.nraddr; \
4741  _argvec[1] = (unsigned long)arg1; \
4742  _argvec[2] = (unsigned long)arg2; \
4743  __asm__ volatile( \
4744  VALGRIND_CFI_PROLOGUE \
4745  "aghi 15,-160\n\t" \
4746  "lg 2, 8(1)\n\t" \
4747  "lg 3,16(1)\n\t" \
4748  "lg 1, 0(1)\n\t" \
4749  VALGRIND_CALL_NOREDIR_R1 \
4750  "lgr %0, 2\n\t" \
4751  "aghi 15,160\n\t" \
4752  VALGRIND_CFI_EPILOGUE \
4753  : /*out*/ "=d" (_res) \
4754  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
4755  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \
4756  ); \
4757  lval = (__typeof__(lval)) _res; \
4758  } while (0)
4759 
4760 #define CALL_FN_W_WWW(lval, orig, arg1, arg2, arg3) \
4761  do { \
4762  volatile OrigFn _orig = (orig); \
4763  volatile unsigned long _argvec[4]; \
4764  volatile unsigned long _res; \
4765  _argvec[0] = (unsigned long)_orig.nraddr; \
4766  _argvec[1] = (unsigned long)arg1; \
4767  _argvec[2] = (unsigned long)arg2; \
4768  _argvec[3] = (unsigned long)arg3; \
4769  __asm__ volatile( \
4770  VALGRIND_CFI_PROLOGUE \
4771  "aghi 15,-160\n\t" \
4772  "lg 2, 8(1)\n\t" \
4773  "lg 3,16(1)\n\t" \
4774  "lg 4,24(1)\n\t" \
4775  "lg 1, 0(1)\n\t" \
4776  VALGRIND_CALL_NOREDIR_R1 \
4777  "lgr %0, 2\n\t" \
4778  "aghi 15,160\n\t" \
4779  VALGRIND_CFI_EPILOGUE \
4780  : /*out*/ "=d" (_res) \
4781  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
4782  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \
4783  ); \
4784  lval = (__typeof__(lval)) _res; \
4785  } while (0)
4786 
4787 #define CALL_FN_W_WWWW(lval, orig, arg1, arg2, arg3, arg4) \
4788  do { \
4789  volatile OrigFn _orig = (orig); \
4790  volatile unsigned long _argvec[5]; \
4791  volatile unsigned long _res; \
4792  _argvec[0] = (unsigned long)_orig.nraddr; \
4793  _argvec[1] = (unsigned long)arg1; \
4794  _argvec[2] = (unsigned long)arg2; \
4795  _argvec[3] = (unsigned long)arg3; \
4796  _argvec[4] = (unsigned long)arg4; \
4797  __asm__ volatile( \
4798  VALGRIND_CFI_PROLOGUE \
4799  "aghi 15,-160\n\t" \
4800  "lg 2, 8(1)\n\t" \
4801  "lg 3,16(1)\n\t" \
4802  "lg 4,24(1)\n\t" \
4803  "lg 5,32(1)\n\t" \
4804  "lg 1, 0(1)\n\t" \
4805  VALGRIND_CALL_NOREDIR_R1 \
4806  "lgr %0, 2\n\t" \
4807  "aghi 15,160\n\t" \
4808  VALGRIND_CFI_EPILOGUE \
4809  : /*out*/ "=d" (_res) \
4810  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
4811  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \
4812  ); \
4813  lval = (__typeof__(lval)) _res; \
4814  } while (0)
4815 
4816 #define CALL_FN_W_5W(lval, orig, arg1, arg2, arg3, arg4, arg5) \
4817  do { \
4818  volatile OrigFn _orig = (orig); \
4819  volatile unsigned long _argvec[6]; \
4820  volatile unsigned long _res; \
4821  _argvec[0] = (unsigned long)_orig.nraddr; \
4822  _argvec[1] = (unsigned long)arg1; \
4823  _argvec[2] = (unsigned long)arg2; \
4824  _argvec[3] = (unsigned long)arg3; \
4825  _argvec[4] = (unsigned long)arg4; \
4826  _argvec[5] = (unsigned long)arg5; \
4827  __asm__ volatile( \
4828  VALGRIND_CFI_PROLOGUE \
4829  "aghi 15,-160\n\t" \
4830  "lg 2, 8(1)\n\t" \
4831  "lg 3,16(1)\n\t" \
4832  "lg 4,24(1)\n\t" \
4833  "lg 5,32(1)\n\t" \
4834  "lg 6,40(1)\n\t" \
4835  "lg 1, 0(1)\n\t" \
4836  VALGRIND_CALL_NOREDIR_R1 \
4837  "lgr %0, 2\n\t" \
4838  "aghi 15,160\n\t" \
4839  VALGRIND_CFI_EPILOGUE \
4840  : /*out*/ "=d" (_res) \
4841  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
4842  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4843  ); \
4844  lval = (__typeof__(lval)) _res; \
4845  } while (0)
4846 
4847 #define CALL_FN_W_6W(lval, orig, arg1, arg2, arg3, arg4, arg5, \
4848  arg6) \
4849  do { \
4850  volatile OrigFn _orig = (orig); \
4851  volatile unsigned long _argvec[7]; \
4852  volatile unsigned long _res; \
4853  _argvec[0] = (unsigned long)_orig.nraddr; \
4854  _argvec[1] = (unsigned long)arg1; \
4855  _argvec[2] = (unsigned long)arg2; \
4856  _argvec[3] = (unsigned long)arg3; \
4857  _argvec[4] = (unsigned long)arg4; \
4858  _argvec[5] = (unsigned long)arg5; \
4859  _argvec[6] = (unsigned long)arg6; \
4860  __asm__ volatile( \
4861  VALGRIND_CFI_PROLOGUE \
4862  "aghi 15,-168\n\t" \
4863  "lg 2, 8(1)\n\t" \
4864  "lg 3,16(1)\n\t" \
4865  "lg 4,24(1)\n\t" \
4866  "lg 5,32(1)\n\t" \
4867  "lg 6,40(1)\n\t" \
4868  "mvc 160(8,15), 48(1)\n\t" \
4869  "lg 1, 0(1)\n\t" \
4870  VALGRIND_CALL_NOREDIR_R1 \
4871  "lgr %0, 2\n\t" \
4872  "aghi 15,168\n\t" \
4873  VALGRIND_CFI_EPILOGUE \
4874  : /*out*/ "=d" (_res) \
4875  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
4876  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4877  ); \
4878  lval = (__typeof__(lval)) _res; \
4879  } while (0)
4880 
4881 #define CALL_FN_W_7W(lval, orig, arg1, arg2, arg3, arg4, arg5, \
4882  arg6, arg7) \
4883  do { \
4884  volatile OrigFn _orig = (orig); \
4885  volatile unsigned long _argvec[8]; \
4886  volatile unsigned long _res; \
4887  _argvec[0] = (unsigned long)_orig.nraddr; \
4888  _argvec[1] = (unsigned long)arg1; \
4889  _argvec[2] = (unsigned long)arg2; \
4890  _argvec[3] = (unsigned long)arg3; \
4891  _argvec[4] = (unsigned long)arg4; \
4892  _argvec[5] = (unsigned long)arg5; \
4893  _argvec[6] = (unsigned long)arg6; \
4894  _argvec[7] = (unsigned long)arg7; \
4895  __asm__ volatile( \
4896  VALGRIND_CFI_PROLOGUE \
4897  "aghi 15,-176\n\t" \
4898  "lg 2, 8(1)\n\t" \
4899  "lg 3,16(1)\n\t" \
4900  "lg 4,24(1)\n\t" \
4901  "lg 5,32(1)\n\t" \
4902  "lg 6,40(1)\n\t" \
4903  "mvc 160(8,15), 48(1)\n\t" \
4904  "mvc 168(8,15), 56(1)\n\t" \
4905  "lg 1, 0(1)\n\t" \
4906  VALGRIND_CALL_NOREDIR_R1 \
4907  "lgr %0, 2\n\t" \
4908  "aghi 15,176\n\t" \
4909  VALGRIND_CFI_EPILOGUE \
4910  : /*out*/ "=d" (_res) \
4911  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
4912  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4913  ); \
4914  lval = (__typeof__(lval)) _res; \
4915  } while (0)
4916 
4917 #define CALL_FN_W_8W(lval, orig, arg1, arg2, arg3, arg4, arg5, \
4918  arg6, arg7 ,arg8) \
4919  do { \
4920  volatile OrigFn _orig = (orig); \
4921  volatile unsigned long _argvec[9]; \
4922  volatile unsigned long _res; \
4923  _argvec[0] = (unsigned long)_orig.nraddr; \
4924  _argvec[1] = (unsigned long)arg1; \
4925  _argvec[2] = (unsigned long)arg2; \
4926  _argvec[3] = (unsigned long)arg3; \
4927  _argvec[4] = (unsigned long)arg4; \
4928  _argvec[5] = (unsigned long)arg5; \
4929  _argvec[6] = (unsigned long)arg6; \
4930  _argvec[7] = (unsigned long)arg7; \
4931  _argvec[8] = (unsigned long)arg8; \
4932  __asm__ volatile( \
4933  VALGRIND_CFI_PROLOGUE \
4934  "aghi 15,-184\n\t" \
4935  "lg 2, 8(1)\n\t" \
4936  "lg 3,16(1)\n\t" \
4937  "lg 4,24(1)\n\t" \
4938  "lg 5,32(1)\n\t" \
4939  "lg 6,40(1)\n\t" \
4940  "mvc 160(8,15), 48(1)\n\t" \
4941  "mvc 168(8,15), 56(1)\n\t" \
4942  "mvc 176(8,15), 64(1)\n\t" \
4943  "lg 1, 0(1)\n\t" \
4944  VALGRIND_CALL_NOREDIR_R1 \
4945  "lgr %0, 2\n\t" \
4946  "aghi 15,184\n\t" \
4947  VALGRIND_CFI_EPILOGUE \
4948  : /*out*/ "=d" (_res) \
4949  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
4950  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4951  ); \
4952  lval = (__typeof__(lval)) _res; \
4953  } while (0)
4954 
4955 #define CALL_FN_W_9W(lval, orig, arg1, arg2, arg3, arg4, arg5, \
4956  arg6, arg7 ,arg8, arg9) \
4957  do { \
4958  volatile OrigFn _orig = (orig); \
4959  volatile unsigned long _argvec[10]; \
4960  volatile unsigned long _res; \
4961  _argvec[0] = (unsigned long)_orig.nraddr; \
4962  _argvec[1] = (unsigned long)arg1; \
4963  _argvec[2] = (unsigned long)arg2; \
4964  _argvec[3] = (unsigned long)arg3; \
4965  _argvec[4] = (unsigned long)arg4; \
4966  _argvec[5] = (unsigned long)arg5; \
4967  _argvec[6] = (unsigned long)arg6; \
4968  _argvec[7] = (unsigned long)arg7; \
4969  _argvec[8] = (unsigned long)arg8; \
4970  _argvec[9] = (unsigned long)arg9; \
4971  __asm__ volatile( \
4972  VALGRIND_CFI_PROLOGUE \
4973  "aghi 15,-192\n\t" \
4974  "lg 2, 8(1)\n\t" \
4975  "lg 3,16(1)\n\t" \
4976  "lg 4,24(1)\n\t" \
4977  "lg 5,32(1)\n\t" \
4978  "lg 6,40(1)\n\t" \
4979  "mvc 160(8,15), 48(1)\n\t" \
4980  "mvc 168(8,15), 56(1)\n\t" \
4981  "mvc 176(8,15), 64(1)\n\t" \
4982  "mvc 184(8,15), 72(1)\n\t" \
4983  "lg 1, 0(1)\n\t" \
4984  VALGRIND_CALL_NOREDIR_R1 \
4985  "lgr %0, 2\n\t" \
4986  "aghi 15,192\n\t" \
4987  VALGRIND_CFI_EPILOGUE \
4988  : /*out*/ "=d" (_res) \
4989  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
4990  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4991  ); \
4992  lval = (__typeof__(lval)) _res; \
4993  } while (0)
4994 
4995 #define CALL_FN_W_10W(lval, orig, arg1, arg2, arg3, arg4, arg5, \
4996  arg6, arg7 ,arg8, arg9, arg10) \
4997  do { \
4998  volatile OrigFn _orig = (orig); \
4999  volatile unsigned long _argvec[11]; \
5000  volatile unsigned long _res; \
5001  _argvec[0] = (unsigned long)_orig.nraddr; \
5002  _argvec[1] = (unsigned long)arg1; \
5003  _argvec[2] = (unsigned long)arg2; \
5004  _argvec[3] = (unsigned long)arg3; \
5005  _argvec[4] = (unsigned long)arg4; \
5006  _argvec[5] = (unsigned long)arg5; \
5007  _argvec[6] = (unsigned long)arg6; \
5008  _argvec[7] = (unsigned long)arg7; \
5009  _argvec[8] = (unsigned long)arg8; \
5010  _argvec[9] = (unsigned long)arg9; \
5011  _argvec[10] = (unsigned long)arg10; \
5012  __asm__ volatile( \
5013  VALGRIND_CFI_PROLOGUE \
5014  "aghi 15,-200\n\t" \
5015  "lg 2, 8(1)\n\t" \
5016  "lg 3,16(1)\n\t" \
5017  "lg 4,24(1)\n\t" \
5018  "lg 5,32(1)\n\t" \
5019  "lg 6,40(1)\n\t" \
5020  "mvc 160(8,15), 48(1)\n\t" \
5021  "mvc 168(8,15), 56(1)\n\t" \
5022  "mvc 176(8,15), 64(1)\n\t" \
5023  "mvc 184(8,15), 72(1)\n\t" \
5024  "mvc 192(8,15), 80(1)\n\t" \
5025  "lg 1, 0(1)\n\t" \
5026  VALGRIND_CALL_NOREDIR_R1 \
5027  "lgr %0, 2\n\t" \
5028  "aghi 15,200\n\t" \
5029  VALGRIND_CFI_EPILOGUE \
5030  : /*out*/ "=d" (_res) \
5031  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
5032  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5033  ); \
5034  lval = (__typeof__(lval)) _res; \
5035  } while (0)
5036 
5037 #define CALL_FN_W_11W(lval, orig, arg1, arg2, arg3, arg4, arg5, \
5038  arg6, arg7 ,arg8, arg9, arg10, arg11) \
5039  do { \
5040  volatile OrigFn _orig = (orig); \
5041  volatile unsigned long _argvec[12]; \
5042  volatile unsigned long _res; \
5043  _argvec[0] = (unsigned long)_orig.nraddr; \
5044  _argvec[1] = (unsigned long)arg1; \
5045  _argvec[2] = (unsigned long)arg2; \
5046  _argvec[3] = (unsigned long)arg3; \
5047  _argvec[4] = (unsigned long)arg4; \
5048  _argvec[5] = (unsigned long)arg5; \
5049  _argvec[6] = (unsigned long)arg6; \
5050  _argvec[7] = (unsigned long)arg7; \
5051  _argvec[8] = (unsigned long)arg8; \
5052  _argvec[9] = (unsigned long)arg9; \
5053  _argvec[10] = (unsigned long)arg10; \
5054  _argvec[11] = (unsigned long)arg11; \
5055  __asm__ volatile( \
5056  VALGRIND_CFI_PROLOGUE \
5057  "aghi 15,-208\n\t" \
5058  "lg 2, 8(1)\n\t" \
5059  "lg 3,16(1)\n\t" \
5060  "lg 4,24(1)\n\t" \
5061  "lg 5,32(1)\n\t" \
5062  "lg 6,40(1)\n\t" \
5063  "mvc 160(8,15), 48(1)\n\t" \
5064  "mvc 168(8,15), 56(1)\n\t" \
5065  "mvc 176(8,15), 64(1)\n\t" \
5066  "mvc 184(8,15), 72(1)\n\t" \
5067  "mvc 192(8,15), 80(1)\n\t" \
5068  "mvc 200(8,15), 88(1)\n\t" \
5069  "lg 1, 0(1)\n\t" \
5070  VALGRIND_CALL_NOREDIR_R1 \
5071  "lgr %0, 2\n\t" \
5072  "aghi 15,208\n\t" \
5073  VALGRIND_CFI_EPILOGUE \
5074  : /*out*/ "=d" (_res) \
5075  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
5076  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5077  ); \
5078  lval = (__typeof__(lval)) _res; \
5079  } while (0)
5080 
5081 #define CALL_FN_W_12W(lval, orig, arg1, arg2, arg3, arg4, arg5, \
5082  arg6, arg7 ,arg8, arg9, arg10, arg11, arg12)\
5083  do { \
5084  volatile OrigFn _orig = (orig); \
5085  volatile unsigned long _argvec[13]; \
5086  volatile unsigned long _res; \
5087  _argvec[0] = (unsigned long)_orig.nraddr; \
5088  _argvec[1] = (unsigned long)arg1; \
5089  _argvec[2] = (unsigned long)arg2; \
5090  _argvec[3] = (unsigned long)arg3; \
5091  _argvec[4] = (unsigned long)arg4; \
5092  _argvec[5] = (unsigned long)arg5; \
5093  _argvec[6] = (unsigned long)arg6; \
5094  _argvec[7] = (unsigned long)arg7; \
5095  _argvec[8] = (unsigned long)arg8; \
5096  _argvec[9] = (unsigned long)arg9; \
5097  _argvec[10] = (unsigned long)arg10; \
5098  _argvec[11] = (unsigned long)arg11; \
5099  _argvec[12] = (unsigned long)arg12; \
5100  __asm__ volatile( \
5101  VALGRIND_CFI_PROLOGUE \
5102  "aghi 15,-216\n\t" \
5103  "lg 2, 8(1)\n\t" \
5104  "lg 3,16(1)\n\t" \
5105  "lg 4,24(1)\n\t" \
5106  "lg 5,32(1)\n\t" \
5107  "lg 6,40(1)\n\t" \
5108  "mvc 160(8,15), 48(1)\n\t" \
5109  "mvc 168(8,15), 56(1)\n\t" \
5110  "mvc 176(8,15), 64(1)\n\t" \
5111  "mvc 184(8,15), 72(1)\n\t" \
5112  "mvc 192(8,15), 80(1)\n\t" \
5113  "mvc 200(8,15), 88(1)\n\t" \
5114  "mvc 208(8,15), 96(1)\n\t" \
5115  "lg 1, 0(1)\n\t" \
5116  VALGRIND_CALL_NOREDIR_R1 \
5117  "lgr %0, 2\n\t" \
5118  "aghi 15,216\n\t" \
5119  VALGRIND_CFI_EPILOGUE \
5120  : /*out*/ "=d" (_res) \
5121  : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \
5122  : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5123  ); \
5124  lval = (__typeof__(lval)) _res; \
5125  } while (0)
5126 
5127 
5128 #endif /* PLAT_s390x_linux */
5129 
5130 /* ------------------------- mips32-linux ----------------------- */
5131 
5132 #if defined(PLAT_mips32_linux)
5133 
5134 /* These regs are trashed by the hidden call. */
5135 #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6", \
5136 "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \
5137 "$25", "$31"
5138 
5139 /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned
5140  long) == 4. */
5141 
5142 #define CALL_FN_W_v(lval, orig) \
5143  do { \
5144  volatile OrigFn _orig = (orig); \
5145  volatile unsigned long _argvec[1]; \
5146  volatile unsigned long _res; \
5147  _argvec[0] = (unsigned long)_orig.nraddr; \
5148  __asm__ volatile( \
5149  "subu $29, $29, 8 \n\t" \
5150  "sw $28, 0($29) \n\t" \
5151  "sw $31, 4($29) \n\t" \
5152  "subu $29, $29, 16 \n\t" \
5153  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5154  VALGRIND_CALL_NOREDIR_T9 \
5155  "addu $29, $29, 16\n\t" \
5156  "lw $28, 0($29) \n\t" \
5157  "lw $31, 4($29) \n\t" \
5158  "addu $29, $29, 8 \n\t" \
5159  "move %0, $2\n" \
5160  : /*out*/ "=r" (_res) \
5161  : /*in*/ "0" (&_argvec[0]) \
5162  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5163  ); \
5164  lval = (__typeof__(lval)) _res; \
5165  } while (0)
5166 
5167 #define CALL_FN_W_W(lval, orig, arg1) \
5168  do { \
5169  volatile OrigFn _orig = (orig); \
5170  volatile unsigned long _argvec[2]; \
5171  volatile unsigned long _res; \
5172  _argvec[0] = (unsigned long)_orig.nraddr; \
5173  _argvec[1] = (unsigned long)(arg1); \
5174  __asm__ volatile( \
5175  "subu $29, $29, 8 \n\t" \
5176  "sw $28, 0($29) \n\t" \
5177  "sw $31, 4($29) \n\t" \
5178  "subu $29, $29, 16 \n\t" \
5179  "lw $4, 4(%1) \n\t" /* arg1*/ \
5180  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5181  VALGRIND_CALL_NOREDIR_T9 \
5182  "addu $29, $29, 16 \n\t" \
5183  "lw $28, 0($29) \n\t" \
5184  "lw $31, 4($29) \n\t" \
5185  "addu $29, $29, 8 \n\t" \
5186  "move %0, $2\n" \
5187  : /*out*/ "=r" (_res) \
5188  : /*in*/ "0" (&_argvec[0]) \
5189  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5190  ); \
5191  lval = (__typeof__(lval)) _res; \
5192  } while (0)
5193 
5194 #define CALL_FN_W_WW(lval, orig, arg1,arg2) \
5195  do { \
5196  volatile OrigFn _orig = (orig); \
5197  volatile unsigned long _argvec[3]; \
5198  volatile unsigned long _res; \
5199  _argvec[0] = (unsigned long)_orig.nraddr; \
5200  _argvec[1] = (unsigned long)(arg1); \
5201  _argvec[2] = (unsigned long)(arg2); \
5202  __asm__ volatile( \
5203  "subu $29, $29, 8 \n\t" \
5204  "sw $28, 0($29) \n\t" \
5205  "sw $31, 4($29) \n\t" \
5206  "subu $29, $29, 16 \n\t" \
5207  "lw $4, 4(%1) \n\t" \
5208  "lw $5, 8(%1) \n\t" \
5209  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5210  VALGRIND_CALL_NOREDIR_T9 \
5211  "addu $29, $29, 16 \n\t" \
5212  "lw $28, 0($29) \n\t" \
5213  "lw $31, 4($29) \n\t" \
5214  "addu $29, $29, 8 \n\t" \
5215  "move %0, $2\n" \
5216  : /*out*/ "=r" (_res) \
5217  : /*in*/ "0" (&_argvec[0]) \
5218  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5219  ); \
5220  lval = (__typeof__(lval)) _res; \
5221  } while (0)
5222 
5223 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
5224  do { \
5225  volatile OrigFn _orig = (orig); \
5226  volatile unsigned long _argvec[4]; \
5227  volatile unsigned long _res; \
5228  _argvec[0] = (unsigned long)_orig.nraddr; \
5229  _argvec[1] = (unsigned long)(arg1); \
5230  _argvec[2] = (unsigned long)(arg2); \
5231  _argvec[3] = (unsigned long)(arg3); \
5232  __asm__ volatile( \
5233  "subu $29, $29, 8 \n\t" \
5234  "sw $28, 0($29) \n\t" \
5235  "sw $31, 4($29) \n\t" \
5236  "subu $29, $29, 16 \n\t" \
5237  "lw $4, 4(%1) \n\t" \
5238  "lw $5, 8(%1) \n\t" \
5239  "lw $6, 12(%1) \n\t" \
5240  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5241  VALGRIND_CALL_NOREDIR_T9 \
5242  "addu $29, $29, 16 \n\t" \
5243  "lw $28, 0($29) \n\t" \
5244  "lw $31, 4($29) \n\t" \
5245  "addu $29, $29, 8 \n\t" \
5246  "move %0, $2\n" \
5247  : /*out*/ "=r" (_res) \
5248  : /*in*/ "0" (&_argvec[0]) \
5249  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5250  ); \
5251  lval = (__typeof__(lval)) _res; \
5252  } while (0)
5253 
5254 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
5255  do { \
5256  volatile OrigFn _orig = (orig); \
5257  volatile unsigned long _argvec[5]; \
5258  volatile unsigned long _res; \
5259  _argvec[0] = (unsigned long)_orig.nraddr; \
5260  _argvec[1] = (unsigned long)(arg1); \
5261  _argvec[2] = (unsigned long)(arg2); \
5262  _argvec[3] = (unsigned long)(arg3); \
5263  _argvec[4] = (unsigned long)(arg4); \
5264  __asm__ volatile( \
5265  "subu $29, $29, 8 \n\t" \
5266  "sw $28, 0($29) \n\t" \
5267  "sw $31, 4($29) \n\t" \
5268  "subu $29, $29, 16 \n\t" \
5269  "lw $4, 4(%1) \n\t" \
5270  "lw $5, 8(%1) \n\t" \
5271  "lw $6, 12(%1) \n\t" \
5272  "lw $7, 16(%1) \n\t" \
5273  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5274  VALGRIND_CALL_NOREDIR_T9 \
5275  "addu $29, $29, 16 \n\t" \
5276  "lw $28, 0($29) \n\t" \
5277  "lw $31, 4($29) \n\t" \
5278  "addu $29, $29, 8 \n\t" \
5279  "move %0, $2\n" \
5280  : /*out*/ "=r" (_res) \
5281  : /*in*/ "0" (&_argvec[0]) \
5282  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5283  ); \
5284  lval = (__typeof__(lval)) _res; \
5285  } while (0)
5286 
5287 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
5288  do { \
5289  volatile OrigFn _orig = (orig); \
5290  volatile unsigned long _argvec[6]; \
5291  volatile unsigned long _res; \
5292  _argvec[0] = (unsigned long)_orig.nraddr; \
5293  _argvec[1] = (unsigned long)(arg1); \
5294  _argvec[2] = (unsigned long)(arg2); \
5295  _argvec[3] = (unsigned long)(arg3); \
5296  _argvec[4] = (unsigned long)(arg4); \
5297  _argvec[5] = (unsigned long)(arg5); \
5298  __asm__ volatile( \
5299  "subu $29, $29, 8 \n\t" \
5300  "sw $28, 0($29) \n\t" \
5301  "sw $31, 4($29) \n\t" \
5302  "lw $4, 20(%1) \n\t" \
5303  "subu $29, $29, 24\n\t" \
5304  "sw $4, 16($29) \n\t" \
5305  "lw $4, 4(%1) \n\t" \
5306  "lw $5, 8(%1) \n\t" \
5307  "lw $6, 12(%1) \n\t" \
5308  "lw $7, 16(%1) \n\t" \
5309  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5310  VALGRIND_CALL_NOREDIR_T9 \
5311  "addu $29, $29, 24 \n\t" \
5312  "lw $28, 0($29) \n\t" \
5313  "lw $31, 4($29) \n\t" \
5314  "addu $29, $29, 8 \n\t" \
5315  "move %0, $2\n" \
5316  : /*out*/ "=r" (_res) \
5317  : /*in*/ "0" (&_argvec[0]) \
5318  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5319  ); \
5320  lval = (__typeof__(lval)) _res; \
5321  } while (0)
5322 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
5323  do { \
5324  volatile OrigFn _orig = (orig); \
5325  volatile unsigned long _argvec[7]; \
5326  volatile unsigned long _res; \
5327  _argvec[0] = (unsigned long)_orig.nraddr; \
5328  _argvec[1] = (unsigned long)(arg1); \
5329  _argvec[2] = (unsigned long)(arg2); \
5330  _argvec[3] = (unsigned long)(arg3); \
5331  _argvec[4] = (unsigned long)(arg4); \
5332  _argvec[5] = (unsigned long)(arg5); \
5333  _argvec[6] = (unsigned long)(arg6); \
5334  __asm__ volatile( \
5335  "subu $29, $29, 8 \n\t" \
5336  "sw $28, 0($29) \n\t" \
5337  "sw $31, 4($29) \n\t" \
5338  "lw $4, 20(%1) \n\t" \
5339  "subu $29, $29, 32\n\t" \
5340  "sw $4, 16($29) \n\t" \
5341  "lw $4, 24(%1) \n\t" \
5342  "nop\n\t" \
5343  "sw $4, 20($29) \n\t" \
5344  "lw $4, 4(%1) \n\t" \
5345  "lw $5, 8(%1) \n\t" \
5346  "lw $6, 12(%1) \n\t" \
5347  "lw $7, 16(%1) \n\t" \
5348  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5349  VALGRIND_CALL_NOREDIR_T9 \
5350  "addu $29, $29, 32 \n\t" \
5351  "lw $28, 0($29) \n\t" \
5352  "lw $31, 4($29) \n\t" \
5353  "addu $29, $29, 8 \n\t" \
5354  "move %0, $2\n" \
5355  : /*out*/ "=r" (_res) \
5356  : /*in*/ "0" (&_argvec[0]) \
5357  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5358  ); \
5359  lval = (__typeof__(lval)) _res; \
5360  } while (0)
5361 
5362 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
5363  arg7) \
5364  do { \
5365  volatile OrigFn _orig = (orig); \
5366  volatile unsigned long _argvec[8]; \
5367  volatile unsigned long _res; \
5368  _argvec[0] = (unsigned long)_orig.nraddr; \
5369  _argvec[1] = (unsigned long)(arg1); \
5370  _argvec[2] = (unsigned long)(arg2); \
5371  _argvec[3] = (unsigned long)(arg3); \
5372  _argvec[4] = (unsigned long)(arg4); \
5373  _argvec[5] = (unsigned long)(arg5); \
5374  _argvec[6] = (unsigned long)(arg6); \
5375  _argvec[7] = (unsigned long)(arg7); \
5376  __asm__ volatile( \
5377  "subu $29, $29, 8 \n\t" \
5378  "sw $28, 0($29) \n\t" \
5379  "sw $31, 4($29) \n\t" \
5380  "lw $4, 20(%1) \n\t" \
5381  "subu $29, $29, 32\n\t" \
5382  "sw $4, 16($29) \n\t" \
5383  "lw $4, 24(%1) \n\t" \
5384  "sw $4, 20($29) \n\t" \
5385  "lw $4, 28(%1) \n\t" \
5386  "sw $4, 24($29) \n\t" \
5387  "lw $4, 4(%1) \n\t" \
5388  "lw $5, 8(%1) \n\t" \
5389  "lw $6, 12(%1) \n\t" \
5390  "lw $7, 16(%1) \n\t" \
5391  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5392  VALGRIND_CALL_NOREDIR_T9 \
5393  "addu $29, $29, 32 \n\t" \
5394  "lw $28, 0($29) \n\t" \
5395  "lw $31, 4($29) \n\t" \
5396  "addu $29, $29, 8 \n\t" \
5397  "move %0, $2\n" \
5398  : /*out*/ "=r" (_res) \
5399  : /*in*/ "0" (&_argvec[0]) \
5400  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5401  ); \
5402  lval = (__typeof__(lval)) _res; \
5403  } while (0)
5404 
5405 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
5406  arg7,arg8) \
5407  do { \
5408  volatile OrigFn _orig = (orig); \
5409  volatile unsigned long _argvec[9]; \
5410  volatile unsigned long _res; \
5411  _argvec[0] = (unsigned long)_orig.nraddr; \
5412  _argvec[1] = (unsigned long)(arg1); \
5413  _argvec[2] = (unsigned long)(arg2); \
5414  _argvec[3] = (unsigned long)(arg3); \
5415  _argvec[4] = (unsigned long)(arg4); \
5416  _argvec[5] = (unsigned long)(arg5); \
5417  _argvec[6] = (unsigned long)(arg6); \
5418  _argvec[7] = (unsigned long)(arg7); \
5419  _argvec[8] = (unsigned long)(arg8); \
5420  __asm__ volatile( \
5421  "subu $29, $29, 8 \n\t" \
5422  "sw $28, 0($29) \n\t" \
5423  "sw $31, 4($29) \n\t" \
5424  "lw $4, 20(%1) \n\t" \
5425  "subu $29, $29, 40\n\t" \
5426  "sw $4, 16($29) \n\t" \
5427  "lw $4, 24(%1) \n\t" \
5428  "sw $4, 20($29) \n\t" \
5429  "lw $4, 28(%1) \n\t" \
5430  "sw $4, 24($29) \n\t" \
5431  "lw $4, 32(%1) \n\t" \
5432  "sw $4, 28($29) \n\t" \
5433  "lw $4, 4(%1) \n\t" \
5434  "lw $5, 8(%1) \n\t" \
5435  "lw $6, 12(%1) \n\t" \
5436  "lw $7, 16(%1) \n\t" \
5437  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5438  VALGRIND_CALL_NOREDIR_T9 \
5439  "addu $29, $29, 40 \n\t" \
5440  "lw $28, 0($29) \n\t" \
5441  "lw $31, 4($29) \n\t" \
5442  "addu $29, $29, 8 \n\t" \
5443  "move %0, $2\n" \
5444  : /*out*/ "=r" (_res) \
5445  : /*in*/ "0" (&_argvec[0]) \
5446  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5447  ); \
5448  lval = (__typeof__(lval)) _res; \
5449  } while (0)
5450 
5451 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
5452  arg7,arg8,arg9) \
5453  do { \
5454  volatile OrigFn _orig = (orig); \
5455  volatile unsigned long _argvec[10]; \
5456  volatile unsigned long _res; \
5457  _argvec[0] = (unsigned long)_orig.nraddr; \
5458  _argvec[1] = (unsigned long)(arg1); \
5459  _argvec[2] = (unsigned long)(arg2); \
5460  _argvec[3] = (unsigned long)(arg3); \
5461  _argvec[4] = (unsigned long)(arg4); \
5462  _argvec[5] = (unsigned long)(arg5); \
5463  _argvec[6] = (unsigned long)(arg6); \
5464  _argvec[7] = (unsigned long)(arg7); \
5465  _argvec[8] = (unsigned long)(arg8); \
5466  _argvec[9] = (unsigned long)(arg9); \
5467  __asm__ volatile( \
5468  "subu $29, $29, 8 \n\t" \
5469  "sw $28, 0($29) \n\t" \
5470  "sw $31, 4($29) \n\t" \
5471  "lw $4, 20(%1) \n\t" \
5472  "subu $29, $29, 40\n\t" \
5473  "sw $4, 16($29) \n\t" \
5474  "lw $4, 24(%1) \n\t" \
5475  "sw $4, 20($29) \n\t" \
5476  "lw $4, 28(%1) \n\t" \
5477  "sw $4, 24($29) \n\t" \
5478  "lw $4, 32(%1) \n\t" \
5479  "sw $4, 28($29) \n\t" \
5480  "lw $4, 36(%1) \n\t" \
5481  "sw $4, 32($29) \n\t" \
5482  "lw $4, 4(%1) \n\t" \
5483  "lw $5, 8(%1) \n\t" \
5484  "lw $6, 12(%1) \n\t" \
5485  "lw $7, 16(%1) \n\t" \
5486  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5487  VALGRIND_CALL_NOREDIR_T9 \
5488  "addu $29, $29, 40 \n\t" \
5489  "lw $28, 0($29) \n\t" \
5490  "lw $31, 4($29) \n\t" \
5491  "addu $29, $29, 8 \n\t" \
5492  "move %0, $2\n" \
5493  : /*out*/ "=r" (_res) \
5494  : /*in*/ "0" (&_argvec[0]) \
5495  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5496  ); \
5497  lval = (__typeof__(lval)) _res; \
5498  } while (0)
5499 
5500 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
5501  arg7,arg8,arg9,arg10) \
5502  do { \
5503  volatile OrigFn _orig = (orig); \
5504  volatile unsigned long _argvec[11]; \
5505  volatile unsigned long _res; \
5506  _argvec[0] = (unsigned long)_orig.nraddr; \
5507  _argvec[1] = (unsigned long)(arg1); \
5508  _argvec[2] = (unsigned long)(arg2); \
5509  _argvec[3] = (unsigned long)(arg3); \
5510  _argvec[4] = (unsigned long)(arg4); \
5511  _argvec[5] = (unsigned long)(arg5); \
5512  _argvec[6] = (unsigned long)(arg6); \
5513  _argvec[7] = (unsigned long)(arg7); \
5514  _argvec[8] = (unsigned long)(arg8); \
5515  _argvec[9] = (unsigned long)(arg9); \
5516  _argvec[10] = (unsigned long)(arg10); \
5517  __asm__ volatile( \
5518  "subu $29, $29, 8 \n\t" \
5519  "sw $28, 0($29) \n\t" \
5520  "sw $31, 4($29) \n\t" \
5521  "lw $4, 20(%1) \n\t" \
5522  "subu $29, $29, 48\n\t" \
5523  "sw $4, 16($29) \n\t" \
5524  "lw $4, 24(%1) \n\t" \
5525  "sw $4, 20($29) \n\t" \
5526  "lw $4, 28(%1) \n\t" \
5527  "sw $4, 24($29) \n\t" \
5528  "lw $4, 32(%1) \n\t" \
5529  "sw $4, 28($29) \n\t" \
5530  "lw $4, 36(%1) \n\t" \
5531  "sw $4, 32($29) \n\t" \
5532  "lw $4, 40(%1) \n\t" \
5533  "sw $4, 36($29) \n\t" \
5534  "lw $4, 4(%1) \n\t" \
5535  "lw $5, 8(%1) \n\t" \
5536  "lw $6, 12(%1) \n\t" \
5537  "lw $7, 16(%1) \n\t" \
5538  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5539  VALGRIND_CALL_NOREDIR_T9 \
5540  "addu $29, $29, 48 \n\t" \
5541  "lw $28, 0($29) \n\t" \
5542  "lw $31, 4($29) \n\t" \
5543  "addu $29, $29, 8 \n\t" \
5544  "move %0, $2\n" \
5545  : /*out*/ "=r" (_res) \
5546  : /*in*/ "0" (&_argvec[0]) \
5547  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5548  ); \
5549  lval = (__typeof__(lval)) _res; \
5550  } while (0)
5551 
5552 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \
5553  arg6,arg7,arg8,arg9,arg10, \
5554  arg11) \
5555  do { \
5556  volatile OrigFn _orig = (orig); \
5557  volatile unsigned long _argvec[12]; \
5558  volatile unsigned long _res; \
5559  _argvec[0] = (unsigned long)_orig.nraddr; \
5560  _argvec[1] = (unsigned long)(arg1); \
5561  _argvec[2] = (unsigned long)(arg2); \
5562  _argvec[3] = (unsigned long)(arg3); \
5563  _argvec[4] = (unsigned long)(arg4); \
5564  _argvec[5] = (unsigned long)(arg5); \
5565  _argvec[6] = (unsigned long)(arg6); \
5566  _argvec[7] = (unsigned long)(arg7); \
5567  _argvec[8] = (unsigned long)(arg8); \
5568  _argvec[9] = (unsigned long)(arg9); \
5569  _argvec[10] = (unsigned long)(arg10); \
5570  _argvec[11] = (unsigned long)(arg11); \
5571  __asm__ volatile( \
5572  "subu $29, $29, 8 \n\t" \
5573  "sw $28, 0($29) \n\t" \
5574  "sw $31, 4($29) \n\t" \
5575  "lw $4, 20(%1) \n\t" \
5576  "subu $29, $29, 48\n\t" \
5577  "sw $4, 16($29) \n\t" \
5578  "lw $4, 24(%1) \n\t" \
5579  "sw $4, 20($29) \n\t" \
5580  "lw $4, 28(%1) \n\t" \
5581  "sw $4, 24($29) \n\t" \
5582  "lw $4, 32(%1) \n\t" \
5583  "sw $4, 28($29) \n\t" \
5584  "lw $4, 36(%1) \n\t" \
5585  "sw $4, 32($29) \n\t" \
5586  "lw $4, 40(%1) \n\t" \
5587  "sw $4, 36($29) \n\t" \
5588  "lw $4, 44(%1) \n\t" \
5589  "sw $4, 40($29) \n\t" \
5590  "lw $4, 4(%1) \n\t" \
5591  "lw $5, 8(%1) \n\t" \
5592  "lw $6, 12(%1) \n\t" \
5593  "lw $7, 16(%1) \n\t" \
5594  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5595  VALGRIND_CALL_NOREDIR_T9 \
5596  "addu $29, $29, 48 \n\t" \
5597  "lw $28, 0($29) \n\t" \
5598  "lw $31, 4($29) \n\t" \
5599  "addu $29, $29, 8 \n\t" \
5600  "move %0, $2\n" \
5601  : /*out*/ "=r" (_res) \
5602  : /*in*/ "0" (&_argvec[0]) \
5603  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5604  ); \
5605  lval = (__typeof__(lval)) _res; \
5606  } while (0)
5607 
5608 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \
5609  arg6,arg7,arg8,arg9,arg10, \
5610  arg11,arg12) \
5611  do { \
5612  volatile OrigFn _orig = (orig); \
5613  volatile unsigned long _argvec[13]; \
5614  volatile unsigned long _res; \
5615  _argvec[0] = (unsigned long)_orig.nraddr; \
5616  _argvec[1] = (unsigned long)(arg1); \
5617  _argvec[2] = (unsigned long)(arg2); \
5618  _argvec[3] = (unsigned long)(arg3); \
5619  _argvec[4] = (unsigned long)(arg4); \
5620  _argvec[5] = (unsigned long)(arg5); \
5621  _argvec[6] = (unsigned long)(arg6); \
5622  _argvec[7] = (unsigned long)(arg7); \
5623  _argvec[8] = (unsigned long)(arg8); \
5624  _argvec[9] = (unsigned long)(arg9); \
5625  _argvec[10] = (unsigned long)(arg10); \
5626  _argvec[11] = (unsigned long)(arg11); \
5627  _argvec[12] = (unsigned long)(arg12); \
5628  __asm__ volatile( \
5629  "subu $29, $29, 8 \n\t" \
5630  "sw $28, 0($29) \n\t" \
5631  "sw $31, 4($29) \n\t" \
5632  "lw $4, 20(%1) \n\t" \
5633  "subu $29, $29, 56\n\t" \
5634  "sw $4, 16($29) \n\t" \
5635  "lw $4, 24(%1) \n\t" \
5636  "sw $4, 20($29) \n\t" \
5637  "lw $4, 28(%1) \n\t" \
5638  "sw $4, 24($29) \n\t" \
5639  "lw $4, 32(%1) \n\t" \
5640  "sw $4, 28($29) \n\t" \
5641  "lw $4, 36(%1) \n\t" \
5642  "sw $4, 32($29) \n\t" \
5643  "lw $4, 40(%1) \n\t" \
5644  "sw $4, 36($29) \n\t" \
5645  "lw $4, 44(%1) \n\t" \
5646  "sw $4, 40($29) \n\t" \
5647  "lw $4, 48(%1) \n\t" \
5648  "sw $4, 44($29) \n\t" \
5649  "lw $4, 4(%1) \n\t" \
5650  "lw $5, 8(%1) \n\t" \
5651  "lw $6, 12(%1) \n\t" \
5652  "lw $7, 16(%1) \n\t" \
5653  "lw $25, 0(%1) \n\t" /* target->t9 */ \
5654  VALGRIND_CALL_NOREDIR_T9 \
5655  "addu $29, $29, 56 \n\t" \
5656  "lw $28, 0($29) \n\t" \
5657  "lw $31, 4($29) \n\t" \
5658  "addu $29, $29, 8 \n\t" \
5659  "move %0, $2\n" \
5660  : /*out*/ "=r" (_res) \
5661  : /*in*/ "r" (&_argvec[0]) \
5662  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5663  ); \
5664  lval = (__typeof__(lval)) _res; \
5665  } while (0)
5666 
5667 #endif /* PLAT_mips32_linux */
5668 
5669 /* ------------------------- mips64-linux ------------------------- */
5670 
5671 #if defined(PLAT_mips64_linux)
5672 
5673 /* These regs are trashed by the hidden call. */
5674 #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6", \
5675 "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \
5676 "$25", "$31"
5677 
5678 /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned
5679  long) == 4. */
5680 
5681 #define CALL_FN_W_v(lval, orig) \
5682  do { \
5683  volatile OrigFn _orig = (orig); \
5684  volatile unsigned long _argvec[1]; \
5685  volatile unsigned long _res; \
5686  _argvec[0] = (unsigned long)_orig.nraddr; \
5687  __asm__ volatile( \
5688  "ld $25, 0(%1)\n\t" /* target->t9 */ \
5689  VALGRIND_CALL_NOREDIR_T9 \
5690  "move %0, $2\n" \
5691  : /*out*/ "=r" (_res) \
5692  : /*in*/ "0" (&_argvec[0]) \
5693  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5694  ); \
5695  lval = (__typeof__(lval)) _res; \
5696  } while (0)
5697 
5698 #define CALL_FN_W_W(lval, orig, arg1) \
5699  do { \
5700  volatile OrigFn _orig = (orig); \
5701  volatile unsigned long _argvec[2]; \
5702  volatile unsigned long _res; \
5703  _argvec[0] = (unsigned long)_orig.nraddr; \
5704  _argvec[1] = (unsigned long)(arg1); \
5705  __asm__ volatile( \
5706  "ld $4, 8(%1)\n\t" /* arg1*/ \
5707  "ld $25, 0(%1)\n\t" /* target->t9 */ \
5708  VALGRIND_CALL_NOREDIR_T9 \
5709  "move %0, $2\n" \
5710  : /*out*/ "=r" (_res) \
5711  : /*in*/ "r" (&_argvec[0]) \
5712  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5713  ); \
5714  lval = (__typeof__(lval)) _res; \
5715  } while (0)
5716 
5717 #define CALL_FN_W_WW(lval, orig, arg1,arg2) \
5718  do { \
5719  volatile OrigFn _orig = (orig); \
5720  volatile unsigned long _argvec[3]; \
5721  volatile unsigned long _res; \
5722  _argvec[0] = (unsigned long)_orig.nraddr; \
5723  _argvec[1] = (unsigned long)(arg1); \
5724  _argvec[2] = (unsigned long)(arg2); \
5725  __asm__ volatile( \
5726  "ld $4, 8(%1)\n\t" \
5727  "ld $5, 16(%1)\n\t" \
5728  "ld $25, 0(%1)\n\t" /* target->t9 */ \
5729  VALGRIND_CALL_NOREDIR_T9 \
5730  "move %0, $2\n" \
5731  : /*out*/ "=r" (_res) \
5732  : /*in*/ "r" (&_argvec[0]) \
5733  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5734  ); \
5735  lval = (__typeof__(lval)) _res; \
5736  } while (0)
5737 
5738 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \
5739  do { \
5740  volatile OrigFn _orig = (orig); \
5741  volatile unsigned long _argvec[4]; \
5742  volatile unsigned long _res; \
5743  _argvec[0] = (unsigned long)_orig.nraddr; \
5744  _argvec[1] = (unsigned long)(arg1); \
5745  _argvec[2] = (unsigned long)(arg2); \
5746  _argvec[3] = (unsigned long)(arg3); \
5747  __asm__ volatile( \
5748  "ld $4, 8(%1)\n\t" \
5749  "ld $5, 16(%1)\n\t" \
5750  "ld $6, 24(%1)\n\t" \
5751  "ld $25, 0(%1)\n\t" /* target->t9 */ \
5752  VALGRIND_CALL_NOREDIR_T9 \
5753  "move %0, $2\n" \
5754  : /*out*/ "=r" (_res) \
5755  : /*in*/ "r" (&_argvec[0]) \
5756  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5757  ); \
5758  lval = (__typeof__(lval)) _res; \
5759  } while (0)
5760 
5761 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
5762  do { \
5763  volatile OrigFn _orig = (orig); \
5764  volatile unsigned long _argvec[5]; \
5765  volatile unsigned long _res; \
5766  _argvec[0] = (unsigned long)_orig.nraddr; \
5767  _argvec[1] = (unsigned long)(arg1); \
5768  _argvec[2] = (unsigned long)(arg2); \
5769  _argvec[3] = (unsigned long)(arg3); \
5770  _argvec[4] = (unsigned long)(arg4); \
5771  __asm__ volatile( \
5772  "ld $4, 8(%1)\n\t" \
5773  "ld $5, 16(%1)\n\t" \
5774  "ld $6, 24(%1)\n\t" \
5775  "ld $7, 32(%1)\n\t" \
5776  "ld $25, 0(%1)\n\t" /* target->t9 */ \
5777  VALGRIND_CALL_NOREDIR_T9 \
5778  "move %0, $2\n" \
5779  : /*out*/ "=r" (_res) \
5780  : /*in*/ "r" (&_argvec[0]) \
5781  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5782  ); \
5783  lval = (__typeof__(lval)) _res; \
5784  } while (0)
5785 
5786 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \
5787  do { \
5788  volatile OrigFn _orig = (orig); \
5789  volatile unsigned long _argvec[6]; \
5790  volatile unsigned long _res; \
5791  _argvec[0] = (unsigned long)_orig.nraddr; \
5792  _argvec[1] = (unsigned long)(arg1); \
5793  _argvec[2] = (unsigned long)(arg2); \
5794  _argvec[3] = (unsigned long)(arg3); \
5795  _argvec[4] = (unsigned long)(arg4); \
5796  _argvec[5] = (unsigned long)(arg5); \
5797  __asm__ volatile( \
5798  "ld $4, 8(%1)\n\t" \
5799  "ld $5, 16(%1)\n\t" \
5800  "ld $6, 24(%1)\n\t" \
5801  "ld $7, 32(%1)\n\t" \
5802  "ld $8, 40(%1)\n\t" \
5803  "ld $25, 0(%1)\n\t" /* target->t9 */ \
5804  VALGRIND_CALL_NOREDIR_T9 \
5805  "move %0, $2\n" \
5806  : /*out*/ "=r" (_res) \
5807  : /*in*/ "r" (&_argvec[0]) \
5808  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5809  ); \
5810  lval = (__typeof__(lval)) _res; \
5811  } while (0)
5812 
5813 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
5814  do { \
5815  volatile OrigFn _orig = (orig); \
5816  volatile unsigned long _argvec[7]; \
5817  volatile unsigned long _res; \
5818  _argvec[0] = (unsigned long)_orig.nraddr; \
5819  _argvec[1] = (unsigned long)(arg1); \
5820  _argvec[2] = (unsigned long)(arg2); \
5821  _argvec[3] = (unsigned long)(arg3); \
5822  _argvec[4] = (unsigned long)(arg4); \
5823  _argvec[5] = (unsigned long)(arg5); \
5824  _argvec[6] = (unsigned long)(arg6); \
5825  __asm__ volatile( \
5826  "ld $4, 8(%1)\n\t" \
5827  "ld $5, 16(%1)\n\t" \
5828  "ld $6, 24(%1)\n\t" \
5829  "ld $7, 32(%1)\n\t" \
5830  "ld $8, 40(%1)\n\t" \
5831  "ld $9, 48(%1)\n\t" \
5832  "ld $25, 0(%1)\n\t" /* target->t9 */ \
5833  VALGRIND_CALL_NOREDIR_T9 \
5834  "move %0, $2\n" \
5835  : /*out*/ "=r" (_res) \
5836  : /*in*/ "r" (&_argvec[0]) \
5837  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5838  ); \
5839  lval = (__typeof__(lval)) _res; \
5840  } while (0)
5841 
5842 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
5843  arg7) \
5844  do { \
5845  volatile OrigFn _orig = (orig); \
5846  volatile unsigned long _argvec[8]; \
5847  volatile unsigned long _res; \
5848  _argvec[0] = (unsigned long)_orig.nraddr; \
5849  _argvec[1] = (unsigned long)(arg1); \
5850  _argvec[2] = (unsigned long)(arg2); \
5851  _argvec[3] = (unsigned long)(arg3); \
5852  _argvec[4] = (unsigned long)(arg4); \
5853  _argvec[5] = (unsigned long)(arg5); \
5854  _argvec[6] = (unsigned long)(arg6); \
5855  _argvec[7] = (unsigned long)(arg7); \
5856  __asm__ volatile( \
5857  "ld $4, 8(%1)\n\t" \
5858  "ld $5, 16(%1)\n\t" \
5859  "ld $6, 24(%1)\n\t" \
5860  "ld $7, 32(%1)\n\t" \
5861  "ld $8, 40(%1)\n\t" \
5862  "ld $9, 48(%1)\n\t" \
5863  "ld $10, 56(%1)\n\t" \
5864  "ld $25, 0(%1) \n\t" /* target->t9 */ \
5865  VALGRIND_CALL_NOREDIR_T9 \
5866  "move %0, $2\n" \
5867  : /*out*/ "=r" (_res) \
5868  : /*in*/ "r" (&_argvec[0]) \
5869  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5870  ); \
5871  lval = (__typeof__(lval)) _res; \
5872  } while (0)
5873 
5874 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
5875  arg7,arg8) \
5876  do { \
5877  volatile OrigFn _orig = (orig); \
5878  volatile unsigned long _argvec[9]; \
5879  volatile unsigned long _res; \
5880  _argvec[0] = (unsigned long)_orig.nraddr; \
5881  _argvec[1] = (unsigned long)(arg1); \
5882  _argvec[2] = (unsigned long)(arg2); \
5883  _argvec[3] = (unsigned long)(arg3); \
5884  _argvec[4] = (unsigned long)(arg4); \
5885  _argvec[5] = (unsigned long)(arg5); \
5886  _argvec[6] = (unsigned long)(arg6); \
5887  _argvec[7] = (unsigned long)(arg7); \
5888  _argvec[8] = (unsigned long)(arg8); \
5889  __asm__ volatile( \
5890  "ld $4, 8(%1)\n\t" \
5891  "ld $5, 16(%1)\n\t" \
5892  "ld $6, 24(%1)\n\t" \
5893  "ld $7, 32(%1)\n\t" \
5894  "ld $8, 40(%1)\n\t" \
5895  "ld $9, 48(%1)\n\t" \
5896  "ld $10, 56(%1)\n\t" \
5897  "ld $11, 64(%1)\n\t" \
5898  "ld $25, 0(%1) \n\t" /* target->t9 */ \
5899  VALGRIND_CALL_NOREDIR_T9 \
5900  "move %0, $2\n" \
5901  : /*out*/ "=r" (_res) \
5902  : /*in*/ "r" (&_argvec[0]) \
5903  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5904  ); \
5905  lval = (__typeof__(lval)) _res; \
5906  } while (0)
5907 
5908 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
5909  arg7,arg8,arg9) \
5910  do { \
5911  volatile OrigFn _orig = (orig); \
5912  volatile unsigned long _argvec[10]; \
5913  volatile unsigned long _res; \
5914  _argvec[0] = (unsigned long)_orig.nraddr; \
5915  _argvec[1] = (unsigned long)(arg1); \
5916  _argvec[2] = (unsigned long)(arg2); \
5917  _argvec[3] = (unsigned long)(arg3); \
5918  _argvec[4] = (unsigned long)(arg4); \
5919  _argvec[5] = (unsigned long)(arg5); \
5920  _argvec[6] = (unsigned long)(arg6); \
5921  _argvec[7] = (unsigned long)(arg7); \
5922  _argvec[8] = (unsigned long)(arg8); \
5923  _argvec[9] = (unsigned long)(arg9); \
5924  __asm__ volatile( \
5925  "dsubu $29, $29, 8\n\t" \
5926  "ld $4, 72(%1)\n\t" \
5927  "sd $4, 0($29)\n\t" \
5928  "ld $4, 8(%1)\n\t" \
5929  "ld $5, 16(%1)\n\t" \
5930  "ld $6, 24(%1)\n\t" \
5931  "ld $7, 32(%1)\n\t" \
5932  "ld $8, 40(%1)\n\t" \
5933  "ld $9, 48(%1)\n\t" \
5934  "ld $10, 56(%1)\n\t" \
5935  "ld $11, 64(%1)\n\t" \
5936  "ld $25, 0(%1)\n\t" /* target->t9 */ \
5937  VALGRIND_CALL_NOREDIR_T9 \
5938  "daddu $29, $29, 8\n\t" \
5939  "move %0, $2\n" \
5940  : /*out*/ "=r" (_res) \
5941  : /*in*/ "r" (&_argvec[0]) \
5942  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5943  ); \
5944  lval = (__typeof__(lval)) _res; \
5945  } while (0)
5946 
5947 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
5948  arg7,arg8,arg9,arg10) \
5949  do { \
5950  volatile OrigFn _orig = (orig); \
5951  volatile unsigned long _argvec[11]; \
5952  volatile unsigned long _res; \
5953  _argvec[0] = (unsigned long)_orig.nraddr; \
5954  _argvec[1] = (unsigned long)(arg1); \
5955  _argvec[2] = (unsigned long)(arg2); \
5956  _argvec[3] = (unsigned long)(arg3); \
5957  _argvec[4] = (unsigned long)(arg4); \
5958  _argvec[5] = (unsigned long)(arg5); \
5959  _argvec[6] = (unsigned long)(arg6); \
5960  _argvec[7] = (unsigned long)(arg7); \
5961  _argvec[8] = (unsigned long)(arg8); \
5962  _argvec[9] = (unsigned long)(arg9); \
5963  _argvec[10] = (unsigned long)(arg10); \
5964  __asm__ volatile( \
5965  "dsubu $29, $29, 16\n\t" \
5966  "ld $4, 72(%1)\n\t" \
5967  "sd $4, 0($29)\n\t" \
5968  "ld $4, 80(%1)\n\t" \
5969  "sd $4, 8($29)\n\t" \
5970  "ld $4, 8(%1)\n\t" \
5971  "ld $5, 16(%1)\n\t" \
5972  "ld $6, 24(%1)\n\t" \
5973  "ld $7, 32(%1)\n\t" \
5974  "ld $8, 40(%1)\n\t" \
5975  "ld $9, 48(%1)\n\t" \
5976  "ld $10, 56(%1)\n\t" \
5977  "ld $11, 64(%1)\n\t" \
5978  "ld $25, 0(%1)\n\t" /* target->t9 */ \
5979  VALGRIND_CALL_NOREDIR_T9 \
5980  "daddu $29, $29, 16\n\t" \
5981  "move %0, $2\n" \
5982  : /*out*/ "=r" (_res) \
5983  : /*in*/ "r" (&_argvec[0]) \
5984  : /*trash*/ "memory", __CALLER_SAVED_REGS \
5985  ); \
5986  lval = (__typeof__(lval)) _res; \
5987  } while (0)
5988 
5989 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \
5990  arg6,arg7,arg8,arg9,arg10, \
5991  arg11) \
5992  do { \
5993  volatile OrigFn _orig = (orig); \
5994  volatile unsigned long _argvec[12]; \
5995  volatile unsigned long _res; \
5996  _argvec[0] = (unsigned long)_orig.nraddr; \
5997  _argvec[1] = (unsigned long)(arg1); \
5998  _argvec[2] = (unsigned long)(arg2); \
5999  _argvec[3] = (unsigned long)(arg3); \
6000  _argvec[4] = (unsigned long)(arg4); \
6001  _argvec[5] = (unsigned long)(arg5); \
6002  _argvec[6] = (unsigned long)(arg6); \
6003  _argvec[7] = (unsigned long)(arg7); \
6004  _argvec[8] = (unsigned long)(arg8); \
6005  _argvec[9] = (unsigned long)(arg9); \
6006  _argvec[10] = (unsigned long)(arg10); \
6007  _argvec[11] = (unsigned long)(arg11); \
6008  __asm__ volatile( \
6009  "dsubu $29, $29, 24\n\t" \
6010  "ld $4, 72(%1)\n\t" \
6011  "sd $4, 0($29)\n\t" \
6012  "ld $4, 80(%1)\n\t" \
6013  "sd $4, 8($29)\n\t" \
6014  "ld $4, 88(%1)\n\t" \
6015  "sd $4, 16($29)\n\t" \
6016  "ld $4, 8(%1)\n\t" \
6017  "ld $5, 16(%1)\n\t" \
6018  "ld $6, 24(%1)\n\t" \
6019  "ld $7, 32(%1)\n\t" \
6020  "ld $8, 40(%1)\n\t" \
6021  "ld $9, 48(%1)\n\t" \
6022  "ld $10, 56(%1)\n\t" \
6023  "ld $11, 64(%1)\n\t" \
6024  "ld $25, 0(%1)\n\t" /* target->t9 */ \
6025  VALGRIND_CALL_NOREDIR_T9 \
6026  "daddu $29, $29, 24\n\t" \
6027  "move %0, $2\n" \
6028  : /*out*/ "=r" (_res) \
6029  : /*in*/ "r" (&_argvec[0]) \
6030  : /*trash*/ "memory", __CALLER_SAVED_REGS \
6031  ); \
6032  lval = (__typeof__(lval)) _res; \
6033  } while (0)
6034 
6035 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \
6036  arg6,arg7,arg8,arg9,arg10, \
6037  arg11,arg12) \
6038  do { \
6039  volatile OrigFn _orig = (orig); \
6040  volatile unsigned long _argvec[13]; \
6041  volatile unsigned long _res; \
6042  _argvec[0] = (unsigned long)_orig.nraddr; \
6043  _argvec[1] = (unsigned long)(arg1); \
6044  _argvec[2] = (unsigned long)(arg2); \
6045  _argvec[3] = (unsigned long)(arg3); \
6046  _argvec[4] = (unsigned long)(arg4); \
6047  _argvec[5] = (unsigned long)(arg5); \
6048  _argvec[6] = (unsigned long)(arg6); \
6049  _argvec[7] = (unsigned long)(arg7); \
6050  _argvec[8] = (unsigned long)(arg8); \
6051  _argvec[9] = (unsigned long)(arg9); \
6052  _argvec[10] = (unsigned long)(arg10); \
6053  _argvec[11] = (unsigned long)(arg11); \
6054  _argvec[12] = (unsigned long)(arg12); \
6055  __asm__ volatile( \
6056  "dsubu $29, $29, 32\n\t" \
6057  "ld $4, 72(%1)\n\t" \
6058  "sd $4, 0($29)\n\t" \
6059  "ld $4, 80(%1)\n\t" \
6060  "sd $4, 8($29)\n\t" \
6061  "ld $4, 88(%1)\n\t" \
6062  "sd $4, 16($29)\n\t" \
6063  "ld $4, 96(%1)\n\t" \
6064  "sd $4, 24($29)\n\t" \
6065  "ld $4, 8(%1)\n\t" \
6066  "ld $5, 16(%1)\n\t" \
6067  "ld $6, 24(%1)\n\t" \
6068  "ld $7, 32(%1)\n\t" \
6069  "ld $8, 40(%1)\n\t" \
6070  "ld $9, 48(%1)\n\t" \
6071  "ld $10, 56(%1)\n\t" \
6072  "ld $11, 64(%1)\n\t" \
6073  "ld $25, 0(%1)\n\t" /* target->t9 */ \
6074  VALGRIND_CALL_NOREDIR_T9 \
6075  "daddu $29, $29, 32\n\t" \
6076  "move %0, $2\n" \
6077  : /*out*/ "=r" (_res) \
6078  : /*in*/ "r" (&_argvec[0]) \
6079  : /*trash*/ "memory", __CALLER_SAVED_REGS \
6080  ); \
6081  lval = (__typeof__(lval)) _res; \
6082  } while (0)
6083 
6084 #endif /* PLAT_mips64_linux */
6085 
6086 
6087 /* ------------------------------------------------------------------ */
6088 /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS. */
6089 /* */
6090 /* ------------------------------------------------------------------ */
6091 
6092 /* Some request codes. There are many more of these, but most are not
6093  exposed to end-user view. These are the public ones, all of the
6094  form 0x1000 + small_number.
6095 
6096  Core ones are in the range 0x00000000--0x0000ffff. The non-public
6097  ones start at 0x2000.
6098 */
6099 
6100 /* These macros are used by tools -- they must be public, but don't
6101  embed them into other programs. */
6102 #define VG_USERREQ_TOOL_BASE(a,b) \
6103  ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
6104 #define VG_IS_TOOL_USERREQ(a, b, v) \
6105  (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
6106 
6107 /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !!
6108  This enum comprises an ABI exported by Valgrind to programs
6109  which use client requests. DO NOT CHANGE THE ORDER OF THESE
6110  ENTRIES, NOR DELETE ANY -- add new ones at the end. */
6111 typedef
6112  enum { VG_USERREQ__RUNNING_ON_VALGRIND = 0x1001,
6113  VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
6114 
6115  /* These allow any function to be called from the simulated
6116  CPU but run on the real CPU. Nb: the first arg passed to
6117  the function is always the ThreadId of the running
6118  thread! So CLIENT_CALL0 actually requires a 1 arg
6119  function, etc. */
6120  VG_USERREQ__CLIENT_CALL0 = 0x1101,
6121  VG_USERREQ__CLIENT_CALL1 = 0x1102,
6122  VG_USERREQ__CLIENT_CALL2 = 0x1103,
6123  VG_USERREQ__CLIENT_CALL3 = 0x1104,
6124 
6125  /* Can be useful in regression testing suites -- eg. can
6126  send Valgrind's output to /dev/null and still count
6127  errors. */
6128  VG_USERREQ__COUNT_ERRORS = 0x1201,
6129 
6130  /* Allows the client program and/or gdbserver to execute a monitor
6131  command. */
6132  VG_USERREQ__GDB_MONITOR_COMMAND = 0x1202,
6133 
6134  /* These are useful and can be interpreted by any tool that
6135  tracks malloc() et al, by using vg_replace_malloc.c. */
6136  VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
6137  VG_USERREQ__RESIZEINPLACE_BLOCK = 0x130b,
6138  VG_USERREQ__FREELIKE_BLOCK = 0x1302,
6139  /* Memory pool support. */
6140  VG_USERREQ__CREATE_MEMPOOL = 0x1303,
6141  VG_USERREQ__DESTROY_MEMPOOL = 0x1304,
6142  VG_USERREQ__MEMPOOL_ALLOC = 0x1305,
6143  VG_USERREQ__MEMPOOL_FREE = 0x1306,
6144  VG_USERREQ__MEMPOOL_TRIM = 0x1307,
6145  VG_USERREQ__MOVE_MEMPOOL = 0x1308,
6146  VG_USERREQ__MEMPOOL_CHANGE = 0x1309,
6147  VG_USERREQ__MEMPOOL_EXISTS = 0x130a,
6148 
6149  /* Allow printfs to valgrind log. */
6150  /* The first two pass the va_list argument by value, which
6151  assumes it is the same size as or smaller than a UWord,
6152  which generally isn't the case. Hence are deprecated.
6153  The second two pass the vargs by reference and so are
6154  immune to this problem. */
6155  /* both :: char* fmt, va_list vargs (DEPRECATED) */
6156  VG_USERREQ__PRINTF = 0x1401,
6157  VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
6158  /* both :: char* fmt, va_list* vargs */
6159  VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403,
6160  VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404,
6161 
6162  /* Stack support. */
6163  VG_USERREQ__STACK_REGISTER = 0x1501,
6164  VG_USERREQ__STACK_DEREGISTER = 0x1502,
6165  VG_USERREQ__STACK_CHANGE = 0x1503,
6166 
6167  /* Wine support */
6168  VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601,
6169 
6170  /* Querying of debug info. */
6171  VG_USERREQ__MAP_IP_TO_SRCLOC = 0x1701,
6172 
6173  /* Disable/enable error reporting level. Takes a single
6174  Word arg which is the delta to this thread's error
6175  disablement indicator. Hence 1 disables or further
6176  disables errors, and -1 moves back towards enablement.
6177  Other values are not allowed. */
6178  VG_USERREQ__CHANGE_ERR_DISABLEMENT = 0x1801,
6179 
6180  /* Initialize IR injection */
6181  VG_USERREQ__VEX_INIT_FOR_IRI = 0x1901
6182  } Vg_ClientRequest;
6183 
6184 #if !defined(__GNUC__)
6185 # define __extension__ /* */
6186 #endif
6187 
6188 
6189 /* Returns the number of Valgrinds this code is running under. That
6190  is, 0 if running natively, 1 if running under Valgrind, 2 if
6191  running under Valgrind which is running under another Valgrind,
6192  etc. */
6193 #define RUNNING_ON_VALGRIND \
6194  (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* if not */, \
6195  VG_USERREQ__RUNNING_ON_VALGRIND, \
6196  0, 0, 0, 0, 0) \
6197 
6198 
6199 /* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
6200  _qzz_len - 1]. Useful if you are debugging a JITter or some such,
6201  since it provides a way to make sure valgrind will retranslate the
6202  invalidated area. Returns no value. */
6203 #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len) \
6204  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DISCARD_TRANSLATIONS, \
6205  _qzz_addr, _qzz_len, 0, 0, 0)
6206 
6207 
6208 /* These requests are for getting Valgrind itself to print something.
6209  Possibly with a backtrace. This is a really ugly hack. The return value
6210  is the number of characters printed, excluding the "**<pid>** " part at the
6211  start and the backtrace (if present). */
6212 
6213 #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER)
6214 /* Modern GCC will optimize the static routine out if unused,
6215  and unused attribute will shut down warnings about it. */
6216 static int VALGRIND_PRINTF(const char *format, ...)
6217  __attribute__((format(__printf__, 1, 2), __unused__));
6218 #endif
6219 static int
6220 #if defined(_MSC_VER)
6221 __inline
6222 #endif
6223 VALGRIND_PRINTF(const char *format, ...)
6224 {
6225 #if defined(NVALGRIND)
6226  return 0;
6227 #else /* NVALGRIND */
6228 #if defined(_MSC_VER) || defined(__MINGW64__)
6229  uintptr_t _qzz_res;
6230 #else
6231  unsigned long _qzz_res;
6232 #endif
6233  va_list vargs;
6234  va_start(vargs, format);
6235 #if defined(_MSC_VER) || defined(__MINGW64__)
6236  _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
6237  VG_USERREQ__PRINTF_VALIST_BY_REF,
6238  (uintptr_t)format,
6239  (uintptr_t)&vargs,
6240  0, 0, 0);
6241 #else
6242  _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
6243  VG_USERREQ__PRINTF_VALIST_BY_REF,
6244  (unsigned long)format,
6245  (unsigned long)&vargs,
6246  0, 0, 0);
6247 #endif
6248  va_end(vargs);
6249  return (int)_qzz_res;
6250 #endif /* NVALGRIND */
6251 }
6252 
6253 #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER)
6254 static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
6255  __attribute__((format(__printf__, 1, 2), __unused__));
6256 #endif
6257 static int
6258 #if defined(_MSC_VER)
6259 __inline
6260 #endif
6261 VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
6262 {
6263 #if defined(NVALGRIND)
6264  return 0;
6265 #else /* NVALGRIND */
6266 #if defined(_MSC_VER) || defined(__MINGW64__)
6267  uintptr_t _qzz_res;
6268 #else
6269  unsigned long _qzz_res;
6270 #endif
6271  va_list vargs;
6272  va_start(vargs, format);
6273 #if defined(_MSC_VER) || defined(__MINGW64__)
6274  _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
6275  VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
6276  (uintptr_t)format,
6277  (uintptr_t)&vargs,
6278  0, 0, 0);
6279 #else
6280  _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
6281  VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
6282  (unsigned long)format,
6283  (unsigned long)&vargs,
6284  0, 0, 0);
6285 #endif
6286  va_end(vargs);
6287  return (int)_qzz_res;
6288 #endif /* NVALGRIND */
6289 }
6290 
6291 
6292 /* These requests allow control to move from the simulated CPU to the
6293  real CPU, calling an arbitrary function.
6294 
6295  Note that the current ThreadId is inserted as the first argument.
6296  So this call:
6297 
6298  VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
6299 
6300  requires f to have this signature:
6301 
6302  Word f(Word tid, Word arg1, Word arg2)
6303 
6304  where "Word" is a word-sized type.
6305 
6306  Note that these client requests are not entirely reliable. For example,
6307  if you call a function with them that subsequently calls printf(),
6308  there's a high chance Valgrind will crash. Generally, your prospects of
6309  these working are made higher if the called function does not refer to
6310  any global variables, and does not refer to any libc or other functions
6311  (printf et al). Any kind of entanglement with libc or dynamic linking is
6312  likely to have a bad outcome, for tricky reasons which we've grappled
6313  with a lot in the past.
6314 */
6315 #define VALGRIND_NON_SIMD_CALL0(_qyy_fn) \
6316  VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \
6317  VG_USERREQ__CLIENT_CALL0, \
6318  _qyy_fn, \
6319  0, 0, 0, 0)
6320 
6321 #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1) \
6322  VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \
6323  VG_USERREQ__CLIENT_CALL1, \
6324  _qyy_fn, \
6325  _qyy_arg1, 0, 0, 0)
6326 
6327 #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2) \
6328  VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \
6329  VG_USERREQ__CLIENT_CALL2, \
6330  _qyy_fn, \
6331  _qyy_arg1, _qyy_arg2, 0, 0)
6332 
6333 #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
6334  VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \
6335  VG_USERREQ__CLIENT_CALL3, \
6336  _qyy_fn, \
6337  _qyy_arg1, _qyy_arg2, \
6338  _qyy_arg3, 0)
6339 
6340 
6341 /* Counts the number of errors that have been recorded by a tool. Nb:
6342  the tool must record the errors with VG_(maybe_record_error)() or
6343  VG_(unique_error)() for them to be counted. */
6344 #define VALGRIND_COUNT_ERRORS \
6345  (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR( \
6346  0 /* default return */, \
6347  VG_USERREQ__COUNT_ERRORS, \
6348  0, 0, 0, 0, 0)
6349 
6350 /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
6351  when heap blocks are allocated in order to give accurate results. This
6352  happens automatically for the standard allocator functions such as
6353  malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
6354  delete[], etc.
6355 
6356  But if your program uses a custom allocator, this doesn't automatically
6357  happen, and Valgrind will not do as well. For example, if you allocate
6358  superblocks with mmap() and then allocates chunks of the superblocks, all
6359  Valgrind's observations will be at the mmap() level and it won't know that
6360  the chunks should be considered separate entities. In Memcheck's case,
6361  that means you probably won't get heap block overrun detection (because
6362  there won't be redzones marked as unaddressable) and you definitely won't
6363  get any leak detection.
6364 
6365  The following client requests allow a custom allocator to be annotated so
6366  that it can be handled accurately by Valgrind.
6367 
6368  VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
6369  by a malloc()-like function. For Memcheck (an illustrative case), this
6370  does two things:
6371 
6372  - It records that the block has been allocated. This means any addresses
6373  within the block mentioned in error messages will be
6374  identified as belonging to the block. It also means that if the block
6375  isn't freed it will be detected by the leak checker.
6376 
6377  - It marks the block as being addressable and undefined (if 'is_zeroed' is
6378  not set), or addressable and defined (if 'is_zeroed' is set). This
6379  controls how accesses to the block by the program are handled.
6380 
6381  'addr' is the start of the usable block (ie. after any
6382  redzone), 'sizeB' is its size. 'rzB' is the redzone size if the allocator
6383  can apply redzones -- these are blocks of padding at the start and end of
6384  each block. Adding redzones is recommended as it makes it much more likely
6385  Valgrind will spot block overruns. `is_zeroed' indicates if the memory is
6386  zeroed (or filled with another predictable value), as is the case for
6387  calloc().
6388 
6389  VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
6390  heap block -- that will be used by the client program -- is allocated.
6391  It's best to put it at the outermost level of the allocator if possible;
6392  for example, if you have a function my_alloc() which calls
6393  internal_alloc(), and the client request is put inside internal_alloc(),
6394  stack traces relating to the heap block will contain entries for both
6395  my_alloc() and internal_alloc(), which is probably not what you want.
6396 
6397  For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
6398  custom blocks from within a heap block, B, that has been allocated with
6399  malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
6400  -- the custom blocks will take precedence.
6401 
6402  VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK. For
6403  Memcheck, it does two things:
6404 
6405  - It records that the block has been deallocated. This assumes that the
6406  block was annotated as having been allocated via
6407  VALGRIND_MALLOCLIKE_BLOCK. Otherwise, an error will be issued.
6408 
6409  - It marks the block as being unaddressable.
6410 
6411  VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
6412  heap block is deallocated.
6413 
6414  VALGRIND_RESIZEINPLACE_BLOCK informs a tool about reallocation. For
6415  Memcheck, it does four things:
6416 
6417  - It records that the size of a block has been changed. This assumes that
6418  the block was annotated as having been allocated via
6419  VALGRIND_MALLOCLIKE_BLOCK. Otherwise, an error will be issued.
6420 
6421  - If the block shrunk, it marks the freed memory as being unaddressable.
6422 
6423  - If the block grew, it marks the new area as undefined and defines a red
6424  zone past the end of the new block.
6425 
6426  - The V-bits of the overlap between the old and the new block are preserved.
6427 
6428  VALGRIND_RESIZEINPLACE_BLOCK should be put after allocation of the new block
6429  and before deallocation of the old block.
6430 
6431  In many cases, these three client requests will not be enough to get your
6432  allocator working well with Memcheck. More specifically, if your allocator
6433  writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
6434  will be necessary to mark the memory as addressable just before the zeroing
6435  occurs, otherwise you'll get a lot of invalid write errors. For example,
6436  you'll need to do this if your allocator recycles freed blocks, but it
6437  zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
6438  Alternatively, if your allocator reuses freed blocks for allocator-internal
6439  data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
6440 
6441  Really, what's happening is a blurring of the lines between the client
6442  program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
6443  memory should be considered unaddressable to the client program, but the
6444  allocator knows more than the rest of the client program and so may be able
6445  to safely access it. Extra client requests are necessary for Valgrind to
6446  understand the distinction between the allocator and the rest of the
6447  program.
6448 
6449  Ignored if addr == 0.
6450 */
6451 #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \
6452  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MALLOCLIKE_BLOCK, \
6453  addr, sizeB, rzB, is_zeroed, 0)
6454 
6455 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
6456  Ignored if addr == 0.
6457 */
6458 #define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \
6459  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__RESIZEINPLACE_BLOCK, \
6460  addr, oldSizeB, newSizeB, rzB, 0)
6461 
6462 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
6463  Ignored if addr == 0.
6464 */
6465 #define VALGRIND_FREELIKE_BLOCK(addr, rzB) \
6466  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__FREELIKE_BLOCK, \
6467  addr, rzB, 0, 0, 0)
6468 
6469 /* Create a memory pool. */
6470 #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed) \
6471  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CREATE_MEMPOOL, \
6472  pool, rzB, is_zeroed, 0, 0)
6473 
6474 /* Destroy a memory pool. */
6475 #define VALGRIND_DESTROY_MEMPOOL(pool) \
6476  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DESTROY_MEMPOOL, \
6477  pool, 0, 0, 0, 0)
6478 
6479 /* Associate a piece of memory with a memory pool. */
6480 #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size) \
6481  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_ALLOC, \
6482  pool, addr, size, 0, 0)
6483 
6484 /* Disassociate a piece of memory from a memory pool. */
6485 #define VALGRIND_MEMPOOL_FREE(pool, addr) \
6486  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_FREE, \
6487  pool, addr, 0, 0, 0)
6488 
6489 /* Disassociate any pieces outside a particular range. */
6490 #define VALGRIND_MEMPOOL_TRIM(pool, addr, size) \
6491  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_TRIM, \
6492  pool, addr, size, 0, 0)
6493 
6494 /* Resize and/or move a piece associated with a memory pool. */
6495 #define VALGRIND_MOVE_MEMPOOL(poolA, poolB) \
6496  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MOVE_MEMPOOL, \
6497  poolA, poolB, 0, 0, 0)
6498 
6499 /* Resize and/or move a piece associated with a memory pool. */
6500 #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size) \
6501  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_CHANGE, \
6502  pool, addrA, addrB, size, 0)
6503 
6504 /* Return 1 if a mempool exists, else 0. */
6505 #define VALGRIND_MEMPOOL_EXISTS(pool) \
6506  (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \
6507  VG_USERREQ__MEMPOOL_EXISTS, \
6508  pool, 0, 0, 0, 0)
6509 
6510 /* Mark a piece of memory as being a stack. Returns a stack id.
6511  start is the lowest addressable stack byte, end is the highest
6512  addressable stack byte. */
6513 #define VALGRIND_STACK_REGISTER(start, end) \
6514  (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \
6515  VG_USERREQ__STACK_REGISTER, \
6516  start, end, 0, 0, 0)
6517 
6518 /* Unmark the piece of memory associated with a stack id as being a
6519  stack. */
6520 #define VALGRIND_STACK_DEREGISTER(id) \
6521  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_DEREGISTER, \
6522  id, 0, 0, 0, 0)
6523 
6524 /* Change the start and end address of the stack id.
6525  start is the new lowest addressable stack byte, end is the new highest
6526  addressable stack byte. */
6527 #define VALGRIND_STACK_CHANGE(id, start, end) \
6528  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_CHANGE, \
6529  id, start, end, 0, 0)
6530 
6531 /* Load PDB debug info for Wine PE image_map. */
6532 #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta) \
6533  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__LOAD_PDB_DEBUGINFO, \
6534  fd, ptr, total_size, delta, 0)
6535 
6536 /* Map a code address to a source file name and line number. buf64
6537  must point to a 64-byte buffer in the caller's address space. The
6538  result will be dumped in there and is guaranteed to be zero
6539  terminated. If no info is found, the first byte is set to zero. */
6540 #define VALGRIND_MAP_IP_TO_SRCLOC(addr, buf64) \
6541  (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \
6542  VG_USERREQ__MAP_IP_TO_SRCLOC, \
6543  addr, buf64, 0, 0, 0)
6544 
6545 /* Disable error reporting for this thread. Behaves in a stack like
6546  way, so you can safely call this multiple times provided that
6547  VALGRIND_ENABLE_ERROR_REPORTING is called the same number of times
6548  to re-enable reporting. The first call of this macro disables
6549  reporting. Subsequent calls have no effect except to increase the
6550  number of VALGRIND_ENABLE_ERROR_REPORTING calls needed to re-enable
6551  reporting. Child threads do not inherit this setting from their
6552  parents -- they are always created with reporting enabled. */
6553 #define VALGRIND_DISABLE_ERROR_REPORTING \
6554  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \
6555  1, 0, 0, 0, 0)
6556 
6557 /* Re-enable error reporting, as per comments on
6558  VALGRIND_DISABLE_ERROR_REPORTING. */
6559 #define VALGRIND_ENABLE_ERROR_REPORTING \
6560  VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \
6561  -1, 0, 0, 0, 0)
6562 
6563 /* Execute a monitor command from the client program.
6564  If a connection is opened with GDB, the output will be sent
6565  according to the output mode set for vgdb.
6566  If no connection is opened, output will go to the log output.
6567  Returns 1 if command not recognized, 0 otherwise. */
6568 #define VALGRIND_MONITOR_COMMAND(command) \
6569  VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__GDB_MONITOR_COMMAND, \
6570  command, 0, 0, 0, 0)
6571 
6572 
6573 #undef PLAT_x86_darwin
6574 #undef PLAT_amd64_darwin
6575 #undef PLAT_x86_win32
6576 #undef PLAT_amd64_win64
6577 #undef PLAT_x86_linux
6578 #undef PLAT_amd64_linux
6579 #undef PLAT_ppc32_linux
6580 #undef PLAT_ppc64be_linux
6581 #undef PLAT_ppc64le_linux
6582 #undef PLAT_arm_linux
6583 #undef PLAT_s390x_linux
6584 #undef PLAT_mips32_linux
6585 #undef PLAT_mips64_linux
6586 
6587 #endif /* __VALGRIND_H */
Generated on Wed Oct 6 2021 20:58:14 for EPICS Base by   doxygen 1.8.5