1 /* Subroutines used for code generation on IA-32.
2 Copyright (C) 1988-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
11 GCC is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
28 #include "double-int.h"
35 #include "fold-const.h"
36 #include "stringpool.h"
39 #include "stor-layout.h"
43 #include "hard-reg-set.h"
44 #include "insn-config.h"
45 #include "conditions.h"
47 #include "insn-codes.h"
48 #include "insn-attr.h"
54 #include "statistics.h"
56 #include "fixed-value.h"
64 #include "diagnostic-core.h"
67 #include "dominance.h"
73 #include "cfgcleanup.h"
74 #include "basic-block.h"
77 #include "target-def.h"
78 #include "common/common-target.h"
79 #include "langhooks.h"
83 #include "plugin-api.h"
86 #include "hash-table.h"
87 #include "tree-ssa-alias.h"
88 #include "internal-fn.h"
89 #include "gimple-fold.h"
91 #include "gimple-expr.h"
97 #include "tm-constrs.h"
101 #include "sched-int.h"
105 #include "diagnostic.h"
106 #include "dumpfile.h"
107 #include "tree-pass.h"
109 #include "pass_manager.h"
110 #include "target-globals.h"
111 #include "tree-vectorizer.h"
112 #include "shrink-wrap.h"
113 #include "builtins.h"
114 #include "rtl-iter.h"
115 #include "tree-iterator.h"
116 #include "tree-chkp.h"
117 #include "rtl-chkp.h"
119 static rtx legitimize_dllimport_symbol (rtx, bool);
120 static rtx legitimize_pe_coff_extern_decl (rtx, bool);
121 static rtx legitimize_pe_coff_symbol (rtx, bool);
123 #ifndef CHECK_STACK_LIMIT
124 #define CHECK_STACK_LIMIT (-1)
127 /* Return index of given mode in mult and division cost tables. */
128 #define MODE_INDEX(mode) \
129 ((mode) == QImode ? 0 \
130 : (mode) == HImode ? 1 \
131 : (mode) == SImode ? 2 \
132 : (mode) == DImode ? 3 \
135 /* Processor costs (relative to an add) */
136 /* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes. */
137 #define COSTS_N_BYTES(N) ((N) * 2)
139 #define DUMMY_STRINGOP_ALGS {libcall, {{-1, libcall, false}}}
141 static stringop_algs ix86_size_memcpy[2] = {
142 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
143 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}}};
144 static stringop_algs ix86_size_memset[2] = {
145 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
146 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}}};
149 struct processor_costs ix86_size_cost = {/* costs for tuning for size */
150 COSTS_N_BYTES (2), /* cost of an add instruction */
151 COSTS_N_BYTES (3), /* cost of a lea instruction */
152 COSTS_N_BYTES (2), /* variable shift costs */
153 COSTS_N_BYTES (3), /* constant shift costs */
154 {COSTS_N_BYTES (3), /* cost of starting multiply for QI */
155 COSTS_N_BYTES (3), /* HI */
156 COSTS_N_BYTES (3), /* SI */
157 COSTS_N_BYTES (3), /* DI */
158 COSTS_N_BYTES (5)}, /* other */
159 0, /* cost of multiply per each bit set */
160 {COSTS_N_BYTES (3), /* cost of a divide/mod for QI */
161 COSTS_N_BYTES (3), /* HI */
162 COSTS_N_BYTES (3), /* SI */
163 COSTS_N_BYTES (3), /* DI */
164 COSTS_N_BYTES (5)}, /* other */
165 COSTS_N_BYTES (3), /* cost of movsx */
166 COSTS_N_BYTES (3), /* cost of movzx */
167 0, /* "large" insn */
169 2, /* cost for loading QImode using movzbl */
170 {2, 2, 2}, /* cost of loading integer registers
171 in QImode, HImode and SImode.
172 Relative to reg-reg move (2). */
173 {2, 2, 2}, /* cost of storing integer registers */
174 2, /* cost of reg,reg fld/fst */
175 {2, 2, 2}, /* cost of loading fp registers
176 in SFmode, DFmode and XFmode */
177 {2, 2, 2}, /* cost of storing fp registers
178 in SFmode, DFmode and XFmode */
179 3, /* cost of moving MMX register */
180 {3, 3}, /* cost of loading MMX registers
181 in SImode and DImode */
182 {3, 3}, /* cost of storing MMX registers
183 in SImode and DImode */
184 3, /* cost of moving SSE register */
185 {3, 3, 3}, /* cost of loading SSE registers
186 in SImode, DImode and TImode */
187 {3, 3, 3}, /* cost of storing SSE registers
188 in SImode, DImode and TImode */
189 3, /* MMX or SSE register to integer */
190 0, /* size of l1 cache */
191 0, /* size of l2 cache */
192 0, /* size of prefetch block */
193 0, /* number of parallel prefetches */
195 COSTS_N_BYTES (2), /* cost of FADD and FSUB insns. */
196 COSTS_N_BYTES (2), /* cost of FMUL instruction. */
197 COSTS_N_BYTES (2), /* cost of FDIV instruction. */
198 COSTS_N_BYTES (2), /* cost of FABS instruction. */
199 COSTS_N_BYTES (2), /* cost of FCHS instruction. */
200 COSTS_N_BYTES (2), /* cost of FSQRT instruction. */
203 1, /* scalar_stmt_cost. */
204 1, /* scalar load_cost. */
205 1, /* scalar_store_cost. */
206 1, /* vec_stmt_cost. */
207 1, /* vec_to_scalar_cost. */
208 1, /* scalar_to_vec_cost. */
209 1, /* vec_align_load_cost. */
210 1, /* vec_unalign_load_cost. */
211 1, /* vec_store_cost. */
212 1, /* cond_taken_branch_cost. */
213 1, /* cond_not_taken_branch_cost. */
216 /* Processor costs (relative to an add) */
217 static stringop_algs i386_memcpy[2] = {
218 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
219 DUMMY_STRINGOP_ALGS};
220 static stringop_algs i386_memset[2] = {
221 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
222 DUMMY_STRINGOP_ALGS};
225 struct processor_costs i386_cost = { /* 386 specific costs */
226 COSTS_N_INSNS (1), /* cost of an add instruction */
227 COSTS_N_INSNS (1), /* cost of a lea instruction */
228 COSTS_N_INSNS (3), /* variable shift costs */
229 COSTS_N_INSNS (2), /* constant shift costs */
230 {COSTS_N_INSNS (6), /* cost of starting multiply for QI */
231 COSTS_N_INSNS (6), /* HI */
232 COSTS_N_INSNS (6), /* SI */
233 COSTS_N_INSNS (6), /* DI */
234 COSTS_N_INSNS (6)}, /* other */
235 COSTS_N_INSNS (1), /* cost of multiply per each bit set */
236 {COSTS_N_INSNS (23), /* cost of a divide/mod for QI */
237 COSTS_N_INSNS (23), /* HI */
238 COSTS_N_INSNS (23), /* SI */
239 COSTS_N_INSNS (23), /* DI */
240 COSTS_N_INSNS (23)}, /* other */
241 COSTS_N_INSNS (3), /* cost of movsx */
242 COSTS_N_INSNS (2), /* cost of movzx */
243 15, /* "large" insn */
245 4, /* cost for loading QImode using movzbl */
246 {2, 4, 2}, /* cost of loading integer registers
247 in QImode, HImode and SImode.
248 Relative to reg-reg move (2). */
249 {2, 4, 2}, /* cost of storing integer registers */
250 2, /* cost of reg,reg fld/fst */
251 {8, 8, 8}, /* cost of loading fp registers
252 in SFmode, DFmode and XFmode */
253 {8, 8, 8}, /* cost of storing fp registers
254 in SFmode, DFmode and XFmode */
255 2, /* cost of moving MMX register */
256 {4, 8}, /* cost of loading MMX registers
257 in SImode and DImode */
258 {4, 8}, /* cost of storing MMX registers
259 in SImode and DImode */
260 2, /* cost of moving SSE register */
261 {4, 8, 16}, /* cost of loading SSE registers
262 in SImode, DImode and TImode */
263 {4, 8, 16}, /* cost of storing SSE registers
264 in SImode, DImode and TImode */
265 3, /* MMX or SSE register to integer */
266 0, /* size of l1 cache */
267 0, /* size of l2 cache */
268 0, /* size of prefetch block */
269 0, /* number of parallel prefetches */
271 COSTS_N_INSNS (23), /* cost of FADD and FSUB insns. */
272 COSTS_N_INSNS (27), /* cost of FMUL instruction. */
273 COSTS_N_INSNS (88), /* cost of FDIV instruction. */
274 COSTS_N_INSNS (22), /* cost of FABS instruction. */
275 COSTS_N_INSNS (24), /* cost of FCHS instruction. */
276 COSTS_N_INSNS (122), /* cost of FSQRT instruction. */
279 1, /* scalar_stmt_cost. */
280 1, /* scalar load_cost. */
281 1, /* scalar_store_cost. */
282 1, /* vec_stmt_cost. */
283 1, /* vec_to_scalar_cost. */
284 1, /* scalar_to_vec_cost. */
285 1, /* vec_align_load_cost. */
286 2, /* vec_unalign_load_cost. */
287 1, /* vec_store_cost. */
288 3, /* cond_taken_branch_cost. */
289 1, /* cond_not_taken_branch_cost. */
292 static stringop_algs i486_memcpy[2] = {
293 {rep_prefix_4_byte, {{-1, rep_prefix_4_byte, false}}},
294 DUMMY_STRINGOP_ALGS};
295 static stringop_algs i486_memset[2] = {
296 {rep_prefix_4_byte, {{-1, rep_prefix_4_byte, false}}},
297 DUMMY_STRINGOP_ALGS};
300 struct processor_costs i486_cost = { /* 486 specific costs */
301 COSTS_N_INSNS (1), /* cost of an add instruction */
302 COSTS_N_INSNS (1), /* cost of a lea instruction */
303 COSTS_N_INSNS (3), /* variable shift costs */
304 COSTS_N_INSNS (2), /* constant shift costs */
305 {COSTS_N_INSNS (12), /* cost of starting multiply for QI */
306 COSTS_N_INSNS (12), /* HI */
307 COSTS_N_INSNS (12), /* SI */
308 COSTS_N_INSNS (12), /* DI */
309 COSTS_N_INSNS (12)}, /* other */
310 1, /* cost of multiply per each bit set */
311 {COSTS_N_INSNS (40), /* cost of a divide/mod for QI */
312 COSTS_N_INSNS (40), /* HI */
313 COSTS_N_INSNS (40), /* SI */
314 COSTS_N_INSNS (40), /* DI */
315 COSTS_N_INSNS (40)}, /* other */
316 COSTS_N_INSNS (3), /* cost of movsx */
317 COSTS_N_INSNS (2), /* cost of movzx */
318 15, /* "large" insn */
320 4, /* cost for loading QImode using movzbl */
321 {2, 4, 2}, /* cost of loading integer registers
322 in QImode, HImode and SImode.
323 Relative to reg-reg move (2). */
324 {2, 4, 2}, /* cost of storing integer registers */
325 2, /* cost of reg,reg fld/fst */
326 {8, 8, 8}, /* cost of loading fp registers
327 in SFmode, DFmode and XFmode */
328 {8, 8, 8}, /* cost of storing fp registers
329 in SFmode, DFmode and XFmode */
330 2, /* cost of moving MMX register */
331 {4, 8}, /* cost of loading MMX registers
332 in SImode and DImode */
333 {4, 8}, /* cost of storing MMX registers
334 in SImode and DImode */
335 2, /* cost of moving SSE register */
336 {4, 8, 16}, /* cost of loading SSE registers
337 in SImode, DImode and TImode */
338 {4, 8, 16}, /* cost of storing SSE registers
339 in SImode, DImode and TImode */
340 3, /* MMX or SSE register to integer */
341 4, /* size of l1 cache. 486 has 8kB cache
342 shared for code and data, so 4kB is
343 not really precise. */
344 4, /* size of l2 cache */
345 0, /* size of prefetch block */
346 0, /* number of parallel prefetches */
348 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
349 COSTS_N_INSNS (16), /* cost of FMUL instruction. */
350 COSTS_N_INSNS (73), /* cost of FDIV instruction. */
351 COSTS_N_INSNS (3), /* cost of FABS instruction. */
352 COSTS_N_INSNS (3), /* cost of FCHS instruction. */
353 COSTS_N_INSNS (83), /* cost of FSQRT instruction. */
356 1, /* scalar_stmt_cost. */
357 1, /* scalar load_cost. */
358 1, /* scalar_store_cost. */
359 1, /* vec_stmt_cost. */
360 1, /* vec_to_scalar_cost. */
361 1, /* scalar_to_vec_cost. */
362 1, /* vec_align_load_cost. */
363 2, /* vec_unalign_load_cost. */
364 1, /* vec_store_cost. */
365 3, /* cond_taken_branch_cost. */
366 1, /* cond_not_taken_branch_cost. */
369 static stringop_algs pentium_memcpy[2] = {
370 {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
371 DUMMY_STRINGOP_ALGS};
372 static stringop_algs pentium_memset[2] = {
373 {libcall, {{-1, rep_prefix_4_byte, false}}},
374 DUMMY_STRINGOP_ALGS};
377 struct processor_costs pentium_cost = {
378 COSTS_N_INSNS (1), /* cost of an add instruction */
379 COSTS_N_INSNS (1), /* cost of a lea instruction */
380 COSTS_N_INSNS (4), /* variable shift costs */
381 COSTS_N_INSNS (1), /* constant shift costs */
382 {COSTS_N_INSNS (11), /* cost of starting multiply for QI */
383 COSTS_N_INSNS (11), /* HI */
384 COSTS_N_INSNS (11), /* SI */
385 COSTS_N_INSNS (11), /* DI */
386 COSTS_N_INSNS (11)}, /* other */
387 0, /* cost of multiply per each bit set */
388 {COSTS_N_INSNS (25), /* cost of a divide/mod for QI */
389 COSTS_N_INSNS (25), /* HI */
390 COSTS_N_INSNS (25), /* SI */
391 COSTS_N_INSNS (25), /* DI */
392 COSTS_N_INSNS (25)}, /* other */
393 COSTS_N_INSNS (3), /* cost of movsx */
394 COSTS_N_INSNS (2), /* cost of movzx */
395 8, /* "large" insn */
397 6, /* cost for loading QImode using movzbl */
398 {2, 4, 2}, /* cost of loading integer registers
399 in QImode, HImode and SImode.
400 Relative to reg-reg move (2). */
401 {2, 4, 2}, /* cost of storing integer registers */
402 2, /* cost of reg,reg fld/fst */
403 {2, 2, 6}, /* cost of loading fp registers
404 in SFmode, DFmode and XFmode */
405 {4, 4, 6}, /* cost of storing fp registers
406 in SFmode, DFmode and XFmode */
407 8, /* cost of moving MMX register */
408 {8, 8}, /* cost of loading MMX registers
409 in SImode and DImode */
410 {8, 8}, /* cost of storing MMX registers
411 in SImode and DImode */
412 2, /* cost of moving SSE register */
413 {4, 8, 16}, /* cost of loading SSE registers
414 in SImode, DImode and TImode */
415 {4, 8, 16}, /* cost of storing SSE registers
416 in SImode, DImode and TImode */
417 3, /* MMX or SSE register to integer */
418 8, /* size of l1 cache. */
419 8, /* size of l2 cache */
420 0, /* size of prefetch block */
421 0, /* number of parallel prefetches */
423 COSTS_N_INSNS (3), /* cost of FADD and FSUB insns. */
424 COSTS_N_INSNS (3), /* cost of FMUL instruction. */
425 COSTS_N_INSNS (39), /* cost of FDIV instruction. */
426 COSTS_N_INSNS (1), /* cost of FABS instruction. */
427 COSTS_N_INSNS (1), /* cost of FCHS instruction. */
428 COSTS_N_INSNS (70), /* cost of FSQRT instruction. */
431 1, /* scalar_stmt_cost. */
432 1, /* scalar load_cost. */
433 1, /* scalar_store_cost. */
434 1, /* vec_stmt_cost. */
435 1, /* vec_to_scalar_cost. */
436 1, /* scalar_to_vec_cost. */
437 1, /* vec_align_load_cost. */
438 2, /* vec_unalign_load_cost. */
439 1, /* vec_store_cost. */
440 3, /* cond_taken_branch_cost. */
441 1, /* cond_not_taken_branch_cost. */
444 /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes
445 (we ensure the alignment). For small blocks inline loop is still a
446 noticeable win, for bigger blocks either rep movsl or rep movsb is
447 way to go. Rep movsb has apparently more expensive startup time in CPU,
448 but after 4K the difference is down in the noise. */
449 static stringop_algs pentiumpro_memcpy[2] = {
450 {rep_prefix_4_byte, {{128, loop, false}, {1024, unrolled_loop, false},
451 {8192, rep_prefix_4_byte, false},
452 {-1, rep_prefix_1_byte, false}}},
453 DUMMY_STRINGOP_ALGS};
454 static stringop_algs pentiumpro_memset[2] = {
455 {rep_prefix_4_byte, {{1024, unrolled_loop, false},
456 {8192, rep_prefix_4_byte, false},
457 {-1, libcall, false}}},
458 DUMMY_STRINGOP_ALGS};
460 struct processor_costs pentiumpro_cost = {
461 COSTS_N_INSNS (1), /* cost of an add instruction */
462 COSTS_N_INSNS (1), /* cost of a lea instruction */
463 COSTS_N_INSNS (1), /* variable shift costs */
464 COSTS_N_INSNS (1), /* constant shift costs */
465 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
466 COSTS_N_INSNS (4), /* HI */
467 COSTS_N_INSNS (4), /* SI */
468 COSTS_N_INSNS (4), /* DI */
469 COSTS_N_INSNS (4)}, /* other */
470 0, /* cost of multiply per each bit set */
471 {COSTS_N_INSNS (17), /* cost of a divide/mod for QI */
472 COSTS_N_INSNS (17), /* HI */
473 COSTS_N_INSNS (17), /* SI */
474 COSTS_N_INSNS (17), /* DI */
475 COSTS_N_INSNS (17)}, /* other */
476 COSTS_N_INSNS (1), /* cost of movsx */
477 COSTS_N_INSNS (1), /* cost of movzx */
478 8, /* "large" insn */
480 2, /* cost for loading QImode using movzbl */
481 {4, 4, 4}, /* cost of loading integer registers
482 in QImode, HImode and SImode.
483 Relative to reg-reg move (2). */
484 {2, 2, 2}, /* cost of storing integer registers */
485 2, /* cost of reg,reg fld/fst */
486 {2, 2, 6}, /* cost of loading fp registers
487 in SFmode, DFmode and XFmode */
488 {4, 4, 6}, /* cost of storing fp registers
489 in SFmode, DFmode and XFmode */
490 2, /* cost of moving MMX register */
491 {2, 2}, /* cost of loading MMX registers
492 in SImode and DImode */
493 {2, 2}, /* cost of storing MMX registers
494 in SImode and DImode */
495 2, /* cost of moving SSE register */
496 {2, 2, 8}, /* cost of loading SSE registers
497 in SImode, DImode and TImode */
498 {2, 2, 8}, /* cost of storing SSE registers
499 in SImode, DImode and TImode */
500 3, /* MMX or SSE register to integer */
501 8, /* size of l1 cache. */
502 256, /* size of l2 cache */
503 32, /* size of prefetch block */
504 6, /* number of parallel prefetches */
506 COSTS_N_INSNS (3), /* cost of FADD and FSUB insns. */
507 COSTS_N_INSNS (5), /* cost of FMUL instruction. */
508 COSTS_N_INSNS (56), /* cost of FDIV instruction. */
509 COSTS_N_INSNS (2), /* cost of FABS instruction. */
510 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
511 COSTS_N_INSNS (56), /* cost of FSQRT instruction. */
514 1, /* scalar_stmt_cost. */
515 1, /* scalar load_cost. */
516 1, /* scalar_store_cost. */
517 1, /* vec_stmt_cost. */
518 1, /* vec_to_scalar_cost. */
519 1, /* scalar_to_vec_cost. */
520 1, /* vec_align_load_cost. */
521 2, /* vec_unalign_load_cost. */
522 1, /* vec_store_cost. */
523 3, /* cond_taken_branch_cost. */
524 1, /* cond_not_taken_branch_cost. */
527 static stringop_algs geode_memcpy[2] = {
528 {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
529 DUMMY_STRINGOP_ALGS};
530 static stringop_algs geode_memset[2] = {
531 {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
532 DUMMY_STRINGOP_ALGS};
534 struct processor_costs geode_cost = {
535 COSTS_N_INSNS (1), /* cost of an add instruction */
536 COSTS_N_INSNS (1), /* cost of a lea instruction */
537 COSTS_N_INSNS (2), /* variable shift costs */
538 COSTS_N_INSNS (1), /* constant shift costs */
539 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
540 COSTS_N_INSNS (4), /* HI */
541 COSTS_N_INSNS (7), /* SI */
542 COSTS_N_INSNS (7), /* DI */
543 COSTS_N_INSNS (7)}, /* other */
544 0, /* cost of multiply per each bit set */
545 {COSTS_N_INSNS (15), /* cost of a divide/mod for QI */
546 COSTS_N_INSNS (23), /* HI */
547 COSTS_N_INSNS (39), /* SI */
548 COSTS_N_INSNS (39), /* DI */
549 COSTS_N_INSNS (39)}, /* other */
550 COSTS_N_INSNS (1), /* cost of movsx */
551 COSTS_N_INSNS (1), /* cost of movzx */
552 8, /* "large" insn */
554 1, /* cost for loading QImode using movzbl */
555 {1, 1, 1}, /* cost of loading integer registers
556 in QImode, HImode and SImode.
557 Relative to reg-reg move (2). */
558 {1, 1, 1}, /* cost of storing integer registers */
559 1, /* cost of reg,reg fld/fst */
560 {1, 1, 1}, /* cost of loading fp registers
561 in SFmode, DFmode and XFmode */
562 {4, 6, 6}, /* cost of storing fp registers
563 in SFmode, DFmode and XFmode */
565 1, /* cost of moving MMX register */
566 {1, 1}, /* cost of loading MMX registers
567 in SImode and DImode */
568 {1, 1}, /* cost of storing MMX registers
569 in SImode and DImode */
570 1, /* cost of moving SSE register */
571 {1, 1, 1}, /* cost of loading SSE registers
572 in SImode, DImode and TImode */
573 {1, 1, 1}, /* cost of storing SSE registers
574 in SImode, DImode and TImode */
575 1, /* MMX or SSE register to integer */
576 64, /* size of l1 cache. */
577 128, /* size of l2 cache. */
578 32, /* size of prefetch block */
579 1, /* number of parallel prefetches */
581 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
582 COSTS_N_INSNS (11), /* cost of FMUL instruction. */
583 COSTS_N_INSNS (47), /* cost of FDIV instruction. */
584 COSTS_N_INSNS (1), /* cost of FABS instruction. */
585 COSTS_N_INSNS (1), /* cost of FCHS instruction. */
586 COSTS_N_INSNS (54), /* cost of FSQRT instruction. */
589 1, /* scalar_stmt_cost. */
590 1, /* scalar load_cost. */
591 1, /* scalar_store_cost. */
592 1, /* vec_stmt_cost. */
593 1, /* vec_to_scalar_cost. */
594 1, /* scalar_to_vec_cost. */
595 1, /* vec_align_load_cost. */
596 2, /* vec_unalign_load_cost. */
597 1, /* vec_store_cost. */
598 3, /* cond_taken_branch_cost. */
599 1, /* cond_not_taken_branch_cost. */
602 static stringop_algs k6_memcpy[2] = {
603 {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
604 DUMMY_STRINGOP_ALGS};
605 static stringop_algs k6_memset[2] = {
606 {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
607 DUMMY_STRINGOP_ALGS};
609 struct processor_costs k6_cost = {
610 COSTS_N_INSNS (1), /* cost of an add instruction */
611 COSTS_N_INSNS (2), /* cost of a lea instruction */
612 COSTS_N_INSNS (1), /* variable shift costs */
613 COSTS_N_INSNS (1), /* constant shift costs */
614 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
615 COSTS_N_INSNS (3), /* HI */
616 COSTS_N_INSNS (3), /* SI */
617 COSTS_N_INSNS (3), /* DI */
618 COSTS_N_INSNS (3)}, /* other */
619 0, /* cost of multiply per each bit set */
620 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
621 COSTS_N_INSNS (18), /* HI */
622 COSTS_N_INSNS (18), /* SI */
623 COSTS_N_INSNS (18), /* DI */
624 COSTS_N_INSNS (18)}, /* other */
625 COSTS_N_INSNS (2), /* cost of movsx */
626 COSTS_N_INSNS (2), /* cost of movzx */
627 8, /* "large" insn */
629 3, /* cost for loading QImode using movzbl */
630 {4, 5, 4}, /* cost of loading integer registers
631 in QImode, HImode and SImode.
632 Relative to reg-reg move (2). */
633 {2, 3, 2}, /* cost of storing integer registers */
634 4, /* cost of reg,reg fld/fst */
635 {6, 6, 6}, /* cost of loading fp registers
636 in SFmode, DFmode and XFmode */
637 {4, 4, 4}, /* cost of storing fp registers
638 in SFmode, DFmode and XFmode */
639 2, /* cost of moving MMX register */
640 {2, 2}, /* cost of loading MMX registers
641 in SImode and DImode */
642 {2, 2}, /* cost of storing MMX registers
643 in SImode and DImode */
644 2, /* cost of moving SSE register */
645 {2, 2, 8}, /* cost of loading SSE registers
646 in SImode, DImode and TImode */
647 {2, 2, 8}, /* cost of storing SSE registers
648 in SImode, DImode and TImode */
649 6, /* MMX or SSE register to integer */
650 32, /* size of l1 cache. */
651 32, /* size of l2 cache. Some models
652 have integrated l2 cache, but
653 optimizing for k6 is not important
654 enough to worry about that. */
655 32, /* size of prefetch block */
656 1, /* number of parallel prefetches */
658 COSTS_N_INSNS (2), /* cost of FADD and FSUB insns. */
659 COSTS_N_INSNS (2), /* cost of FMUL instruction. */
660 COSTS_N_INSNS (56), /* cost of FDIV instruction. */
661 COSTS_N_INSNS (2), /* cost of FABS instruction. */
662 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
663 COSTS_N_INSNS (56), /* cost of FSQRT instruction. */
666 1, /* scalar_stmt_cost. */
667 1, /* scalar load_cost. */
668 1, /* scalar_store_cost. */
669 1, /* vec_stmt_cost. */
670 1, /* vec_to_scalar_cost. */
671 1, /* scalar_to_vec_cost. */
672 1, /* vec_align_load_cost. */
673 2, /* vec_unalign_load_cost. */
674 1, /* vec_store_cost. */
675 3, /* cond_taken_branch_cost. */
676 1, /* cond_not_taken_branch_cost. */
679 /* For some reason, Athlon deals better with REP prefix (relative to loops)
680 compared to K8. Alignment becomes important after 8 bytes for memcpy and
681 128 bytes for memset. */
682 static stringop_algs athlon_memcpy[2] = {
683 {libcall, {{2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
684 DUMMY_STRINGOP_ALGS};
685 static stringop_algs athlon_memset[2] = {
686 {libcall, {{2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
687 DUMMY_STRINGOP_ALGS};
689 struct processor_costs athlon_cost = {
690 COSTS_N_INSNS (1), /* cost of an add instruction */
691 COSTS_N_INSNS (2), /* cost of a lea instruction */
692 COSTS_N_INSNS (1), /* variable shift costs */
693 COSTS_N_INSNS (1), /* constant shift costs */
694 {COSTS_N_INSNS (5), /* cost of starting multiply for QI */
695 COSTS_N_INSNS (5), /* HI */
696 COSTS_N_INSNS (5), /* SI */
697 COSTS_N_INSNS (5), /* DI */
698 COSTS_N_INSNS (5)}, /* other */
699 0, /* cost of multiply per each bit set */
700 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
701 COSTS_N_INSNS (26), /* HI */
702 COSTS_N_INSNS (42), /* SI */
703 COSTS_N_INSNS (74), /* DI */
704 COSTS_N_INSNS (74)}, /* other */
705 COSTS_N_INSNS (1), /* cost of movsx */
706 COSTS_N_INSNS (1), /* cost of movzx */
707 8, /* "large" insn */
709 4, /* cost for loading QImode using movzbl */
710 {3, 4, 3}, /* cost of loading integer registers
711 in QImode, HImode and SImode.
712 Relative to reg-reg move (2). */
713 {3, 4, 3}, /* cost of storing integer registers */
714 4, /* cost of reg,reg fld/fst */
715 {4, 4, 12}, /* cost of loading fp registers
716 in SFmode, DFmode and XFmode */
717 {6, 6, 8}, /* cost of storing fp registers
718 in SFmode, DFmode and XFmode */
719 2, /* cost of moving MMX register */
720 {4, 4}, /* cost of loading MMX registers
721 in SImode and DImode */
722 {4, 4}, /* cost of storing MMX registers
723 in SImode and DImode */
724 2, /* cost of moving SSE register */
725 {4, 4, 6}, /* cost of loading SSE registers
726 in SImode, DImode and TImode */
727 {4, 4, 5}, /* cost of storing SSE registers
728 in SImode, DImode and TImode */
729 5, /* MMX or SSE register to integer */
730 64, /* size of l1 cache. */
731 256, /* size of l2 cache. */
732 64, /* size of prefetch block */
733 6, /* number of parallel prefetches */
735 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
736 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
737 COSTS_N_INSNS (24), /* cost of FDIV instruction. */
738 COSTS_N_INSNS (2), /* cost of FABS instruction. */
739 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
740 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
743 1, /* scalar_stmt_cost. */
744 1, /* scalar load_cost. */
745 1, /* scalar_store_cost. */
746 1, /* vec_stmt_cost. */
747 1, /* vec_to_scalar_cost. */
748 1, /* scalar_to_vec_cost. */
749 1, /* vec_align_load_cost. */
750 2, /* vec_unalign_load_cost. */
751 1, /* vec_store_cost. */
752 3, /* cond_taken_branch_cost. */
753 1, /* cond_not_taken_branch_cost. */
756 /* K8 has optimized REP instruction for medium sized blocks, but for very
757 small blocks it is better to use loop. For large blocks, libcall can
758 do nontemporary accesses and beat inline considerably. */
759 static stringop_algs k8_memcpy[2] = {
760 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
761 {-1, rep_prefix_4_byte, false}}},
762 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
763 {-1, libcall, false}}}};
764 static stringop_algs k8_memset[2] = {
765 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
766 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
767 {libcall, {{48, unrolled_loop, false},
768 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
770 struct processor_costs k8_cost = {
771 COSTS_N_INSNS (1), /* cost of an add instruction */
772 COSTS_N_INSNS (2), /* cost of a lea instruction */
773 COSTS_N_INSNS (1), /* variable shift costs */
774 COSTS_N_INSNS (1), /* constant shift costs */
775 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
776 COSTS_N_INSNS (4), /* HI */
777 COSTS_N_INSNS (3), /* SI */
778 COSTS_N_INSNS (4), /* DI */
779 COSTS_N_INSNS (5)}, /* other */
780 0, /* cost of multiply per each bit set */
781 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
782 COSTS_N_INSNS (26), /* HI */
783 COSTS_N_INSNS (42), /* SI */
784 COSTS_N_INSNS (74), /* DI */
785 COSTS_N_INSNS (74)}, /* other */
786 COSTS_N_INSNS (1), /* cost of movsx */
787 COSTS_N_INSNS (1), /* cost of movzx */
788 8, /* "large" insn */
790 4, /* cost for loading QImode using movzbl */
791 {3, 4, 3}, /* cost of loading integer registers
792 in QImode, HImode and SImode.
793 Relative to reg-reg move (2). */
794 {3, 4, 3}, /* cost of storing integer registers */
795 4, /* cost of reg,reg fld/fst */
796 {4, 4, 12}, /* cost of loading fp registers
797 in SFmode, DFmode and XFmode */
798 {6, 6, 8}, /* cost of storing fp registers
799 in SFmode, DFmode and XFmode */
800 2, /* cost of moving MMX register */
801 {3, 3}, /* cost of loading MMX registers
802 in SImode and DImode */
803 {4, 4}, /* cost of storing MMX registers
804 in SImode and DImode */
805 2, /* cost of moving SSE register */
806 {4, 3, 6}, /* cost of loading SSE registers
807 in SImode, DImode and TImode */
808 {4, 4, 5}, /* cost of storing SSE registers
809 in SImode, DImode and TImode */
810 5, /* MMX or SSE register to integer */
811 64, /* size of l1 cache. */
812 512, /* size of l2 cache. */
813 64, /* size of prefetch block */
814 /* New AMD processors never drop prefetches; if they cannot be performed
815 immediately, they are queued. We set number of simultaneous prefetches
816 to a large constant to reflect this (it probably is not a good idea not
817 to limit number of prefetches at all, as their execution also takes some
819 100, /* number of parallel prefetches */
821 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
822 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
823 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
824 COSTS_N_INSNS (2), /* cost of FABS instruction. */
825 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
826 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
830 4, /* scalar_stmt_cost. */
831 2, /* scalar load_cost. */
832 2, /* scalar_store_cost. */
833 5, /* vec_stmt_cost. */
834 0, /* vec_to_scalar_cost. */
835 2, /* scalar_to_vec_cost. */
836 2, /* vec_align_load_cost. */
837 3, /* vec_unalign_load_cost. */
838 3, /* vec_store_cost. */
839 3, /* cond_taken_branch_cost. */
840 2, /* cond_not_taken_branch_cost. */
843 /* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
844 very small blocks it is better to use loop. For large blocks, libcall can
845 do nontemporary accesses and beat inline considerably. */
846 static stringop_algs amdfam10_memcpy[2] = {
847 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
848 {-1, rep_prefix_4_byte, false}}},
849 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
850 {-1, libcall, false}}}};
851 static stringop_algs amdfam10_memset[2] = {
852 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
853 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
854 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
855 {-1, libcall, false}}}};
856 struct processor_costs amdfam10_cost = {
857 COSTS_N_INSNS (1), /* cost of an add instruction */
858 COSTS_N_INSNS (2), /* cost of a lea instruction */
859 COSTS_N_INSNS (1), /* variable shift costs */
860 COSTS_N_INSNS (1), /* constant shift costs */
861 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
862 COSTS_N_INSNS (4), /* HI */
863 COSTS_N_INSNS (3), /* SI */
864 COSTS_N_INSNS (4), /* DI */
865 COSTS_N_INSNS (5)}, /* other */
866 0, /* cost of multiply per each bit set */
867 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
868 COSTS_N_INSNS (35), /* HI */
869 COSTS_N_INSNS (51), /* SI */
870 COSTS_N_INSNS (83), /* DI */
871 COSTS_N_INSNS (83)}, /* other */
872 COSTS_N_INSNS (1), /* cost of movsx */
873 COSTS_N_INSNS (1), /* cost of movzx */
874 8, /* "large" insn */
876 4, /* cost for loading QImode using movzbl */
877 {3, 4, 3}, /* cost of loading integer registers
878 in QImode, HImode and SImode.
879 Relative to reg-reg move (2). */
880 {3, 4, 3}, /* cost of storing integer registers */
881 4, /* cost of reg,reg fld/fst */
882 {4, 4, 12}, /* cost of loading fp registers
883 in SFmode, DFmode and XFmode */
884 {6, 6, 8}, /* cost of storing fp registers
885 in SFmode, DFmode and XFmode */
886 2, /* cost of moving MMX register */
887 {3, 3}, /* cost of loading MMX registers
888 in SImode and DImode */
889 {4, 4}, /* cost of storing MMX registers
890 in SImode and DImode */
891 2, /* cost of moving SSE register */
892 {4, 4, 3}, /* cost of loading SSE registers
893 in SImode, DImode and TImode */
894 {4, 4, 5}, /* cost of storing SSE registers
895 in SImode, DImode and TImode */
896 3, /* MMX or SSE register to integer */
898 MOVD reg64, xmmreg Double FSTORE 4
899 MOVD reg32, xmmreg Double FSTORE 4
901 MOVD reg64, xmmreg Double FADD 3
903 MOVD reg32, xmmreg Double FADD 3
905 64, /* size of l1 cache. */
906 512, /* size of l2 cache. */
907 64, /* size of prefetch block */
908 /* New AMD processors never drop prefetches; if they cannot be performed
909 immediately, they are queued. We set number of simultaneous prefetches
910 to a large constant to reflect this (it probably is not a good idea not
911 to limit number of prefetches at all, as their execution also takes some
913 100, /* number of parallel prefetches */
915 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
916 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
917 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
918 COSTS_N_INSNS (2), /* cost of FABS instruction. */
919 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
920 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
924 4, /* scalar_stmt_cost. */
925 2, /* scalar load_cost. */
926 2, /* scalar_store_cost. */
927 6, /* vec_stmt_cost. */
928 0, /* vec_to_scalar_cost. */
929 2, /* scalar_to_vec_cost. */
930 2, /* vec_align_load_cost. */
931 2, /* vec_unalign_load_cost. */
932 2, /* vec_store_cost. */
933 2, /* cond_taken_branch_cost. */
934 1, /* cond_not_taken_branch_cost. */
937 /* BDVER1 has optimized REP instruction for medium sized blocks, but for
938 very small blocks it is better to use loop. For large blocks, libcall
939 can do nontemporary accesses and beat inline considerably. */
940 static stringop_algs bdver1_memcpy[2] = {
941 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
942 {-1, rep_prefix_4_byte, false}}},
943 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
944 {-1, libcall, false}}}};
945 static stringop_algs bdver1_memset[2] = {
946 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
947 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
948 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
949 {-1, libcall, false}}}};
951 const struct processor_costs bdver1_cost = {
952 COSTS_N_INSNS (1), /* cost of an add instruction */
953 COSTS_N_INSNS (1), /* cost of a lea instruction */
954 COSTS_N_INSNS (1), /* variable shift costs */
955 COSTS_N_INSNS (1), /* constant shift costs */
956 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
957 COSTS_N_INSNS (4), /* HI */
958 COSTS_N_INSNS (4), /* SI */
959 COSTS_N_INSNS (6), /* DI */
960 COSTS_N_INSNS (6)}, /* other */
961 0, /* cost of multiply per each bit set */
962 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
963 COSTS_N_INSNS (35), /* HI */
964 COSTS_N_INSNS (51), /* SI */
965 COSTS_N_INSNS (83), /* DI */
966 COSTS_N_INSNS (83)}, /* other */
967 COSTS_N_INSNS (1), /* cost of movsx */
968 COSTS_N_INSNS (1), /* cost of movzx */
969 8, /* "large" insn */
971 4, /* cost for loading QImode using movzbl */
972 {5, 5, 4}, /* cost of loading integer registers
973 in QImode, HImode and SImode.
974 Relative to reg-reg move (2). */
975 {4, 4, 4}, /* cost of storing integer registers */
976 2, /* cost of reg,reg fld/fst */
977 {5, 5, 12}, /* cost of loading fp registers
978 in SFmode, DFmode and XFmode */
979 {4, 4, 8}, /* cost of storing fp registers
980 in SFmode, DFmode and XFmode */
981 2, /* cost of moving MMX register */
982 {4, 4}, /* cost of loading MMX registers
983 in SImode and DImode */
984 {4, 4}, /* cost of storing MMX registers
985 in SImode and DImode */
986 2, /* cost of moving SSE register */
987 {4, 4, 4}, /* cost of loading SSE registers
988 in SImode, DImode and TImode */
989 {4, 4, 4}, /* cost of storing SSE registers
990 in SImode, DImode and TImode */
991 2, /* MMX or SSE register to integer */
993 MOVD reg64, xmmreg Double FSTORE 4
994 MOVD reg32, xmmreg Double FSTORE 4
996 MOVD reg64, xmmreg Double FADD 3
998 MOVD reg32, xmmreg Double FADD 3
1000 16, /* size of l1 cache. */
1001 2048, /* size of l2 cache. */
1002 64, /* size of prefetch block */
1003 /* New AMD processors never drop prefetches; if they cannot be performed
1004 immediately, they are queued. We set number of simultaneous prefetches
1005 to a large constant to reflect this (it probably is not a good idea not
1006 to limit number of prefetches at all, as their execution also takes some
1008 100, /* number of parallel prefetches */
1009 2, /* Branch cost */
1010 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
1011 COSTS_N_INSNS (6), /* cost of FMUL instruction. */
1012 COSTS_N_INSNS (42), /* cost of FDIV instruction. */
1013 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1014 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1015 COSTS_N_INSNS (52), /* cost of FSQRT instruction. */
1019 6, /* scalar_stmt_cost. */
1020 4, /* scalar load_cost. */
1021 4, /* scalar_store_cost. */
1022 6, /* vec_stmt_cost. */
1023 0, /* vec_to_scalar_cost. */
1024 2, /* scalar_to_vec_cost. */
1025 4, /* vec_align_load_cost. */
1026 4, /* vec_unalign_load_cost. */
1027 4, /* vec_store_cost. */
1028 2, /* cond_taken_branch_cost. */
1029 1, /* cond_not_taken_branch_cost. */
1032 /* BDVER2 has optimized REP instruction for medium sized blocks, but for
1033 very small blocks it is better to use loop. For large blocks, libcall
1034 can do nontemporary accesses and beat inline considerably. */
1036 static stringop_algs bdver2_memcpy[2] = {
1037 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
1038 {-1, rep_prefix_4_byte, false}}},
1039 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
1040 {-1, libcall, false}}}};
1041 static stringop_algs bdver2_memset[2] = {
1042 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
1043 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1044 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
1045 {-1, libcall, false}}}};
1047 const struct processor_costs bdver2_cost = {
1048 COSTS_N_INSNS (1), /* cost of an add instruction */
1049 COSTS_N_INSNS (1), /* cost of a lea instruction */
1050 COSTS_N_INSNS (1), /* variable shift costs */
1051 COSTS_N_INSNS (1), /* constant shift costs */
1052 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
1053 COSTS_N_INSNS (4), /* HI */
1054 COSTS_N_INSNS (4), /* SI */
1055 COSTS_N_INSNS (6), /* DI */
1056 COSTS_N_INSNS (6)}, /* other */
1057 0, /* cost of multiply per each bit set */
1058 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
1059 COSTS_N_INSNS (35), /* HI */
1060 COSTS_N_INSNS (51), /* SI */
1061 COSTS_N_INSNS (83), /* DI */
1062 COSTS_N_INSNS (83)}, /* other */
1063 COSTS_N_INSNS (1), /* cost of movsx */
1064 COSTS_N_INSNS (1), /* cost of movzx */
1065 8, /* "large" insn */
1067 4, /* cost for loading QImode using movzbl */
1068 {5, 5, 4}, /* cost of loading integer registers
1069 in QImode, HImode and SImode.
1070 Relative to reg-reg move (2). */
1071 {4, 4, 4}, /* cost of storing integer registers */
1072 2, /* cost of reg,reg fld/fst */
1073 {5, 5, 12}, /* cost of loading fp registers
1074 in SFmode, DFmode and XFmode */
1075 {4, 4, 8}, /* cost of storing fp registers
1076 in SFmode, DFmode and XFmode */
1077 2, /* cost of moving MMX register */
1078 {4, 4}, /* cost of loading MMX registers
1079 in SImode and DImode */
1080 {4, 4}, /* cost of storing MMX registers
1081 in SImode and DImode */
1082 2, /* cost of moving SSE register */
1083 {4, 4, 4}, /* cost of loading SSE registers
1084 in SImode, DImode and TImode */
1085 {4, 4, 4}, /* cost of storing SSE registers
1086 in SImode, DImode and TImode */
1087 2, /* MMX or SSE register to integer */
1089 MOVD reg64, xmmreg Double FSTORE 4
1090 MOVD reg32, xmmreg Double FSTORE 4
1092 MOVD reg64, xmmreg Double FADD 3
1094 MOVD reg32, xmmreg Double FADD 3
1096 16, /* size of l1 cache. */
1097 2048, /* size of l2 cache. */
1098 64, /* size of prefetch block */
1099 /* New AMD processors never drop prefetches; if they cannot be performed
1100 immediately, they are queued. We set number of simultaneous prefetches
1101 to a large constant to reflect this (it probably is not a good idea not
1102 to limit number of prefetches at all, as their execution also takes some
1104 100, /* number of parallel prefetches */
1105 2, /* Branch cost */
1106 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
1107 COSTS_N_INSNS (6), /* cost of FMUL instruction. */
1108 COSTS_N_INSNS (42), /* cost of FDIV instruction. */
1109 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1110 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1111 COSTS_N_INSNS (52), /* cost of FSQRT instruction. */
1115 6, /* scalar_stmt_cost. */
1116 4, /* scalar load_cost. */
1117 4, /* scalar_store_cost. */
1118 6, /* vec_stmt_cost. */
1119 0, /* vec_to_scalar_cost. */
1120 2, /* scalar_to_vec_cost. */
1121 4, /* vec_align_load_cost. */
1122 4, /* vec_unalign_load_cost. */
1123 4, /* vec_store_cost. */
1124 2, /* cond_taken_branch_cost. */
1125 1, /* cond_not_taken_branch_cost. */
1129 /* BDVER3 has optimized REP instruction for medium sized blocks, but for
1130 very small blocks it is better to use loop. For large blocks, libcall
1131 can do nontemporary accesses and beat inline considerably. */
1132 static stringop_algs bdver3_memcpy[2] = {
1133 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
1134 {-1, rep_prefix_4_byte, false}}},
1135 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
1136 {-1, libcall, false}}}};
1137 static stringop_algs bdver3_memset[2] = {
1138 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
1139 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1140 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
1141 {-1, libcall, false}}}};
1142 struct processor_costs bdver3_cost = {
1143 COSTS_N_INSNS (1), /* cost of an add instruction */
1144 COSTS_N_INSNS (1), /* cost of a lea instruction */
1145 COSTS_N_INSNS (1), /* variable shift costs */
1146 COSTS_N_INSNS (1), /* constant shift costs */
1147 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
1148 COSTS_N_INSNS (4), /* HI */
1149 COSTS_N_INSNS (4), /* SI */
1150 COSTS_N_INSNS (6), /* DI */
1151 COSTS_N_INSNS (6)}, /* other */
1152 0, /* cost of multiply per each bit set */
1153 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
1154 COSTS_N_INSNS (35), /* HI */
1155 COSTS_N_INSNS (51), /* SI */
1156 COSTS_N_INSNS (83), /* DI */
1157 COSTS_N_INSNS (83)}, /* other */
1158 COSTS_N_INSNS (1), /* cost of movsx */
1159 COSTS_N_INSNS (1), /* cost of movzx */
1160 8, /* "large" insn */
1162 4, /* cost for loading QImode using movzbl */
1163 {5, 5, 4}, /* cost of loading integer registers
1164 in QImode, HImode and SImode.
1165 Relative to reg-reg move (2). */
1166 {4, 4, 4}, /* cost of storing integer registers */
1167 2, /* cost of reg,reg fld/fst */
1168 {5, 5, 12}, /* cost of loading fp registers
1169 in SFmode, DFmode and XFmode */
1170 {4, 4, 8}, /* cost of storing fp registers
1171 in SFmode, DFmode and XFmode */
1172 2, /* cost of moving MMX register */
1173 {4, 4}, /* cost of loading MMX registers
1174 in SImode and DImode */
1175 {4, 4}, /* cost of storing MMX registers
1176 in SImode and DImode */
1177 2, /* cost of moving SSE register */
1178 {4, 4, 4}, /* cost of loading SSE registers
1179 in SImode, DImode and TImode */
1180 {4, 4, 4}, /* cost of storing SSE registers
1181 in SImode, DImode and TImode */
1182 2, /* MMX or SSE register to integer */
1183 16, /* size of l1 cache. */
1184 2048, /* size of l2 cache. */
1185 64, /* size of prefetch block */
1186 /* New AMD processors never drop prefetches; if they cannot be performed
1187 immediately, they are queued. We set number of simultaneous prefetches
1188 to a large constant to reflect this (it probably is not a good idea not
1189 to limit number of prefetches at all, as their execution also takes some
1191 100, /* number of parallel prefetches */
1192 2, /* Branch cost */
1193 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
1194 COSTS_N_INSNS (6), /* cost of FMUL instruction. */
1195 COSTS_N_INSNS (42), /* cost of FDIV instruction. */
1196 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1197 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1198 COSTS_N_INSNS (52), /* cost of FSQRT instruction. */
1202 6, /* scalar_stmt_cost. */
1203 4, /* scalar load_cost. */
1204 4, /* scalar_store_cost. */
1205 6, /* vec_stmt_cost. */
1206 0, /* vec_to_scalar_cost. */
1207 2, /* scalar_to_vec_cost. */
1208 4, /* vec_align_load_cost. */
1209 4, /* vec_unalign_load_cost. */
1210 4, /* vec_store_cost. */
1211 2, /* cond_taken_branch_cost. */
1212 1, /* cond_not_taken_branch_cost. */
1215 /* BDVER4 has optimized REP instruction for medium sized blocks, but for
1216 very small blocks it is better to use loop. For large blocks, libcall
1217 can do nontemporary accesses and beat inline considerably. */
1218 static stringop_algs bdver4_memcpy[2] = {
1219 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
1220 {-1, rep_prefix_4_byte, false}}},
1221 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
1222 {-1, libcall, false}}}};
1223 static stringop_algs bdver4_memset[2] = {
1224 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
1225 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1226 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
1227 {-1, libcall, false}}}};
1228 struct processor_costs bdver4_cost = {
1229 COSTS_N_INSNS (1), /* cost of an add instruction */
1230 COSTS_N_INSNS (1), /* cost of a lea instruction */
1231 COSTS_N_INSNS (1), /* variable shift costs */
1232 COSTS_N_INSNS (1), /* constant shift costs */
1233 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
1234 COSTS_N_INSNS (4), /* HI */
1235 COSTS_N_INSNS (4), /* SI */
1236 COSTS_N_INSNS (6), /* DI */
1237 COSTS_N_INSNS (6)}, /* other */
1238 0, /* cost of multiply per each bit set */
1239 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
1240 COSTS_N_INSNS (35), /* HI */
1241 COSTS_N_INSNS (51), /* SI */
1242 COSTS_N_INSNS (83), /* DI */
1243 COSTS_N_INSNS (83)}, /* other */
1244 COSTS_N_INSNS (1), /* cost of movsx */
1245 COSTS_N_INSNS (1), /* cost of movzx */
1246 8, /* "large" insn */
1248 4, /* cost for loading QImode using movzbl */
1249 {5, 5, 4}, /* cost of loading integer registers
1250 in QImode, HImode and SImode.
1251 Relative to reg-reg move (2). */
1252 {4, 4, 4}, /* cost of storing integer registers */
1253 2, /* cost of reg,reg fld/fst */
1254 {5, 5, 12}, /* cost of loading fp registers
1255 in SFmode, DFmode and XFmode */
1256 {4, 4, 8}, /* cost of storing fp registers
1257 in SFmode, DFmode and XFmode */
1258 2, /* cost of moving MMX register */
1259 {4, 4}, /* cost of loading MMX registers
1260 in SImode and DImode */
1261 {4, 4}, /* cost of storing MMX registers
1262 in SImode and DImode */
1263 2, /* cost of moving SSE register */
1264 {4, 4, 4}, /* cost of loading SSE registers
1265 in SImode, DImode and TImode */
1266 {4, 4, 4}, /* cost of storing SSE registers
1267 in SImode, DImode and TImode */
1268 2, /* MMX or SSE register to integer */
1269 16, /* size of l1 cache. */
1270 2048, /* size of l2 cache. */
1271 64, /* size of prefetch block */
1272 /* New AMD processors never drop prefetches; if they cannot be performed
1273 immediately, they are queued. We set number of simultaneous prefetches
1274 to a large constant to reflect this (it probably is not a good idea not
1275 to limit number of prefetches at all, as their execution also takes some
1277 100, /* number of parallel prefetches */
1278 2, /* Branch cost */
1279 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
1280 COSTS_N_INSNS (6), /* cost of FMUL instruction. */
1281 COSTS_N_INSNS (42), /* cost of FDIV instruction. */
1282 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1283 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1284 COSTS_N_INSNS (52), /* cost of FSQRT instruction. */
1288 6, /* scalar_stmt_cost. */
1289 4, /* scalar load_cost. */
1290 4, /* scalar_store_cost. */
1291 6, /* vec_stmt_cost. */
1292 0, /* vec_to_scalar_cost. */
1293 2, /* scalar_to_vec_cost. */
1294 4, /* vec_align_load_cost. */
1295 4, /* vec_unalign_load_cost. */
1296 4, /* vec_store_cost. */
1297 2, /* cond_taken_branch_cost. */
1298 1, /* cond_not_taken_branch_cost. */
1301 /* BTVER1 has optimized REP instruction for medium sized blocks, but for
1302 very small blocks it is better to use loop. For large blocks, libcall can
1303 do nontemporary accesses and beat inline considerably. */
1304 static stringop_algs btver1_memcpy[2] = {
1305 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
1306 {-1, rep_prefix_4_byte, false}}},
1307 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
1308 {-1, libcall, false}}}};
1309 static stringop_algs btver1_memset[2] = {
1310 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
1311 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1312 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
1313 {-1, libcall, false}}}};
1314 const struct processor_costs btver1_cost = {
1315 COSTS_N_INSNS (1), /* cost of an add instruction */
1316 COSTS_N_INSNS (2), /* cost of a lea instruction */
1317 COSTS_N_INSNS (1), /* variable shift costs */
1318 COSTS_N_INSNS (1), /* constant shift costs */
1319 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1320 COSTS_N_INSNS (4), /* HI */
1321 COSTS_N_INSNS (3), /* SI */
1322 COSTS_N_INSNS (4), /* DI */
1323 COSTS_N_INSNS (5)}, /* other */
1324 0, /* cost of multiply per each bit set */
1325 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
1326 COSTS_N_INSNS (35), /* HI */
1327 COSTS_N_INSNS (51), /* SI */
1328 COSTS_N_INSNS (83), /* DI */
1329 COSTS_N_INSNS (83)}, /* other */
1330 COSTS_N_INSNS (1), /* cost of movsx */
1331 COSTS_N_INSNS (1), /* cost of movzx */
1332 8, /* "large" insn */
1334 4, /* cost for loading QImode using movzbl */
1335 {3, 4, 3}, /* cost of loading integer registers
1336 in QImode, HImode and SImode.
1337 Relative to reg-reg move (2). */
1338 {3, 4, 3}, /* cost of storing integer registers */
1339 4, /* cost of reg,reg fld/fst */
1340 {4, 4, 12}, /* cost of loading fp registers
1341 in SFmode, DFmode and XFmode */
1342 {6, 6, 8}, /* cost of storing fp registers
1343 in SFmode, DFmode and XFmode */
1344 2, /* cost of moving MMX register */
1345 {3, 3}, /* cost of loading MMX registers
1346 in SImode and DImode */
1347 {4, 4}, /* cost of storing MMX registers
1348 in SImode and DImode */
1349 2, /* cost of moving SSE register */
1350 {4, 4, 3}, /* cost of loading SSE registers
1351 in SImode, DImode and TImode */
1352 {4, 4, 5}, /* cost of storing SSE registers
1353 in SImode, DImode and TImode */
1354 3, /* MMX or SSE register to integer */
1356 MOVD reg64, xmmreg Double FSTORE 4
1357 MOVD reg32, xmmreg Double FSTORE 4
1359 MOVD reg64, xmmreg Double FADD 3
1361 MOVD reg32, xmmreg Double FADD 3
1363 32, /* size of l1 cache. */
1364 512, /* size of l2 cache. */
1365 64, /* size of prefetch block */
1366 100, /* number of parallel prefetches */
1367 2, /* Branch cost */
1368 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
1369 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
1370 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
1371 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1372 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1373 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
1377 4, /* scalar_stmt_cost. */
1378 2, /* scalar load_cost. */
1379 2, /* scalar_store_cost. */
1380 6, /* vec_stmt_cost. */
1381 0, /* vec_to_scalar_cost. */
1382 2, /* scalar_to_vec_cost. */
1383 2, /* vec_align_load_cost. */
1384 2, /* vec_unalign_load_cost. */
1385 2, /* vec_store_cost. */
1386 2, /* cond_taken_branch_cost. */
1387 1, /* cond_not_taken_branch_cost. */
1390 static stringop_algs btver2_memcpy[2] = {
1391 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
1392 {-1, rep_prefix_4_byte, false}}},
1393 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
1394 {-1, libcall, false}}}};
1395 static stringop_algs btver2_memset[2] = {
1396 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
1397 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1398 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
1399 {-1, libcall, false}}}};
1400 const struct processor_costs btver2_cost = {
1401 COSTS_N_INSNS (1), /* cost of an add instruction */
1402 COSTS_N_INSNS (2), /* cost of a lea instruction */
1403 COSTS_N_INSNS (1), /* variable shift costs */
1404 COSTS_N_INSNS (1), /* constant shift costs */
1405 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1406 COSTS_N_INSNS (4), /* HI */
1407 COSTS_N_INSNS (3), /* SI */
1408 COSTS_N_INSNS (4), /* DI */
1409 COSTS_N_INSNS (5)}, /* other */
1410 0, /* cost of multiply per each bit set */
1411 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
1412 COSTS_N_INSNS (35), /* HI */
1413 COSTS_N_INSNS (51), /* SI */
1414 COSTS_N_INSNS (83), /* DI */
1415 COSTS_N_INSNS (83)}, /* other */
1416 COSTS_N_INSNS (1), /* cost of movsx */
1417 COSTS_N_INSNS (1), /* cost of movzx */
1418 8, /* "large" insn */
1420 4, /* cost for loading QImode using movzbl */
1421 {3, 4, 3}, /* cost of loading integer registers
1422 in QImode, HImode and SImode.
1423 Relative to reg-reg move (2). */
1424 {3, 4, 3}, /* cost of storing integer registers */
1425 4, /* cost of reg,reg fld/fst */
1426 {4, 4, 12}, /* cost of loading fp registers
1427 in SFmode, DFmode and XFmode */
1428 {6, 6, 8}, /* cost of storing fp registers
1429 in SFmode, DFmode and XFmode */
1430 2, /* cost of moving MMX register */
1431 {3, 3}, /* cost of loading MMX registers
1432 in SImode and DImode */
1433 {4, 4}, /* cost of storing MMX registers
1434 in SImode and DImode */
1435 2, /* cost of moving SSE register */
1436 {4, 4, 3}, /* cost of loading SSE registers
1437 in SImode, DImode and TImode */
1438 {4, 4, 5}, /* cost of storing SSE registers
1439 in SImode, DImode and TImode */
1440 3, /* MMX or SSE register to integer */
1442 MOVD reg64, xmmreg Double FSTORE 4
1443 MOVD reg32, xmmreg Double FSTORE 4
1445 MOVD reg64, xmmreg Double FADD 3
1447 MOVD reg32, xmmreg Double FADD 3
1449 32, /* size of l1 cache. */
1450 2048, /* size of l2 cache. */
1451 64, /* size of prefetch block */
1452 100, /* number of parallel prefetches */
1453 2, /* Branch cost */
1454 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
1455 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
1456 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
1457 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1458 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1459 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
1462 4, /* scalar_stmt_cost. */
1463 2, /* scalar load_cost. */
1464 2, /* scalar_store_cost. */
1465 6, /* vec_stmt_cost. */
1466 0, /* vec_to_scalar_cost. */
1467 2, /* scalar_to_vec_cost. */
1468 2, /* vec_align_load_cost. */
1469 2, /* vec_unalign_load_cost. */
1470 2, /* vec_store_cost. */
1471 2, /* cond_taken_branch_cost. */
1472 1, /* cond_not_taken_branch_cost. */
1475 static stringop_algs pentium4_memcpy[2] = {
1476 {libcall, {{12, loop_1_byte, false}, {-1, rep_prefix_4_byte, false}}},
1477 DUMMY_STRINGOP_ALGS};
1478 static stringop_algs pentium4_memset[2] = {
1479 {libcall, {{6, loop_1_byte, false}, {48, loop, false},
1480 {20480, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1481 DUMMY_STRINGOP_ALGS};
1484 struct processor_costs pentium4_cost = {
1485 COSTS_N_INSNS (1), /* cost of an add instruction */
1486 COSTS_N_INSNS (3), /* cost of a lea instruction */
1487 COSTS_N_INSNS (4), /* variable shift costs */
1488 COSTS_N_INSNS (4), /* constant shift costs */
1489 {COSTS_N_INSNS (15), /* cost of starting multiply for QI */
1490 COSTS_N_INSNS (15), /* HI */
1491 COSTS_N_INSNS (15), /* SI */
1492 COSTS_N_INSNS (15), /* DI */
1493 COSTS_N_INSNS (15)}, /* other */
1494 0, /* cost of multiply per each bit set */
1495 {COSTS_N_INSNS (56), /* cost of a divide/mod for QI */
1496 COSTS_N_INSNS (56), /* HI */
1497 COSTS_N_INSNS (56), /* SI */
1498 COSTS_N_INSNS (56), /* DI */
1499 COSTS_N_INSNS (56)}, /* other */
1500 COSTS_N_INSNS (1), /* cost of movsx */
1501 COSTS_N_INSNS (1), /* cost of movzx */
1502 16, /* "large" insn */
1504 2, /* cost for loading QImode using movzbl */
1505 {4, 5, 4}, /* cost of loading integer registers
1506 in QImode, HImode and SImode.
1507 Relative to reg-reg move (2). */
1508 {2, 3, 2}, /* cost of storing integer registers */
1509 2, /* cost of reg,reg fld/fst */
1510 {2, 2, 6}, /* cost of loading fp registers
1511 in SFmode, DFmode and XFmode */
1512 {4, 4, 6}, /* cost of storing fp registers
1513 in SFmode, DFmode and XFmode */
1514 2, /* cost of moving MMX register */
1515 {2, 2}, /* cost of loading MMX registers
1516 in SImode and DImode */
1517 {2, 2}, /* cost of storing MMX registers
1518 in SImode and DImode */
1519 12, /* cost of moving SSE register */
1520 {12, 12, 12}, /* cost of loading SSE registers
1521 in SImode, DImode and TImode */
1522 {2, 2, 8}, /* cost of storing SSE registers
1523 in SImode, DImode and TImode */
1524 10, /* MMX or SSE register to integer */
1525 8, /* size of l1 cache. */
1526 256, /* size of l2 cache. */
1527 64, /* size of prefetch block */
1528 6, /* number of parallel prefetches */
1529 2, /* Branch cost */
1530 COSTS_N_INSNS (5), /* cost of FADD and FSUB insns. */
1531 COSTS_N_INSNS (7), /* cost of FMUL instruction. */
1532 COSTS_N_INSNS (43), /* cost of FDIV instruction. */
1533 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1534 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1535 COSTS_N_INSNS (43), /* cost of FSQRT instruction. */
1538 1, /* scalar_stmt_cost. */
1539 1, /* scalar load_cost. */
1540 1, /* scalar_store_cost. */
1541 1, /* vec_stmt_cost. */
1542 1, /* vec_to_scalar_cost. */
1543 1, /* scalar_to_vec_cost. */
1544 1, /* vec_align_load_cost. */
1545 2, /* vec_unalign_load_cost. */
1546 1, /* vec_store_cost. */
1547 3, /* cond_taken_branch_cost. */
1548 1, /* cond_not_taken_branch_cost. */
1551 static stringop_algs nocona_memcpy[2] = {
1552 {libcall, {{12, loop_1_byte, false}, {-1, rep_prefix_4_byte, false}}},
1553 {libcall, {{32, loop, false}, {20000, rep_prefix_8_byte, false},
1554 {100000, unrolled_loop, false}, {-1, libcall, false}}}};
1556 static stringop_algs nocona_memset[2] = {
1557 {libcall, {{6, loop_1_byte, false}, {48, loop, false},
1558 {20480, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1559 {libcall, {{24, loop, false}, {64, unrolled_loop, false},
1560 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1563 struct processor_costs nocona_cost = {
1564 COSTS_N_INSNS (1), /* cost of an add instruction */
1565 COSTS_N_INSNS (1), /* cost of a lea instruction */
1566 COSTS_N_INSNS (1), /* variable shift costs */
1567 COSTS_N_INSNS (1), /* constant shift costs */
1568 {COSTS_N_INSNS (10), /* cost of starting multiply for QI */
1569 COSTS_N_INSNS (10), /* HI */
1570 COSTS_N_INSNS (10), /* SI */
1571 COSTS_N_INSNS (10), /* DI */
1572 COSTS_N_INSNS (10)}, /* other */
1573 0, /* cost of multiply per each bit set */
1574 {COSTS_N_INSNS (66), /* cost of a divide/mod for QI */
1575 COSTS_N_INSNS (66), /* HI */
1576 COSTS_N_INSNS (66), /* SI */
1577 COSTS_N_INSNS (66), /* DI */
1578 COSTS_N_INSNS (66)}, /* other */
1579 COSTS_N_INSNS (1), /* cost of movsx */
1580 COSTS_N_INSNS (1), /* cost of movzx */
1581 16, /* "large" insn */
1582 17, /* MOVE_RATIO */
1583 4, /* cost for loading QImode using movzbl */
1584 {4, 4, 4}, /* cost of loading integer registers
1585 in QImode, HImode and SImode.
1586 Relative to reg-reg move (2). */
1587 {4, 4, 4}, /* cost of storing integer registers */
1588 3, /* cost of reg,reg fld/fst */
1589 {12, 12, 12}, /* cost of loading fp registers
1590 in SFmode, DFmode and XFmode */
1591 {4, 4, 4}, /* cost of storing fp registers
1592 in SFmode, DFmode and XFmode */
1593 6, /* cost of moving MMX register */
1594 {12, 12}, /* cost of loading MMX registers
1595 in SImode and DImode */
1596 {12, 12}, /* cost of storing MMX registers
1597 in SImode and DImode */
1598 6, /* cost of moving SSE register */
1599 {12, 12, 12}, /* cost of loading SSE registers
1600 in SImode, DImode and TImode */
1601 {12, 12, 12}, /* cost of storing SSE registers
1602 in SImode, DImode and TImode */
1603 8, /* MMX or SSE register to integer */
1604 8, /* size of l1 cache. */
1605 1024, /* size of l2 cache. */
1606 64, /* size of prefetch block */
1607 8, /* number of parallel prefetches */
1608 1, /* Branch cost */
1609 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
1610 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1611 COSTS_N_INSNS (40), /* cost of FDIV instruction. */
1612 COSTS_N_INSNS (3), /* cost of FABS instruction. */
1613 COSTS_N_INSNS (3), /* cost of FCHS instruction. */
1614 COSTS_N_INSNS (44), /* cost of FSQRT instruction. */
1617 1, /* scalar_stmt_cost. */
1618 1, /* scalar load_cost. */
1619 1, /* scalar_store_cost. */
1620 1, /* vec_stmt_cost. */
1621 1, /* vec_to_scalar_cost. */
1622 1, /* scalar_to_vec_cost. */
1623 1, /* vec_align_load_cost. */
1624 2, /* vec_unalign_load_cost. */
1625 1, /* vec_store_cost. */
1626 3, /* cond_taken_branch_cost. */
1627 1, /* cond_not_taken_branch_cost. */
1630 static stringop_algs atom_memcpy[2] = {
1631 {libcall, {{11, loop, false}, {-1, rep_prefix_4_byte, false}}},
1632 {libcall, {{32, loop, false}, {64, rep_prefix_4_byte, false},
1633 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1634 static stringop_algs atom_memset[2] = {
1635 {libcall, {{8, loop, false}, {15, unrolled_loop, false},
1636 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1637 {libcall, {{24, loop, false}, {32, unrolled_loop, false},
1638 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1640 struct processor_costs atom_cost = {
1641 COSTS_N_INSNS (1), /* cost of an add instruction */
1642 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1643 COSTS_N_INSNS (1), /* variable shift costs */
1644 COSTS_N_INSNS (1), /* constant shift costs */
1645 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1646 COSTS_N_INSNS (4), /* HI */
1647 COSTS_N_INSNS (3), /* SI */
1648 COSTS_N_INSNS (4), /* DI */
1649 COSTS_N_INSNS (2)}, /* other */
1650 0, /* cost of multiply per each bit set */
1651 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1652 COSTS_N_INSNS (26), /* HI */
1653 COSTS_N_INSNS (42), /* SI */
1654 COSTS_N_INSNS (74), /* DI */
1655 COSTS_N_INSNS (74)}, /* other */
1656 COSTS_N_INSNS (1), /* cost of movsx */
1657 COSTS_N_INSNS (1), /* cost of movzx */
1658 8, /* "large" insn */
1659 17, /* MOVE_RATIO */
1660 4, /* cost for loading QImode using movzbl */
1661 {4, 4, 4}, /* cost of loading integer registers
1662 in QImode, HImode and SImode.
1663 Relative to reg-reg move (2). */
1664 {4, 4, 4}, /* cost of storing integer registers */
1665 4, /* cost of reg,reg fld/fst */
1666 {12, 12, 12}, /* cost of loading fp registers
1667 in SFmode, DFmode and XFmode */
1668 {6, 6, 8}, /* cost of storing fp registers
1669 in SFmode, DFmode and XFmode */
1670 2, /* cost of moving MMX register */
1671 {8, 8}, /* cost of loading MMX registers
1672 in SImode and DImode */
1673 {8, 8}, /* cost of storing MMX registers
1674 in SImode and DImode */
1675 2, /* cost of moving SSE register */
1676 {8, 8, 8}, /* cost of loading SSE registers
1677 in SImode, DImode and TImode */
1678 {8, 8, 8}, /* cost of storing SSE registers
1679 in SImode, DImode and TImode */
1680 5, /* MMX or SSE register to integer */
1681 32, /* size of l1 cache. */
1682 256, /* size of l2 cache. */
1683 64, /* size of prefetch block */
1684 6, /* number of parallel prefetches */
1685 3, /* Branch cost */
1686 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1687 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1688 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1689 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1690 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1691 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1694 1, /* scalar_stmt_cost. */
1695 1, /* scalar load_cost. */
1696 1, /* scalar_store_cost. */
1697 1, /* vec_stmt_cost. */
1698 1, /* vec_to_scalar_cost. */
1699 1, /* scalar_to_vec_cost. */
1700 1, /* vec_align_load_cost. */
1701 2, /* vec_unalign_load_cost. */
1702 1, /* vec_store_cost. */
1703 3, /* cond_taken_branch_cost. */
1704 1, /* cond_not_taken_branch_cost. */
1707 static stringop_algs slm_memcpy[2] = {
1708 {libcall, {{11, loop, false}, {-1, rep_prefix_4_byte, false}}},
1709 {libcall, {{32, loop, false}, {64, rep_prefix_4_byte, false},
1710 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1711 static stringop_algs slm_memset[2] = {
1712 {libcall, {{8, loop, false}, {15, unrolled_loop, false},
1713 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1714 {libcall, {{24, loop, false}, {32, unrolled_loop, false},
1715 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1717 struct processor_costs slm_cost = {
1718 COSTS_N_INSNS (1), /* cost of an add instruction */
1719 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1720 COSTS_N_INSNS (1), /* variable shift costs */
1721 COSTS_N_INSNS (1), /* constant shift costs */
1722 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1723 COSTS_N_INSNS (3), /* HI */
1724 COSTS_N_INSNS (3), /* SI */
1725 COSTS_N_INSNS (4), /* DI */
1726 COSTS_N_INSNS (2)}, /* other */
1727 0, /* cost of multiply per each bit set */
1728 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1729 COSTS_N_INSNS (26), /* HI */
1730 COSTS_N_INSNS (42), /* SI */
1731 COSTS_N_INSNS (74), /* DI */
1732 COSTS_N_INSNS (74)}, /* other */
1733 COSTS_N_INSNS (1), /* cost of movsx */
1734 COSTS_N_INSNS (1), /* cost of movzx */
1735 8, /* "large" insn */
1736 17, /* MOVE_RATIO */
1737 4, /* cost for loading QImode using movzbl */
1738 {4, 4, 4}, /* cost of loading integer registers
1739 in QImode, HImode and SImode.
1740 Relative to reg-reg move (2). */
1741 {4, 4, 4}, /* cost of storing integer registers */
1742 4, /* cost of reg,reg fld/fst */
1743 {12, 12, 12}, /* cost of loading fp registers
1744 in SFmode, DFmode and XFmode */
1745 {6, 6, 8}, /* cost of storing fp registers
1746 in SFmode, DFmode and XFmode */
1747 2, /* cost of moving MMX register */
1748 {8, 8}, /* cost of loading MMX registers
1749 in SImode and DImode */
1750 {8, 8}, /* cost of storing MMX registers
1751 in SImode and DImode */
1752 2, /* cost of moving SSE register */
1753 {8, 8, 8}, /* cost of loading SSE registers
1754 in SImode, DImode and TImode */
1755 {8, 8, 8}, /* cost of storing SSE registers
1756 in SImode, DImode and TImode */
1757 5, /* MMX or SSE register to integer */
1758 32, /* size of l1 cache. */
1759 256, /* size of l2 cache. */
1760 64, /* size of prefetch block */
1761 6, /* number of parallel prefetches */
1762 3, /* Branch cost */
1763 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1764 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1765 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1766 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1767 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1768 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1771 1, /* scalar_stmt_cost. */
1772 1, /* scalar load_cost. */
1773 1, /* scalar_store_cost. */
1774 1, /* vec_stmt_cost. */
1775 4, /* vec_to_scalar_cost. */
1776 1, /* scalar_to_vec_cost. */
1777 1, /* vec_align_load_cost. */
1778 2, /* vec_unalign_load_cost. */
1779 1, /* vec_store_cost. */
1780 3, /* cond_taken_branch_cost. */
1781 1, /* cond_not_taken_branch_cost. */
1784 static stringop_algs intel_memcpy[2] = {
1785 {libcall, {{11, loop, false}, {-1, rep_prefix_4_byte, false}}},
1786 {libcall, {{32, loop, false}, {64, rep_prefix_4_byte, false},
1787 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1788 static stringop_algs intel_memset[2] = {
1789 {libcall, {{8, loop, false}, {15, unrolled_loop, false},
1790 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1791 {libcall, {{24, loop, false}, {32, unrolled_loop, false},
1792 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1794 struct processor_costs intel_cost = {
1795 COSTS_N_INSNS (1), /* cost of an add instruction */
1796 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1797 COSTS_N_INSNS (1), /* variable shift costs */
1798 COSTS_N_INSNS (1), /* constant shift costs */
1799 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1800 COSTS_N_INSNS (3), /* HI */
1801 COSTS_N_INSNS (3), /* SI */
1802 COSTS_N_INSNS (4), /* DI */
1803 COSTS_N_INSNS (2)}, /* other */
1804 0, /* cost of multiply per each bit set */
1805 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1806 COSTS_N_INSNS (26), /* HI */
1807 COSTS_N_INSNS (42), /* SI */
1808 COSTS_N_INSNS (74), /* DI */
1809 COSTS_N_INSNS (74)}, /* other */
1810 COSTS_N_INSNS (1), /* cost of movsx */
1811 COSTS_N_INSNS (1), /* cost of movzx */
1812 8, /* "large" insn */
1813 17, /* MOVE_RATIO */
1814 4, /* cost for loading QImode using movzbl */
1815 {4, 4, 4}, /* cost of loading integer registers
1816 in QImode, HImode and SImode.
1817 Relative to reg-reg move (2). */
1818 {4, 4, 4}, /* cost of storing integer registers */
1819 4, /* cost of reg,reg fld/fst */
1820 {12, 12, 12}, /* cost of loading fp registers
1821 in SFmode, DFmode and XFmode */
1822 {6, 6, 8}, /* cost of storing fp registers
1823 in SFmode, DFmode and XFmode */
1824 2, /* cost of moving MMX register */
1825 {8, 8}, /* cost of loading MMX registers
1826 in SImode and DImode */
1827 {8, 8}, /* cost of storing MMX registers
1828 in SImode and DImode */
1829 2, /* cost of moving SSE register */
1830 {8, 8, 8}, /* cost of loading SSE registers
1831 in SImode, DImode and TImode */
1832 {8, 8, 8}, /* cost of storing SSE registers
1833 in SImode, DImode and TImode */
1834 5, /* MMX or SSE register to integer */
1835 32, /* size of l1 cache. */
1836 256, /* size of l2 cache. */
1837 64, /* size of prefetch block */
1838 6, /* number of parallel prefetches */
1839 3, /* Branch cost */
1840 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1841 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1842 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1843 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1844 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1845 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1848 1, /* scalar_stmt_cost. */
1849 1, /* scalar load_cost. */
1850 1, /* scalar_store_cost. */
1851 1, /* vec_stmt_cost. */
1852 4, /* vec_to_scalar_cost. */
1853 1, /* scalar_to_vec_cost. */
1854 1, /* vec_align_load_cost. */
1855 2, /* vec_unalign_load_cost. */
1856 1, /* vec_store_cost. */
1857 3, /* cond_taken_branch_cost. */
1858 1, /* cond_not_taken_branch_cost. */
1861 /* Generic should produce code tuned for Core-i7 (and newer chips)
1862 and btver1 (and newer chips). */
1864 static stringop_algs generic_memcpy[2] = {
1865 {libcall, {{32, loop, false}, {8192, rep_prefix_4_byte, false},
1866 {-1, libcall, false}}},
1867 {libcall, {{32, loop, false}, {8192, rep_prefix_8_byte, false},
1868 {-1, libcall, false}}}};
1869 static stringop_algs generic_memset[2] = {
1870 {libcall, {{32, loop, false}, {8192, rep_prefix_4_byte, false},
1871 {-1, libcall, false}}},
1872 {libcall, {{32, loop, false}, {8192, rep_prefix_8_byte, false},
1873 {-1, libcall, false}}}};
1875 struct processor_costs generic_cost = {
1876 COSTS_N_INSNS (1), /* cost of an add instruction */
1877 /* On all chips taken into consideration lea is 2 cycles and more. With
1878 this cost however our current implementation of synth_mult results in
1879 use of unnecessary temporary registers causing regression on several
1880 SPECfp benchmarks. */
1881 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1882 COSTS_N_INSNS (1), /* variable shift costs */
1883 COSTS_N_INSNS (1), /* constant shift costs */
1884 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1885 COSTS_N_INSNS (4), /* HI */
1886 COSTS_N_INSNS (3), /* SI */
1887 COSTS_N_INSNS (4), /* DI */
1888 COSTS_N_INSNS (2)}, /* other */
1889 0, /* cost of multiply per each bit set */
1890 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1891 COSTS_N_INSNS (26), /* HI */
1892 COSTS_N_INSNS (42), /* SI */
1893 COSTS_N_INSNS (74), /* DI */
1894 COSTS_N_INSNS (74)}, /* other */
1895 COSTS_N_INSNS (1), /* cost of movsx */
1896 COSTS_N_INSNS (1), /* cost of movzx */
1897 8, /* "large" insn */
1898 17, /* MOVE_RATIO */
1899 4, /* cost for loading QImode using movzbl */
1900 {4, 4, 4}, /* cost of loading integer registers
1901 in QImode, HImode and SImode.
1902 Relative to reg-reg move (2). */
1903 {4, 4, 4}, /* cost of storing integer registers */
1904 4, /* cost of reg,reg fld/fst */
1905 {12, 12, 12}, /* cost of loading fp registers
1906 in SFmode, DFmode and XFmode */
1907 {6, 6, 8}, /* cost of storing fp registers
1908 in SFmode, DFmode and XFmode */
1909 2, /* cost of moving MMX register */
1910 {8, 8}, /* cost of loading MMX registers
1911 in SImode and DImode */
1912 {8, 8}, /* cost of storing MMX registers
1913 in SImode and DImode */
1914 2, /* cost of moving SSE register */
1915 {8, 8, 8}, /* cost of loading SSE registers
1916 in SImode, DImode and TImode */
1917 {8, 8, 8}, /* cost of storing SSE registers
1918 in SImode, DImode and TImode */
1919 5, /* MMX or SSE register to integer */
1920 32, /* size of l1 cache. */
1921 512, /* size of l2 cache. */
1922 64, /* size of prefetch block */
1923 6, /* number of parallel prefetches */
1924 /* Benchmarks shows large regressions on K8 sixtrack benchmark when this
1925 value is increased to perhaps more appropriate value of 5. */
1926 3, /* Branch cost */
1927 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1928 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1929 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1930 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1931 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1932 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1935 1, /* scalar_stmt_cost. */
1936 1, /* scalar load_cost. */
1937 1, /* scalar_store_cost. */
1938 1, /* vec_stmt_cost. */
1939 1, /* vec_to_scalar_cost. */
1940 1, /* scalar_to_vec_cost. */
1941 1, /* vec_align_load_cost. */
1942 2, /* vec_unalign_load_cost. */
1943 1, /* vec_store_cost. */
1944 3, /* cond_taken_branch_cost. */
1945 1, /* cond_not_taken_branch_cost. */
1948 /* core_cost should produce code tuned for Core familly of CPUs. */
1949 static stringop_algs core_memcpy[2] = {
1950 {libcall, {{1024, rep_prefix_4_byte, true}, {-1, libcall, false}}},
1951 {libcall, {{24, loop, true}, {128, rep_prefix_8_byte, true},
1952 {-1, libcall, false}}}};
1953 static stringop_algs core_memset[2] = {
1954 {libcall, {{6, loop_1_byte, true},
1956 {8192, rep_prefix_4_byte, true},
1957 {-1, libcall, false}}},
1958 {libcall, {{24, loop, true}, {512, rep_prefix_8_byte, true},
1959 {-1, libcall, false}}}};
1962 struct processor_costs core_cost = {
1963 COSTS_N_INSNS (1), /* cost of an add instruction */
1964 /* On all chips taken into consideration lea is 2 cycles and more. With
1965 this cost however our current implementation of synth_mult results in
1966 use of unnecessary temporary registers causing regression on several
1967 SPECfp benchmarks. */
1968 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1969 COSTS_N_INSNS (1), /* variable shift costs */
1970 COSTS_N_INSNS (1), /* constant shift costs */
1971 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1972 COSTS_N_INSNS (4), /* HI */
1973 COSTS_N_INSNS (3), /* SI */
1974 COSTS_N_INSNS (4), /* DI */
1975 COSTS_N_INSNS (2)}, /* other */
1976 0, /* cost of multiply per each bit set */
1977 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1978 COSTS_N_INSNS (26), /* HI */
1979 COSTS_N_INSNS (42), /* SI */
1980 COSTS_N_INSNS (74), /* DI */
1981 COSTS_N_INSNS (74)}, /* other */
1982 COSTS_N_INSNS (1), /* cost of movsx */
1983 COSTS_N_INSNS (1), /* cost of movzx */
1984 8, /* "large" insn */
1985 17, /* MOVE_RATIO */
1986 4, /* cost for loading QImode using movzbl */
1987 {4, 4, 4}, /* cost of loading integer registers
1988 in QImode, HImode and SImode.
1989 Relative to reg-reg move (2). */
1990 {4, 4, 4}, /* cost of storing integer registers */
1991 4, /* cost of reg,reg fld/fst */
1992 {12, 12, 12}, /* cost of loading fp registers
1993 in SFmode, DFmode and XFmode */
1994 {6, 6, 8}, /* cost of storing fp registers
1995 in SFmode, DFmode and XFmode */
1996 2, /* cost of moving MMX register */
1997 {8, 8}, /* cost of loading MMX registers
1998 in SImode and DImode */
1999 {8, 8}, /* cost of storing MMX registers
2000 in SImode and DImode */
2001 2, /* cost of moving SSE register */
2002 {8, 8, 8}, /* cost of loading SSE registers
2003 in SImode, DImode and TImode */
2004 {8, 8, 8}, /* cost of storing SSE registers
2005 in SImode, DImode and TImode */
2006 5, /* MMX or SSE register to integer */
2007 64, /* size of l1 cache. */
2008 512, /* size of l2 cache. */
2009 64, /* size of prefetch block */
2010 6, /* number of parallel prefetches */
2011 /* FIXME perhaps more appropriate value is 5. */
2012 3, /* Branch cost */
2013 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
2014 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
2015 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
2016 COSTS_N_INSNS (8), /* cost of FABS instruction. */
2017 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
2018 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
2021 1, /* scalar_stmt_cost. */
2022 1, /* scalar load_cost. */
2023 1, /* scalar_store_cost. */
2024 1, /* vec_stmt_cost. */
2025 1, /* vec_to_scalar_cost. */
2026 1, /* scalar_to_vec_cost. */
2027 1, /* vec_align_load_cost. */
2028 2, /* vec_unalign_load_cost. */
2029 1, /* vec_store_cost. */
2030 3, /* cond_taken_branch_cost. */
2031 1, /* cond_not_taken_branch_cost. */
2035 /* Set by -mtune. */
2036 const struct processor_costs *ix86_tune_cost = &pentium_cost;
2038 /* Set by -mtune or -Os. */
2039 const struct processor_costs *ix86_cost = &pentium_cost;
2041 /* Processor feature/optimization bitmasks. */
2042 #define m_386 (1<<PROCESSOR_I386)
2043 #define m_486 (1<<PROCESSOR_I486)
2044 #define m_PENT (1<<PROCESSOR_PENTIUM)
2045 #define m_PPRO (1<<PROCESSOR_PENTIUMPRO)
2046 #define m_PENT4 (1<<PROCESSOR_PENTIUM4)
2047 #define m_NOCONA (1<<PROCESSOR_NOCONA)
2048 #define m_P4_NOCONA (m_PENT4 | m_NOCONA)
2049 #define m_CORE2 (1<<PROCESSOR_CORE2)
2050 #define m_NEHALEM (1<<PROCESSOR_NEHALEM)
2051 #define m_SANDYBRIDGE (1<<PROCESSOR_SANDYBRIDGE)
2052 #define m_HASWELL (1<<PROCESSOR_HASWELL)
2053 #define m_CORE_ALL (m_CORE2 | m_NEHALEM | m_SANDYBRIDGE | m_HASWELL)
2054 #define m_BONNELL (1<<PROCESSOR_BONNELL)
2055 #define m_SILVERMONT (1<<PROCESSOR_SILVERMONT)
2056 #define m_KNL (1<<PROCESSOR_KNL)
2057 #define m_INTEL (1<<PROCESSOR_INTEL)
2059 #define m_GEODE (1<<PROCESSOR_GEODE)
2060 #define m_K6 (1<<PROCESSOR_K6)
2061 #define m_K6_GEODE (m_K6 | m_GEODE)
2062 #define m_K8 (1<<PROCESSOR_K8)
2063 #define m_ATHLON (1<<PROCESSOR_ATHLON)
2064 #define m_ATHLON_K8 (m_K8 | m_ATHLON)
2065 #define m_AMDFAM10 (1<<PROCESSOR_AMDFAM10)
2066 #define m_BDVER1 (1<<PROCESSOR_BDVER1)
2067 #define m_BDVER2 (1<<PROCESSOR_BDVER2)
2068 #define m_BDVER3 (1<<PROCESSOR_BDVER3)
2069 #define m_BDVER4 (1<<PROCESSOR_BDVER4)
2070 #define m_BTVER1 (1<<PROCESSOR_BTVER1)
2071 #define m_BTVER2 (1<<PROCESSOR_BTVER2)
2072 #define m_BDVER (m_BDVER1 | m_BDVER2 | m_BDVER3 | m_BDVER4)
2073 #define m_BTVER (m_BTVER1 | m_BTVER2)
2074 #define m_AMD_MULTIPLE (m_ATHLON_K8 | m_AMDFAM10 | m_BDVER | m_BTVER)
2076 #define m_GENERIC (1<<PROCESSOR_GENERIC)
2078 const char* ix86_tune_feature_names[X86_TUNE_LAST] = {
2080 #define DEF_TUNE(tune, name, selector) name,
2081 #include "x86-tune.def"
2085 /* Feature tests against the various tunings. */
2086 unsigned char ix86_tune_features[X86_TUNE_LAST];
2088 /* Feature tests against the various tunings used to create ix86_tune_features
2089 based on the processor mask. */
2090 static unsigned int initial_ix86_tune_features[X86_TUNE_LAST] = {
2092 #define DEF_TUNE(tune, name, selector) selector,
2093 #include "x86-tune.def"
2097 /* Feature tests against the various architecture variations. */
2098 unsigned char ix86_arch_features[X86_ARCH_LAST];
2100 /* Feature tests against the various architecture variations, used to create
2101 ix86_arch_features based on the processor mask. */
2102 static unsigned int initial_ix86_arch_features[X86_ARCH_LAST] = {
2103 /* X86_ARCH_CMOV: Conditional move was added for pentiumpro. */
2104 ~(m_386 | m_486 | m_PENT | m_K6),
2106 /* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486. */
2109 /* X86_ARCH_CMPXCHG8B: Compare and exchange 8 bytes was added for pentium. */
2112 /* X86_ARCH_XADD: Exchange and add was added for 80486. */
2115 /* X86_ARCH_BSWAP: Byteswap was added for 80486. */
2119 /* In case the average insn count for single function invocation is
2120 lower than this constant, emit fast (but longer) prologue and
2122 #define FAST_PROLOGUE_INSN_COUNT 20
2124 /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
2125 static const char *const qi_reg_name[] = QI_REGISTER_NAMES;
2126 static const char *const qi_high_reg_name[] = QI_HIGH_REGISTER_NAMES;
2127 static const char *const hi_reg_name[] = HI_REGISTER_NAMES;
2129 /* Array of the smallest class containing reg number REGNO, indexed by
2130 REGNO. Used by REGNO_REG_CLASS in i386.h. */
2132 enum reg_class const regclass_map[FIRST_PSEUDO_REGISTER] =
2134 /* ax, dx, cx, bx */
2135 AREG, DREG, CREG, BREG,
2136 /* si, di, bp, sp */
2137 SIREG, DIREG, NON_Q_REGS, NON_Q_REGS,
2139 FP_TOP_REG, FP_SECOND_REG, FLOAT_REGS, FLOAT_REGS,
2140 FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS,
2143 /* flags, fpsr, fpcr, frame */
2144 NO_REGS, NO_REGS, NO_REGS, NON_Q_REGS,
2146 SSE_FIRST_REG, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
2149 MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS,
2152 NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS,
2153 NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS,
2154 /* SSE REX registers */
2155 SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
2157 /* AVX-512 SSE registers */
2158 EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
2159 EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
2160 EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
2161 EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
2162 /* Mask registers. */
2163 MASK_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS,
2164 MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS,
2165 /* MPX bound registers */
2166 BND_REGS, BND_REGS, BND_REGS, BND_REGS,
2169 /* The "default" register map used in 32bit mode. */
2171 int const dbx_register_map[FIRST_PSEUDO_REGISTER] =
2173 0, 2, 1, 3, 6, 7, 4, 5, /* general regs */
2174 12, 13, 14, 15, 16, 17, 18, 19, /* fp regs */
2175 -1, -1, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
2176 21, 22, 23, 24, 25, 26, 27, 28, /* SSE */
2177 29, 30, 31, 32, 33, 34, 35, 36, /* MMX */
2178 -1, -1, -1, -1, -1, -1, -1, -1, /* extended integer registers */
2179 -1, -1, -1, -1, -1, -1, -1, -1, /* extended SSE registers */
2180 -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 16-23*/
2181 -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 24-31*/
2182 93, 94, 95, 96, 97, 98, 99, 100, /* Mask registers */
2183 101, 102, 103, 104, /* bound registers */
2186 /* The "default" register map used in 64bit mode. */
2188 int const dbx64_register_map[FIRST_PSEUDO_REGISTER] =
2190 0, 1, 2, 3, 4, 5, 6, 7, /* general regs */
2191 33, 34, 35, 36, 37, 38, 39, 40, /* fp regs */
2192 -1, -1, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
2193 17, 18, 19, 20, 21, 22, 23, 24, /* SSE */
2194 41, 42, 43, 44, 45, 46, 47, 48, /* MMX */
2195 8,9,10,11,12,13,14,15, /* extended integer registers */
2196 25, 26, 27, 28, 29, 30, 31, 32, /* extended SSE registers */
2197 67, 68, 69, 70, 71, 72, 73, 74, /* AVX-512 registers 16-23 */
2198 75, 76, 77, 78, 79, 80, 81, 82, /* AVX-512 registers 24-31 */
2199 118, 119, 120, 121, 122, 123, 124, 125, /* Mask registers */
2200 126, 127, 128, 129, /* bound registers */
2203 /* Define the register numbers to be used in Dwarf debugging information.
2204 The SVR4 reference port C compiler uses the following register numbers
2205 in its Dwarf output code:
2206 0 for %eax (gcc regno = 0)
2207 1 for %ecx (gcc regno = 2)
2208 2 for %edx (gcc regno = 1)
2209 3 for %ebx (gcc regno = 3)
2210 4 for %esp (gcc regno = 7)
2211 5 for %ebp (gcc regno = 6)
2212 6 for %esi (gcc regno = 4)
2213 7 for %edi (gcc regno = 5)
2214 The following three DWARF register numbers are never generated by
2215 the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
2216 believes these numbers have these meanings.
2217 8 for %eip (no gcc equivalent)
2218 9 for %eflags (gcc regno = 17)
2219 10 for %trapno (no gcc equivalent)
2220 It is not at all clear how we should number the FP stack registers
2221 for the x86 architecture. If the version of SDB on x86/svr4 were
2222 a bit less brain dead with respect to floating-point then we would
2223 have a precedent to follow with respect to DWARF register numbers
2224 for x86 FP registers, but the SDB on x86/svr4 is so completely
2225 broken with respect to FP registers that it is hardly worth thinking
2226 of it as something to strive for compatibility with.
2227 The version of x86/svr4 SDB I have at the moment does (partially)
2228 seem to believe that DWARF register number 11 is associated with
2229 the x86 register %st(0), but that's about all. Higher DWARF
2230 register numbers don't seem to be associated with anything in
2231 particular, and even for DWARF regno 11, SDB only seems to under-
2232 stand that it should say that a variable lives in %st(0) (when
2233 asked via an `=' command) if we said it was in DWARF regno 11,
2234 but SDB still prints garbage when asked for the value of the
2235 variable in question (via a `/' command).
2236 (Also note that the labels SDB prints for various FP stack regs
2237 when doing an `x' command are all wrong.)
2238 Note that these problems generally don't affect the native SVR4
2239 C compiler because it doesn't allow the use of -O with -g and
2240 because when it is *not* optimizing, it allocates a memory
2241 location for each floating-point variable, and the memory
2242 location is what gets described in the DWARF AT_location
2243 attribute for the variable in question.
2244 Regardless of the severe mental illness of the x86/svr4 SDB, we
2245 do something sensible here and we use the following DWARF
2246 register numbers. Note that these are all stack-top-relative
2248 11 for %st(0) (gcc regno = 8)
2249 12 for %st(1) (gcc regno = 9)
2250 13 for %st(2) (gcc regno = 10)
2251 14 for %st(3) (gcc regno = 11)
2252 15 for %st(4) (gcc regno = 12)
2253 16 for %st(5) (gcc regno = 13)
2254 17 for %st(6) (gcc regno = 14)
2255 18 for %st(7) (gcc regno = 15)
2257 int const svr4_dbx_register_map[FIRST_PSEUDO_REGISTER] =
2259 0, 2, 1, 3, 6, 7, 5, 4, /* general regs */
2260 11, 12, 13, 14, 15, 16, 17, 18, /* fp regs */
2261 -1, 9, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
2262 21, 22, 23, 24, 25, 26, 27, 28, /* SSE registers */
2263 29, 30, 31, 32, 33, 34, 35, 36, /* MMX registers */
2264 -1, -1, -1, -1, -1, -1, -1, -1, /* extended integer registers */
2265 -1, -1, -1, -1, -1, -1, -1, -1, /* extended SSE registers */
2266 -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 16-23*/
2267 -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 24-31*/
2268 93, 94, 95, 96, 97, 98, 99, 100, /* Mask registers */
2269 101, 102, 103, 104, /* bound registers */
2272 /* Define parameter passing and return registers. */
2274 static int const x86_64_int_parameter_registers[6] =
2276 DI_REG, SI_REG, DX_REG, CX_REG, R8_REG, R9_REG
2279 static int const x86_64_ms_abi_int_parameter_registers[4] =
2281 CX_REG, DX_REG, R8_REG, R9_REG
2284 static int const x86_64_int_return_registers[4] =
2286 AX_REG, DX_REG, DI_REG, SI_REG
2289 /* Additional registers that are clobbered by SYSV calls. */
2291 int const x86_64_ms_sysv_extra_clobbered_registers[12] =
2295 XMM8_REG, XMM9_REG, XMM10_REG, XMM11_REG,
2296 XMM12_REG, XMM13_REG, XMM14_REG, XMM15_REG
2299 /* Define the structure for the machine field in struct function. */
2301 struct GTY(()) stack_local_entry {
2302 unsigned short mode;
2305 struct stack_local_entry *next;
2308 /* Structure describing stack frame layout.
2309 Stack grows downward:
2315 saved static chain if ix86_static_chain_on_stack
2317 saved frame pointer if frame_pointer_needed
2318 <- HARD_FRAME_POINTER
2324 <- sse_regs_save_offset
2327 [va_arg registers] |
2331 [padding2] | = to_allocate
2340 int outgoing_arguments_size;
2342 /* The offsets relative to ARG_POINTER. */
2343 HOST_WIDE_INT frame_pointer_offset;
2344 HOST_WIDE_INT hard_frame_pointer_offset;
2345 HOST_WIDE_INT stack_pointer_offset;
2346 HOST_WIDE_INT hfp_save_offset;
2347 HOST_WIDE_INT reg_save_offset;
2348 HOST_WIDE_INT sse_reg_save_offset;
2350 /* When save_regs_using_mov is set, emit prologue using
2351 move instead of push instructions. */
2352 bool save_regs_using_mov;
2355 /* Which cpu are we scheduling for. */
2356 enum attr_cpu ix86_schedule;
2358 /* Which cpu are we optimizing for. */
2359 enum processor_type ix86_tune;
2361 /* Which instruction set architecture to use. */
2362 enum processor_type ix86_arch;
2364 /* True if processor has SSE prefetch instruction. */
2365 unsigned char x86_prefetch_sse;
2367 /* -mstackrealign option */
2368 static const char ix86_force_align_arg_pointer_string[]
2369 = "force_align_arg_pointer";
2371 static rtx (*ix86_gen_leave) (void);
2372 static rtx (*ix86_gen_add3) (rtx, rtx, rtx);
2373 static rtx (*ix86_gen_sub3) (rtx, rtx, rtx);
2374 static rtx (*ix86_gen_sub3_carry) (rtx, rtx, rtx, rtx, rtx);
2375 static rtx (*ix86_gen_one_cmpl2) (rtx, rtx);
2376 static rtx (*ix86_gen_monitor) (rtx, rtx, rtx);
2377 static rtx (*ix86_gen_monitorx) (rtx, rtx, rtx);
2378 static rtx (*ix86_gen_andsp) (rtx, rtx, rtx);
2379 static rtx (*ix86_gen_allocate_stack_worker) (rtx, rtx);
2380 static rtx (*ix86_gen_adjust_stack_and_probe) (rtx, rtx, rtx);
2381 static rtx (*ix86_gen_probe_stack_range) (rtx, rtx, rtx);
2382 static rtx (*ix86_gen_tls_global_dynamic_64) (rtx, rtx, rtx);
2383 static rtx (*ix86_gen_tls_local_dynamic_base_64) (rtx, rtx);
2385 /* Preferred alignment for stack boundary in bits. */
2386 unsigned int ix86_preferred_stack_boundary;
2388 /* Alignment for incoming stack boundary in bits specified at
2390 static unsigned int ix86_user_incoming_stack_boundary;
2392 /* Default alignment for incoming stack boundary in bits. */
2393 static unsigned int ix86_default_incoming_stack_boundary;
2395 /* Alignment for incoming stack boundary in bits. */
2396 unsigned int ix86_incoming_stack_boundary;
2398 /* Calling abi specific va_list type nodes. */
2399 static GTY(()) tree sysv_va_list_type_node;
2400 static GTY(()) tree ms_va_list_type_node;
2402 /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
2403 char internal_label_prefix[16];
2404 int internal_label_prefix_len;
2406 /* Fence to use after loop using movnt. */
2409 /* Register class used for passing given 64bit part of the argument.
2410 These represent classes as documented by the PS ABI, with the exception
2411 of SSESF, SSEDF classes, that are basically SSE class, just gcc will
2412 use SF or DFmode move instead of DImode to avoid reformatting penalties.
2414 Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
2415 whenever possible (upper half does contain padding). */
2416 enum x86_64_reg_class
2419 X86_64_INTEGER_CLASS,
2420 X86_64_INTEGERSI_CLASS,
2427 X86_64_COMPLEX_X87_CLASS,
2431 #define MAX_CLASSES 8
2433 /* Table of constants used by fldpi, fldln2, etc.... */
2434 static REAL_VALUE_TYPE ext_80387_constants_table [5];
2435 static bool ext_80387_constants_init = 0;
2438 static struct machine_function * ix86_init_machine_status (void);
2439 static rtx ix86_function_value (const_tree, const_tree, bool);
2440 static bool ix86_function_value_regno_p (const unsigned int);
2441 static unsigned int ix86_function_arg_boundary (machine_mode,
2443 static rtx ix86_static_chain (const_tree, bool);
2444 static int ix86_function_regparm (const_tree, const_tree);
2445 static void ix86_compute_frame_layout (struct ix86_frame *);
2446 static bool ix86_expand_vector_init_one_nonzero (bool, machine_mode,
2448 static void ix86_add_new_builtins (HOST_WIDE_INT);
2449 static tree ix86_canonical_va_list_type (tree);
2450 static void predict_jump (int);
2451 static unsigned int split_stack_prologue_scratch_regno (void);
2452 static bool i386_asm_output_addr_const_extra (FILE *, rtx);
2454 enum ix86_function_specific_strings
2456 IX86_FUNCTION_SPECIFIC_ARCH,
2457 IX86_FUNCTION_SPECIFIC_TUNE,
2458 IX86_FUNCTION_SPECIFIC_MAX
2461 static char *ix86_target_string (HOST_WIDE_INT, int, const char *,
2462 const char *, enum fpmath_unit, bool);
2463 static void ix86_function_specific_save (struct cl_target_option *,
2464 struct gcc_options *opts);
2465 static void ix86_function_specific_restore (struct gcc_options *opts,
2466 struct cl_target_option *);
2467 static void ix86_function_specific_post_stream_in (struct cl_target_option *);
2468 static void ix86_function_specific_print (FILE *, int,
2469 struct cl_target_option *);
2470 static bool ix86_valid_target_attribute_p (tree, tree, tree, int);
2471 static bool ix86_valid_target_attribute_inner_p (tree, char *[],
2472 struct gcc_options *,
2473 struct gcc_options *,
2474 struct gcc_options *);
2475 static bool ix86_can_inline_p (tree, tree);
2476 static void ix86_set_current_function (tree);
2477 static unsigned int ix86_minimum_incoming_stack_boundary (bool);
2479 static enum calling_abi ix86_function_abi (const_tree);
2482 #ifndef SUBTARGET32_DEFAULT_CPU
2483 #define SUBTARGET32_DEFAULT_CPU "i386"
2486 /* Whether -mtune= or -march= were specified */
2487 static int ix86_tune_defaulted;
2488 static int ix86_arch_specified;
2490 /* Vectorization library interface and handlers. */
2491 static tree (*ix86_veclib_handler) (enum built_in_function, tree, tree);
2493 static tree ix86_veclibabi_svml (enum built_in_function, tree, tree);
2494 static tree ix86_veclibabi_acml (enum built_in_function, tree, tree);
2496 /* Processor target table, indexed by processor number */
2499 const char *const name; /* processor name */
2500 const struct processor_costs *cost; /* Processor costs */
2501 const int align_loop; /* Default alignments. */
2502 const int align_loop_max_skip;
2503 const int align_jump;
2504 const int align_jump_max_skip;
2505 const int align_func;
2508 /* This table must be in sync with enum processor_type in i386.h. */
2509 static const struct ptt processor_target_table[PROCESSOR_max] =
2511 {"generic", &generic_cost, 16, 10, 16, 10, 16},
2512 {"i386", &i386_cost, 4, 3, 4, 3, 4},
2513 {"i486", &i486_cost, 16, 15, 16, 15, 16},
2514 {"pentium", &pentium_cost, 16, 7, 16, 7, 16},
2515 {"pentiumpro", &pentiumpro_cost, 16, 15, 16, 10, 16},
2516 {"pentium4", &pentium4_cost, 0, 0, 0, 0, 0},
2517 {"nocona", &nocona_cost, 0, 0, 0, 0, 0},
2518 {"core2", &core_cost, 16, 10, 16, 10, 16},
2519 {"nehalem", &core_cost, 16, 10, 16, 10, 16},
2520 {"sandybridge", &core_cost, 16, 10, 16, 10, 16},
2521 {"haswell", &core_cost, 16, 10, 16, 10, 16},
2522 {"bonnell", &atom_cost, 16, 15, 16, 7, 16},
2523 {"silvermont", &slm_cost, 16, 15, 16, 7, 16},
2524 {"knl", &slm_cost, 16, 15, 16, 7, 16},
2525 {"intel", &intel_cost, 16, 15, 16, 7, 16},
2526 {"geode", &geode_cost, 0, 0, 0, 0, 0},
2527 {"k6", &k6_cost, 32, 7, 32, 7, 32},
2528 {"athlon", &athlon_cost, 16, 7, 16, 7, 16},
2529 {"k8", &k8_cost, 16, 7, 16, 7, 16},
2530 {"amdfam10", &amdfam10_cost, 32, 24, 32, 7, 32},
2531 {"bdver1", &bdver1_cost, 16, 10, 16, 7, 11},
2532 {"bdver2", &bdver2_cost, 16, 10, 16, 7, 11},
2533 {"bdver3", &bdver3_cost, 16, 10, 16, 7, 11},
2534 {"bdver4", &bdver4_cost, 16, 10, 16, 7, 11},
2535 {"btver1", &btver1_cost, 16, 10, 16, 7, 11},
2536 {"btver2", &btver2_cost, 16, 10, 16, 7, 11}
2540 rest_of_handle_insert_vzeroupper (void)
2544 /* vzeroupper instructions are inserted immediately after reload to
2545 account for possible spills from 256bit registers. The pass
2546 reuses mode switching infrastructure by re-running mode insertion
2547 pass, so disable entities that have already been processed. */
2548 for (i = 0; i < MAX_386_ENTITIES; i++)
2549 ix86_optimize_mode_switching[i] = 0;
2551 ix86_optimize_mode_switching[AVX_U128] = 1;
2553 /* Call optimize_mode_switching. */
2554 g->get_passes ()->execute_pass_mode_switching ();
2560 const pass_data pass_data_insert_vzeroupper =
2562 RTL_PASS, /* type */
2563 "vzeroupper", /* name */
2564 OPTGROUP_NONE, /* optinfo_flags */
2565 TV_NONE, /* tv_id */
2566 0, /* properties_required */
2567 0, /* properties_provided */
2568 0, /* properties_destroyed */
2569 0, /* todo_flags_start */
2570 TODO_df_finish, /* todo_flags_finish */
2573 class pass_insert_vzeroupper : public rtl_opt_pass
2576 pass_insert_vzeroupper(gcc::context *ctxt)
2577 : rtl_opt_pass(pass_data_insert_vzeroupper, ctxt)
2580 /* opt_pass methods: */
2581 virtual bool gate (function *)
2583 return TARGET_AVX && !TARGET_AVX512F
2584 && TARGET_VZEROUPPER && flag_expensive_optimizations
2588 virtual unsigned int execute (function *)
2590 return rest_of_handle_insert_vzeroupper ();
2593 }; // class pass_insert_vzeroupper
2598 make_pass_insert_vzeroupper (gcc::context *ctxt)
2600 return new pass_insert_vzeroupper (ctxt);
2603 /* Return true if a red-zone is in use. */
2606 ix86_using_red_zone (void)
2608 return TARGET_RED_ZONE && !TARGET_64BIT_MS_ABI;
2611 /* Return a string that documents the current -m options. The caller is
2612 responsible for freeing the string. */
2615 ix86_target_string (HOST_WIDE_INT isa, int flags, const char *arch,
2616 const char *tune, enum fpmath_unit fpmath,
2619 struct ix86_target_opts
2621 const char *option; /* option string */
2622 HOST_WIDE_INT mask; /* isa mask options */
2625 /* This table is ordered so that options like -msse4.2 that imply
2626 preceding options while match those first. */
2627 static struct ix86_target_opts isa_opts[] =
2629 { "-mfma4", OPTION_MASK_ISA_FMA4 },
2630 { "-mfma", OPTION_MASK_ISA_FMA },
2631 { "-mxop", OPTION_MASK_ISA_XOP },
2632 { "-mlwp", OPTION_MASK_ISA_LWP },
2633 { "-mavx512f", OPTION_MASK_ISA_AVX512F },
2634 { "-mavx512er", OPTION_MASK_ISA_AVX512ER },
2635 { "-mavx512cd", OPTION_MASK_ISA_AVX512CD },
2636 { "-mavx512pf", OPTION_MASK_ISA_AVX512PF },
2637 { "-mavx512dq", OPTION_MASK_ISA_AVX512DQ },
2638 { "-mavx512bw", OPTION_MASK_ISA_AVX512BW },
2639 { "-mavx512vl", OPTION_MASK_ISA_AVX512VL },
2640 { "-mavx512ifma", OPTION_MASK_ISA_AVX512IFMA },
2641 { "-mavx512vbmi", OPTION_MASK_ISA_AVX512VBMI },
2642 { "-msse4a", OPTION_MASK_ISA_SSE4A },
2643 { "-msse4.2", OPTION_MASK_ISA_SSE4_2 },
2644 { "-msse4.1", OPTION_MASK_ISA_SSE4_1 },
2645 { "-mssse3", OPTION_MASK_ISA_SSSE3 },
2646 { "-msse3", OPTION_MASK_ISA_SSE3 },
2647 { "-msse2", OPTION_MASK_ISA_SSE2 },
2648 { "-msse", OPTION_MASK_ISA_SSE },
2649 { "-m3dnow", OPTION_MASK_ISA_3DNOW },
2650 { "-m3dnowa", OPTION_MASK_ISA_3DNOW_A },
2651 { "-mmmx", OPTION_MASK_ISA_MMX },
2652 { "-mabm", OPTION_MASK_ISA_ABM },
2653 { "-mbmi", OPTION_MASK_ISA_BMI },
2654 { "-mbmi2", OPTION_MASK_ISA_BMI2 },
2655 { "-mlzcnt", OPTION_MASK_ISA_LZCNT },
2656 { "-mhle", OPTION_MASK_ISA_HLE },
2657 { "-mfxsr", OPTION_MASK_ISA_FXSR },
2658 { "-mrdseed", OPTION_MASK_ISA_RDSEED },
2659 { "-mprfchw", OPTION_MASK_ISA_PRFCHW },
2660 { "-madx", OPTION_MASK_ISA_ADX },
2661 { "-mtbm", OPTION_MASK_ISA_TBM },
2662 { "-mpopcnt", OPTION_MASK_ISA_POPCNT },
2663 { "-mmovbe", OPTION_MASK_ISA_MOVBE },
2664 { "-mcrc32", OPTION_MASK_ISA_CRC32 },
2665 { "-maes", OPTION_MASK_ISA_AES },
2666 { "-msha", OPTION_MASK_ISA_SHA },
2667 { "-mpclmul", OPTION_MASK_ISA_PCLMUL },
2668 { "-mfsgsbase", OPTION_MASK_ISA_FSGSBASE },
2669 { "-mrdrnd", OPTION_MASK_ISA_RDRND },
2670 { "-mf16c", OPTION_MASK_ISA_F16C },
2671 { "-mrtm", OPTION_MASK_ISA_RTM },
2672 { "-mxsave", OPTION_MASK_ISA_XSAVE },
2673 { "-mxsaveopt", OPTION_MASK_ISA_XSAVEOPT },
2674 { "-mprefetchwt1", OPTION_MASK_ISA_PREFETCHWT1 },
2675 { "-mclflushopt", OPTION_MASK_ISA_CLFLUSHOPT },
2676 { "-mxsavec", OPTION_MASK_ISA_XSAVEC },
2677 { "-mxsaves", OPTION_MASK_ISA_XSAVES },
2678 { "-mmpx", OPTION_MASK_ISA_MPX },
2679 { "-mclwb", OPTION_MASK_ISA_CLWB },
2680 { "-mpcommit", OPTION_MASK_ISA_PCOMMIT },
2681 { "-mmwaitx", OPTION_MASK_ISA_MWAITX },
2685 static struct ix86_target_opts flag_opts[] =
2687 { "-m128bit-long-double", MASK_128BIT_LONG_DOUBLE },
2688 { "-mlong-double-128", MASK_LONG_DOUBLE_128 },
2689 { "-mlong-double-64", MASK_LONG_DOUBLE_64 },
2690 { "-m80387", MASK_80387 },
2691 { "-maccumulate-outgoing-args", MASK_ACCUMULATE_OUTGOING_ARGS },
2692 { "-malign-double", MASK_ALIGN_DOUBLE },
2693 { "-mcld", MASK_CLD },
2694 { "-mfp-ret-in-387", MASK_FLOAT_RETURNS },
2695 { "-mieee-fp", MASK_IEEE_FP },
2696 { "-minline-all-stringops", MASK_INLINE_ALL_STRINGOPS },
2697 { "-minline-stringops-dynamically", MASK_INLINE_STRINGOPS_DYNAMICALLY },
2698 { "-mms-bitfields", MASK_MS_BITFIELD_LAYOUT },
2699 { "-mno-align-stringops", MASK_NO_ALIGN_STRINGOPS },
2700 { "-mno-fancy-math-387", MASK_NO_FANCY_MATH_387 },
2701 { "-mno-push-args", MASK_NO_PUSH_ARGS },
2702 { "-mno-red-zone", MASK_NO_RED_ZONE },
2703 { "-momit-leaf-frame-pointer", MASK_OMIT_LEAF_FRAME_POINTER },
2704 { "-mrecip", MASK_RECIP },
2705 { "-mrtd", MASK_RTD },
2706 { "-msseregparm", MASK_SSEREGPARM },
2707 { "-mstack-arg-probe", MASK_STACK_PROBE },
2708 { "-mtls-direct-seg-refs", MASK_TLS_DIRECT_SEG_REFS },
2709 { "-mvect8-ret-in-mem", MASK_VECT8_RETURNS },
2710 { "-m8bit-idiv", MASK_USE_8BIT_IDIV },
2711 { "-mvzeroupper", MASK_VZEROUPPER },
2712 { "-mavx256-split-unaligned-load", MASK_AVX256_SPLIT_UNALIGNED_LOAD},
2713 { "-mavx256-split-unaligned-store", MASK_AVX256_SPLIT_UNALIGNED_STORE},
2714 { "-mprefer-avx128", MASK_PREFER_AVX128},
2717 const char *opts[ARRAY_SIZE (isa_opts) + ARRAY_SIZE (flag_opts) + 6][2];
2720 char target_other[40];
2730 memset (opts, '\0', sizeof (opts));
2732 /* Add -march= option. */
2735 opts[num][0] = "-march=";
2736 opts[num++][1] = arch;
2739 /* Add -mtune= option. */
2742 opts[num][0] = "-mtune=";
2743 opts[num++][1] = tune;
2746 /* Add -m32/-m64/-mx32. */
2747 if ((isa & OPTION_MASK_ISA_64BIT) != 0)
2749 if ((isa & OPTION_MASK_ABI_64) != 0)
2753 isa &= ~ (OPTION_MASK_ISA_64BIT
2754 | OPTION_MASK_ABI_64
2755 | OPTION_MASK_ABI_X32);
2759 opts[num++][0] = abi;
2761 /* Pick out the options in isa options. */
2762 for (i = 0; i < ARRAY_SIZE (isa_opts); i++)
2764 if ((isa & isa_opts[i].mask) != 0)
2766 opts[num++][0] = isa_opts[i].option;
2767 isa &= ~ isa_opts[i].mask;
2771 if (isa && add_nl_p)
2773 opts[num++][0] = isa_other;
2774 sprintf (isa_other, "(other isa: %#" HOST_WIDE_INT_PRINT "x)",
2778 /* Add flag options. */
2779 for (i = 0; i < ARRAY_SIZE (flag_opts); i++)
2781 if ((flags & flag_opts[i].mask) != 0)
2783 opts[num++][0] = flag_opts[i].option;
2784 flags &= ~ flag_opts[i].mask;
2788 if (flags && add_nl_p)
2790 opts[num++][0] = target_other;
2791 sprintf (target_other, "(other flags: %#x)", flags);
2794 /* Add -fpmath= option. */
2797 opts[num][0] = "-mfpmath=";
2798 switch ((int) fpmath)
2801 opts[num++][1] = "387";
2805 opts[num++][1] = "sse";
2808 case FPMATH_387 | FPMATH_SSE:
2809 opts[num++][1] = "sse+387";
2821 gcc_assert (num < ARRAY_SIZE (opts));
2823 /* Size the string. */
2825 sep_len = (add_nl_p) ? 3 : 1;
2826 for (i = 0; i < num; i++)
2829 for (j = 0; j < 2; j++)
2831 len += strlen (opts[i][j]);
2834 /* Build the string. */
2835 ret = ptr = (char *) xmalloc (len);
2838 for (i = 0; i < num; i++)
2842 for (j = 0; j < 2; j++)
2843 len2[j] = (opts[i][j]) ? strlen (opts[i][j]) : 0;
2850 if (add_nl_p && line_len + len2[0] + len2[1] > 70)
2858 for (j = 0; j < 2; j++)
2861 memcpy (ptr, opts[i][j], len2[j]);
2863 line_len += len2[j];
2868 gcc_assert (ret + len >= ptr);
2873 /* Return true, if profiling code should be emitted before
2874 prologue. Otherwise it returns false.
2875 Note: For x86 with "hotfix" it is sorried. */
2877 ix86_profile_before_prologue (void)
2879 return flag_fentry != 0;
2882 /* Function that is callable from the debugger to print the current
2884 void ATTRIBUTE_UNUSED
2885 ix86_debug_options (void)
2887 char *opts = ix86_target_string (ix86_isa_flags, target_flags,
2888 ix86_arch_string, ix86_tune_string,
2893 fprintf (stderr, "%s\n\n", opts);
2897 fputs ("<no options>\n\n", stderr);
2902 static const char *stringop_alg_names[] = {
2904 #define DEF_ALG(alg, name) #name,
2905 #include "stringop.def"
2910 /* Parse parameter string passed to -mmemcpy-strategy= or -mmemset-strategy=.
2911 The string is of the following form (or comma separated list of it):
2913 strategy_alg:max_size:[align|noalign]
2915 where the full size range for the strategy is either [0, max_size] or
2916 [min_size, max_size], in which min_size is the max_size + 1 of the
2917 preceding range. The last size range must have max_size == -1.
2922 -mmemcpy-strategy=libcall:-1:noalign
2924 this is equivalent to (for known size memcpy) -mstringop-strategy=libcall
2928 -mmemset-strategy=rep_8byte:16:noalign,vector_loop:2048:align,libcall:-1:noalign
2930 This is to tell the compiler to use the following strategy for memset
2931 1) when the expected size is between [1, 16], use rep_8byte strategy;
2932 2) when the size is between [17, 2048], use vector_loop;
2933 3) when the size is > 2048, use libcall. */
2935 struct stringop_size_range
2943 ix86_parse_stringop_strategy_string (char *strategy_str, bool is_memset)
2945 const struct stringop_algs *default_algs;
2946 stringop_size_range input_ranges[MAX_STRINGOP_ALGS];
2947 char *curr_range_str, *next_range_str;
2951 default_algs = &ix86_cost->memset[TARGET_64BIT != 0];
2953 default_algs = &ix86_cost->memcpy[TARGET_64BIT != 0];
2955 curr_range_str = strategy_str;
2962 next_range_str = strchr (curr_range_str, ',');
2964 *next_range_str++ = '\0';
2966 if (3 != sscanf (curr_range_str, "%20[^:]:%d:%10s",
2967 alg_name, &maxs, align))
2969 error ("wrong arg %s to option %s", curr_range_str,
2970 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
2974 if (n > 0 && (maxs < (input_ranges[n - 1].max + 1) && maxs != -1))
2976 error ("size ranges of option %s should be increasing",
2977 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
2981 for (i = 0; i < last_alg; i++)
2982 if (!strcmp (alg_name, stringop_alg_names[i]))
2987 error ("wrong stringop strategy name %s specified for option %s",
2989 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
2993 if ((stringop_alg) i == rep_prefix_8_byte
2996 /* rep; movq isn't available in 32-bit code. */
2997 error ("stringop strategy name %s specified for option %s "
2998 "not supported for 32-bit code",
3000 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
3004 input_ranges[n].max = maxs;
3005 input_ranges[n].alg = (stringop_alg) i;
3006 if (!strcmp (align, "align"))
3007 input_ranges[n].noalign = false;
3008 else if (!strcmp (align, "noalign"))
3009 input_ranges[n].noalign = true;
3012 error ("unknown alignment %s specified for option %s",
3013 align, is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
3017 curr_range_str = next_range_str;
3019 while (curr_range_str);
3021 if (input_ranges[n - 1].max != -1)
3023 error ("the max value for the last size range should be -1"
3025 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
3029 if (n > MAX_STRINGOP_ALGS)
3031 error ("too many size ranges specified in option %s",
3032 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
3036 /* Now override the default algs array. */
3037 for (i = 0; i < n; i++)
3039 *const_cast<int *>(&default_algs->size[i].max) = input_ranges[i].max;
3040 *const_cast<stringop_alg *>(&default_algs->size[i].alg)
3041 = input_ranges[i].alg;
3042 *const_cast<int *>(&default_algs->size[i].noalign)
3043 = input_ranges[i].noalign;
3048 /* parse -mtune-ctrl= option. When DUMP is true,
3049 print the features that are explicitly set. */
3052 parse_mtune_ctrl_str (bool dump)
3054 if (!ix86_tune_ctrl_string)
3057 char *next_feature_string = NULL;
3058 char *curr_feature_string = xstrdup (ix86_tune_ctrl_string);
3059 char *orig = curr_feature_string;
3065 next_feature_string = strchr (curr_feature_string, ',');
3066 if (next_feature_string)
3067 *next_feature_string++ = '\0';
3068 if (*curr_feature_string == '^')
3070 curr_feature_string++;
3073 for (i = 0; i < X86_TUNE_LAST; i++)
3075 if (!strcmp (curr_feature_string, ix86_tune_feature_names[i]))
3077 ix86_tune_features[i] = !clear;
3079 fprintf (stderr, "Explicitly %s feature %s\n",
3080 clear ? "clear" : "set", ix86_tune_feature_names[i]);
3084 if (i == X86_TUNE_LAST)
3085 error ("Unknown parameter to option -mtune-ctrl: %s",
3086 clear ? curr_feature_string - 1 : curr_feature_string);
3087 curr_feature_string = next_feature_string;
3089 while (curr_feature_string);
3093 /* Helper function to set ix86_tune_features. IX86_TUNE is the
3097 set_ix86_tune_features (enum processor_type ix86_tune, bool dump)
3099 unsigned int ix86_tune_mask = 1u << ix86_tune;
3102 for (i = 0; i < X86_TUNE_LAST; ++i)
3104 if (ix86_tune_no_default)
3105 ix86_tune_features[i] = 0;
3107 ix86_tune_features[i] = !!(initial_ix86_tune_features[i] & ix86_tune_mask);
3112 fprintf (stderr, "List of x86 specific tuning parameter names:\n");
3113 for (i = 0; i < X86_TUNE_LAST; i++)
3114 fprintf (stderr, "%s : %s\n", ix86_tune_feature_names[i],
3115 ix86_tune_features[i] ? "on" : "off");
3118 parse_mtune_ctrl_str (dump);
3122 /* Override various settings based on options. If MAIN_ARGS_P, the
3123 options are from the command line, otherwise they are from
3127 ix86_option_override_internal (bool main_args_p,
3128 struct gcc_options *opts,
3129 struct gcc_options *opts_set)
3132 unsigned int ix86_arch_mask;
3133 const bool ix86_tune_specified = (opts->x_ix86_tune_string != NULL);
3138 #define PTA_3DNOW (HOST_WIDE_INT_1 << 0)
3139 #define PTA_3DNOW_A (HOST_WIDE_INT_1 << 1)
3140 #define PTA_64BIT (HOST_WIDE_INT_1 << 2)
3141 #define PTA_ABM (HOST_WIDE_INT_1 << 3)
3142 #define PTA_AES (HOST_WIDE_INT_1 << 4)
3143 #define PTA_AVX (HOST_WIDE_INT_1 << 5)
3144 #define PTA_BMI (HOST_WIDE_INT_1 << 6)
3145 #define PTA_CX16 (HOST_WIDE_INT_1 << 7)
3146 #define PTA_F16C (HOST_WIDE_INT_1 << 8)
3147 #define PTA_FMA (HOST_WIDE_INT_1 << 9)
3148 #define PTA_FMA4 (HOST_WIDE_INT_1 << 10)
3149 #define PTA_FSGSBASE (HOST_WIDE_INT_1 << 11)
3150 #define PTA_LWP (HOST_WIDE_INT_1 << 12)
3151 #define PTA_LZCNT (HOST_WIDE_INT_1 << 13)
3152 #define PTA_MMX (HOST_WIDE_INT_1 << 14)
3153 #define PTA_MOVBE (HOST_WIDE_INT_1 << 15)
3154 #define PTA_NO_SAHF (HOST_WIDE_INT_1 << 16)
3155 #define PTA_PCLMUL (HOST_WIDE_INT_1 << 17)
3156 #define PTA_POPCNT (HOST_WIDE_INT_1 << 18)
3157 #define PTA_PREFETCH_SSE (HOST_WIDE_INT_1 << 19)
3158 #define PTA_RDRND (HOST_WIDE_INT_1 << 20)
3159 #define PTA_SSE (HOST_WIDE_INT_1 << 21)
3160 #define PTA_SSE2 (HOST_WIDE_INT_1 << 22)
3161 #define PTA_SSE3 (HOST_WIDE_INT_1 << 23)
3162 #define PTA_SSE4_1 (HOST_WIDE_INT_1 << 24)
3163 #define PTA_SSE4_2 (HOST_WIDE_INT_1 << 25)
3164 #define PTA_SSE4A (HOST_WIDE_INT_1 << 26)
3165 #define PTA_SSSE3 (HOST_WIDE_INT_1 << 27)
3166 #define PTA_TBM (HOST_WIDE_INT_1 << 28)
3167 #define PTA_XOP (HOST_WIDE_INT_1 << 29)
3168 #define PTA_AVX2 (HOST_WIDE_INT_1 << 30)
3169 #define PTA_BMI2 (HOST_WIDE_INT_1 << 31)
3170 #define PTA_RTM (HOST_WIDE_INT_1 << 32)
3171 #define PTA_HLE (HOST_WIDE_INT_1 << 33)
3172 #define PTA_PRFCHW (HOST_WIDE_INT_1 << 34)
3173 #define PTA_RDSEED (HOST_WIDE_INT_1 << 35)
3174 #define PTA_ADX (HOST_WIDE_INT_1 << 36)
3175 #define PTA_FXSR (HOST_WIDE_INT_1 << 37)
3176 #define PTA_XSAVE (HOST_WIDE_INT_1 << 38)
3177 #define PTA_XSAVEOPT (HOST_WIDE_INT_1 << 39)
3178 #define PTA_AVX512F (HOST_WIDE_INT_1 << 40)
3179 #define PTA_AVX512ER (HOST_WIDE_INT_1 << 41)
3180 #define PTA_AVX512PF (HOST_WIDE_INT_1 << 42)
3181 #define PTA_AVX512CD (HOST_WIDE_INT_1 << 43)
3182 #define PTA_MPX (HOST_WIDE_INT_1 << 44)
3183 #define PTA_SHA (HOST_WIDE_INT_1 << 45)
3184 #define PTA_PREFETCHWT1 (HOST_WIDE_INT_1 << 46)
3185 #define PTA_CLFLUSHOPT (HOST_WIDE_INT_1 << 47)
3186 #define PTA_XSAVEC (HOST_WIDE_INT_1 << 48)
3187 #define PTA_XSAVES (HOST_WIDE_INT_1 << 49)
3188 #define PTA_AVX512DQ (HOST_WIDE_INT_1 << 50)
3189 #define PTA_AVX512BW (HOST_WIDE_INT_1 << 51)
3190 #define PTA_AVX512VL (HOST_WIDE_INT_1 << 52)
3191 #define PTA_AVX512IFMA (HOST_WIDE_INT_1 << 53)
3192 #define PTA_AVX512VBMI (HOST_WIDE_INT_1 << 54)
3193 #define PTA_CLWB (HOST_WIDE_INT_1 << 55)
3194 #define PTA_PCOMMIT (HOST_WIDE_INT_1 << 56)
3195 #define PTA_MWAITX (HOST_WIDE_INT_1 << 57)
3198 (PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_SSSE3 \
3199 | PTA_CX16 | PTA_FXSR)
3200 #define PTA_NEHALEM \
3201 (PTA_CORE2 | PTA_SSE4_1 | PTA_SSE4_2 | PTA_POPCNT)
3202 #define PTA_WESTMERE \
3203 (PTA_NEHALEM | PTA_AES | PTA_PCLMUL)
3204 #define PTA_SANDYBRIDGE \
3205 (PTA_WESTMERE | PTA_AVX | PTA_XSAVE | PTA_XSAVEOPT)
3206 #define PTA_IVYBRIDGE \
3207 (PTA_SANDYBRIDGE | PTA_FSGSBASE | PTA_RDRND | PTA_F16C)
3208 #define PTA_HASWELL \
3209 (PTA_IVYBRIDGE | PTA_AVX2 | PTA_BMI | PTA_BMI2 | PTA_LZCNT \
3210 | PTA_FMA | PTA_MOVBE | PTA_HLE)
3211 #define PTA_BROADWELL \
3212 (PTA_HASWELL | PTA_ADX | PTA_PRFCHW | PTA_RDSEED)
3214 (PTA_BROADWELL | PTA_AVX512PF | PTA_AVX512ER | PTA_AVX512F | PTA_AVX512CD)
3215 #define PTA_BONNELL \
3216 (PTA_CORE2 | PTA_MOVBE)
3217 #define PTA_SILVERMONT \
3218 (PTA_WESTMERE | PTA_MOVBE)
3220 /* if this reaches 64, need to widen struct pta flags below */
3224 const char *const name; /* processor name or nickname. */
3225 const enum processor_type processor;
3226 const enum attr_cpu schedule;
3227 const unsigned HOST_WIDE_INT flags;
3229 const processor_alias_table[] =
3231 {"i386", PROCESSOR_I386, CPU_NONE, 0},
3232 {"i486", PROCESSOR_I486, CPU_NONE, 0},
3233 {"i586", PROCESSOR_PENTIUM, CPU_PENTIUM, 0},
3234 {"pentium", PROCESSOR_PENTIUM, CPU_PENTIUM, 0},
3235 {"pentium-mmx", PROCESSOR_PENTIUM, CPU_PENTIUM, PTA_MMX},
3236 {"winchip-c6", PROCESSOR_I486, CPU_NONE, PTA_MMX},
3237 {"winchip2", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW | PTA_PRFCHW},
3238 {"c3", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW | PTA_PRFCHW},
3239 {"c3-2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
3240 PTA_MMX | PTA_SSE | PTA_FXSR},
3241 {"i686", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0},
3242 {"pentiumpro", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0},
3243 {"pentium2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, PTA_MMX | PTA_FXSR},
3244 {"pentium3", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
3245 PTA_MMX | PTA_SSE | PTA_FXSR},
3246 {"pentium3m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
3247 PTA_MMX | PTA_SSE | PTA_FXSR},
3248 {"pentium-m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
3249 PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_FXSR},
3250 {"pentium4", PROCESSOR_PENTIUM4, CPU_NONE,
3251 PTA_MMX |PTA_SSE | PTA_SSE2 | PTA_FXSR},
3252 {"pentium4m", PROCESSOR_PENTIUM4, CPU_NONE,
3253 PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_FXSR},
3254 {"prescott", PROCESSOR_NOCONA, CPU_NONE,
3255 PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_FXSR},
3256 {"nocona", PROCESSOR_NOCONA, CPU_NONE,
3257 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3258 | PTA_CX16 | PTA_NO_SAHF | PTA_FXSR},
3259 {"core2", PROCESSOR_CORE2, CPU_CORE2, PTA_CORE2},
3260 {"nehalem", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_NEHALEM},
3261 {"corei7", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_NEHALEM},
3262 {"westmere", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_WESTMERE},
3263 {"sandybridge", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
3265 {"corei7-avx", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
3267 {"ivybridge", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
3269 {"core-avx-i", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
3271 {"haswell", PROCESSOR_HASWELL, CPU_NEHALEM, PTA_HASWELL},
3272 {"core-avx2", PROCESSOR_HASWELL, CPU_NEHALEM, PTA_HASWELL},
3273 {"broadwell", PROCESSOR_HASWELL, CPU_NEHALEM, PTA_BROADWELL},
3274 {"bonnell", PROCESSOR_BONNELL, CPU_ATOM, PTA_BONNELL},
3275 {"atom", PROCESSOR_BONNELL, CPU_ATOM, PTA_BONNELL},
3276 {"silvermont", PROCESSOR_SILVERMONT, CPU_SLM, PTA_SILVERMONT},
3277 {"slm", PROCESSOR_SILVERMONT, CPU_SLM, PTA_SILVERMONT},
3278 {"knl", PROCESSOR_KNL, CPU_KNL, PTA_KNL},
3279 {"intel", PROCESSOR_INTEL, CPU_SLM, PTA_NEHALEM},
3280 {"geode", PROCESSOR_GEODE, CPU_GEODE,
3281 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE | PTA_PRFCHW},
3282 {"k6", PROCESSOR_K6, CPU_K6, PTA_MMX},
3283 {"k6-2", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW | PTA_PRFCHW},
3284 {"k6-3", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW | PTA_PRFCHW},
3285 {"athlon", PROCESSOR_ATHLON, CPU_ATHLON,
3286 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE | PTA_PRFCHW},
3287 {"athlon-tbird", PROCESSOR_ATHLON, CPU_ATHLON,
3288 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE | PTA_PRFCHW},
3289 {"athlon-4", PROCESSOR_ATHLON, CPU_ATHLON,
3290 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_PRFCHW | PTA_FXSR},
3291 {"athlon-xp", PROCESSOR_ATHLON, CPU_ATHLON,
3292 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_PRFCHW | PTA_FXSR},
3293 {"athlon-mp", PROCESSOR_ATHLON, CPU_ATHLON,
3294 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_PRFCHW | PTA_FXSR},
3295 {"x86-64", PROCESSOR_K8, CPU_K8,
3296 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR},
3297 {"k8", PROCESSOR_K8, CPU_K8,
3298 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3299 | PTA_SSE2 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3300 {"k8-sse3", PROCESSOR_K8, CPU_K8,
3301 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3302 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3303 {"opteron", PROCESSOR_K8, CPU_K8,
3304 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3305 | PTA_SSE2 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3306 {"opteron-sse3", PROCESSOR_K8, CPU_K8,
3307 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3308 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3309 {"athlon64", PROCESSOR_K8, CPU_K8,
3310 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3311 | PTA_SSE2 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3312 {"athlon64-sse3", PROCESSOR_K8, CPU_K8,
3313 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3314 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3315 {"athlon-fx", PROCESSOR_K8, CPU_K8,
3316 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3317 | PTA_SSE2 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3318 {"amdfam10", PROCESSOR_AMDFAM10, CPU_AMDFAM10,
3319 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_SSE2
3320 | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_PRFCHW | PTA_FXSR},
3321 {"barcelona", PROCESSOR_AMDFAM10, CPU_AMDFAM10,
3322 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_SSE2
3323 | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_PRFCHW | PTA_FXSR},
3324 {"bdver1", PROCESSOR_BDVER1, CPU_BDVER1,
3325 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3326 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
3327 | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4
3328 | PTA_XOP | PTA_LWP | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE},
3329 {"bdver2", PROCESSOR_BDVER2, CPU_BDVER2,
3330 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3331 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
3332 | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4
3333 | PTA_XOP | PTA_LWP | PTA_BMI | PTA_TBM | PTA_F16C
3334 | PTA_FMA | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE},
3335 {"bdver3", PROCESSOR_BDVER3, CPU_BDVER3,
3336 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3337 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
3338 | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4
3339 | PTA_XOP | PTA_LWP | PTA_BMI | PTA_TBM | PTA_F16C
3340 | PTA_FMA | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE
3341 | PTA_XSAVEOPT | PTA_FSGSBASE},
3342 {"bdver4", PROCESSOR_BDVER4, CPU_BDVER4,
3343 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3344 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
3345 | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_AVX2
3346 | PTA_FMA4 | PTA_XOP | PTA_LWP | PTA_BMI | PTA_BMI2
3347 | PTA_TBM | PTA_F16C | PTA_FMA | PTA_PRFCHW | PTA_FXSR
3348 | PTA_XSAVE | PTA_XSAVEOPT | PTA_FSGSBASE | PTA_RDRND
3349 | PTA_MOVBE | PTA_MWAITX},
3350 {"btver1", PROCESSOR_BTVER1, CPU_GENERIC,
3351 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3352 | PTA_SSSE3 | PTA_SSE4A |PTA_ABM | PTA_CX16 | PTA_PRFCHW
3353 | PTA_FXSR | PTA_XSAVE},
3354 {"btver2", PROCESSOR_BTVER2, CPU_BTVER2,
3355 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3356 | PTA_SSSE3 | PTA_SSE4A |PTA_ABM | PTA_CX16 | PTA_SSE4_1
3357 | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX
3358 | PTA_BMI | PTA_F16C | PTA_MOVBE | PTA_PRFCHW
3359 | PTA_FXSR | PTA_XSAVE | PTA_XSAVEOPT},
3361 {"generic", PROCESSOR_GENERIC, CPU_GENERIC,
3363 | PTA_HLE /* flags are only used for -march switch. */ },
3366 /* -mrecip options. */
3369 const char *string; /* option name */
3370 unsigned int mask; /* mask bits to set */
3372 const recip_options[] =
3374 { "all", RECIP_MASK_ALL },
3375 { "none", RECIP_MASK_NONE },
3376 { "div", RECIP_MASK_DIV },
3377 { "sqrt", RECIP_MASK_SQRT },
3378 { "vec-div", RECIP_MASK_VEC_DIV },
3379 { "vec-sqrt", RECIP_MASK_VEC_SQRT },
3382 int const pta_size = ARRAY_SIZE (processor_alias_table);
3384 /* Set up prefix/suffix so the error messages refer to either the command
3385 line argument, or the attribute(target). */
3394 prefix = "option(\"";
3399 /* Turn off both OPTION_MASK_ABI_64 and OPTION_MASK_ABI_X32 if
3400 TARGET_64BIT_DEFAULT is true and TARGET_64BIT is false. */
3401 if (TARGET_64BIT_DEFAULT && !TARGET_64BIT_P (opts->x_ix86_isa_flags))
3402 opts->x_ix86_isa_flags &= ~(OPTION_MASK_ABI_64 | OPTION_MASK_ABI_X32);
3403 #ifdef TARGET_BI_ARCH
3406 #if TARGET_BI_ARCH == 1
3407 /* When TARGET_BI_ARCH == 1, by default, OPTION_MASK_ABI_64
3408 is on and OPTION_MASK_ABI_X32 is off. We turn off
3409 OPTION_MASK_ABI_64 if OPTION_MASK_ABI_X32 is turned on by
3411 if (TARGET_X32_P (opts->x_ix86_isa_flags))
3412 opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_64;
3414 /* When TARGET_BI_ARCH == 2, by default, OPTION_MASK_ABI_X32 is
3415 on and OPTION_MASK_ABI_64 is off. We turn off
3416 OPTION_MASK_ABI_X32 if OPTION_MASK_ABI_64 is turned on by
3417 -m64 or OPTION_MASK_CODE16 is turned on by -m16. */
3418 if (TARGET_LP64_P (opts->x_ix86_isa_flags)
3419 || TARGET_16BIT_P (opts->x_ix86_isa_flags))
3420 opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
3425 if (TARGET_X32_P (opts->x_ix86_isa_flags))
3427 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3428 OPTION_MASK_ABI_64 for TARGET_X32. */
3429 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
3430 opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_64;
3432 else if (TARGET_16BIT_P (opts->x_ix86_isa_flags))
3433 opts->x_ix86_isa_flags &= ~(OPTION_MASK_ISA_64BIT
3434 | OPTION_MASK_ABI_X32
3435 | OPTION_MASK_ABI_64);
3436 else if (TARGET_LP64_P (opts->x_ix86_isa_flags))
3438 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3439 OPTION_MASK_ABI_X32 for TARGET_LP64. */
3440 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
3441 opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
3444 #ifdef SUBTARGET_OVERRIDE_OPTIONS
3445 SUBTARGET_OVERRIDE_OPTIONS;
3448 #ifdef SUBSUBTARGET_OVERRIDE_OPTIONS
3449 SUBSUBTARGET_OVERRIDE_OPTIONS;
3452 /* -fPIC is the default for x86_64. */
3453 if (TARGET_MACHO && TARGET_64BIT_P (opts->x_ix86_isa_flags))
3454 opts->x_flag_pic = 2;
3456 /* Need to check -mtune=generic first. */
3457 if (opts->x_ix86_tune_string)
3459 /* As special support for cross compilers we read -mtune=native
3460 as -mtune=generic. With native compilers we won't see the
3461 -mtune=native, as it was changed by the driver. */
3462 if (!strcmp (opts->x_ix86_tune_string, "native"))
3464 opts->x_ix86_tune_string = "generic";
3466 else if (!strcmp (opts->x_ix86_tune_string, "x86-64"))
3467 warning (OPT_Wdeprecated, "%stune=x86-64%s is deprecated; use "
3468 "%stune=k8%s or %stune=generic%s instead as appropriate",
3469 prefix, suffix, prefix, suffix, prefix, suffix);
3473 if (opts->x_ix86_arch_string)
3474 opts->x_ix86_tune_string = opts->x_ix86_arch_string;
3475 if (!opts->x_ix86_tune_string)
3477 opts->x_ix86_tune_string
3478 = processor_target_table[TARGET_CPU_DEFAULT].name;
3479 ix86_tune_defaulted = 1;
3482 /* opts->x_ix86_tune_string is set to opts->x_ix86_arch_string
3483 or defaulted. We need to use a sensible tune option. */
3484 if (!strcmp (opts->x_ix86_tune_string, "x86-64"))
3486 opts->x_ix86_tune_string = "generic";
3490 if (opts->x_ix86_stringop_alg == rep_prefix_8_byte
3491 && !TARGET_64BIT_P (opts->x_ix86_isa_flags))
3493 /* rep; movq isn't available in 32-bit code. */
3494 error ("-mstringop-strategy=rep_8byte not supported for 32-bit code");
3495 opts->x_ix86_stringop_alg = no_stringop;
3498 if (!opts->x_ix86_arch_string)
3499 opts->x_ix86_arch_string
3500 = TARGET_64BIT_P (opts->x_ix86_isa_flags)
3501 ? "x86-64" : SUBTARGET32_DEFAULT_CPU;
3503 ix86_arch_specified = 1;
3505 if (opts_set->x_ix86_pmode)
3507 if ((TARGET_LP64_P (opts->x_ix86_isa_flags)
3508 && opts->x_ix86_pmode == PMODE_SI)
3509 || (!TARGET_64BIT_P (opts->x_ix86_isa_flags)
3510 && opts->x_ix86_pmode == PMODE_DI))
3511 error ("address mode %qs not supported in the %s bit mode",
3512 TARGET_64BIT_P (opts->x_ix86_isa_flags) ? "short" : "long",
3513 TARGET_64BIT_P (opts->x_ix86_isa_flags) ? "64" : "32");
3516 opts->x_ix86_pmode = TARGET_LP64_P (opts->x_ix86_isa_flags)
3517 ? PMODE_DI : PMODE_SI;
3519 if (!opts_set->x_ix86_abi)
3520 opts->x_ix86_abi = DEFAULT_ABI;
3522 /* For targets using ms ABI enable ms-extensions, if not
3523 explicit turned off. For non-ms ABI we turn off this
3525 if (!opts_set->x_flag_ms_extensions)
3526 opts->x_flag_ms_extensions = (MS_ABI == DEFAULT_ABI);
3528 if (opts_set->x_ix86_cmodel)
3530 switch (opts->x_ix86_cmodel)
3534 if (opts->x_flag_pic)
3535 opts->x_ix86_cmodel = CM_SMALL_PIC;
3536 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
3537 error ("code model %qs not supported in the %s bit mode",
3543 if (opts->x_flag_pic)
3544 opts->x_ix86_cmodel = CM_MEDIUM_PIC;
3545 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
3546 error ("code model %qs not supported in the %s bit mode",
3548 else if (TARGET_X32_P (opts->x_ix86_isa_flags))
3549 error ("code model %qs not supported in x32 mode",
3555 if (opts->x_flag_pic)
3556 opts->x_ix86_cmodel = CM_LARGE_PIC;
3557 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
3558 error ("code model %qs not supported in the %s bit mode",
3560 else if (TARGET_X32_P (opts->x_ix86_isa_flags))
3561 error ("code model %qs not supported in x32 mode",
3566 if (opts->x_flag_pic)
3567 error ("code model %s does not support PIC mode", "32");
3568 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3569 error ("code model %qs not supported in the %s bit mode",
3574 if (opts->x_flag_pic)
3576 error ("code model %s does not support PIC mode", "kernel");
3577 opts->x_ix86_cmodel = CM_32;
3579 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
3580 error ("code model %qs not supported in the %s bit mode",
3590 /* For TARGET_64BIT and MS_ABI, force pic on, in order to enable the
3591 use of rip-relative addressing. This eliminates fixups that
3592 would otherwise be needed if this object is to be placed in a
3593 DLL, and is essentially just as efficient as direct addressing. */
3594 if (TARGET_64BIT_P (opts->x_ix86_isa_flags)
3595 && (TARGET_RDOS || TARGET_PECOFF))
3596 opts->x_ix86_cmodel = CM_MEDIUM_PIC, opts->x_flag_pic = 1;
3597 else if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3598 opts->x_ix86_cmodel = opts->x_flag_pic ? CM_SMALL_PIC : CM_SMALL;
3600 opts->x_ix86_cmodel = CM_32;
3602 if (TARGET_MACHO && opts->x_ix86_asm_dialect == ASM_INTEL)
3604 error ("-masm=intel not supported in this configuration");
3605 opts->x_ix86_asm_dialect = ASM_ATT;
3607 if ((TARGET_64BIT_P (opts->x_ix86_isa_flags) != 0)
3608 != ((opts->x_ix86_isa_flags & OPTION_MASK_ISA_64BIT) != 0))
3609 sorry ("%i-bit mode not compiled in",
3610 (opts->x_ix86_isa_flags & OPTION_MASK_ISA_64BIT) ? 64 : 32);
3612 for (i = 0; i < pta_size; i++)
3613 if (! strcmp (opts->x_ix86_arch_string, processor_alias_table[i].name))
3615 ix86_schedule = processor_alias_table[i].schedule;
3616 ix86_arch = processor_alias_table[i].processor;
3617 /* Default cpu tuning to the architecture. */
3618 ix86_tune = ix86_arch;
3620 if (TARGET_64BIT_P (opts->x_ix86_isa_flags)
3621 && !(processor_alias_table[i].flags & PTA_64BIT))
3622 error ("CPU you selected does not support x86-64 "
3625 if (processor_alias_table[i].flags & PTA_MMX
3626 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MMX))
3627 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MMX;
3628 if (processor_alias_table[i].flags & PTA_3DNOW
3629 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW))
3630 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_3DNOW;
3631 if (processor_alias_table[i].flags & PTA_3DNOW_A
3632 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW_A))
3633 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_3DNOW_A;
3634 if (processor_alias_table[i].flags & PTA_SSE
3635 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE))
3636 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE;
3637 if (processor_alias_table[i].flags & PTA_SSE2
3638 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE2))
3639 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE2;
3640 if (processor_alias_table[i].flags & PTA_SSE3
3641 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE3))
3642 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE3;
3643 if (processor_alias_table[i].flags & PTA_SSSE3
3644 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSSE3))
3645 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSSE3;
3646 if (processor_alias_table[i].flags & PTA_SSE4_1
3647 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_1))
3648 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4_1;
3649 if (processor_alias_table[i].flags & PTA_SSE4_2
3650 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_2))
3651 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4_2;
3652 if (processor_alias_table[i].flags & PTA_AVX
3653 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX))
3654 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX;
3655 if (processor_alias_table[i].flags & PTA_AVX2
3656 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX2))
3657 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX2;
3658 if (processor_alias_table[i].flags & PTA_FMA
3659 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA))
3660 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FMA;
3661 if (processor_alias_table[i].flags & PTA_SSE4A
3662 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4A))
3663 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4A;
3664 if (processor_alias_table[i].flags & PTA_FMA4
3665 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA4))
3666 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FMA4;
3667 if (processor_alias_table[i].flags & PTA_XOP
3668 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XOP))
3669 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XOP;
3670 if (processor_alias_table[i].flags & PTA_LWP
3671 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_LWP))
3672 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_LWP;
3673 if (processor_alias_table[i].flags & PTA_ABM
3674 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_ABM))
3675 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_ABM;
3676 if (processor_alias_table[i].flags & PTA_BMI
3677 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI))
3678 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_BMI;
3679 if (processor_alias_table[i].flags & (PTA_LZCNT | PTA_ABM)
3680 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_LZCNT))
3681 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_LZCNT;
3682 if (processor_alias_table[i].flags & PTA_TBM
3683 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_TBM))
3684 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_TBM;
3685 if (processor_alias_table[i].flags & PTA_BMI2
3686 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI2))
3687 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_BMI2;
3688 if (processor_alias_table[i].flags & PTA_CX16
3689 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CX16))
3690 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CX16;
3691 if (processor_alias_table[i].flags & (PTA_POPCNT | PTA_ABM)
3692 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_POPCNT))
3693 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_POPCNT;
3694 if (!(TARGET_64BIT_P (opts->x_ix86_isa_flags)
3695 && (processor_alias_table[i].flags & PTA_NO_SAHF))
3696 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SAHF))
3697 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SAHF;
3698 if (processor_alias_table[i].flags & PTA_MOVBE
3699 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MOVBE))
3700 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MOVBE;
3701 if (processor_alias_table[i].flags & PTA_AES
3702 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AES))
3703 ix86_isa_flags |= OPTION_MASK_ISA_AES;
3704 if (processor_alias_table[i].flags & PTA_SHA
3705 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SHA))
3706 ix86_isa_flags |= OPTION_MASK_ISA_SHA;
3707 if (processor_alias_table[i].flags & PTA_PCLMUL
3708 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PCLMUL))
3709 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PCLMUL;
3710 if (processor_alias_table[i].flags & PTA_FSGSBASE
3711 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FSGSBASE))
3712 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FSGSBASE;
3713 if (processor_alias_table[i].flags & PTA_RDRND
3714 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RDRND))
3715 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RDRND;
3716 if (processor_alias_table[i].flags & PTA_F16C
3717 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_F16C))
3718 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_F16C;
3719 if (processor_alias_table[i].flags & PTA_RTM
3720 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RTM))
3721 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RTM;
3722 if (processor_alias_table[i].flags & PTA_HLE
3723 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_HLE))
3724 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_HLE;
3725 if (processor_alias_table[i].flags & PTA_PRFCHW
3726 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PRFCHW))
3727 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PRFCHW;
3728 if (processor_alias_table[i].flags & PTA_RDSEED
3729 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RDSEED))
3730 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RDSEED;
3731 if (processor_alias_table[i].flags & PTA_ADX
3732 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_ADX))
3733 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_ADX;
3734 if (processor_alias_table[i].flags & PTA_FXSR
3735 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FXSR))
3736 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FXSR;
3737 if (processor_alias_table[i].flags & PTA_XSAVE
3738 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVE))
3739 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVE;
3740 if (processor_alias_table[i].flags & PTA_XSAVEOPT
3741 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVEOPT))
3742 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVEOPT;
3743 if (processor_alias_table[i].flags & PTA_AVX512F
3744 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512F))
3745 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512F;
3746 if (processor_alias_table[i].flags & PTA_AVX512ER
3747 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512ER))
3748 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512ER;
3749 if (processor_alias_table[i].flags & PTA_AVX512PF
3750 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512PF))
3751 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512PF;
3752 if (processor_alias_table[i].flags & PTA_AVX512CD
3753 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512CD))
3754 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512CD;
3755 if (processor_alias_table[i].flags & PTA_PREFETCHWT1
3756 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PREFETCHWT1))
3757 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PREFETCHWT1;
3758 if (processor_alias_table[i].flags & PTA_PCOMMIT
3759 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PCOMMIT))
3760 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PCOMMIT;
3761 if (processor_alias_table[i].flags & PTA_CLWB
3762 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CLWB))
3763 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CLWB;
3764 if (processor_alias_table[i].flags & PTA_CLFLUSHOPT
3765 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CLFLUSHOPT))
3766 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CLFLUSHOPT;
3767 if (processor_alias_table[i].flags & PTA_XSAVEC
3768 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVEC))
3769 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVEC;
3770 if (processor_alias_table[i].flags & PTA_XSAVES
3771 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVES))
3772 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVES;
3773 if (processor_alias_table[i].flags & PTA_AVX512DQ
3774 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512DQ))
3775 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512DQ;
3776 if (processor_alias_table[i].flags & PTA_AVX512BW
3777 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512BW))
3778 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512BW;
3779 if (processor_alias_table[i].flags & PTA_AVX512VL
3780 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512VL))
3781 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512VL;
3782 if (processor_alias_table[i].flags & PTA_MPX
3783 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MPX))
3784 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MPX;
3785 if (processor_alias_table[i].flags & PTA_AVX512VBMI
3786 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512VBMI))
3787 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512VBMI;
3788 if (processor_alias_table[i].flags & PTA_AVX512IFMA
3789 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512IFMA))
3790 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512IFMA;
3791 if (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE))
3792 x86_prefetch_sse = true;
3793 if (processor_alias_table[i].flags & PTA_MWAITX
3794 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MWAITX))
3795 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MWAITX;
3800 if (TARGET_X32 && (opts->x_ix86_isa_flags & OPTION_MASK_ISA_MPX))
3801 error ("Intel MPX does not support x32");
3803 if (TARGET_X32 && (ix86_isa_flags & OPTION_MASK_ISA_MPX))
3804 error ("Intel MPX does not support x32");
3806 if (!strcmp (opts->x_ix86_arch_string, "generic"))
3807 error ("generic CPU can be used only for %stune=%s %s",
3808 prefix, suffix, sw);
3809 else if (!strcmp (opts->x_ix86_arch_string, "intel"))
3810 error ("intel CPU can be used only for %stune=%s %s",
3811 prefix, suffix, sw);
3812 else if (i == pta_size)
3813 error ("bad value (%s) for %sarch=%s %s",
3814 opts->x_ix86_arch_string, prefix, suffix, sw);
3816 ix86_arch_mask = 1u << ix86_arch;
3817 for (i = 0; i < X86_ARCH_LAST; ++i)
3818 ix86_arch_features[i] = !!(initial_ix86_arch_features[i] & ix86_arch_mask);
3820 for (i = 0; i < pta_size; i++)
3821 if (! strcmp (opts->x_ix86_tune_string, processor_alias_table[i].name))
3823 ix86_schedule = processor_alias_table[i].schedule;
3824 ix86_tune = processor_alias_table[i].processor;
3825 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3827 if (!(processor_alias_table[i].flags & PTA_64BIT))
3829 if (ix86_tune_defaulted)
3831 opts->x_ix86_tune_string = "x86-64";
3832 for (i = 0; i < pta_size; i++)
3833 if (! strcmp (opts->x_ix86_tune_string,
3834 processor_alias_table[i].name))
3836 ix86_schedule = processor_alias_table[i].schedule;
3837 ix86_tune = processor_alias_table[i].processor;
3840 error ("CPU you selected does not support x86-64 "
3844 /* Intel CPUs have always interpreted SSE prefetch instructions as
3845 NOPs; so, we can enable SSE prefetch instructions even when
3846 -mtune (rather than -march) points us to a processor that has them.
3847 However, the VIA C3 gives a SIGILL, so we only do that for i686 and
3848 higher processors. */
3850 && (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE)))
3851 x86_prefetch_sse = true;
3855 if (ix86_tune_specified && i == pta_size)
3856 error ("bad value (%s) for %stune=%s %s",
3857 opts->x_ix86_tune_string, prefix, suffix, sw);
3859 set_ix86_tune_features (ix86_tune, opts->x_ix86_dump_tunes);
3861 #ifndef USE_IX86_FRAME_POINTER
3862 #define USE_IX86_FRAME_POINTER 0
3865 #ifndef USE_X86_64_FRAME_POINTER
3866 #define USE_X86_64_FRAME_POINTER 0
3869 /* Set the default values for switches whose default depends on TARGET_64BIT
3870 in case they weren't overwritten by command line options. */
3871 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3873 if (opts->x_optimize >= 1 && !opts_set->x_flag_omit_frame_pointer)
3874 opts->x_flag_omit_frame_pointer = !USE_X86_64_FRAME_POINTER;
3875 if (opts->x_flag_asynchronous_unwind_tables
3876 && !opts_set->x_flag_unwind_tables
3877 && TARGET_64BIT_MS_ABI)
3878 opts->x_flag_unwind_tables = 1;
3879 if (opts->x_flag_asynchronous_unwind_tables == 2)
3880 opts->x_flag_unwind_tables
3881 = opts->x_flag_asynchronous_unwind_tables = 1;
3882 if (opts->x_flag_pcc_struct_return == 2)
3883 opts->x_flag_pcc_struct_return = 0;
3887 if (opts->x_optimize >= 1 && !opts_set->x_flag_omit_frame_pointer)
3888 opts->x_flag_omit_frame_pointer
3889 = !(USE_IX86_FRAME_POINTER || opts->x_optimize_size);
3890 if (opts->x_flag_asynchronous_unwind_tables == 2)
3891 opts->x_flag_asynchronous_unwind_tables = !USE_IX86_FRAME_POINTER;
3892 if (opts->x_flag_pcc_struct_return == 2)
3893 opts->x_flag_pcc_struct_return = DEFAULT_PCC_STRUCT_RETURN;
3896 ix86_tune_cost = processor_target_table[ix86_tune].cost;
3897 /* TODO: ix86_cost should be chosen at instruction or function granuality
3898 so for cold code we use size_cost even in !optimize_size compilation. */
3899 if (opts->x_optimize_size)
3900 ix86_cost = &ix86_size_cost;
3902 ix86_cost = ix86_tune_cost;
3904 /* Arrange to set up i386_stack_locals for all functions. */
3905 init_machine_status = ix86_init_machine_status;
3907 /* Validate -mregparm= value. */
3908 if (opts_set->x_ix86_regparm)
3910 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3911 warning (0, "-mregparm is ignored in 64-bit mode");
3912 if (opts->x_ix86_regparm > REGPARM_MAX)
3914 error ("-mregparm=%d is not between 0 and %d",
3915 opts->x_ix86_regparm, REGPARM_MAX);
3916 opts->x_ix86_regparm = 0;
3919 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3920 opts->x_ix86_regparm = REGPARM_MAX;
3922 /* Default align_* from the processor table. */
3923 if (opts->x_align_loops == 0)
3925 opts->x_align_loops = processor_target_table[ix86_tune].align_loop;
3926 align_loops_max_skip = processor_target_table[ix86_tune].align_loop_max_skip;
3928 if (opts->x_align_jumps == 0)
3930 opts->x_align_jumps = processor_target_table[ix86_tune].align_jump;
3931 align_jumps_max_skip = processor_target_table[ix86_tune].align_jump_max_skip;
3933 if (opts->x_align_functions == 0)
3935 opts->x_align_functions = processor_target_table[ix86_tune].align_func;
3938 /* Provide default for -mbranch-cost= value. */
3939 if (!opts_set->x_ix86_branch_cost)
3940 opts->x_ix86_branch_cost = ix86_tune_cost->branch_cost;
3942 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3944 opts->x_target_flags
3945 |= TARGET_SUBTARGET64_DEFAULT & ~opts_set->x_target_flags;
3947 /* Enable by default the SSE and MMX builtins. Do allow the user to
3948 explicitly disable any of these. In particular, disabling SSE and
3949 MMX for kernel code is extremely useful. */
3950 if (!ix86_arch_specified)
3951 opts->x_ix86_isa_flags
3952 |= ((OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_MMX
3953 | TARGET_SUBTARGET64_ISA_DEFAULT)
3954 & ~opts->x_ix86_isa_flags_explicit);
3956 if (TARGET_RTD_P (opts->x_target_flags))
3957 warning (0, "%srtd%s is ignored in 64bit mode", prefix, suffix);
3961 opts->x_target_flags
3962 |= TARGET_SUBTARGET32_DEFAULT & ~opts_set->x_target_flags;
3964 if (!ix86_arch_specified)
3965 opts->x_ix86_isa_flags
3966 |= TARGET_SUBTARGET32_ISA_DEFAULT & ~opts->x_ix86_isa_flags_explicit;
3968 /* i386 ABI does not specify red zone. It still makes sense to use it
3969 when programmer takes care to stack from being destroyed. */
3970 if (!(opts_set->x_target_flags & MASK_NO_RED_ZONE))
3971 opts->x_target_flags |= MASK_NO_RED_ZONE;
3974 /* Keep nonleaf frame pointers. */
3975 if (opts->x_flag_omit_frame_pointer)
3976 opts->x_target_flags &= ~MASK_OMIT_LEAF_FRAME_POINTER;
3977 else if (TARGET_OMIT_LEAF_FRAME_POINTER_P (opts->x_target_flags))
3978 opts->x_flag_omit_frame_pointer = 1;
3980 /* If we're doing fast math, we don't care about comparison order
3981 wrt NaNs. This lets us use a shorter comparison sequence. */
3982 if (opts->x_flag_finite_math_only)
3983 opts->x_target_flags &= ~MASK_IEEE_FP;
3985 /* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
3986 since the insns won't need emulation. */
3987 if (ix86_tune_features [X86_TUNE_ALWAYS_FANCY_MATH_387])
3988 opts->x_target_flags &= ~MASK_NO_FANCY_MATH_387;
3990 /* Likewise, if the target doesn't have a 387, or we've specified
3991 software floating point, don't use 387 inline intrinsics. */
3992 if (!TARGET_80387_P (opts->x_target_flags))
3993 opts->x_target_flags |= MASK_NO_FANCY_MATH_387;
3995 /* Turn on MMX builtins for -msse. */
3996 if (TARGET_SSE_P (opts->x_ix86_isa_flags))
3997 opts->x_ix86_isa_flags
3998 |= OPTION_MASK_ISA_MMX & ~opts->x_ix86_isa_flags_explicit;
4000 /* Enable SSE prefetch. */
4001 if (TARGET_SSE_P (opts->x_ix86_isa_flags)
4002 || (TARGET_PRFCHW && !TARGET_3DNOW_P (opts->x_ix86_isa_flags)))
4003 x86_prefetch_sse = true;
4005 /* Enable prefetch{,w} instructions for -m3dnow and -mprefetchwt1. */
4006 if (TARGET_3DNOW_P (opts->x_ix86_isa_flags)
4007 || TARGET_PREFETCHWT1_P (opts->x_ix86_isa_flags))
4008 opts->x_ix86_isa_flags
4009 |= OPTION_MASK_ISA_PRFCHW & ~opts->x_ix86_isa_flags_explicit;
4011 /* Enable popcnt instruction for -msse4.2 or -mabm. */
4012 if (TARGET_SSE4_2_P (opts->x_ix86_isa_flags)
4013 || TARGET_ABM_P (opts->x_ix86_isa_flags))
4014 opts->x_ix86_isa_flags
4015 |= OPTION_MASK_ISA_POPCNT & ~opts->x_ix86_isa_flags_explicit;
4017 /* Enable lzcnt instruction for -mabm. */
4018 if (TARGET_ABM_P(opts->x_ix86_isa_flags))
4019 opts->x_ix86_isa_flags
4020 |= OPTION_MASK_ISA_LZCNT & ~opts->x_ix86_isa_flags_explicit;
4022 /* Validate -mpreferred-stack-boundary= value or default it to
4023 PREFERRED_STACK_BOUNDARY_DEFAULT. */
4024 ix86_preferred_stack_boundary = PREFERRED_STACK_BOUNDARY_DEFAULT;
4025 if (opts_set->x_ix86_preferred_stack_boundary_arg)
4027 int min = (TARGET_64BIT_P (opts->x_ix86_isa_flags)
4028 ? (TARGET_SSE_P (opts->x_ix86_isa_flags) ? 4 : 3) : 2);
4029 int max = (TARGET_SEH ? 4 : 12);
4031 if (opts->x_ix86_preferred_stack_boundary_arg < min
4032 || opts->x_ix86_preferred_stack_boundary_arg > max)
4035 error ("-mpreferred-stack-boundary is not supported "
4038 error ("-mpreferred-stack-boundary=%d is not between %d and %d",
4039 opts->x_ix86_preferred_stack_boundary_arg, min, max);
4042 ix86_preferred_stack_boundary
4043 = (1 << opts->x_ix86_preferred_stack_boundary_arg) * BITS_PER_UNIT;
4046 /* Set the default value for -mstackrealign. */
4047 if (opts->x_ix86_force_align_arg_pointer == -1)
4048 opts->x_ix86_force_align_arg_pointer = STACK_REALIGN_DEFAULT;
4050 ix86_default_incoming_stack_boundary = PREFERRED_STACK_BOUNDARY;
4052 /* Validate -mincoming-stack-boundary= value or default it to
4053 MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY. */
4054 ix86_incoming_stack_boundary = ix86_default_incoming_stack_boundary;
4055 if (opts_set->x_ix86_incoming_stack_boundary_arg)
4057 if (opts->x_ix86_incoming_stack_boundary_arg
4058 < (TARGET_64BIT_P (opts->x_ix86_isa_flags) ? 4 : 2)
4059 || opts->x_ix86_incoming_stack_boundary_arg > 12)
4060 error ("-mincoming-stack-boundary=%d is not between %d and 12",
4061 opts->x_ix86_incoming_stack_boundary_arg,
4062 TARGET_64BIT_P (opts->x_ix86_isa_flags) ? 4 : 2);
4065 ix86_user_incoming_stack_boundary
4066 = (1 << opts->x_ix86_incoming_stack_boundary_arg) * BITS_PER_UNIT;
4067 ix86_incoming_stack_boundary
4068 = ix86_user_incoming_stack_boundary;
4072 #ifndef NO_PROFILE_COUNTERS
4073 if (flag_nop_mcount)
4074 error ("-mnop-mcount is not compatible with this target");
4076 if (flag_nop_mcount && flag_pic)
4077 error ("-mnop-mcount is not implemented for -fPIC");
4079 /* Accept -msseregparm only if at least SSE support is enabled. */
4080 if (TARGET_SSEREGPARM_P (opts->x_target_flags)
4081 && ! TARGET_SSE_P (opts->x_ix86_isa_flags))
4082 error ("%ssseregparm%s used without SSE enabled", prefix, suffix);
4084 if (opts_set->x_ix86_fpmath)
4086 if (opts->x_ix86_fpmath & FPMATH_SSE)
4088 if (!TARGET_SSE_P (opts->x_ix86_isa_flags))
4090 warning (0, "SSE instruction set disabled, using 387 arithmetics");
4091 opts->x_ix86_fpmath = FPMATH_387;
4093 else if ((opts->x_ix86_fpmath & FPMATH_387)
4094 && !TARGET_80387_P (opts->x_target_flags))
4096 warning (0, "387 instruction set disabled, using SSE arithmetics");
4097 opts->x_ix86_fpmath = FPMATH_SSE;
4101 /* For all chips supporting SSE2, -mfpmath=sse performs better than
4102 fpmath=387. The second is however default at many targets since the
4103 extra 80bit precision of temporaries is considered to be part of ABI.
4104 Overwrite the default at least for -ffast-math.
4105 TODO: -mfpmath=both seems to produce same performing code with bit
4106 smaller binaries. It is however not clear if register allocation is
4107 ready for this setting.
4108 Also -mfpmath=387 is overall a lot more compact (bout 4-5%) than SSE
4109 codegen. We may switch to 387 with -ffast-math for size optimized
4111 else if (fast_math_flags_set_p (&global_options)
4112 && TARGET_SSE2_P (opts->x_ix86_isa_flags))
4113 opts->x_ix86_fpmath = FPMATH_SSE;
4115 opts->x_ix86_fpmath = TARGET_FPMATH_DEFAULT_P (opts->x_ix86_isa_flags);
4117 /* If the i387 is disabled, then do not return values in it. */
4118 if (!TARGET_80387_P (opts->x_target_flags))
4119 opts->x_target_flags &= ~MASK_FLOAT_RETURNS;
4121 /* Use external vectorized library in vectorizing intrinsics. */
4122 if (opts_set->x_ix86_veclibabi_type)
4123 switch (opts->x_ix86_veclibabi_type)
4125 case ix86_veclibabi_type_svml:
4126 ix86_veclib_handler = ix86_veclibabi_svml;
4129 case ix86_veclibabi_type_acml:
4130 ix86_veclib_handler = ix86_veclibabi_acml;
4137 if (ix86_tune_features [X86_TUNE_ACCUMULATE_OUTGOING_ARGS]
4138 && !(opts_set->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
4139 opts->x_target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
4141 /* If stack probes are required, the space used for large function
4142 arguments on the stack must also be probed, so enable
4143 -maccumulate-outgoing-args so this happens in the prologue. */
4144 if (TARGET_STACK_PROBE_P (opts->x_target_flags)
4145 && !(opts->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
4147 if (opts_set->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS)
4148 warning (0, "stack probing requires %saccumulate-outgoing-args%s "
4149 "for correctness", prefix, suffix);
4150 opts->x_target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
4153 /* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
4156 ASM_GENERATE_INTERNAL_LABEL (internal_label_prefix, "LX", 0);
4157 p = strchr (internal_label_prefix, 'X');
4158 internal_label_prefix_len = p - internal_label_prefix;
4162 /* When scheduling description is not available, disable scheduler pass
4163 so it won't slow down the compilation and make x87 code slower. */
4164 if (!TARGET_SCHEDULE)
4165 opts->x_flag_schedule_insns_after_reload = opts->x_flag_schedule_insns = 0;
4167 maybe_set_param_value (PARAM_SIMULTANEOUS_PREFETCHES,
4168 ix86_tune_cost->simultaneous_prefetches,
4169 opts->x_param_values,
4170 opts_set->x_param_values);
4171 maybe_set_param_value (PARAM_L1_CACHE_LINE_SIZE,
4172 ix86_tune_cost->prefetch_block,
4173 opts->x_param_values,
4174 opts_set->x_param_values);
4175 maybe_set_param_value (PARAM_L1_CACHE_SIZE,
4176 ix86_tune_cost->l1_cache_size,
4177 opts->x_param_values,
4178 opts_set->x_param_values);
4179 maybe_set_param_value (PARAM_L2_CACHE_SIZE,
4180 ix86_tune_cost->l2_cache_size,
4181 opts->x_param_values,
4182 opts_set->x_param_values);
4184 /* Enable sw prefetching at -O3 for CPUS that prefetching is helpful. */
4185 if (opts->x_flag_prefetch_loop_arrays < 0
4187 && (opts->x_optimize >= 3 || opts->x_flag_profile_use)
4188 && TARGET_SOFTWARE_PREFETCHING_BENEFICIAL)
4189 opts->x_flag_prefetch_loop_arrays = 1;
4191 /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
4192 can be opts->x_optimized to ap = __builtin_next_arg (0). */
4193 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags) && !opts->x_flag_split_stack)
4194 targetm.expand_builtin_va_start = NULL;
4196 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
4198 ix86_gen_leave = gen_leave_rex64;
4199 if (Pmode == DImode)
4201 ix86_gen_tls_global_dynamic_64 = gen_tls_global_dynamic_64_di;
4202 ix86_gen_tls_local_dynamic_base_64
4203 = gen_tls_local_dynamic_base_64_di;
4207 ix86_gen_tls_global_dynamic_64 = gen_tls_global_dynamic_64_si;
4208 ix86_gen_tls_local_dynamic_base_64
4209 = gen_tls_local_dynamic_base_64_si;
4213 ix86_gen_leave = gen_leave;
4215 if (Pmode == DImode)
4217 ix86_gen_add3 = gen_adddi3;
4218 ix86_gen_sub3 = gen_subdi3;
4219 ix86_gen_sub3_carry = gen_subdi3_carry;
4220 ix86_gen_one_cmpl2 = gen_one_cmpldi2;
4221 ix86_gen_andsp = gen_anddi3;
4222 ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_probe_di;
4223 ix86_gen_adjust_stack_and_probe = gen_adjust_stack_and_probedi;
4224 ix86_gen_probe_stack_range = gen_probe_stack_rangedi;
4225 ix86_gen_monitor = gen_sse3_monitor_di;
4226 ix86_gen_monitorx = gen_monitorx_di;
4230 ix86_gen_add3 = gen_addsi3;
4231 ix86_gen_sub3 = gen_subsi3;
4232 ix86_gen_sub3_carry = gen_subsi3_carry;
4233 ix86_gen_one_cmpl2 = gen_one_cmplsi2;
4234 ix86_gen_andsp = gen_andsi3;
4235 ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_probe_si;
4236 ix86_gen_adjust_stack_and_probe = gen_adjust_stack_and_probesi;
4237 ix86_gen_probe_stack_range = gen_probe_stack_rangesi;
4238 ix86_gen_monitor = gen_sse3_monitor_si;
4239 ix86_gen_monitorx = gen_monitorx_si;
4243 /* Use -mcld by default for 32-bit code if configured with --enable-cld. */
4244 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
4245 opts->x_target_flags |= MASK_CLD & ~opts_set->x_target_flags;
4248 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags) && opts->x_flag_pic)
4250 if (opts->x_flag_fentry > 0)
4251 sorry ("-mfentry isn%'t supported for 32-bit in combination "
4253 opts->x_flag_fentry = 0;
4255 else if (TARGET_SEH)
4257 if (opts->x_flag_fentry == 0)
4258 sorry ("-mno-fentry isn%'t compatible with SEH");
4259 opts->x_flag_fentry = 1;
4261 else if (opts->x_flag_fentry < 0)
4263 #if defined(PROFILE_BEFORE_PROLOGUE)
4264 opts->x_flag_fentry = 1;
4266 opts->x_flag_fentry = 0;
4270 if (!(opts_set->x_target_flags & MASK_VZEROUPPER))
4271 opts->x_target_flags |= MASK_VZEROUPPER;
4272 if (!ix86_tune_features[X86_TUNE_AVX256_UNALIGNED_LOAD_OPTIMAL]
4273 && !(opts_set->x_target_flags & MASK_AVX256_SPLIT_UNALIGNED_LOAD))
4274 opts->x_target_flags |= MASK_AVX256_SPLIT_UNALIGNED_LOAD;
4275 if (!ix86_tune_features[X86_TUNE_AVX256_UNALIGNED_STORE_OPTIMAL]
4276 && !(opts_set->x_target_flags & MASK_AVX256_SPLIT_UNALIGNED_STORE))
4277 opts->x_target_flags |= MASK_AVX256_SPLIT_UNALIGNED_STORE;
4278 /* Enable 128-bit AVX instruction generation
4279 for the auto-vectorizer. */
4280 if (TARGET_AVX128_OPTIMAL
4281 && !(opts_set->x_target_flags & MASK_PREFER_AVX128))
4282 opts->x_target_flags |= MASK_PREFER_AVX128;
4284 if (opts->x_ix86_recip_name)
4286 char *p = ASTRDUP (opts->x_ix86_recip_name);
4288 unsigned int mask, i;
4291 while ((q = strtok (p, ",")) != NULL)
4302 if (!strcmp (q, "default"))
4303 mask = RECIP_MASK_ALL;
4306 for (i = 0; i < ARRAY_SIZE (recip_options); i++)
4307 if (!strcmp (q, recip_options[i].string))
4309 mask = recip_options[i].mask;
4313 if (i == ARRAY_SIZE (recip_options))
4315 error ("unknown option for -mrecip=%s", q);
4317 mask = RECIP_MASK_NONE;
4321 opts->x_recip_mask_explicit |= mask;
4323 opts->x_recip_mask &= ~mask;
4325 opts->x_recip_mask |= mask;
4329 if (TARGET_RECIP_P (opts->x_target_flags))
4330 opts->x_recip_mask |= RECIP_MASK_ALL & ~opts->x_recip_mask_explicit;
4331 else if (opts_set->x_target_flags & MASK_RECIP)
4332 opts->x_recip_mask &= ~(RECIP_MASK_ALL & ~opts->x_recip_mask_explicit);
4334 /* Default long double to 64-bit for 32-bit Bionic and to __float128
4335 for 64-bit Bionic. */
4336 if (TARGET_HAS_BIONIC
4337 && !(opts_set->x_target_flags
4338 & (MASK_LONG_DOUBLE_64 | MASK_LONG_DOUBLE_128)))
4339 opts->x_target_flags |= (TARGET_64BIT
4340 ? MASK_LONG_DOUBLE_128
4341 : MASK_LONG_DOUBLE_64);
4343 /* Only one of them can be active. */
4344 gcc_assert ((opts->x_target_flags & MASK_LONG_DOUBLE_64) == 0
4345 || (opts->x_target_flags & MASK_LONG_DOUBLE_128) == 0);
4347 /* Save the initial options in case the user does function specific
4350 target_option_default_node = target_option_current_node
4351 = build_target_option_node (opts);
4353 /* Handle stack protector */
4354 if (!opts_set->x_ix86_stack_protector_guard)
4355 opts->x_ix86_stack_protector_guard
4356 = TARGET_HAS_BIONIC ? SSP_GLOBAL : SSP_TLS;
4358 /* Handle -mmemcpy-strategy= and -mmemset-strategy= */
4359 if (opts->x_ix86_tune_memcpy_strategy)
4361 char *str = xstrdup (opts->x_ix86_tune_memcpy_strategy);
4362 ix86_parse_stringop_strategy_string (str, false);
4366 if (opts->x_ix86_tune_memset_strategy)
4368 char *str = xstrdup (opts->x_ix86_tune_memset_strategy);
4369 ix86_parse_stringop_strategy_string (str, true);
4374 /* Implement the TARGET_OPTION_OVERRIDE hook. */
4377 ix86_option_override (void)
4379 opt_pass *pass_insert_vzeroupper = make_pass_insert_vzeroupper (g);
4380 struct register_pass_info insert_vzeroupper_info
4381 = { pass_insert_vzeroupper, "reload",
4382 1, PASS_POS_INSERT_AFTER
4385 ix86_option_override_internal (true, &global_options, &global_options_set);
4388 /* This needs to be done at start up. It's convenient to do it here. */
4389 register_pass (&insert_vzeroupper_info);
4392 /* Implement the TARGET_OFFLOAD_OPTIONS hook. */
4394 ix86_offload_options (void)
4397 return xstrdup ("-foffload-abi=lp64");
4398 return xstrdup ("-foffload-abi=ilp32");
4401 /* Update register usage after having seen the compiler flags. */
4404 ix86_conditional_register_usage (void)
4408 /* For 32-bit targets, squash the REX registers. */
4411 for (i = FIRST_REX_INT_REG; i <= LAST_REX_INT_REG; i++)
4412 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4413 for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
4414 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4415 for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
4416 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4419 /* See the definition of CALL_USED_REGISTERS in i386.h. */
4420 c_mask = (TARGET_64BIT_MS_ABI ? (1 << 3)
4421 : TARGET_64BIT ? (1 << 2)
4424 CLEAR_HARD_REG_SET (reg_class_contents[(int)CLOBBERED_REGS]);
4426 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
4428 /* Set/reset conditionally defined registers from
4429 CALL_USED_REGISTERS initializer. */
4430 if (call_used_regs[i] > 1)
4431 call_used_regs[i] = !!(call_used_regs[i] & c_mask);
4433 /* Calculate registers of CLOBBERED_REGS register set
4434 as call used registers from GENERAL_REGS register set. */
4435 if (TEST_HARD_REG_BIT (reg_class_contents[(int)GENERAL_REGS], i)
4436 && call_used_regs[i])
4437 SET_HARD_REG_BIT (reg_class_contents[(int)CLOBBERED_REGS], i);
4440 /* If MMX is disabled, squash the registers. */
4442 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
4443 if (TEST_HARD_REG_BIT (reg_class_contents[(int)MMX_REGS], i))
4444 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4446 /* If SSE is disabled, squash the registers. */
4448 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
4449 if (TEST_HARD_REG_BIT (reg_class_contents[(int)SSE_REGS], i))
4450 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4452 /* If the FPU is disabled, squash the registers. */
4453 if (! (TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387))
4454 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
4455 if (TEST_HARD_REG_BIT (reg_class_contents[(int)FLOAT_REGS], i))
4456 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4458 /* If AVX512F is disabled, squash the registers. */
4459 if (! TARGET_AVX512F)
4461 for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
4462 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4464 for (i = FIRST_MASK_REG; i <= LAST_MASK_REG; i++)
4465 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4468 /* If MPX is disabled, squash the registers. */
4470 for (i = FIRST_BND_REG; i <= LAST_BND_REG; i++)
4471 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4475 /* Save the current options */
4478 ix86_function_specific_save (struct cl_target_option *ptr,
4479 struct gcc_options *opts)
4481 ptr->arch = ix86_arch;
4482 ptr->schedule = ix86_schedule;
4483 ptr->prefetch_sse = x86_prefetch_sse;
4484 ptr->tune = ix86_tune;
4485 ptr->branch_cost = ix86_branch_cost;
4486 ptr->tune_defaulted = ix86_tune_defaulted;
4487 ptr->arch_specified = ix86_arch_specified;
4488 ptr->x_ix86_isa_flags_explicit = opts->x_ix86_isa_flags_explicit;
4489 ptr->x_ix86_target_flags_explicit = opts->x_ix86_target_flags_explicit;
4490 ptr->x_recip_mask_explicit = opts->x_recip_mask_explicit;
4491 ptr->x_ix86_arch_string = opts->x_ix86_arch_string;
4492 ptr->x_ix86_tune_string = opts->x_ix86_tune_string;
4493 ptr->x_ix86_cmodel = opts->x_ix86_cmodel;
4494 ptr->x_ix86_abi = opts->x_ix86_abi;
4495 ptr->x_ix86_asm_dialect = opts->x_ix86_asm_dialect;
4496 ptr->x_ix86_branch_cost = opts->x_ix86_branch_cost;
4497 ptr->x_ix86_dump_tunes = opts->x_ix86_dump_tunes;
4498 ptr->x_ix86_force_align_arg_pointer = opts->x_ix86_force_align_arg_pointer;
4499 ptr->x_ix86_force_drap = opts->x_ix86_force_drap;
4500 ptr->x_ix86_incoming_stack_boundary_arg = opts->x_ix86_incoming_stack_boundary_arg;
4501 ptr->x_ix86_pmode = opts->x_ix86_pmode;
4502 ptr->x_ix86_preferred_stack_boundary_arg = opts->x_ix86_preferred_stack_boundary_arg;
4503 ptr->x_ix86_recip_name = opts->x_ix86_recip_name;
4504 ptr->x_ix86_regparm = opts->x_ix86_regparm;
4505 ptr->x_ix86_section_threshold = opts->x_ix86_section_threshold;
4506 ptr->x_ix86_sse2avx = opts->x_ix86_sse2avx;
4507 ptr->x_ix86_stack_protector_guard = opts->x_ix86_stack_protector_guard;
4508 ptr->x_ix86_stringop_alg = opts->x_ix86_stringop_alg;
4509 ptr->x_ix86_tls_dialect = opts->x_ix86_tls_dialect;
4510 ptr->x_ix86_tune_ctrl_string = opts->x_ix86_tune_ctrl_string;
4511 ptr->x_ix86_tune_memcpy_strategy = opts->x_ix86_tune_memcpy_strategy;
4512 ptr->x_ix86_tune_memset_strategy = opts->x_ix86_tune_memset_strategy;
4513 ptr->x_ix86_tune_no_default = opts->x_ix86_tune_no_default;
4514 ptr->x_ix86_veclibabi_type = opts->x_ix86_veclibabi_type;
4516 /* The fields are char but the variables are not; make sure the
4517 values fit in the fields. */
4518 gcc_assert (ptr->arch == ix86_arch);
4519 gcc_assert (ptr->schedule == ix86_schedule);
4520 gcc_assert (ptr->tune == ix86_tune);
4521 gcc_assert (ptr->branch_cost == ix86_branch_cost);
4524 /* Restore the current options */
4527 ix86_function_specific_restore (struct gcc_options *opts,
4528 struct cl_target_option *ptr)
4530 enum processor_type old_tune = ix86_tune;
4531 enum processor_type old_arch = ix86_arch;
4532 unsigned int ix86_arch_mask;
4535 /* We don't change -fPIC. */
4536 opts->x_flag_pic = flag_pic;
4538 ix86_arch = (enum processor_type) ptr->arch;
4539 ix86_schedule = (enum attr_cpu) ptr->schedule;
4540 ix86_tune = (enum processor_type) ptr->tune;
4541 x86_prefetch_sse = ptr->prefetch_sse;
4542 opts->x_ix86_branch_cost = ptr->branch_cost;
4543 ix86_tune_defaulted = ptr->tune_defaulted;
4544 ix86_arch_specified = ptr->arch_specified;
4545 opts->x_ix86_isa_flags_explicit = ptr->x_ix86_isa_flags_explicit;
4546 opts->x_ix86_target_flags_explicit = ptr->x_ix86_target_flags_explicit;
4547 opts->x_recip_mask_explicit = ptr->x_recip_mask_explicit;
4548 opts->x_ix86_arch_string = ptr->x_ix86_arch_string;
4549 opts->x_ix86_tune_string = ptr->x_ix86_tune_string;
4550 opts->x_ix86_cmodel = ptr->x_ix86_cmodel;
4551 opts->x_ix86_abi = ptr->x_ix86_abi;
4552 opts->x_ix86_asm_dialect = ptr->x_ix86_asm_dialect;
4553 opts->x_ix86_branch_cost = ptr->x_ix86_branch_cost;
4554 opts->x_ix86_dump_tunes = ptr->x_ix86_dump_tunes;
4555 opts->x_ix86_force_align_arg_pointer = ptr->x_ix86_force_align_arg_pointer;
4556 opts->x_ix86_force_drap = ptr->x_ix86_force_drap;
4557 opts->x_ix86_incoming_stack_boundary_arg = ptr->x_ix86_incoming_stack_boundary_arg;
4558 opts->x_ix86_pmode = ptr->x_ix86_pmode;
4559 opts->x_ix86_preferred_stack_boundary_arg = ptr->x_ix86_preferred_stack_boundary_arg;
4560 opts->x_ix86_recip_name = ptr->x_ix86_recip_name;
4561 opts->x_ix86_regparm = ptr->x_ix86_regparm;
4562 opts->x_ix86_section_threshold = ptr->x_ix86_section_threshold;
4563 opts->x_ix86_sse2avx = ptr->x_ix86_sse2avx;
4564 opts->x_ix86_stack_protector_guard = ptr->x_ix86_stack_protector_guard;
4565 opts->x_ix86_stringop_alg = ptr->x_ix86_stringop_alg;
4566 opts->x_ix86_tls_dialect = ptr->x_ix86_tls_dialect;
4567 opts->x_ix86_tune_ctrl_string = ptr->x_ix86_tune_ctrl_string;
4568 opts->x_ix86_tune_memcpy_strategy = ptr->x_ix86_tune_memcpy_strategy;
4569 opts->x_ix86_tune_memset_strategy = ptr->x_ix86_tune_memset_strategy;
4570 opts->x_ix86_tune_no_default = ptr->x_ix86_tune_no_default;
4571 opts->x_ix86_veclibabi_type = ptr->x_ix86_veclibabi_type;
4572 ix86_tune_cost = processor_target_table[ix86_tune].cost;
4573 /* TODO: ix86_cost should be chosen at instruction or function granuality
4574 so for cold code we use size_cost even in !optimize_size compilation. */
4575 if (opts->x_optimize_size)
4576 ix86_cost = &ix86_size_cost;
4578 ix86_cost = ix86_tune_cost;
4580 /* Recreate the arch feature tests if the arch changed */
4581 if (old_arch != ix86_arch)
4583 ix86_arch_mask = 1u << ix86_arch;
4584 for (i = 0; i < X86_ARCH_LAST; ++i)
4585 ix86_arch_features[i]
4586 = !!(initial_ix86_arch_features[i] & ix86_arch_mask);
4589 /* Recreate the tune optimization tests */
4590 if (old_tune != ix86_tune)
4591 set_ix86_tune_features (ix86_tune, false);
4594 /* Adjust target options after streaming them in. This is mainly about
4595 reconciling them with global options. */
4598 ix86_function_specific_post_stream_in (struct cl_target_option *ptr)
4600 /* flag_pic is a global option, but ix86_cmodel is target saved option
4601 partly computed from flag_pic. If flag_pic is on, adjust x_ix86_cmodel
4602 for PIC, or error out. */
4604 switch (ptr->x_ix86_cmodel)
4607 ptr->x_ix86_cmodel = CM_SMALL_PIC;
4611 ptr->x_ix86_cmodel = CM_MEDIUM_PIC;
4615 ptr->x_ix86_cmodel = CM_LARGE_PIC;
4619 error ("code model %s does not support PIC mode", "kernel");
4626 switch (ptr->x_ix86_cmodel)
4629 ptr->x_ix86_cmodel = CM_SMALL;
4633 ptr->x_ix86_cmodel = CM_MEDIUM;
4637 ptr->x_ix86_cmodel = CM_LARGE;
4645 /* Print the current options */
4648 ix86_function_specific_print (FILE *file, int indent,
4649 struct cl_target_option *ptr)
4652 = ix86_target_string (ptr->x_ix86_isa_flags, ptr->x_target_flags,
4653 NULL, NULL, ptr->x_ix86_fpmath, false);
4655 gcc_assert (ptr->arch < PROCESSOR_max);
4656 fprintf (file, "%*sarch = %d (%s)\n",
4658 ptr->arch, processor_target_table[ptr->arch].name);
4660 gcc_assert (ptr->tune < PROCESSOR_max);
4661 fprintf (file, "%*stune = %d (%s)\n",
4663 ptr->tune, processor_target_table[ptr->tune].name);
4665 fprintf (file, "%*sbranch_cost = %d\n", indent, "", ptr->branch_cost);
4669 fprintf (file, "%*s%s\n", indent, "", target_string);
4670 free (target_string);
4675 /* Inner function to process the attribute((target(...))), take an argument and
4676 set the current options from the argument. If we have a list, recursively go
4680 ix86_valid_target_attribute_inner_p (tree args, char *p_strings[],
4681 struct gcc_options *opts,
4682 struct gcc_options *opts_set,
4683 struct gcc_options *enum_opts_set)
4688 #define IX86_ATTR_ISA(S,O) { S, sizeof (S)-1, ix86_opt_isa, O, 0 }
4689 #define IX86_ATTR_STR(S,O) { S, sizeof (S)-1, ix86_opt_str, O, 0 }
4690 #define IX86_ATTR_ENUM(S,O) { S, sizeof (S)-1, ix86_opt_enum, O, 0 }
4691 #define IX86_ATTR_YES(S,O,M) { S, sizeof (S)-1, ix86_opt_yes, O, M }
4692 #define IX86_ATTR_NO(S,O,M) { S, sizeof (S)-1, ix86_opt_no, O, M }
4708 enum ix86_opt_type type;
4713 IX86_ATTR_ISA ("3dnow", OPT_m3dnow),
4714 IX86_ATTR_ISA ("abm", OPT_mabm),
4715 IX86_ATTR_ISA ("bmi", OPT_mbmi),
4716 IX86_ATTR_ISA ("bmi2", OPT_mbmi2),
4717 IX86_ATTR_ISA ("lzcnt", OPT_mlzcnt),
4718 IX86_ATTR_ISA ("tbm", OPT_mtbm),
4719 IX86_ATTR_ISA ("aes", OPT_maes),
4720 IX86_ATTR_ISA ("sha", OPT_msha),
4721 IX86_ATTR_ISA ("avx", OPT_mavx),
4722 IX86_ATTR_ISA ("avx2", OPT_mavx2),
4723 IX86_ATTR_ISA ("avx512f", OPT_mavx512f),
4724 IX86_ATTR_ISA ("avx512pf", OPT_mavx512pf),
4725 IX86_ATTR_ISA ("avx512er", OPT_mavx512er),
4726 IX86_ATTR_ISA ("avx512cd", OPT_mavx512cd),
4727 IX86_ATTR_ISA ("avx512dq", OPT_mavx512dq),
4728 IX86_ATTR_ISA ("avx512bw", OPT_mavx512bw),
4729 IX86_ATTR_ISA ("avx512vl", OPT_mavx512vl),
4730 IX86_ATTR_ISA ("mmx", OPT_mmmx),
4731 IX86_ATTR_ISA ("pclmul", OPT_mpclmul),
4732 IX86_ATTR_ISA ("popcnt", OPT_mpopcnt),
4733 IX86_ATTR_ISA ("sse", OPT_msse),
4734 IX86_ATTR_ISA ("sse2", OPT_msse2),
4735 IX86_ATTR_ISA ("sse3", OPT_msse3),
4736 IX86_ATTR_ISA ("sse4", OPT_msse4),
4737 IX86_ATTR_ISA ("sse4.1", OPT_msse4_1),
4738 IX86_ATTR_ISA ("sse4.2", OPT_msse4_2),
4739 IX86_ATTR_ISA ("sse4a", OPT_msse4a),
4740 IX86_ATTR_ISA ("ssse3", OPT_mssse3),
4741 IX86_ATTR_ISA ("fma4", OPT_mfma4),
4742 IX86_ATTR_ISA ("fma", OPT_mfma),
4743 IX86_ATTR_ISA ("xop", OPT_mxop),
4744 IX86_ATTR_ISA ("lwp", OPT_mlwp),
4745 IX86_ATTR_ISA ("fsgsbase", OPT_mfsgsbase),
4746 IX86_ATTR_ISA ("rdrnd", OPT_mrdrnd),
4747 IX86_ATTR_ISA ("f16c", OPT_mf16c),
4748 IX86_ATTR_ISA ("rtm", OPT_mrtm),
4749 IX86_ATTR_ISA ("hle", OPT_mhle),
4750 IX86_ATTR_ISA ("prfchw", OPT_mprfchw),
4751 IX86_ATTR_ISA ("rdseed", OPT_mrdseed),
4752 IX86_ATTR_ISA ("adx", OPT_madx),
4753 IX86_ATTR_ISA ("fxsr", OPT_mfxsr),
4754 IX86_ATTR_ISA ("xsave", OPT_mxsave),
4755 IX86_ATTR_ISA ("xsaveopt", OPT_mxsaveopt),
4756 IX86_ATTR_ISA ("prefetchwt1", OPT_mprefetchwt1),
4757 IX86_ATTR_ISA ("clflushopt", OPT_mclflushopt),
4758 IX86_ATTR_ISA ("xsavec", OPT_mxsavec),
4759 IX86_ATTR_ISA ("xsaves", OPT_mxsaves),
4760 IX86_ATTR_ISA ("avx512vbmi", OPT_mavx512vbmi),
4761 IX86_ATTR_ISA ("avx512ifma", OPT_mavx512ifma),
4762 IX86_ATTR_ISA ("clwb", OPT_mclwb),
4763 IX86_ATTR_ISA ("pcommit", OPT_mpcommit),
4764 IX86_ATTR_ISA ("mwaitx", OPT_mmwaitx),
4767 IX86_ATTR_ENUM ("fpmath=", OPT_mfpmath_),
4769 /* string options */
4770 IX86_ATTR_STR ("arch=", IX86_FUNCTION_SPECIFIC_ARCH),
4771 IX86_ATTR_STR ("tune=", IX86_FUNCTION_SPECIFIC_TUNE),
4774 IX86_ATTR_YES ("cld",
4778 IX86_ATTR_NO ("fancy-math-387",
4779 OPT_mfancy_math_387,
4780 MASK_NO_FANCY_MATH_387),
4782 IX86_ATTR_YES ("ieee-fp",
4786 IX86_ATTR_YES ("inline-all-stringops",
4787 OPT_minline_all_stringops,
4788 MASK_INLINE_ALL_STRINGOPS),
4790 IX86_ATTR_YES ("inline-stringops-dynamically",
4791 OPT_minline_stringops_dynamically,
4792 MASK_INLINE_STRINGOPS_DYNAMICALLY),
4794 IX86_ATTR_NO ("align-stringops",
4795 OPT_mno_align_stringops,
4796 MASK_NO_ALIGN_STRINGOPS),
4798 IX86_ATTR_YES ("recip",
4804 /* If this is a list, recurse to get the options. */
4805 if (TREE_CODE (args) == TREE_LIST)
4809 for (; args; args = TREE_CHAIN (args))
4810 if (TREE_VALUE (args)
4811 && !ix86_valid_target_attribute_inner_p (TREE_VALUE (args),
4812 p_strings, opts, opts_set,
4819 else if (TREE_CODE (args) != STRING_CST)
4821 error ("attribute %<target%> argument not a string");
4825 /* Handle multiple arguments separated by commas. */
4826 next_optstr = ASTRDUP (TREE_STRING_POINTER (args));
4828 while (next_optstr && *next_optstr != '\0')
4830 char *p = next_optstr;
4832 char *comma = strchr (next_optstr, ',');
4833 const char *opt_string;
4834 size_t len, opt_len;
4839 enum ix86_opt_type type = ix86_opt_unknown;
4845 len = comma - next_optstr;
4846 next_optstr = comma + 1;
4854 /* Recognize no-xxx. */
4855 if (len > 3 && p[0] == 'n' && p[1] == 'o' && p[2] == '-')
4864 /* Find the option. */
4867 for (i = 0; i < ARRAY_SIZE (attrs); i++)
4869 type = attrs[i].type;
4870 opt_len = attrs[i].len;
4871 if (ch == attrs[i].string[0]
4872 && ((type != ix86_opt_str && type != ix86_opt_enum)
4875 && memcmp (p, attrs[i].string, opt_len) == 0)
4878 mask = attrs[i].mask;
4879 opt_string = attrs[i].string;
4884 /* Process the option. */
4887 error ("attribute(target(\"%s\")) is unknown", orig_p);
4891 else if (type == ix86_opt_isa)
4893 struct cl_decoded_option decoded;
4895 generate_option (opt, NULL, opt_set_p, CL_TARGET, &decoded);
4896 ix86_handle_option (opts, opts_set,
4897 &decoded, input_location);
4900 else if (type == ix86_opt_yes || type == ix86_opt_no)
4902 if (type == ix86_opt_no)
4903 opt_set_p = !opt_set_p;
4906 opts->x_target_flags |= mask;
4908 opts->x_target_flags &= ~mask;
4911 else if (type == ix86_opt_str)
4915 error ("option(\"%s\") was already specified", opt_string);
4919 p_strings[opt] = xstrdup (p + opt_len);
4922 else if (type == ix86_opt_enum)
4927 arg_ok = opt_enum_arg_to_value (opt, p + opt_len, &value, CL_TARGET);
4929 set_option (opts, enum_opts_set, opt, value,
4930 p + opt_len, DK_UNSPECIFIED, input_location,
4934 error ("attribute(target(\"%s\")) is unknown", orig_p);
4946 /* Return a TARGET_OPTION_NODE tree of the target options listed or NULL. */
4949 ix86_valid_target_attribute_tree (tree args,
4950 struct gcc_options *opts,
4951 struct gcc_options *opts_set)
4953 const char *orig_arch_string = opts->x_ix86_arch_string;
4954 const char *orig_tune_string = opts->x_ix86_tune_string;
4955 enum fpmath_unit orig_fpmath_set = opts_set->x_ix86_fpmath;
4956 int orig_tune_defaulted = ix86_tune_defaulted;
4957 int orig_arch_specified = ix86_arch_specified;
4958 char *option_strings[IX86_FUNCTION_SPECIFIC_MAX] = { NULL, NULL };
4961 struct cl_target_option *def
4962 = TREE_TARGET_OPTION (target_option_default_node);
4963 struct gcc_options enum_opts_set;
4965 memset (&enum_opts_set, 0, sizeof (enum_opts_set));
4967 /* Process each of the options on the chain. */
4968 if (! ix86_valid_target_attribute_inner_p (args, option_strings, opts,
4969 opts_set, &enum_opts_set))
4970 return error_mark_node;
4972 /* If the changed options are different from the default, rerun
4973 ix86_option_override_internal, and then save the options away.
4974 The string options are are attribute options, and will be undone
4975 when we copy the save structure. */
4976 if (opts->x_ix86_isa_flags != def->x_ix86_isa_flags
4977 || opts->x_target_flags != def->x_target_flags
4978 || option_strings[IX86_FUNCTION_SPECIFIC_ARCH]
4979 || option_strings[IX86_FUNCTION_SPECIFIC_TUNE]
4980 || enum_opts_set.x_ix86_fpmath)
4982 /* If we are using the default tune= or arch=, undo the string assigned,
4983 and use the default. */
4984 if (option_strings[IX86_FUNCTION_SPECIFIC_ARCH])
4985 opts->x_ix86_arch_string = option_strings[IX86_FUNCTION_SPECIFIC_ARCH];
4986 else if (!orig_arch_specified)
4987 opts->x_ix86_arch_string = NULL;
4989 if (option_strings[IX86_FUNCTION_SPECIFIC_TUNE])
4990 opts->x_ix86_tune_string = option_strings[IX86_FUNCTION_SPECIFIC_TUNE];
4991 else if (orig_tune_defaulted)
4992 opts->x_ix86_tune_string = NULL;
4994 /* If fpmath= is not set, and we now have sse2 on 32-bit, use it. */
4995 if (enum_opts_set.x_ix86_fpmath)
4996 opts_set->x_ix86_fpmath = (enum fpmath_unit) 1;
4997 else if (!TARGET_64BIT_P (opts->x_ix86_isa_flags)
4998 && TARGET_SSE_P (opts->x_ix86_isa_flags))
5000 opts->x_ix86_fpmath = (enum fpmath_unit) (FPMATH_SSE | FPMATH_387);
5001 opts_set->x_ix86_fpmath = (enum fpmath_unit) 1;
5004 /* Do any overrides, such as arch=xxx, or tune=xxx support. */
5005 ix86_option_override_internal (false, opts, opts_set);
5007 /* Add any builtin functions with the new isa if any. */
5008 ix86_add_new_builtins (opts->x_ix86_isa_flags);
5010 /* Save the current options unless we are validating options for
5012 t = build_target_option_node (opts);
5014 opts->x_ix86_arch_string = orig_arch_string;
5015 opts->x_ix86_tune_string = orig_tune_string;
5016 opts_set->x_ix86_fpmath = orig_fpmath_set;
5018 /* Free up memory allocated to hold the strings */
5019 for (i = 0; i < IX86_FUNCTION_SPECIFIC_MAX; i++)
5020 free (option_strings[i]);
5026 /* Hook to validate attribute((target("string"))). */
5029 ix86_valid_target_attribute_p (tree fndecl,
5030 tree ARG_UNUSED (name),
5032 int ARG_UNUSED (flags))
5034 struct gcc_options func_options;
5035 tree new_target, new_optimize;
5038 /* attribute((target("default"))) does nothing, beyond
5039 affecting multi-versioning. */
5040 if (TREE_VALUE (args)
5041 && TREE_CODE (TREE_VALUE (args)) == STRING_CST
5042 && TREE_CHAIN (args) == NULL_TREE
5043 && strcmp (TREE_STRING_POINTER (TREE_VALUE (args)), "default") == 0)
5046 tree old_optimize = build_optimization_node (&global_options);
5048 /* Get the optimization options of the current function. */
5049 tree func_optimize = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl);
5052 func_optimize = old_optimize;
5054 /* Init func_options. */
5055 memset (&func_options, 0, sizeof (func_options));
5056 init_options_struct (&func_options, NULL);
5057 lang_hooks.init_options_struct (&func_options);
5059 cl_optimization_restore (&func_options,
5060 TREE_OPTIMIZATION (func_optimize));
5062 /* Initialize func_options to the default before its target options can
5064 cl_target_option_restore (&func_options,
5065 TREE_TARGET_OPTION (target_option_default_node));
5067 new_target = ix86_valid_target_attribute_tree (args, &func_options,
5068 &global_options_set);
5070 new_optimize = build_optimization_node (&func_options);
5072 if (new_target == error_mark_node)
5075 else if (fndecl && new_target)
5077 DECL_FUNCTION_SPECIFIC_TARGET (fndecl) = new_target;
5079 if (old_optimize != new_optimize)
5080 DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl) = new_optimize;
5087 /* Hook to determine if one function can safely inline another. */
5090 ix86_can_inline_p (tree caller, tree callee)
5093 tree caller_tree = DECL_FUNCTION_SPECIFIC_TARGET (caller);
5094 tree callee_tree = DECL_FUNCTION_SPECIFIC_TARGET (callee);
5096 /* If callee has no option attributes, then it is ok to inline. */
5100 /* If caller has no option attributes, but callee does then it is not ok to
5102 else if (!caller_tree)
5107 struct cl_target_option *caller_opts = TREE_TARGET_OPTION (caller_tree);
5108 struct cl_target_option *callee_opts = TREE_TARGET_OPTION (callee_tree);
5110 /* Callee's isa options should a subset of the caller's, i.e. a SSE4 function
5111 can inline a SSE2 function but a SSE2 function can't inline a SSE4
5113 if ((caller_opts->x_ix86_isa_flags & callee_opts->x_ix86_isa_flags)
5114 != callee_opts->x_ix86_isa_flags)
5117 /* See if we have the same non-isa options. */
5118 else if (caller_opts->x_target_flags != callee_opts->x_target_flags)
5121 /* See if arch, tune, etc. are the same. */
5122 else if (caller_opts->arch != callee_opts->arch)
5125 else if (caller_opts->tune != callee_opts->tune)
5128 else if (caller_opts->x_ix86_fpmath != callee_opts->x_ix86_fpmath)
5131 else if (caller_opts->branch_cost != callee_opts->branch_cost)
5142 /* Remember the last target of ix86_set_current_function. */
5143 static GTY(()) tree ix86_previous_fndecl;
5145 /* Set targets globals to the default (or current #pragma GCC target
5146 if active). Invalidate ix86_previous_fndecl cache. */
5149 ix86_reset_previous_fndecl (void)
5151 tree new_tree = target_option_current_node;
5152 cl_target_option_restore (&global_options, TREE_TARGET_OPTION (new_tree));
5153 if (TREE_TARGET_GLOBALS (new_tree))
5154 restore_target_globals (TREE_TARGET_GLOBALS (new_tree));
5155 else if (new_tree == target_option_default_node)
5156 restore_target_globals (&default_target_globals);
5158 TREE_TARGET_GLOBALS (new_tree) = save_target_globals_default_opts ();
5159 ix86_previous_fndecl = NULL_TREE;
5162 /* Establish appropriate back-end context for processing the function
5163 FNDECL. The argument might be NULL to indicate processing at top
5164 level, outside of any function scope. */
5166 ix86_set_current_function (tree fndecl)
5168 /* Only change the context if the function changes. This hook is called
5169 several times in the course of compiling a function, and we don't want to
5170 slow things down too much or call target_reinit when it isn't safe. */
5171 if (fndecl == ix86_previous_fndecl)
5175 if (ix86_previous_fndecl == NULL_TREE)
5176 old_tree = target_option_current_node;
5177 else if (DECL_FUNCTION_SPECIFIC_TARGET (ix86_previous_fndecl))
5178 old_tree = DECL_FUNCTION_SPECIFIC_TARGET (ix86_previous_fndecl);
5180 old_tree = target_option_default_node;
5182 if (fndecl == NULL_TREE)
5184 if (old_tree != target_option_current_node)
5185 ix86_reset_previous_fndecl ();
5189 tree new_tree = DECL_FUNCTION_SPECIFIC_TARGET (fndecl);
5190 if (new_tree == NULL_TREE)
5191 new_tree = target_option_default_node;
5193 if (old_tree != new_tree)
5195 cl_target_option_restore (&global_options, TREE_TARGET_OPTION (new_tree));
5196 if (TREE_TARGET_GLOBALS (new_tree))
5197 restore_target_globals (TREE_TARGET_GLOBALS (new_tree));
5198 else if (new_tree == target_option_default_node)
5199 restore_target_globals (&default_target_globals);
5201 TREE_TARGET_GLOBALS (new_tree) = save_target_globals_default_opts ();
5203 ix86_previous_fndecl = fndecl;
5207 /* Return true if this goes in large data/bss. */
5210 ix86_in_large_data_p (tree exp)
5212 if (ix86_cmodel != CM_MEDIUM && ix86_cmodel != CM_MEDIUM_PIC)
5215 /* Functions are never large data. */
5216 if (TREE_CODE (exp) == FUNCTION_DECL)
5219 /* Automatic variables are never large data. */
5220 if (TREE_CODE (exp) == VAR_DECL && !is_global_var (exp))
5223 if (TREE_CODE (exp) == VAR_DECL && DECL_SECTION_NAME (exp))
5225 const char *section = DECL_SECTION_NAME (exp);
5226 if (strcmp (section, ".ldata") == 0
5227 || strcmp (section, ".lbss") == 0)
5233 HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
5235 /* If this is an incomplete type with size 0, then we can't put it
5236 in data because it might be too big when completed. Also,
5237 int_size_in_bytes returns -1 if size can vary or is larger than
5238 an integer in which case also it is safer to assume that it goes in
5240 if (size <= 0 || size > ix86_section_threshold)
5247 /* Switch to the appropriate section for output of DECL.
5248 DECL is either a `VAR_DECL' node or a constant of some sort.
5249 RELOC indicates whether forming the initial value of DECL requires
5250 link-time relocations. */
5252 ATTRIBUTE_UNUSED static section *
5253 x86_64_elf_select_section (tree decl, int reloc,
5254 unsigned HOST_WIDE_INT align)
5256 if (ix86_in_large_data_p (decl))
5258 const char *sname = NULL;
5259 unsigned int flags = SECTION_WRITE;
5260 switch (categorize_decl_for_section (decl, reloc))
5265 case SECCAT_DATA_REL:
5266 sname = ".ldata.rel";
5268 case SECCAT_DATA_REL_LOCAL:
5269 sname = ".ldata.rel.local";
5271 case SECCAT_DATA_REL_RO:
5272 sname = ".ldata.rel.ro";
5274 case SECCAT_DATA_REL_RO_LOCAL:
5275 sname = ".ldata.rel.ro.local";
5279 flags |= SECTION_BSS;
5282 case SECCAT_RODATA_MERGE_STR:
5283 case SECCAT_RODATA_MERGE_STR_INIT:
5284 case SECCAT_RODATA_MERGE_CONST:
5288 case SECCAT_SRODATA:
5295 /* We don't split these for medium model. Place them into
5296 default sections and hope for best. */
5301 /* We might get called with string constants, but get_named_section
5302 doesn't like them as they are not DECLs. Also, we need to set
5303 flags in that case. */
5305 return get_section (sname, flags, NULL);
5306 return get_named_section (decl, sname, reloc);
5309 return default_elf_select_section (decl, reloc, align);
5312 /* Select a set of attributes for section NAME based on the properties
5313 of DECL and whether or not RELOC indicates that DECL's initializer
5314 might contain runtime relocations. */
5316 static unsigned int ATTRIBUTE_UNUSED
5317 x86_64_elf_section_type_flags (tree decl, const char *name, int reloc)
5319 unsigned int flags = default_section_type_flags (decl, name, reloc);
5321 if (decl == NULL_TREE
5322 && (strcmp (name, ".ldata.rel.ro") == 0
5323 || strcmp (name, ".ldata.rel.ro.local") == 0))
5324 flags |= SECTION_RELRO;
5326 if (strcmp (name, ".lbss") == 0
5327 || strncmp (name, ".lbss.", 5) == 0
5328 || strncmp (name, ".gnu.linkonce.lb.", 16) == 0)
5329 flags |= SECTION_BSS;
5334 /* Build up a unique section name, expressed as a
5335 STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
5336 RELOC indicates whether the initial value of EXP requires
5337 link-time relocations. */
5339 static void ATTRIBUTE_UNUSED
5340 x86_64_elf_unique_section (tree decl, int reloc)
5342 if (ix86_in_large_data_p (decl))
5344 const char *prefix = NULL;
5345 /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
5346 bool one_only = DECL_COMDAT_GROUP (decl) && !HAVE_COMDAT_GROUP;
5348 switch (categorize_decl_for_section (decl, reloc))
5351 case SECCAT_DATA_REL:
5352 case SECCAT_DATA_REL_LOCAL:
5353 case SECCAT_DATA_REL_RO:
5354 case SECCAT_DATA_REL_RO_LOCAL:
5355 prefix = one_only ? ".ld" : ".ldata";
5358 prefix = one_only ? ".lb" : ".lbss";
5361 case SECCAT_RODATA_MERGE_STR:
5362 case SECCAT_RODATA_MERGE_STR_INIT:
5363 case SECCAT_RODATA_MERGE_CONST:
5364 prefix = one_only ? ".lr" : ".lrodata";
5366 case SECCAT_SRODATA:
5373 /* We don't split these for medium model. Place them into
5374 default sections and hope for best. */
5379 const char *name, *linkonce;
5382 name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
5383 name = targetm.strip_name_encoding (name);
5385 /* If we're using one_only, then there needs to be a .gnu.linkonce
5386 prefix to the section name. */
5387 linkonce = one_only ? ".gnu.linkonce" : "";
5389 string = ACONCAT ((linkonce, prefix, ".", name, NULL));
5391 set_decl_section_name (decl, string);
5395 default_unique_section (decl, reloc);
5398 #ifdef COMMON_ASM_OP
5399 /* This says how to output assembler code to declare an
5400 uninitialized external linkage data object.
5402 For medium model x86-64 we need to use .largecomm opcode for
5405 x86_elf_aligned_common (FILE *file,
5406 const char *name, unsigned HOST_WIDE_INT size,
5409 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
5410 && size > (unsigned int)ix86_section_threshold)
5411 fputs ("\t.largecomm\t", file);
5413 fputs (COMMON_ASM_OP, file);
5414 assemble_name (file, name);
5415 fprintf (file, "," HOST_WIDE_INT_PRINT_UNSIGNED ",%u\n",
5416 size, align / BITS_PER_UNIT);
5420 /* Utility function for targets to use in implementing
5421 ASM_OUTPUT_ALIGNED_BSS. */
5424 x86_output_aligned_bss (FILE *file, tree decl, const char *name,
5425 unsigned HOST_WIDE_INT size, int align)
5427 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
5428 && size > (unsigned int)ix86_section_threshold)
5429 switch_to_section (get_named_section (decl, ".lbss", 0));
5431 switch_to_section (bss_section);
5432 ASM_OUTPUT_ALIGN (file, floor_log2 (align / BITS_PER_UNIT));
5433 #ifdef ASM_DECLARE_OBJECT_NAME
5434 last_assemble_variable_decl = decl;
5435 ASM_DECLARE_OBJECT_NAME (file, name, decl);
5437 /* Standard thing is just output label for the object. */
5438 ASM_OUTPUT_LABEL (file, name);
5439 #endif /* ASM_DECLARE_OBJECT_NAME */
5440 ASM_OUTPUT_SKIP (file, size ? size : 1);
5443 /* Decide whether we must probe the stack before any space allocation
5444 on this target. It's essentially TARGET_STACK_PROBE except when
5445 -fstack-check causes the stack to be already probed differently. */
5448 ix86_target_stack_probe (void)
5450 /* Do not probe the stack twice if static stack checking is enabled. */
5451 if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
5454 return TARGET_STACK_PROBE;
5457 /* Decide whether we can make a sibling call to a function. DECL is the
5458 declaration of the function being targeted by the call and EXP is the
5459 CALL_EXPR representing the call. */
5462 ix86_function_ok_for_sibcall (tree decl, tree exp)
5464 tree type, decl_or_type;
5467 /* If we are generating position-independent code, we cannot sibcall
5468 optimize any indirect call, or a direct call to a global function,
5469 as the PLT requires %ebx be live. (Darwin does not have a PLT.) */
5473 && (!decl || !targetm.binds_local_p (decl)))
5476 /* If we need to align the outgoing stack, then sibcalling would
5477 unalign the stack, which may break the called function. */
5478 if (ix86_minimum_incoming_stack_boundary (true)
5479 < PREFERRED_STACK_BOUNDARY)
5484 decl_or_type = decl;
5485 type = TREE_TYPE (decl);
5489 /* We're looking at the CALL_EXPR, we need the type of the function. */
5490 type = CALL_EXPR_FN (exp); /* pointer expression */
5491 type = TREE_TYPE (type); /* pointer type */
5492 type = TREE_TYPE (type); /* function type */
5493 decl_or_type = type;
5496 /* Check that the return value locations are the same. Like
5497 if we are returning floats on the 80387 register stack, we cannot
5498 make a sibcall from a function that doesn't return a float to a
5499 function that does or, conversely, from a function that does return
5500 a float to a function that doesn't; the necessary stack adjustment
5501 would not be executed. This is also the place we notice
5502 differences in the return value ABI. Note that it is ok for one
5503 of the functions to have void return type as long as the return
5504 value of the other is passed in a register. */
5505 a = ix86_function_value (TREE_TYPE (exp), decl_or_type, false);
5506 b = ix86_function_value (TREE_TYPE (DECL_RESULT (cfun->decl)),
5508 if (STACK_REG_P (a) || STACK_REG_P (b))
5510 if (!rtx_equal_p (a, b))
5513 else if (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (cfun->decl))))
5515 else if (!rtx_equal_p (a, b))
5520 /* The SYSV ABI has more call-clobbered registers;
5521 disallow sibcalls from MS to SYSV. */
5522 if (cfun->machine->call_abi == MS_ABI
5523 && ix86_function_type_abi (type) == SYSV_ABI)
5528 /* If this call is indirect, we'll need to be able to use a
5529 call-clobbered register for the address of the target function.
5530 Make sure that all such registers are not used for passing
5531 parameters. Note that DLLIMPORT functions are indirect. */
5533 || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && DECL_DLLIMPORT_P (decl)))
5535 if (ix86_function_regparm (type, NULL) >= 3)
5537 /* ??? Need to count the actual number of registers to be used,
5538 not the possible number of registers. Fix later. */
5544 /* Otherwise okay. That also includes certain types of indirect calls. */
5548 /* Handle "cdecl", "stdcall", "fastcall", "regparm", "thiscall",
5549 and "sseregparm" calling convention attributes;
5550 arguments as in struct attribute_spec.handler. */
5553 ix86_handle_cconv_attribute (tree *node, tree name,
5558 if (TREE_CODE (*node) != FUNCTION_TYPE
5559 && TREE_CODE (*node) != METHOD_TYPE
5560 && TREE_CODE (*node) != FIELD_DECL
5561 && TREE_CODE (*node) != TYPE_DECL)
5563 warning (OPT_Wattributes, "%qE attribute only applies to functions",
5565 *no_add_attrs = true;
5569 /* Can combine regparm with all attributes but fastcall, and thiscall. */
5570 if (is_attribute_p ("regparm", name))
5574 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
5576 error ("fastcall and regparm attributes are not compatible");
5579 if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
5581 error ("regparam and thiscall attributes are not compatible");
5584 cst = TREE_VALUE (args);
5585 if (TREE_CODE (cst) != INTEGER_CST)
5587 warning (OPT_Wattributes,
5588 "%qE attribute requires an integer constant argument",
5590 *no_add_attrs = true;
5592 else if (compare_tree_int (cst, REGPARM_MAX) > 0)
5594 warning (OPT_Wattributes, "argument to %qE attribute larger than %d",
5596 *no_add_attrs = true;
5604 /* Do not warn when emulating the MS ABI. */
5605 if ((TREE_CODE (*node) != FUNCTION_TYPE
5606 && TREE_CODE (*node) != METHOD_TYPE)
5607 || ix86_function_type_abi (*node) != MS_ABI)
5608 warning (OPT_Wattributes, "%qE attribute ignored",
5610 *no_add_attrs = true;
5614 /* Can combine fastcall with stdcall (redundant) and sseregparm. */
5615 if (is_attribute_p ("fastcall", name))
5617 if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
5619 error ("fastcall and cdecl attributes are not compatible");
5621 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
5623 error ("fastcall and stdcall attributes are not compatible");
5625 if (lookup_attribute ("regparm", TYPE_ATTRIBUTES (*node)))
5627 error ("fastcall and regparm attributes are not compatible");
5629 if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
5631 error ("fastcall and thiscall attributes are not compatible");
5635 /* Can combine stdcall with fastcall (redundant), regparm and
5637 else if (is_attribute_p ("stdcall", name))
5639 if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
5641 error ("stdcall and cdecl attributes are not compatible");
5643 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
5645 error ("stdcall and fastcall attributes are not compatible");
5647 if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
5649 error ("stdcall and thiscall attributes are not compatible");
5653 /* Can combine cdecl with regparm and sseregparm. */
5654 else if (is_attribute_p ("cdecl", name))
5656 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
5658 error ("stdcall and cdecl attributes are not compatible");
5660 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
5662 error ("fastcall and cdecl attributes are not compatible");
5664 if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
5666 error ("cdecl and thiscall attributes are not compatible");
5669 else if (is_attribute_p ("thiscall", name))
5671 if (TREE_CODE (*node) != METHOD_TYPE && pedantic)
5672 warning (OPT_Wattributes, "%qE attribute is used for non-class method",
5674 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
5676 error ("stdcall and thiscall attributes are not compatible");
5678 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
5680 error ("fastcall and thiscall attributes are not compatible");
5682 if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
5684 error ("cdecl and thiscall attributes are not compatible");
5688 /* Can combine sseregparm with all attributes. */
5693 /* The transactional memory builtins are implicitly regparm or fastcall
5694 depending on the ABI. Override the generic do-nothing attribute that
5695 these builtins were declared with, and replace it with one of the two
5696 attributes that we expect elsewhere. */
5699 ix86_handle_tm_regparm_attribute (tree *node, tree, tree,
5700 int flags, bool *no_add_attrs)
5704 /* In no case do we want to add the placeholder attribute. */
5705 *no_add_attrs = true;
5707 /* The 64-bit ABI is unchanged for transactional memory. */
5711 /* ??? Is there a better way to validate 32-bit windows? We have
5712 cfun->machine->call_abi, but that seems to be set only for 64-bit. */
5713 if (CHECK_STACK_LIMIT > 0)
5714 alt = tree_cons (get_identifier ("fastcall"), NULL, NULL);
5717 alt = tree_cons (NULL, build_int_cst (NULL, 2), NULL);
5718 alt = tree_cons (get_identifier ("regparm"), alt, NULL);
5720 decl_attributes (node, alt, flags);
5725 /* This function determines from TYPE the calling-convention. */
5728 ix86_get_callcvt (const_tree type)
5730 unsigned int ret = 0;
5735 return IX86_CALLCVT_CDECL;
5737 attrs = TYPE_ATTRIBUTES (type);
5738 if (attrs != NULL_TREE)
5740 if (lookup_attribute ("cdecl", attrs))
5741 ret |= IX86_CALLCVT_CDECL;
5742 else if (lookup_attribute ("stdcall", attrs))
5743 ret |= IX86_CALLCVT_STDCALL;
5744 else if (lookup_attribute ("fastcall", attrs))
5745 ret |= IX86_CALLCVT_FASTCALL;
5746 else if (lookup_attribute ("thiscall", attrs))
5747 ret |= IX86_CALLCVT_THISCALL;
5749 /* Regparam isn't allowed for thiscall and fastcall. */
5750 if ((ret & (IX86_CALLCVT_THISCALL | IX86_CALLCVT_FASTCALL)) == 0)
5752 if (lookup_attribute ("regparm", attrs))
5753 ret |= IX86_CALLCVT_REGPARM;
5754 if (lookup_attribute ("sseregparm", attrs))
5755 ret |= IX86_CALLCVT_SSEREGPARM;
5758 if (IX86_BASE_CALLCVT(ret) != 0)
5762 is_stdarg = stdarg_p (type);
5763 if (TARGET_RTD && !is_stdarg)
5764 return IX86_CALLCVT_STDCALL | ret;
5768 || TREE_CODE (type) != METHOD_TYPE
5769 || ix86_function_type_abi (type) != MS_ABI)
5770 return IX86_CALLCVT_CDECL | ret;
5772 return IX86_CALLCVT_THISCALL;
5775 /* Return 0 if the attributes for two types are incompatible, 1 if they
5776 are compatible, and 2 if they are nearly compatible (which causes a
5777 warning to be generated). */
5780 ix86_comp_type_attributes (const_tree type1, const_tree type2)
5782 unsigned int ccvt1, ccvt2;
5784 if (TREE_CODE (type1) != FUNCTION_TYPE
5785 && TREE_CODE (type1) != METHOD_TYPE)
5788 ccvt1 = ix86_get_callcvt (type1);
5789 ccvt2 = ix86_get_callcvt (type2);
5792 if (ix86_function_regparm (type1, NULL)
5793 != ix86_function_regparm (type2, NULL))
5799 /* Return the regparm value for a function with the indicated TYPE and DECL.
5800 DECL may be NULL when calling function indirectly
5801 or considering a libcall. */
5804 ix86_function_regparm (const_tree type, const_tree decl)
5811 return (ix86_function_type_abi (type) == SYSV_ABI
5812 ? X86_64_REGPARM_MAX : X86_64_MS_REGPARM_MAX);
5813 ccvt = ix86_get_callcvt (type);
5814 regparm = ix86_regparm;
5816 if ((ccvt & IX86_CALLCVT_REGPARM) != 0)
5818 attr = lookup_attribute ("regparm", TYPE_ATTRIBUTES (type));
5821 regparm = TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr)));
5825 else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
5827 else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
5830 /* Use register calling convention for local functions when possible. */
5832 && TREE_CODE (decl) == FUNCTION_DECL)
5834 cgraph_node *target = cgraph_node::get (decl);
5836 target = target->function_symbol ();
5838 /* Caller and callee must agree on the calling convention, so
5839 checking here just optimize means that with
5840 __attribute__((optimize (...))) caller could use regparm convention
5841 and callee not, or vice versa. Instead look at whether the callee
5842 is optimized or not. */
5843 if (target && opt_for_fn (target->decl, optimize)
5844 && !(profile_flag && !flag_fentry))
5846 cgraph_local_info *i = &target->local;
5847 if (i && i->local && i->can_change_signature)
5849 int local_regparm, globals = 0, regno;
5851 /* Make sure no regparm register is taken by a
5852 fixed register variable. */
5853 for (local_regparm = 0; local_regparm < REGPARM_MAX;
5855 if (fixed_regs[local_regparm])
5858 /* We don't want to use regparm(3) for nested functions as
5859 these use a static chain pointer in the third argument. */
5860 if (local_regparm == 3 && DECL_STATIC_CHAIN (target->decl))
5863 /* Save a register for the split stack. */
5864 if (local_regparm == 3 && flag_split_stack)
5867 /* Each fixed register usage increases register pressure,
5868 so less registers should be used for argument passing.
5869 This functionality can be overriden by an explicit
5871 for (regno = AX_REG; regno <= DI_REG; regno++)
5872 if (fixed_regs[regno])
5876 = globals < local_regparm ? local_regparm - globals : 0;
5878 if (local_regparm > regparm)
5879 regparm = local_regparm;
5887 /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
5888 DFmode (2) arguments in SSE registers for a function with the
5889 indicated TYPE and DECL. DECL may be NULL when calling function
5890 indirectly or considering a libcall. Return -1 if any FP parameter
5891 should be rejected by error. This is used in siutation we imply SSE
5892 calling convetion but the function is called from another function with
5893 SSE disabled. Otherwise return 0. */
5896 ix86_function_sseregparm (const_tree type, const_tree decl, bool warn)
5898 gcc_assert (!TARGET_64BIT);
5900 /* Use SSE registers to pass SFmode and DFmode arguments if requested
5901 by the sseregparm attribute. */
5902 if (TARGET_SSEREGPARM
5903 || (type && lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type))))
5910 error ("calling %qD with attribute sseregparm without "
5911 "SSE/SSE2 enabled", decl);
5913 error ("calling %qT with attribute sseregparm without "
5914 "SSE/SSE2 enabled", type);
5925 cgraph_node *target = cgraph_node::get (decl);
5927 target = target->function_symbol ();
5929 /* For local functions, pass up to SSE_REGPARM_MAX SFmode
5930 (and DFmode for SSE2) arguments in SSE registers. */
5932 /* TARGET_SSE_MATH */
5933 && (target_opts_for_fn (target->decl)->x_ix86_fpmath & FPMATH_SSE)
5934 && opt_for_fn (target->decl, optimize)
5935 && !(profile_flag && !flag_fentry))
5937 cgraph_local_info *i = &target->local;
5938 if (i && i->local && i->can_change_signature)
5940 /* Refuse to produce wrong code when local function with SSE enabled
5941 is called from SSE disabled function.
5942 FIXME: We need a way to detect these cases cross-ltrans partition
5943 and avoid using SSE calling conventions on local functions called
5944 from function with SSE disabled. For now at least delay the
5945 warning until we know we are going to produce wrong code.
5947 if (!TARGET_SSE && warn)
5949 return TARGET_SSE2_P (target_opts_for_fn (target->decl)
5950 ->x_ix86_isa_flags) ? 2 : 1;
5957 /* Return true if EAX is live at the start of the function. Used by
5958 ix86_expand_prologue to determine if we need special help before
5959 calling allocate_stack_worker. */
5962 ix86_eax_live_at_start_p (void)
5964 /* Cheat. Don't bother working forward from ix86_function_regparm
5965 to the function type to whether an actual argument is located in
5966 eax. Instead just look at cfg info, which is still close enough
5967 to correct at this point. This gives false positives for broken
5968 functions that might use uninitialized data that happens to be
5969 allocated in eax, but who cares? */
5970 return REGNO_REG_SET_P (df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun)), 0);
5974 ix86_keep_aggregate_return_pointer (tree fntype)
5980 attr = lookup_attribute ("callee_pop_aggregate_return",
5981 TYPE_ATTRIBUTES (fntype));
5983 return (TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr))) == 0);
5985 /* For 32-bit MS-ABI the default is to keep aggregate
5987 if (ix86_function_type_abi (fntype) == MS_ABI)
5990 return KEEP_AGGREGATE_RETURN_POINTER != 0;
5993 /* Value is the number of bytes of arguments automatically
5994 popped when returning from a subroutine call.
5995 FUNDECL is the declaration node of the function (as a tree),
5996 FUNTYPE is the data type of the function (as a tree),
5997 or for a library call it is an identifier node for the subroutine name.
5998 SIZE is the number of bytes of arguments passed on the stack.
6000 On the 80386, the RTD insn may be used to pop them if the number
6001 of args is fixed, but if the number is variable then the caller
6002 must pop them all. RTD can't be used for library calls now
6003 because the library is compiled with the Unix compiler.
6004 Use of RTD is a selectable option, since it is incompatible with
6005 standard Unix calling sequences. If the option is not selected,
6006 the caller must always pop the args.
6008 The attribute stdcall is equivalent to RTD on a per module basis. */
6011 ix86_return_pops_args (tree fundecl, tree funtype, int size)
6015 /* None of the 64-bit ABIs pop arguments. */
6019 ccvt = ix86_get_callcvt (funtype);
6021 if ((ccvt & (IX86_CALLCVT_STDCALL | IX86_CALLCVT_FASTCALL
6022 | IX86_CALLCVT_THISCALL)) != 0
6023 && ! stdarg_p (funtype))
6026 /* Lose any fake structure return argument if it is passed on the stack. */
6027 if (aggregate_value_p (TREE_TYPE (funtype), fundecl)
6028 && !ix86_keep_aggregate_return_pointer (funtype))
6030 int nregs = ix86_function_regparm (funtype, fundecl);
6032 return GET_MODE_SIZE (Pmode);
6038 /* Implement the TARGET_LEGITIMATE_COMBINED_INSN hook. */
6041 ix86_legitimate_combined_insn (rtx_insn *insn)
6043 /* Check operand constraints in case hard registers were propagated
6044 into insn pattern. This check prevents combine pass from
6045 generating insn patterns with invalid hard register operands.
6046 These invalid insns can eventually confuse reload to error out
6047 with a spill failure. See also PRs 46829 and 46843. */
6048 if ((INSN_CODE (insn) = recog (PATTERN (insn), insn, 0)) >= 0)
6052 extract_insn (insn);
6053 preprocess_constraints (insn);
6055 int n_operands = recog_data.n_operands;
6056 int n_alternatives = recog_data.n_alternatives;
6057 for (i = 0; i < n_operands; i++)
6059 rtx op = recog_data.operand[i];
6060 machine_mode mode = GET_MODE (op);
6061 const operand_alternative *op_alt;
6066 /* For pre-AVX disallow unaligned loads/stores where the
6067 instructions don't support it. */
6069 && VECTOR_MODE_P (GET_MODE (op))
6070 && misaligned_operand (op, GET_MODE (op)))
6072 int min_align = get_attr_ssememalign (insn);
6077 /* A unary operator may be accepted by the predicate, but it
6078 is irrelevant for matching constraints. */
6082 if (GET_CODE (op) == SUBREG)
6084 if (REG_P (SUBREG_REG (op))
6085 && REGNO (SUBREG_REG (op)) < FIRST_PSEUDO_REGISTER)
6086 offset = subreg_regno_offset (REGNO (SUBREG_REG (op)),
6087 GET_MODE (SUBREG_REG (op)),
6090 op = SUBREG_REG (op);
6093 if (!(REG_P (op) && HARD_REGISTER_P (op)))
6096 op_alt = recog_op_alt;
6098 /* Operand has no constraints, anything is OK. */
6099 win = !n_alternatives;
6101 alternative_mask preferred = get_preferred_alternatives (insn);
6102 for (j = 0; j < n_alternatives; j++, op_alt += n_operands)
6104 if (!TEST_BIT (preferred, j))
6106 if (op_alt[i].anything_ok
6107 || (op_alt[i].matches != -1
6109 (recog_data.operand[i],
6110 recog_data.operand[op_alt[i].matches]))
6111 || reg_fits_class_p (op, op_alt[i].cl, offset, mode))
6126 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
6128 static unsigned HOST_WIDE_INT
6129 ix86_asan_shadow_offset (void)
6131 return TARGET_LP64 ? (TARGET_MACHO ? (HOST_WIDE_INT_1 << 44)
6132 : HOST_WIDE_INT_C (0x7fff8000))
6133 : (HOST_WIDE_INT_1 << 29);
6136 /* Argument support functions. */
6138 /* Return true when register may be used to pass function parameters. */
6140 ix86_function_arg_regno_p (int regno)
6143 enum calling_abi call_abi;
6144 const int *parm_regs;
6146 if (TARGET_MPX && BND_REGNO_P (regno))
6152 return (regno < REGPARM_MAX
6153 || (TARGET_SSE && SSE_REGNO_P (regno) && !fixed_regs[regno]));
6155 return (regno < REGPARM_MAX
6156 || (TARGET_MMX && MMX_REGNO_P (regno)
6157 && (regno < FIRST_MMX_REG + MMX_REGPARM_MAX))
6158 || (TARGET_SSE && SSE_REGNO_P (regno)
6159 && (regno < FIRST_SSE_REG + SSE_REGPARM_MAX)));
6162 if (TARGET_SSE && SSE_REGNO_P (regno)
6163 && (regno < FIRST_SSE_REG + SSE_REGPARM_MAX))
6166 /* TODO: The function should depend on current function ABI but
6167 builtins.c would need updating then. Therefore we use the
6169 call_abi = ix86_cfun_abi ();
6171 /* RAX is used as hidden argument to va_arg functions. */
6172 if (call_abi == SYSV_ABI && regno == AX_REG)
6175 if (call_abi == MS_ABI)
6176 parm_regs = x86_64_ms_abi_int_parameter_registers;
6178 parm_regs = x86_64_int_parameter_registers;
6180 for (i = 0; i < (call_abi == MS_ABI
6181 ? X86_64_MS_REGPARM_MAX : X86_64_REGPARM_MAX); i++)
6182 if (regno == parm_regs[i])
6187 /* Return if we do not know how to pass TYPE solely in registers. */
6190 ix86_must_pass_in_stack (machine_mode mode, const_tree type)
6192 if (must_pass_in_stack_var_size_or_pad (mode, type))
6195 /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
6196 The layout_type routine is crafty and tries to trick us into passing
6197 currently unsupported vector types on the stack by using TImode. */
6198 return (!TARGET_64BIT && mode == TImode
6199 && type && TREE_CODE (type) != VECTOR_TYPE);
6202 /* It returns the size, in bytes, of the area reserved for arguments passed
6203 in registers for the function represented by fndecl dependent to the used
6206 ix86_reg_parm_stack_space (const_tree fndecl)
6208 enum calling_abi call_abi = SYSV_ABI;
6209 if (fndecl != NULL_TREE && TREE_CODE (fndecl) == FUNCTION_DECL)
6210 call_abi = ix86_function_abi (fndecl);
6212 call_abi = ix86_function_type_abi (fndecl);
6213 if (TARGET_64BIT && call_abi == MS_ABI)
6218 /* Returns value SYSV_ABI, MS_ABI dependent on fntype, specifying the
6221 ix86_function_type_abi (const_tree fntype)
6223 if (fntype != NULL_TREE && TYPE_ATTRIBUTES (fntype) != NULL_TREE)
6225 enum calling_abi abi = ix86_abi;
6226 if (abi == SYSV_ABI)
6228 if (lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (fntype)))
6232 static bool warned = false;
6235 error ("X32 does not support ms_abi attribute");
6242 else if (lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (fntype)))
6249 /* We add this as a workaround in order to use libc_has_function
6252 ix86_libc_has_function (enum function_class fn_class)
6254 return targetm.libc_has_function (fn_class);
6258 ix86_function_ms_hook_prologue (const_tree fn)
6260 if (fn && lookup_attribute ("ms_hook_prologue", DECL_ATTRIBUTES (fn)))
6262 if (decl_function_context (fn) != NULL_TREE)
6263 error_at (DECL_SOURCE_LOCATION (fn),
6264 "ms_hook_prologue is not compatible with nested function");
6271 static enum calling_abi
6272 ix86_function_abi (const_tree fndecl)
6276 return ix86_function_type_abi (TREE_TYPE (fndecl));
6279 /* Returns value SYSV_ABI, MS_ABI dependent on cfun, specifying the
6282 ix86_cfun_abi (void)
6286 return cfun->machine->call_abi;
6289 /* Write the extra assembler code needed to declare a function properly. */
6292 ix86_asm_output_function_label (FILE *asm_out_file, const char *fname,
6295 bool is_ms_hook = ix86_function_ms_hook_prologue (decl);
6299 int i, filler_count = (TARGET_64BIT ? 32 : 16);
6300 unsigned int filler_cc = 0xcccccccc;
6302 for (i = 0; i < filler_count; i += 4)
6303 fprintf (asm_out_file, ASM_LONG " %#x\n", filler_cc);
6306 #ifdef SUBTARGET_ASM_UNWIND_INIT
6307 SUBTARGET_ASM_UNWIND_INIT (asm_out_file);
6310 ASM_OUTPUT_LABEL (asm_out_file, fname);
6312 /* Output magic byte marker, if hot-patch attribute is set. */
6317 /* leaq [%rsp + 0], %rsp */
6318 asm_fprintf (asm_out_file, ASM_BYTE
6319 "0x48, 0x8d, 0xa4, 0x24, 0x00, 0x00, 0x00, 0x00\n");
6323 /* movl.s %edi, %edi
6325 movl.s %esp, %ebp */
6326 asm_fprintf (asm_out_file, ASM_BYTE
6327 "0x8b, 0xff, 0x55, 0x8b, 0xec\n");
6333 extern void init_regs (void);
6335 /* Implementation of call abi switching target hook. Specific to FNDECL
6336 the specific call register sets are set. See also
6337 ix86_conditional_register_usage for more details. */
6339 ix86_call_abi_override (const_tree fndecl)
6341 if (fndecl == NULL_TREE)
6342 cfun->machine->call_abi = ix86_abi;
6344 cfun->machine->call_abi = ix86_function_type_abi (TREE_TYPE (fndecl));
6347 /* 64-bit MS and SYSV ABI have different set of call used registers. Avoid
6348 expensive re-initialization of init_regs each time we switch function context
6349 since this is needed only during RTL expansion. */
6351 ix86_maybe_switch_abi (void)
6354 call_used_regs[SI_REG] == (cfun->machine->call_abi == MS_ABI))
6358 /* Return 1 if pseudo register should be created and used to hold
6359 GOT address for PIC code. */
6361 ix86_use_pseudo_pic_reg (void)
6364 && (ix86_cmodel == CM_SMALL_PIC
6371 /* Initialize large model PIC register. */
6374 ix86_init_large_pic_reg (unsigned int tmp_regno)
6376 rtx_code_label *label;
6379 gcc_assert (Pmode == DImode);
6380 label = gen_label_rtx ();
6382 LABEL_PRESERVE_P (label) = 1;
6383 tmp_reg = gen_rtx_REG (Pmode, tmp_regno);
6384 gcc_assert (REGNO (pic_offset_table_rtx) != tmp_regno);
6385 emit_insn (gen_set_rip_rex64 (pic_offset_table_rtx,
6387 emit_insn (gen_set_got_offset_rex64 (tmp_reg, label));
6388 emit_insn (ix86_gen_add3 (pic_offset_table_rtx,
6389 pic_offset_table_rtx, tmp_reg));
6392 /* Create and initialize PIC register if required. */
6394 ix86_init_pic_reg (void)
6399 if (!ix86_use_pseudo_pic_reg ())
6406 if (ix86_cmodel == CM_LARGE_PIC)
6407 ix86_init_large_pic_reg (R11_REG);
6409 emit_insn (gen_set_got_rex64 (pic_offset_table_rtx));
6413 /* If there is future mcount call in the function it is more profitable
6414 to emit SET_GOT into ABI defined REAL_PIC_OFFSET_TABLE_REGNUM. */
6415 rtx reg = crtl->profile
6416 ? gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM)
6417 : pic_offset_table_rtx;
6418 rtx insn = emit_insn (gen_set_got (reg));
6419 RTX_FRAME_RELATED_P (insn) = 1;
6421 emit_move_insn (pic_offset_table_rtx, reg);
6422 add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
6428 entry_edge = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
6429 insert_insn_on_edge (seq, entry_edge);
6430 commit_one_edge_insertion (entry_edge);
6433 /* Initialize a variable CUM of type CUMULATIVE_ARGS
6434 for a call to a function whose data type is FNTYPE.
6435 For a library call, FNTYPE is 0. */
6438 init_cumulative_args (CUMULATIVE_ARGS *cum, /* Argument info to initialize */
6439 tree fntype, /* tree ptr for function decl */
6440 rtx libname, /* SYMBOL_REF of library name or 0 */
6444 struct cgraph_local_info *i = NULL;
6445 struct cgraph_node *target = NULL;
6447 memset (cum, 0, sizeof (*cum));
6451 target = cgraph_node::get (fndecl);
6454 target = target->function_symbol ();
6455 i = cgraph_node::local_info (target->decl);
6456 cum->call_abi = ix86_function_abi (target->decl);
6459 cum->call_abi = ix86_function_abi (fndecl);
6462 cum->call_abi = ix86_function_type_abi (fntype);
6464 cum->caller = caller;
6466 /* Set up the number of registers to use for passing arguments. */
6467 cum->nregs = ix86_regparm;
6470 cum->nregs = (cum->call_abi == SYSV_ABI
6471 ? X86_64_REGPARM_MAX
6472 : X86_64_MS_REGPARM_MAX);
6476 cum->sse_nregs = SSE_REGPARM_MAX;
6479 cum->sse_nregs = (cum->call_abi == SYSV_ABI
6480 ? X86_64_SSE_REGPARM_MAX
6481 : X86_64_MS_SSE_REGPARM_MAX);
6485 cum->mmx_nregs = MMX_REGPARM_MAX;
6486 cum->warn_avx512f = true;
6487 cum->warn_avx = true;
6488 cum->warn_sse = true;
6489 cum->warn_mmx = true;
6491 /* Because type might mismatch in between caller and callee, we need to
6492 use actual type of function for local calls.
6493 FIXME: cgraph_analyze can be told to actually record if function uses
6494 va_start so for local functions maybe_vaarg can be made aggressive
6496 FIXME: once typesytem is fixed, we won't need this code anymore. */
6497 if (i && i->local && i->can_change_signature)
6498 fntype = TREE_TYPE (target->decl);
6499 cum->stdarg = stdarg_p (fntype);
6500 cum->maybe_vaarg = (fntype
6501 ? (!prototype_p (fntype) || stdarg_p (fntype))
6504 cum->bnd_regno = FIRST_BND_REG;
6505 cum->bnds_in_bt = 0;
6506 cum->force_bnd_pass = 0;
6511 /* If there are variable arguments, then we won't pass anything
6512 in registers in 32-bit mode. */
6513 if (stdarg_p (fntype))
6518 cum->warn_avx512f = false;
6519 cum->warn_avx = false;
6520 cum->warn_sse = false;
6521 cum->warn_mmx = false;
6525 /* Use ecx and edx registers if function has fastcall attribute,
6526 else look for regparm information. */
6529 unsigned int ccvt = ix86_get_callcvt (fntype);
6530 if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
6533 cum->fastcall = 1; /* Same first register as in fastcall. */
6535 else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
6541 cum->nregs = ix86_function_regparm (fntype, fndecl);
6544 /* Set up the number of SSE registers used for passing SFmode
6545 and DFmode arguments. Warn for mismatching ABI. */
6546 cum->float_in_sse = ix86_function_sseregparm (fntype, fndecl, true);
6550 /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
6551 But in the case of vector types, it is some vector mode.
6553 When we have only some of our vector isa extensions enabled, then there
6554 are some modes for which vector_mode_supported_p is false. For these
6555 modes, the generic vector support in gcc will choose some non-vector mode
6556 in order to implement the type. By computing the natural mode, we'll
6557 select the proper ABI location for the operand and not depend on whatever
6558 the middle-end decides to do with these vector types.
6560 The midde-end can't deal with the vector types > 16 bytes. In this
6561 case, we return the original mode and warn ABI change if CUM isn't
6564 If INT_RETURN is true, warn ABI change if the vector mode isn't
6565 available for function return value. */
6568 type_natural_mode (const_tree type, const CUMULATIVE_ARGS *cum,
6571 machine_mode mode = TYPE_MODE (type);
6573 if (TREE_CODE (type) == VECTOR_TYPE && !VECTOR_MODE_P (mode))
6575 HOST_WIDE_INT size = int_size_in_bytes (type);
6576 if ((size == 8 || size == 16 || size == 32 || size == 64)
6577 /* ??? Generic code allows us to create width 1 vectors. Ignore. */
6578 && TYPE_VECTOR_SUBPARTS (type) > 1)
6580 machine_mode innermode = TYPE_MODE (TREE_TYPE (type));
6582 if (TREE_CODE (TREE_TYPE (type)) == REAL_TYPE)
6583 mode = MIN_MODE_VECTOR_FLOAT;
6585 mode = MIN_MODE_VECTOR_INT;
6587 /* Get the mode which has this inner mode and number of units. */
6588 for (; mode != VOIDmode; mode = GET_MODE_WIDER_MODE (mode))
6589 if (GET_MODE_NUNITS (mode) == TYPE_VECTOR_SUBPARTS (type)
6590 && GET_MODE_INNER (mode) == innermode)
6592 if (size == 64 && !TARGET_AVX512F)
6594 static bool warnedavx512f;
6595 static bool warnedavx512f_ret;
6597 if (cum && cum->warn_avx512f && !warnedavx512f)
6599 if (warning (OPT_Wpsabi, "AVX512F vector argument "
6600 "without AVX512F enabled changes the ABI"))
6601 warnedavx512f = true;
6603 else if (in_return && !warnedavx512f_ret)
6605 if (warning (OPT_Wpsabi, "AVX512F vector return "
6606 "without AVX512F enabled changes the ABI"))
6607 warnedavx512f_ret = true;
6610 return TYPE_MODE (type);
6612 else if (size == 32 && !TARGET_AVX)
6614 static bool warnedavx;
6615 static bool warnedavx_ret;
6617 if (cum && cum->warn_avx && !warnedavx)
6619 if (warning (OPT_Wpsabi, "AVX vector argument "
6620 "without AVX enabled changes the ABI"))
6623 else if (in_return && !warnedavx_ret)
6625 if (warning (OPT_Wpsabi, "AVX vector return "
6626 "without AVX enabled changes the ABI"))
6627 warnedavx_ret = true;
6630 return TYPE_MODE (type);
6632 else if (((size == 8 && TARGET_64BIT) || size == 16)
6635 static bool warnedsse;
6636 static bool warnedsse_ret;
6638 if (cum && cum->warn_sse && !warnedsse)
6640 if (warning (OPT_Wpsabi, "SSE vector argument "
6641 "without SSE enabled changes the ABI"))
6644 else if (!TARGET_64BIT && in_return && !warnedsse_ret)
6646 if (warning (OPT_Wpsabi, "SSE vector return "
6647 "without SSE enabled changes the ABI"))
6648 warnedsse_ret = true;
6651 else if ((size == 8 && !TARGET_64BIT) && !TARGET_MMX)
6653 static bool warnedmmx;
6654 static bool warnedmmx_ret;
6656 if (cum && cum->warn_mmx && !warnedmmx)
6658 if (warning (OPT_Wpsabi, "MMX vector argument "
6659 "without MMX enabled changes the ABI"))
6662 else if (in_return && !warnedmmx_ret)
6664 if (warning (OPT_Wpsabi, "MMX vector return "
6665 "without MMX enabled changes the ABI"))
6666 warnedmmx_ret = true;
6679 /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
6680 this may not agree with the mode that the type system has chosen for the
6681 register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
6682 go ahead and use it. Otherwise we have to build a PARALLEL instead. */
6685 gen_reg_or_parallel (machine_mode mode, machine_mode orig_mode,
6690 if (orig_mode != BLKmode)
6691 tmp = gen_rtx_REG (orig_mode, regno);
6694 tmp = gen_rtx_REG (mode, regno);
6695 tmp = gen_rtx_EXPR_LIST (VOIDmode, tmp, const0_rtx);
6696 tmp = gen_rtx_PARALLEL (orig_mode, gen_rtvec (1, tmp));
6702 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
6703 of this code is to classify each 8bytes of incoming argument by the register
6704 class and assign registers accordingly. */
6706 /* Return the union class of CLASS1 and CLASS2.
6707 See the x86-64 PS ABI for details. */
6709 static enum x86_64_reg_class
6710 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
6712 /* Rule #1: If both classes are equal, this is the resulting class. */
6713 if (class1 == class2)
6716 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
6718 if (class1 == X86_64_NO_CLASS)
6720 if (class2 == X86_64_NO_CLASS)
6723 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
6724 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
6725 return X86_64_MEMORY_CLASS;
6727 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
6728 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
6729 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
6730 return X86_64_INTEGERSI_CLASS;
6731 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
6732 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
6733 return X86_64_INTEGER_CLASS;
6735 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
6737 if (class1 == X86_64_X87_CLASS
6738 || class1 == X86_64_X87UP_CLASS
6739 || class1 == X86_64_COMPLEX_X87_CLASS
6740 || class2 == X86_64_X87_CLASS
6741 || class2 == X86_64_X87UP_CLASS
6742 || class2 == X86_64_COMPLEX_X87_CLASS)
6743 return X86_64_MEMORY_CLASS;
6745 /* Rule #6: Otherwise class SSE is used. */
6746 return X86_64_SSE_CLASS;
6749 /* Classify the argument of type TYPE and mode MODE.
6750 CLASSES will be filled by the register class used to pass each word
6751 of the operand. The number of words is returned. In case the parameter
6752 should be passed in memory, 0 is returned. As a special case for zero
6753 sized containers, classes[0] will be NO_CLASS and 1 is returned.
6755 BIT_OFFSET is used internally for handling records and specifies offset
6756 of the offset in bits modulo 512 to avoid overflow cases.
6758 See the x86-64 PS ABI for details.
6762 classify_argument (machine_mode mode, const_tree type,
6763 enum x86_64_reg_class classes[MAX_CLASSES], int bit_offset)
6765 HOST_WIDE_INT bytes =
6766 (mode == BLKmode) ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
6768 = (bytes + (bit_offset % 64) / 8 + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
6770 /* Variable sized entities are always passed/returned in memory. */
6774 if (mode != VOIDmode
6775 && targetm.calls.must_pass_in_stack (mode, type))
6778 if (type && AGGREGATE_TYPE_P (type))
6782 enum x86_64_reg_class subclasses[MAX_CLASSES];
6784 /* On x86-64 we pass structures larger than 64 bytes on the stack. */
6788 for (i = 0; i < words; i++)
6789 classes[i] = X86_64_NO_CLASS;
6791 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
6792 signalize memory class, so handle it as special case. */
6795 classes[0] = X86_64_NO_CLASS;
6799 /* Classify each field of record and merge classes. */
6800 switch (TREE_CODE (type))
6803 /* And now merge the fields of structure. */
6804 for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
6806 if (TREE_CODE (field) == FIELD_DECL)
6810 if (TREE_TYPE (field) == error_mark_node)
6813 /* Bitfields are always classified as integer. Handle them
6814 early, since later code would consider them to be
6815 misaligned integers. */
6816 if (DECL_BIT_FIELD (field))
6818 for (i = (int_bit_position (field)
6819 + (bit_offset % 64)) / 8 / 8;
6820 i < ((int_bit_position (field) + (bit_offset % 64))
6821 + tree_to_shwi (DECL_SIZE (field))
6824 merge_classes (X86_64_INTEGER_CLASS,
6831 type = TREE_TYPE (field);
6833 /* Flexible array member is ignored. */
6834 if (TYPE_MODE (type) == BLKmode
6835 && TREE_CODE (type) == ARRAY_TYPE
6836 && TYPE_SIZE (type) == NULL_TREE
6837 && TYPE_DOMAIN (type) != NULL_TREE
6838 && (TYPE_MAX_VALUE (TYPE_DOMAIN (type))
6843 if (!warned && warn_psabi)
6846 inform (input_location,
6847 "the ABI of passing struct with"
6848 " a flexible array member has"
6849 " changed in GCC 4.4");
6853 num = classify_argument (TYPE_MODE (type), type,
6855 (int_bit_position (field)
6856 + bit_offset) % 512);
6859 pos = (int_bit_position (field)
6860 + (bit_offset % 64)) / 8 / 8;
6861 for (i = 0; i < num && (i + pos) < words; i++)
6863 merge_classes (subclasses[i], classes[i + pos]);
6870 /* Arrays are handled as small records. */
6873 num = classify_argument (TYPE_MODE (TREE_TYPE (type)),
6874 TREE_TYPE (type), subclasses, bit_offset);
6878 /* The partial classes are now full classes. */
6879 if (subclasses[0] == X86_64_SSESF_CLASS && bytes != 4)
6880 subclasses[0] = X86_64_SSE_CLASS;
6881 if (subclasses[0] == X86_64_INTEGERSI_CLASS
6882 && !((bit_offset % 64) == 0 && bytes == 4))
6883 subclasses[0] = X86_64_INTEGER_CLASS;
6885 for (i = 0; i < words; i++)
6886 classes[i] = subclasses[i % num];
6891 case QUAL_UNION_TYPE:
6892 /* Unions are similar to RECORD_TYPE but offset is always 0.
6894 for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
6896 if (TREE_CODE (field) == FIELD_DECL)
6900 if (TREE_TYPE (field) == error_mark_node)
6903 num = classify_argument (TYPE_MODE (TREE_TYPE (field)),
6904 TREE_TYPE (field), subclasses,
6908 for (i = 0; i < num && i < words; i++)
6909 classes[i] = merge_classes (subclasses[i], classes[i]);
6920 /* When size > 16 bytes, if the first one isn't
6921 X86_64_SSE_CLASS or any other ones aren't
6922 X86_64_SSEUP_CLASS, everything should be passed in
6924 if (classes[0] != X86_64_SSE_CLASS)
6927 for (i = 1; i < words; i++)
6928 if (classes[i] != X86_64_SSEUP_CLASS)
6932 /* Final merger cleanup. */
6933 for (i = 0; i < words; i++)
6935 /* If one class is MEMORY, everything should be passed in
6937 if (classes[i] == X86_64_MEMORY_CLASS)
6940 /* The X86_64_SSEUP_CLASS should be always preceded by
6941 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
6942 if (classes[i] == X86_64_SSEUP_CLASS
6943 && classes[i - 1] != X86_64_SSE_CLASS
6944 && classes[i - 1] != X86_64_SSEUP_CLASS)
6946 /* The first one should never be X86_64_SSEUP_CLASS. */
6947 gcc_assert (i != 0);
6948 classes[i] = X86_64_SSE_CLASS;
6951 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
6952 everything should be passed in memory. */
6953 if (classes[i] == X86_64_X87UP_CLASS
6954 && (classes[i - 1] != X86_64_X87_CLASS))
6958 /* The first one should never be X86_64_X87UP_CLASS. */
6959 gcc_assert (i != 0);
6960 if (!warned && warn_psabi)
6963 inform (input_location,
6964 "the ABI of passing union with long double"
6965 " has changed in GCC 4.4");
6973 /* Compute alignment needed. We align all types to natural boundaries with
6974 exception of XFmode that is aligned to 64bits. */
6975 if (mode != VOIDmode && mode != BLKmode)
6977 int mode_alignment = GET_MODE_BITSIZE (mode);
6980 mode_alignment = 128;
6981 else if (mode == XCmode)
6982 mode_alignment = 256;
6983 if (COMPLEX_MODE_P (mode))
6984 mode_alignment /= 2;
6985 /* Misaligned fields are always returned in memory. */
6986 if (bit_offset % mode_alignment)
6990 /* for V1xx modes, just use the base mode */
6991 if (VECTOR_MODE_P (mode) && mode != V1DImode && mode != V1TImode
6992 && GET_MODE_SIZE (GET_MODE_INNER (mode)) == bytes)
6993 mode = GET_MODE_INNER (mode);
6995 /* Classification of atomic types. */
7000 classes[0] = X86_64_SSE_CLASS;
7003 classes[0] = X86_64_SSE_CLASS;
7004 classes[1] = X86_64_SSEUP_CLASS;
7014 int size = bit_offset + (int) GET_MODE_BITSIZE (mode);
7016 /* Analyze last 128 bits only. */
7017 size = (size - 1) & 0x7f;
7021 classes[0] = X86_64_INTEGERSI_CLASS;
7026 classes[0] = X86_64_INTEGER_CLASS;
7029 else if (size < 64+32)
7031 classes[0] = X86_64_INTEGER_CLASS;
7032 classes[1] = X86_64_INTEGERSI_CLASS;
7035 else if (size < 64+64)
7037 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
7045 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
7049 /* OImode shouldn't be used directly. */
7054 if (!(bit_offset % 64))
7055 classes[0] = X86_64_SSESF_CLASS;
7057 classes[0] = X86_64_SSE_CLASS;
7060 classes[0] = X86_64_SSEDF_CLASS;
7063 classes[0] = X86_64_X87_CLASS;
7064 classes[1] = X86_64_X87UP_CLASS;
7067 classes[0] = X86_64_SSE_CLASS;
7068 classes[1] = X86_64_SSEUP_CLASS;
7071 classes[0] = X86_64_SSE_CLASS;
7072 if (!(bit_offset % 64))
7078 if (!warned && warn_psabi)
7081 inform (input_location,
7082 "the ABI of passing structure with complex float"
7083 " member has changed in GCC 4.4");
7085 classes[1] = X86_64_SSESF_CLASS;
7089 classes[0] = X86_64_SSEDF_CLASS;
7090 classes[1] = X86_64_SSEDF_CLASS;
7093 classes[0] = X86_64_COMPLEX_X87_CLASS;
7096 /* This modes is larger than 16 bytes. */
7104 classes[0] = X86_64_SSE_CLASS;
7105 classes[1] = X86_64_SSEUP_CLASS;
7106 classes[2] = X86_64_SSEUP_CLASS;
7107 classes[3] = X86_64_SSEUP_CLASS;
7115 classes[0] = X86_64_SSE_CLASS;
7116 classes[1] = X86_64_SSEUP_CLASS;
7117 classes[2] = X86_64_SSEUP_CLASS;
7118 classes[3] = X86_64_SSEUP_CLASS;
7119 classes[4] = X86_64_SSEUP_CLASS;
7120 classes[5] = X86_64_SSEUP_CLASS;
7121 classes[6] = X86_64_SSEUP_CLASS;
7122 classes[7] = X86_64_SSEUP_CLASS;
7130 classes[0] = X86_64_SSE_CLASS;
7131 classes[1] = X86_64_SSEUP_CLASS;
7139 classes[0] = X86_64_SSE_CLASS;
7145 gcc_assert (VECTOR_MODE_P (mode));
7150 gcc_assert (GET_MODE_CLASS (GET_MODE_INNER (mode)) == MODE_INT);
7152 if (bit_offset + GET_MODE_BITSIZE (mode) <= 32)
7153 classes[0] = X86_64_INTEGERSI_CLASS;
7155 classes[0] = X86_64_INTEGER_CLASS;
7156 classes[1] = X86_64_INTEGER_CLASS;
7157 return 1 + (bytes > 8);
7161 /* Examine the argument and return set number of register required in each
7162 class. Return true iff parameter should be passed in memory. */
7165 examine_argument (machine_mode mode, const_tree type, int in_return,
7166 int *int_nregs, int *sse_nregs)
7168 enum x86_64_reg_class regclass[MAX_CLASSES];
7169 int n = classify_argument (mode, type, regclass, 0);
7176 for (n--; n >= 0; n--)
7177 switch (regclass[n])
7179 case X86_64_INTEGER_CLASS:
7180 case X86_64_INTEGERSI_CLASS:
7183 case X86_64_SSE_CLASS:
7184 case X86_64_SSESF_CLASS:
7185 case X86_64_SSEDF_CLASS:
7188 case X86_64_NO_CLASS:
7189 case X86_64_SSEUP_CLASS:
7191 case X86_64_X87_CLASS:
7192 case X86_64_X87UP_CLASS:
7193 case X86_64_COMPLEX_X87_CLASS:
7197 case X86_64_MEMORY_CLASS:
7204 /* Construct container for the argument used by GCC interface. See
7205 FUNCTION_ARG for the detailed description. */
7208 construct_container (machine_mode mode, machine_mode orig_mode,
7209 const_tree type, int in_return, int nintregs, int nsseregs,
7210 const int *intreg, int sse_regno)
7212 /* The following variables hold the static issued_error state. */
7213 static bool issued_sse_arg_error;
7214 static bool issued_sse_ret_error;
7215 static bool issued_x87_ret_error;
7217 machine_mode tmpmode;
7219 (mode == BLKmode) ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
7220 enum x86_64_reg_class regclass[MAX_CLASSES];
7224 int needed_sseregs, needed_intregs;
7225 rtx exp[MAX_CLASSES];
7228 n = classify_argument (mode, type, regclass, 0);
7231 if (examine_argument (mode, type, in_return, &needed_intregs,
7234 if (needed_intregs > nintregs || needed_sseregs > nsseregs)
7237 /* We allowed the user to turn off SSE for kernel mode. Don't crash if
7238 some less clueful developer tries to use floating-point anyway. */
7239 if (needed_sseregs && !TARGET_SSE)
7243 if (!issued_sse_ret_error)
7245 error ("SSE register return with SSE disabled");
7246 issued_sse_ret_error = true;
7249 else if (!issued_sse_arg_error)
7251 error ("SSE register argument with SSE disabled");
7252 issued_sse_arg_error = true;
7257 /* Likewise, error if the ABI requires us to return values in the
7258 x87 registers and the user specified -mno-80387. */
7259 if (!TARGET_FLOAT_RETURNS_IN_80387 && in_return)
7260 for (i = 0; i < n; i++)
7261 if (regclass[i] == X86_64_X87_CLASS
7262 || regclass[i] == X86_64_X87UP_CLASS
7263 || regclass[i] == X86_64_COMPLEX_X87_CLASS)
7265 if (!issued_x87_ret_error)
7267 error ("x87 register return with x87 disabled");
7268 issued_x87_ret_error = true;
7273 /* First construct simple cases. Avoid SCmode, since we want to use
7274 single register to pass this type. */
7275 if (n == 1 && mode != SCmode)
7276 switch (regclass[0])
7278 case X86_64_INTEGER_CLASS:
7279 case X86_64_INTEGERSI_CLASS:
7280 return gen_rtx_REG (mode, intreg[0]);
7281 case X86_64_SSE_CLASS:
7282 case X86_64_SSESF_CLASS:
7283 case X86_64_SSEDF_CLASS:
7284 if (mode != BLKmode)
7285 return gen_reg_or_parallel (mode, orig_mode,
7286 SSE_REGNO (sse_regno));
7288 case X86_64_X87_CLASS:
7289 case X86_64_COMPLEX_X87_CLASS:
7290 return gen_rtx_REG (mode, FIRST_STACK_REG);
7291 case X86_64_NO_CLASS:
7292 /* Zero sized array, struct or class. */
7298 && regclass[0] == X86_64_SSE_CLASS
7299 && regclass[1] == X86_64_SSEUP_CLASS
7301 return gen_reg_or_parallel (mode, orig_mode,
7302 SSE_REGNO (sse_regno));
7304 && regclass[0] == X86_64_SSE_CLASS
7305 && regclass[1] == X86_64_SSEUP_CLASS
7306 && regclass[2] == X86_64_SSEUP_CLASS
7307 && regclass[3] == X86_64_SSEUP_CLASS
7309 return gen_reg_or_parallel (mode, orig_mode,
7310 SSE_REGNO (sse_regno));
7312 && regclass[0] == X86_64_SSE_CLASS
7313 && regclass[1] == X86_64_SSEUP_CLASS
7314 && regclass[2] == X86_64_SSEUP_CLASS
7315 && regclass[3] == X86_64_SSEUP_CLASS
7316 && regclass[4] == X86_64_SSEUP_CLASS
7317 && regclass[5] == X86_64_SSEUP_CLASS
7318 && regclass[6] == X86_64_SSEUP_CLASS
7319 && regclass[7] == X86_64_SSEUP_CLASS
7321 return gen_reg_or_parallel (mode, orig_mode,
7322 SSE_REGNO (sse_regno));
7324 && regclass[0] == X86_64_X87_CLASS
7325 && regclass[1] == X86_64_X87UP_CLASS)
7326 return gen_rtx_REG (XFmode, FIRST_STACK_REG);
7329 && regclass[0] == X86_64_INTEGER_CLASS
7330 && regclass[1] == X86_64_INTEGER_CLASS
7331 && (mode == CDImode || mode == TImode)
7332 && intreg[0] + 1 == intreg[1])
7333 return gen_rtx_REG (mode, intreg[0]);
7335 /* Otherwise figure out the entries of the PARALLEL. */
7336 for (i = 0; i < n; i++)
7340 switch (regclass[i])
7342 case X86_64_NO_CLASS:
7344 case X86_64_INTEGER_CLASS:
7345 case X86_64_INTEGERSI_CLASS:
7346 /* Merge TImodes on aligned occasions here too. */
7347 if (i * 8 + 8 > bytes)
7349 = mode_for_size ((bytes - i * 8) * BITS_PER_UNIT, MODE_INT, 0);
7350 else if (regclass[i] == X86_64_INTEGERSI_CLASS)
7354 /* We've requested 24 bytes we
7355 don't have mode for. Use DImode. */
7356 if (tmpmode == BLKmode)
7359 = gen_rtx_EXPR_LIST (VOIDmode,
7360 gen_rtx_REG (tmpmode, *intreg),
7364 case X86_64_SSESF_CLASS:
7366 = gen_rtx_EXPR_LIST (VOIDmode,
7367 gen_rtx_REG (SFmode,
7368 SSE_REGNO (sse_regno)),
7372 case X86_64_SSEDF_CLASS:
7374 = gen_rtx_EXPR_LIST (VOIDmode,
7375 gen_rtx_REG (DFmode,
7376 SSE_REGNO (sse_regno)),
7380 case X86_64_SSE_CLASS:
7388 if (i == 0 && regclass[1] == X86_64_SSEUP_CLASS)
7398 && regclass[1] == X86_64_SSEUP_CLASS
7399 && regclass[2] == X86_64_SSEUP_CLASS
7400 && regclass[3] == X86_64_SSEUP_CLASS);
7406 && regclass[1] == X86_64_SSEUP_CLASS
7407 && regclass[2] == X86_64_SSEUP_CLASS
7408 && regclass[3] == X86_64_SSEUP_CLASS
7409 && regclass[4] == X86_64_SSEUP_CLASS
7410 && regclass[5] == X86_64_SSEUP_CLASS
7411 && regclass[6] == X86_64_SSEUP_CLASS
7412 && regclass[7] == X86_64_SSEUP_CLASS);
7420 = gen_rtx_EXPR_LIST (VOIDmode,
7421 gen_rtx_REG (tmpmode,
7422 SSE_REGNO (sse_regno)),
7431 /* Empty aligned struct, union or class. */
7435 ret = gen_rtx_PARALLEL (mode, rtvec_alloc (nexps));
7436 for (i = 0; i < nexps; i++)
7437 XVECEXP (ret, 0, i) = exp [i];
7441 /* Update the data in CUM to advance over an argument of mode MODE
7442 and data type TYPE. (TYPE is null for libcalls where that information
7443 may not be available.)
7445 Return a number of integer regsiters advanced over. */
7448 function_arg_advance_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
7449 const_tree type, HOST_WIDE_INT bytes,
7450 HOST_WIDE_INT words)
7453 bool error_p = NULL;
7469 cum->words += words;
7470 cum->nregs -= words;
7471 cum->regno += words;
7472 if (cum->nregs >= 0)
7474 if (cum->nregs <= 0)
7482 /* OImode shouldn't be used directly. */
7486 if (cum->float_in_sse == -1)
7488 if (cum->float_in_sse < 2)
7491 if (cum->float_in_sse == -1)
7493 if (cum->float_in_sse < 1)
7516 if (!type || !AGGREGATE_TYPE_P (type))
7518 cum->sse_words += words;
7519 cum->sse_nregs -= 1;
7520 cum->sse_regno += 1;
7521 if (cum->sse_nregs <= 0)
7535 if (!type || !AGGREGATE_TYPE_P (type))
7537 cum->mmx_words += words;
7538 cum->mmx_nregs -= 1;
7539 cum->mmx_regno += 1;
7540 if (cum->mmx_nregs <= 0)
7550 cum->float_in_sse = 0;
7551 error ("calling %qD with SSE calling convention without "
7552 "SSE/SSE2 enabled", cum->decl);
7553 sorry ("this is a GCC bug that can be worked around by adding "
7554 "attribute used to function called");
7561 function_arg_advance_64 (CUMULATIVE_ARGS *cum, machine_mode mode,
7562 const_tree type, HOST_WIDE_INT words, bool named)
7564 int int_nregs, sse_nregs;
7566 /* Unnamed 512 and 256bit vector mode parameters are passed on stack. */
7567 if (!named && (VALID_AVX512F_REG_MODE (mode)
7568 || VALID_AVX256_REG_MODE (mode)))
7571 if (!examine_argument (mode, type, 0, &int_nregs, &sse_nregs)
7572 && sse_nregs <= cum->sse_nregs && int_nregs <= cum->nregs)
7574 cum->nregs -= int_nregs;
7575 cum->sse_nregs -= sse_nregs;
7576 cum->regno += int_nregs;
7577 cum->sse_regno += sse_nregs;
7582 int align = ix86_function_arg_boundary (mode, type) / BITS_PER_WORD;
7583 cum->words = (cum->words + align - 1) & ~(align - 1);
7584 cum->words += words;
7590 function_arg_advance_ms_64 (CUMULATIVE_ARGS *cum, HOST_WIDE_INT bytes,
7591 HOST_WIDE_INT words)
7593 /* Otherwise, this should be passed indirect. */
7594 gcc_assert (bytes == 1 || bytes == 2 || bytes == 4 || bytes == 8);
7596 cum->words += words;
7606 /* Update the data in CUM to advance over an argument of mode MODE and
7607 data type TYPE. (TYPE is null for libcalls where that information
7608 may not be available.) */
7611 ix86_function_arg_advance (cumulative_args_t cum_v, machine_mode mode,
7612 const_tree type, bool named)
7614 CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
7615 HOST_WIDE_INT bytes, words;
7618 if (mode == BLKmode)
7619 bytes = int_size_in_bytes (type);
7621 bytes = GET_MODE_SIZE (mode);
7622 words = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
7625 mode = type_natural_mode (type, NULL, false);
7627 if ((type && POINTER_BOUNDS_TYPE_P (type))
7628 || POINTER_BOUNDS_MODE_P (mode))
7630 /* If we pass bounds in BT then just update remained bounds count. */
7631 if (cum->bnds_in_bt)
7637 /* Update remained number of bounds to force. */
7638 if (cum->force_bnd_pass)
7639 cum->force_bnd_pass--;
7646 /* The first arg not going to Bounds Tables resets this counter. */
7647 cum->bnds_in_bt = 0;
7648 /* For unnamed args we always pass bounds to avoid bounds mess when
7649 passed and received types do not match. If bounds do not follow
7650 unnamed arg, still pretend required number of bounds were passed. */
7651 if (cum->force_bnd_pass)
7653 cum->bnd_regno += cum->force_bnd_pass;
7654 cum->force_bnd_pass = 0;
7657 if (TARGET_64BIT && (cum ? cum->call_abi : ix86_abi) == MS_ABI)
7658 nregs = function_arg_advance_ms_64 (cum, bytes, words);
7659 else if (TARGET_64BIT)
7660 nregs = function_arg_advance_64 (cum, mode, type, words, named);
7662 nregs = function_arg_advance_32 (cum, mode, type, bytes, words);
7664 /* For stdarg we expect bounds to be passed for each value passed
7667 cum->force_bnd_pass = nregs;
7668 /* For pointers passed in memory we expect bounds passed in Bounds
7671 cum->bnds_in_bt = chkp_type_bounds_count (type);
7674 /* Define where to put the arguments to a function.
7675 Value is zero to push the argument on the stack,
7676 or a hard register in which to store the argument.
7678 MODE is the argument's machine mode.
7679 TYPE is the data type of the argument (as a tree).
7680 This is null for libcalls where that information may
7682 CUM is a variable of type CUMULATIVE_ARGS which gives info about
7683 the preceding args and about the function being called.
7684 NAMED is nonzero if this argument is a named parameter
7685 (otherwise it is an extra parameter matching an ellipsis). */
7688 function_arg_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
7689 machine_mode orig_mode, const_tree type,
7690 HOST_WIDE_INT bytes, HOST_WIDE_INT words)
7692 bool error_p = false;
7693 /* Avoid the AL settings for the Unix64 ABI. */
7694 if (mode == VOIDmode)
7710 if (words <= cum->nregs)
7712 int regno = cum->regno;
7714 /* Fastcall allocates the first two DWORD (SImode) or
7715 smaller arguments to ECX and EDX if it isn't an
7721 || (type && AGGREGATE_TYPE_P (type)))
7724 /* ECX not EAX is the first allocated register. */
7725 if (regno == AX_REG)
7728 return gen_rtx_REG (mode, regno);
7733 if (cum->float_in_sse == -1)
7735 if (cum->float_in_sse < 2)
7738 if (cum->float_in_sse == -1)
7740 if (cum->float_in_sse < 1)
7744 /* In 32bit, we pass TImode in xmm registers. */
7751 if (!type || !AGGREGATE_TYPE_P (type))
7754 return gen_reg_or_parallel (mode, orig_mode,
7755 cum->sse_regno + FIRST_SSE_REG);
7761 /* OImode and XImode shouldn't be used directly. */
7776 if (!type || !AGGREGATE_TYPE_P (type))
7779 return gen_reg_or_parallel (mode, orig_mode,
7780 cum->sse_regno + FIRST_SSE_REG);
7790 if (!type || !AGGREGATE_TYPE_P (type))
7793 return gen_reg_or_parallel (mode, orig_mode,
7794 cum->mmx_regno + FIRST_MMX_REG);
7800 cum->float_in_sse = 0;
7801 error ("calling %qD with SSE calling convention without "
7802 "SSE/SSE2 enabled", cum->decl);
7803 sorry ("this is a GCC bug that can be worked around by adding "
7804 "attribute used to function called");
7811 function_arg_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
7812 machine_mode orig_mode, const_tree type, bool named)
7814 /* Handle a hidden AL argument containing number of registers
7815 for varargs x86-64 functions. */
7816 if (mode == VOIDmode)
7817 return GEN_INT (cum->maybe_vaarg
7818 ? (cum->sse_nregs < 0
7819 ? X86_64_SSE_REGPARM_MAX
7840 /* Unnamed 256 and 512bit vector mode parameters are passed on stack. */
7846 return construct_container (mode, orig_mode, type, 0, cum->nregs,
7848 &x86_64_int_parameter_registers [cum->regno],
7853 function_arg_ms_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
7854 machine_mode orig_mode, bool named,
7855 HOST_WIDE_INT bytes)
7859 /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
7860 We use value of -2 to specify that current function call is MSABI. */
7861 if (mode == VOIDmode)
7862 return GEN_INT (-2);
7864 /* If we've run out of registers, it goes on the stack. */
7865 if (cum->nregs == 0)
7868 regno = x86_64_ms_abi_int_parameter_registers[cum->regno];
7870 /* Only floating point modes are passed in anything but integer regs. */
7871 if (TARGET_SSE && (mode == SFmode || mode == DFmode))
7874 regno = cum->regno + FIRST_SSE_REG;
7879 /* Unnamed floating parameters are passed in both the
7880 SSE and integer registers. */
7881 t1 = gen_rtx_REG (mode, cum->regno + FIRST_SSE_REG);
7882 t2 = gen_rtx_REG (mode, regno);
7883 t1 = gen_rtx_EXPR_LIST (VOIDmode, t1, const0_rtx);
7884 t2 = gen_rtx_EXPR_LIST (VOIDmode, t2, const0_rtx);
7885 return gen_rtx_PARALLEL (mode, gen_rtvec (2, t1, t2));
7888 /* Handle aggregated types passed in register. */
7889 if (orig_mode == BLKmode)
7891 if (bytes > 0 && bytes <= 8)
7892 mode = (bytes > 4 ? DImode : SImode);
7893 if (mode == BLKmode)
7897 return gen_reg_or_parallel (mode, orig_mode, regno);
7900 /* Return where to put the arguments to a function.
7901 Return zero to push the argument on the stack, or a hard register in which to store the argument.
7903 MODE is the argument's machine mode. TYPE is the data type of the
7904 argument. It is null for libcalls where that information may not be
7905 available. CUM gives information about the preceding args and about
7906 the function being called. NAMED is nonzero if this argument is a
7907 named parameter (otherwise it is an extra parameter matching an
7911 ix86_function_arg (cumulative_args_t cum_v, machine_mode omode,
7912 const_tree type, bool named)
7914 CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
7915 machine_mode mode = omode;
7916 HOST_WIDE_INT bytes, words;
7919 /* All pointer bounds argumntas are handled separately here. */
7920 if ((type && POINTER_BOUNDS_TYPE_P (type))
7921 || POINTER_BOUNDS_MODE_P (mode))
7923 /* Return NULL if bounds are forced to go in Bounds Table. */
7924 if (cum->bnds_in_bt)
7926 /* Return the next available bound reg if any. */
7927 else if (cum->bnd_regno <= LAST_BND_REG)
7928 arg = gen_rtx_REG (BNDmode, cum->bnd_regno);
7929 /* Return the next special slot number otherwise. */
7931 arg = GEN_INT (cum->bnd_regno - LAST_BND_REG - 1);
7936 if (mode == BLKmode)
7937 bytes = int_size_in_bytes (type);
7939 bytes = GET_MODE_SIZE (mode);
7940 words = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
7942 /* To simplify the code below, represent vector types with a vector mode
7943 even if MMX/SSE are not active. */
7944 if (type && TREE_CODE (type) == VECTOR_TYPE)
7945 mode = type_natural_mode (type, cum, false);
7947 if (TARGET_64BIT && (cum ? cum->call_abi : ix86_abi) == MS_ABI)
7948 arg = function_arg_ms_64 (cum, mode, omode, named, bytes);
7949 else if (TARGET_64BIT)
7950 arg = function_arg_64 (cum, mode, omode, type, named);
7952 arg = function_arg_32 (cum, mode, omode, type, bytes, words);
7957 /* A C expression that indicates when an argument must be passed by
7958 reference. If nonzero for an argument, a copy of that argument is
7959 made in memory and a pointer to the argument is passed instead of
7960 the argument itself. The pointer is passed in whatever way is
7961 appropriate for passing a pointer to that type. */
7964 ix86_pass_by_reference (cumulative_args_t cum_v, machine_mode mode,
7965 const_tree type, bool)
7967 CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
7969 /* Bounds are never passed by reference. */
7970 if ((type && POINTER_BOUNDS_TYPE_P (type))
7971 || POINTER_BOUNDS_MODE_P (mode))
7974 /* See Windows x64 Software Convention. */
7975 if (TARGET_64BIT && (cum ? cum->call_abi : ix86_abi) == MS_ABI)
7977 int msize = (int) GET_MODE_SIZE (mode);
7980 /* Arrays are passed by reference. */
7981 if (TREE_CODE (type) == ARRAY_TYPE)
7984 if (AGGREGATE_TYPE_P (type))
7986 /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
7987 are passed by reference. */
7988 msize = int_size_in_bytes (type);
7992 /* __m128 is passed by reference. */
7994 case 1: case 2: case 4: case 8:
8000 else if (TARGET_64BIT && type && int_size_in_bytes (type) == -1)
8006 /* Return true when TYPE should be 128bit aligned for 32bit argument
8007 passing ABI. XXX: This function is obsolete and is only used for
8008 checking psABI compatibility with previous versions of GCC. */
8011 ix86_compat_aligned_value_p (const_tree type)
8013 machine_mode mode = TYPE_MODE (type);
8014 if (((TARGET_SSE && SSE_REG_MODE_P (mode))
8018 && (!TYPE_USER_ALIGN (type) || TYPE_ALIGN (type) > 128))
8020 if (TYPE_ALIGN (type) < 128)
8023 if (AGGREGATE_TYPE_P (type))
8025 /* Walk the aggregates recursively. */
8026 switch (TREE_CODE (type))
8030 case QUAL_UNION_TYPE:
8034 /* Walk all the structure fields. */
8035 for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
8037 if (TREE_CODE (field) == FIELD_DECL
8038 && ix86_compat_aligned_value_p (TREE_TYPE (field)))
8045 /* Just for use if some languages passes arrays by value. */
8046 if (ix86_compat_aligned_value_p (TREE_TYPE (type)))
8057 /* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
8058 XXX: This function is obsolete and is only used for checking psABI
8059 compatibility with previous versions of GCC. */
8062 ix86_compat_function_arg_boundary (machine_mode mode,
8063 const_tree type, unsigned int align)
8065 /* In 32bit, only _Decimal128 and __float128 are aligned to their
8066 natural boundaries. */
8067 if (!TARGET_64BIT && mode != TDmode && mode != TFmode)
8069 /* i386 ABI defines all arguments to be 4 byte aligned. We have to
8070 make an exception for SSE modes since these require 128bit
8073 The handling here differs from field_alignment. ICC aligns MMX
8074 arguments to 4 byte boundaries, while structure fields are aligned
8075 to 8 byte boundaries. */
8078 if (!(TARGET_SSE && SSE_REG_MODE_P (mode)))
8079 align = PARM_BOUNDARY;
8083 if (!ix86_compat_aligned_value_p (type))
8084 align = PARM_BOUNDARY;
8087 if (align > BIGGEST_ALIGNMENT)
8088 align = BIGGEST_ALIGNMENT;
8092 /* Return true when TYPE should be 128bit aligned for 32bit argument
8096 ix86_contains_aligned_value_p (const_tree type)
8098 machine_mode mode = TYPE_MODE (type);
8100 if (mode == XFmode || mode == XCmode)
8103 if (TYPE_ALIGN (type) < 128)
8106 if (AGGREGATE_TYPE_P (type))
8108 /* Walk the aggregates recursively. */
8109 switch (TREE_CODE (type))
8113 case QUAL_UNION_TYPE:
8117 /* Walk all the structure fields. */
8118 for (field = TYPE_FIELDS (type);
8120 field = DECL_CHAIN (field))
8122 if (TREE_CODE (field) == FIELD_DECL
8123 && ix86_contains_aligned_value_p (TREE_TYPE (field)))
8130 /* Just for use if some languages passes arrays by value. */
8131 if (ix86_contains_aligned_value_p (TREE_TYPE (type)))
8140 return TYPE_ALIGN (type) >= 128;
8145 /* Gives the alignment boundary, in bits, of an argument with the
8146 specified mode and type. */
8149 ix86_function_arg_boundary (machine_mode mode, const_tree type)
8154 /* Since the main variant type is used for call, we convert it to
8155 the main variant type. */
8156 type = TYPE_MAIN_VARIANT (type);
8157 align = TYPE_ALIGN (type);
8160 align = GET_MODE_ALIGNMENT (mode);
8161 if (align < PARM_BOUNDARY)
8162 align = PARM_BOUNDARY;
8166 unsigned int saved_align = align;
8170 /* i386 ABI defines XFmode arguments to be 4 byte aligned. */
8173 if (mode == XFmode || mode == XCmode)
8174 align = PARM_BOUNDARY;
8176 else if (!ix86_contains_aligned_value_p (type))
8177 align = PARM_BOUNDARY;
8180 align = PARM_BOUNDARY;
8185 && align != ix86_compat_function_arg_boundary (mode, type,
8189 inform (input_location,
8190 "The ABI for passing parameters with %d-byte"
8191 " alignment has changed in GCC 4.6",
8192 align / BITS_PER_UNIT);
8199 /* Return true if N is a possible register number of function value. */
8202 ix86_function_value_regno_p (const unsigned int regno)
8209 return (!TARGET_64BIT || ix86_cfun_abi () != MS_ABI);
8212 return TARGET_64BIT && ix86_cfun_abi () != MS_ABI;
8215 return chkp_function_instrumented_p (current_function_decl);
8217 /* Complex values are returned in %st(0)/%st(1) pair. */
8220 /* TODO: The function should depend on current function ABI but
8221 builtins.c would need updating then. Therefore we use the
8223 if (TARGET_64BIT && ix86_cfun_abi () == MS_ABI)
8225 return TARGET_FLOAT_RETURNS_IN_80387;
8227 /* Complex values are returned in %xmm0/%xmm1 pair. */
8233 if (TARGET_MACHO || TARGET_64BIT)
8241 /* Define how to find the value returned by a function.
8242 VALTYPE is the data type of the value (as a tree).
8243 If the precise function being called is known, FUNC is its FUNCTION_DECL;
8244 otherwise, FUNC is 0. */
8247 function_value_32 (machine_mode orig_mode, machine_mode mode,
8248 const_tree fntype, const_tree fn)
8252 /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
8253 we normally prevent this case when mmx is not available. However
8254 some ABIs may require the result to be returned like DImode. */
8255 if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 8)
8256 regno = FIRST_MMX_REG;
8258 /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
8259 we prevent this case when sse is not available. However some ABIs
8260 may require the result to be returned like integer TImode. */
8261 else if (mode == TImode
8262 || (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 16))
8263 regno = FIRST_SSE_REG;
8265 /* 32-byte vector modes in %ymm0. */
8266 else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 32)
8267 regno = FIRST_SSE_REG;
8269 /* 64-byte vector modes in %zmm0. */
8270 else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
8271 regno = FIRST_SSE_REG;
8273 /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
8274 else if (X87_FLOAT_MODE_P (mode) && TARGET_FLOAT_RETURNS_IN_80387)
8275 regno = FIRST_FLOAT_REG;
8277 /* Most things go in %eax. */
8280 /* Override FP return register with %xmm0 for local functions when
8281 SSE math is enabled or for functions with sseregparm attribute. */
8282 if ((fn || fntype) && (mode == SFmode || mode == DFmode))
8284 int sse_level = ix86_function_sseregparm (fntype, fn, false);
8285 if (sse_level == -1)
8287 error ("calling %qD with SSE caling convention without "
8288 "SSE/SSE2 enabled", fn);
8289 sorry ("this is a GCC bug that can be worked around by adding "
8290 "attribute used to function called");
8292 else if ((sse_level >= 1 && mode == SFmode)
8293 || (sse_level == 2 && mode == DFmode))
8294 regno = FIRST_SSE_REG;
8297 /* OImode shouldn't be used directly. */
8298 gcc_assert (mode != OImode);
8300 return gen_rtx_REG (orig_mode, regno);
8304 function_value_64 (machine_mode orig_mode, machine_mode mode,
8309 /* Handle libcalls, which don't provide a type node. */
8310 if (valtype == NULL)
8324 regno = FIRST_SSE_REG;
8328 regno = FIRST_FLOAT_REG;
8336 return gen_rtx_REG (mode, regno);
8338 else if (POINTER_TYPE_P (valtype))
8340 /* Pointers are always returned in word_mode. */
8344 ret = construct_container (mode, orig_mode, valtype, 1,
8345 X86_64_REGPARM_MAX, X86_64_SSE_REGPARM_MAX,
8346 x86_64_int_return_registers, 0);
8348 /* For zero sized structures, construct_container returns NULL, but we
8349 need to keep rest of compiler happy by returning meaningful value. */
8351 ret = gen_rtx_REG (orig_mode, AX_REG);
8357 function_value_ms_64 (machine_mode orig_mode, machine_mode mode,
8360 unsigned int regno = AX_REG;
8364 switch (GET_MODE_SIZE (mode))
8367 if (valtype != NULL_TREE
8368 && !VECTOR_INTEGER_TYPE_P (valtype)
8369 && !VECTOR_INTEGER_TYPE_P (valtype)
8370 && !INTEGRAL_TYPE_P (valtype)
8371 && !VECTOR_FLOAT_TYPE_P (valtype))
8373 if ((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
8374 && !COMPLEX_MODE_P (mode))
8375 regno = FIRST_SSE_REG;
8379 if (mode == SFmode || mode == DFmode)
8380 regno = FIRST_SSE_REG;
8386 return gen_rtx_REG (orig_mode, regno);
8390 ix86_function_value_1 (const_tree valtype, const_tree fntype_or_decl,
8391 machine_mode orig_mode, machine_mode mode)
8393 const_tree fn, fntype;
8396 if (fntype_or_decl && DECL_P (fntype_or_decl))
8397 fn = fntype_or_decl;
8398 fntype = fn ? TREE_TYPE (fn) : fntype_or_decl;
8400 if ((valtype && POINTER_BOUNDS_TYPE_P (valtype))
8401 || POINTER_BOUNDS_MODE_P (mode))
8402 return gen_rtx_REG (BNDmode, FIRST_BND_REG);
8403 else if (TARGET_64BIT && ix86_function_type_abi (fntype) == MS_ABI)
8404 return function_value_ms_64 (orig_mode, mode, valtype);
8405 else if (TARGET_64BIT)
8406 return function_value_64 (orig_mode, mode, valtype);
8408 return function_value_32 (orig_mode, mode, fntype, fn);
8412 ix86_function_value (const_tree valtype, const_tree fntype_or_decl, bool)
8414 machine_mode mode, orig_mode;
8416 orig_mode = TYPE_MODE (valtype);
8417 mode = type_natural_mode (valtype, NULL, true);
8418 return ix86_function_value_1 (valtype, fntype_or_decl, orig_mode, mode);
8421 /* Return an RTX representing a place where a function returns
8422 or recieves pointer bounds or NULL if no bounds are returned.
8424 VALTYPE is a data type of a value returned by the function.
8426 FN_DECL_OR_TYPE is a tree node representing FUNCTION_DECL
8427 or FUNCTION_TYPE of the function.
8429 If OUTGOING is false, return a place in which the caller will
8430 see the return value. Otherwise, return a place where a
8431 function returns a value. */
8434 ix86_function_value_bounds (const_tree valtype,
8435 const_tree fntype_or_decl ATTRIBUTE_UNUSED,
8436 bool outgoing ATTRIBUTE_UNUSED)
8440 if (BOUNDED_TYPE_P (valtype))
8441 res = gen_rtx_REG (BNDmode, FIRST_BND_REG);
8442 else if (chkp_type_has_pointer (valtype))
8447 unsigned i, bnd_no = 0;
8449 bitmap_obstack_initialize (NULL);
8450 slots = BITMAP_ALLOC (NULL);
8451 chkp_find_bound_slots (valtype, slots);
8453 EXECUTE_IF_SET_IN_BITMAP (slots, 0, i, bi)
8455 rtx reg = gen_rtx_REG (BNDmode, FIRST_BND_REG + bnd_no);
8456 rtx offs = GEN_INT (i * POINTER_SIZE / BITS_PER_UNIT);
8457 gcc_assert (bnd_no < 2);
8458 bounds[bnd_no++] = gen_rtx_EXPR_LIST (VOIDmode, reg, offs);
8461 res = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (bnd_no, bounds));
8463 BITMAP_FREE (slots);
8464 bitmap_obstack_release (NULL);
8472 /* Pointer function arguments and return values are promoted to
8476 ix86_promote_function_mode (const_tree type, machine_mode mode,
8477 int *punsignedp, const_tree fntype,
8480 if (type != NULL_TREE && POINTER_TYPE_P (type))
8482 *punsignedp = POINTERS_EXTEND_UNSIGNED;
8485 return default_promote_function_mode (type, mode, punsignedp, fntype,
8489 /* Return true if a structure, union or array with MODE containing FIELD
8490 should be accessed using BLKmode. */
8493 ix86_member_type_forces_blk (const_tree field, machine_mode mode)
8495 /* Union with XFmode must be in BLKmode. */
8496 return (mode == XFmode
8497 && (TREE_CODE (DECL_FIELD_CONTEXT (field)) == UNION_TYPE
8498 || TREE_CODE (DECL_FIELD_CONTEXT (field)) == QUAL_UNION_TYPE));
8502 ix86_libcall_value (machine_mode mode)
8504 return ix86_function_value_1 (NULL, NULL, mode, mode);
8507 /* Return true iff type is returned in memory. */
8510 ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
8512 #ifdef SUBTARGET_RETURN_IN_MEMORY
8513 return SUBTARGET_RETURN_IN_MEMORY (type, fntype);
8515 const machine_mode mode = type_natural_mode (type, NULL, true);
8518 if (POINTER_BOUNDS_TYPE_P (type))
8523 if (ix86_function_type_abi (fntype) == MS_ABI)
8525 size = int_size_in_bytes (type);
8527 /* __m128 is returned in xmm0. */
8528 if ((!type || VECTOR_INTEGER_TYPE_P (type)
8529 || INTEGRAL_TYPE_P (type)
8530 || VECTOR_FLOAT_TYPE_P (type))
8531 && (SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
8532 && !COMPLEX_MODE_P (mode)
8533 && (GET_MODE_SIZE (mode) == 16 || size == 16))
8536 /* Otherwise, the size must be exactly in [1248]. */
8537 return size != 1 && size != 2 && size != 4 && size != 8;
8541 int needed_intregs, needed_sseregs;
8543 return examine_argument (mode, type, 1,
8544 &needed_intregs, &needed_sseregs);
8549 if (mode == BLKmode)
8552 size = int_size_in_bytes (type);
8554 if (MS_AGGREGATE_RETURN && AGGREGATE_TYPE_P (type) && size <= 8)
8557 if (VECTOR_MODE_P (mode) || mode == TImode)
8559 /* User-created vectors small enough to fit in EAX. */
8563 /* Unless ABI prescibes otherwise,
8564 MMX/3dNow values are returned in MM0 if available. */
8567 return TARGET_VECT8_RETURNS || !TARGET_MMX;
8569 /* SSE values are returned in XMM0 if available. */
8573 /* AVX values are returned in YMM0 if available. */
8577 /* AVX512F values are returned in ZMM0 if available. */
8579 return !TARGET_AVX512F;
8588 /* OImode shouldn't be used directly. */
8589 gcc_assert (mode != OImode);
8597 /* Create the va_list data type. */
8599 /* Returns the calling convention specific va_list date type.
8600 The argument ABI can be DEFAULT_ABI, MS_ABI, or SYSV_ABI. */
8603 ix86_build_builtin_va_list_abi (enum calling_abi abi)
8605 tree f_gpr, f_fpr, f_ovf, f_sav, record, type_decl;
8607 /* For i386 we use plain pointer to argument area. */
8608 if (!TARGET_64BIT || abi == MS_ABI)
8609 return build_pointer_type (char_type_node);
8611 record = lang_hooks.types.make_type (RECORD_TYPE);
8612 type_decl = build_decl (BUILTINS_LOCATION,
8613 TYPE_DECL, get_identifier ("__va_list_tag"), record);
8615 f_gpr = build_decl (BUILTINS_LOCATION,
8616 FIELD_DECL, get_identifier ("gp_offset"),
8617 unsigned_type_node);
8618 f_fpr = build_decl (BUILTINS_LOCATION,
8619 FIELD_DECL, get_identifier ("fp_offset"),
8620 unsigned_type_node);
8621 f_ovf = build_decl (BUILTINS_LOCATION,
8622 FIELD_DECL, get_identifier ("overflow_arg_area"),
8624 f_sav = build_decl (BUILTINS_LOCATION,
8625 FIELD_DECL, get_identifier ("reg_save_area"),
8628 va_list_gpr_counter_field = f_gpr;
8629 va_list_fpr_counter_field = f_fpr;
8631 DECL_FIELD_CONTEXT (f_gpr) = record;
8632 DECL_FIELD_CONTEXT (f_fpr) = record;
8633 DECL_FIELD_CONTEXT (f_ovf) = record;
8634 DECL_FIELD_CONTEXT (f_sav) = record;
8636 TYPE_STUB_DECL (record) = type_decl;
8637 TYPE_NAME (record) = type_decl;
8638 TYPE_FIELDS (record) = f_gpr;
8639 DECL_CHAIN (f_gpr) = f_fpr;
8640 DECL_CHAIN (f_fpr) = f_ovf;
8641 DECL_CHAIN (f_ovf) = f_sav;
8643 layout_type (record);
8645 /* The correct type is an array type of one element. */
8646 return build_array_type (record, build_index_type (size_zero_node));
8649 /* Setup the builtin va_list data type and for 64-bit the additional
8650 calling convention specific va_list data types. */
8653 ix86_build_builtin_va_list (void)
8655 tree ret = ix86_build_builtin_va_list_abi (ix86_abi);
8657 /* Initialize abi specific va_list builtin types. */
8661 if (ix86_abi == MS_ABI)
8663 t = ix86_build_builtin_va_list_abi (SYSV_ABI);
8664 if (TREE_CODE (t) != RECORD_TYPE)
8665 t = build_variant_type_copy (t);
8666 sysv_va_list_type_node = t;
8671 if (TREE_CODE (t) != RECORD_TYPE)
8672 t = build_variant_type_copy (t);
8673 sysv_va_list_type_node = t;
8675 if (ix86_abi != MS_ABI)
8677 t = ix86_build_builtin_va_list_abi (MS_ABI);
8678 if (TREE_CODE (t) != RECORD_TYPE)
8679 t = build_variant_type_copy (t);
8680 ms_va_list_type_node = t;
8685 if (TREE_CODE (t) != RECORD_TYPE)
8686 t = build_variant_type_copy (t);
8687 ms_va_list_type_node = t;
8694 /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
8697 setup_incoming_varargs_64 (CUMULATIVE_ARGS *cum)
8703 /* GPR size of varargs save area. */
8704 if (cfun->va_list_gpr_size)
8705 ix86_varargs_gpr_size = X86_64_REGPARM_MAX * UNITS_PER_WORD;
8707 ix86_varargs_gpr_size = 0;
8709 /* FPR size of varargs save area. We don't need it if we don't pass
8710 anything in SSE registers. */
8711 if (TARGET_SSE && cfun->va_list_fpr_size)
8712 ix86_varargs_fpr_size = X86_64_SSE_REGPARM_MAX * 16;
8714 ix86_varargs_fpr_size = 0;
8716 if (! ix86_varargs_gpr_size && ! ix86_varargs_fpr_size)
8719 save_area = frame_pointer_rtx;
8720 set = get_varargs_alias_set ();
8722 max = cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD;
8723 if (max > X86_64_REGPARM_MAX)
8724 max = X86_64_REGPARM_MAX;
8726 for (i = cum->regno; i < max; i++)
8728 mem = gen_rtx_MEM (word_mode,
8729 plus_constant (Pmode, save_area, i * UNITS_PER_WORD));
8730 MEM_NOTRAP_P (mem) = 1;
8731 set_mem_alias_set (mem, set);
8732 emit_move_insn (mem,
8733 gen_rtx_REG (word_mode,
8734 x86_64_int_parameter_registers[i]));
8737 if (ix86_varargs_fpr_size)
8740 rtx_code_label *label;
8743 /* Now emit code to save SSE registers. The AX parameter contains number
8744 of SSE parameter registers used to call this function, though all we
8745 actually check here is the zero/non-zero status. */
8747 label = gen_label_rtx ();
8748 test = gen_rtx_EQ (VOIDmode, gen_rtx_REG (QImode, AX_REG), const0_rtx);
8749 emit_jump_insn (gen_cbranchqi4 (test, XEXP (test, 0), XEXP (test, 1),
8752 /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
8753 we used movdqa (i.e. TImode) instead? Perhaps even better would
8754 be if we could determine the real mode of the data, via a hook
8755 into pass_stdarg. Ignore all that for now. */
8757 if (crtl->stack_alignment_needed < GET_MODE_ALIGNMENT (smode))
8758 crtl->stack_alignment_needed = GET_MODE_ALIGNMENT (smode);
8760 max = cum->sse_regno + cfun->va_list_fpr_size / 16;
8761 if (max > X86_64_SSE_REGPARM_MAX)
8762 max = X86_64_SSE_REGPARM_MAX;
8764 for (i = cum->sse_regno; i < max; ++i)
8766 mem = plus_constant (Pmode, save_area,
8767 i * 16 + ix86_varargs_gpr_size);
8768 mem = gen_rtx_MEM (smode, mem);
8769 MEM_NOTRAP_P (mem) = 1;
8770 set_mem_alias_set (mem, set);
8771 set_mem_align (mem, GET_MODE_ALIGNMENT (smode));
8773 emit_move_insn (mem, gen_rtx_REG (smode, SSE_REGNO (i)));
8781 setup_incoming_varargs_ms_64 (CUMULATIVE_ARGS *cum)
8783 alias_set_type set = get_varargs_alias_set ();
8786 /* Reset to zero, as there might be a sysv vaarg used
8788 ix86_varargs_gpr_size = 0;
8789 ix86_varargs_fpr_size = 0;
8791 for (i = cum->regno; i < X86_64_MS_REGPARM_MAX; i++)
8795 mem = gen_rtx_MEM (Pmode,
8796 plus_constant (Pmode, virtual_incoming_args_rtx,
8797 i * UNITS_PER_WORD));
8798 MEM_NOTRAP_P (mem) = 1;
8799 set_mem_alias_set (mem, set);
8801 reg = gen_rtx_REG (Pmode, x86_64_ms_abi_int_parameter_registers[i]);
8802 emit_move_insn (mem, reg);
8807 ix86_setup_incoming_varargs (cumulative_args_t cum_v, machine_mode mode,
8808 tree type, int *, int no_rtl)
8810 CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
8811 CUMULATIVE_ARGS next_cum;
8814 /* This argument doesn't appear to be used anymore. Which is good,
8815 because the old code here didn't suppress rtl generation. */
8816 gcc_assert (!no_rtl);
8821 fntype = TREE_TYPE (current_function_decl);
8823 /* For varargs, we do not want to skip the dummy va_dcl argument.
8824 For stdargs, we do want to skip the last named argument. */
8826 if (stdarg_p (fntype))
8827 ix86_function_arg_advance (pack_cumulative_args (&next_cum), mode, type,
8830 if (cum->call_abi == MS_ABI)
8831 setup_incoming_varargs_ms_64 (&next_cum);
8833 setup_incoming_varargs_64 (&next_cum);
8837 ix86_setup_incoming_vararg_bounds (cumulative_args_t cum_v,
8838 enum machine_mode mode,
8840 int *pretend_size ATTRIBUTE_UNUSED,
8843 CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
8844 CUMULATIVE_ARGS next_cum;
8847 int bnd_reg, i, max;
8849 gcc_assert (!no_rtl);
8851 /* Do nothing if we use plain pointer to argument area. */
8852 if (!TARGET_64BIT || cum->call_abi == MS_ABI)
8855 fntype = TREE_TYPE (current_function_decl);
8857 /* For varargs, we do not want to skip the dummy va_dcl argument.
8858 For stdargs, we do want to skip the last named argument. */
8860 if (stdarg_p (fntype))
8861 ix86_function_arg_advance (pack_cumulative_args (&next_cum), mode, type,
8863 save_area = frame_pointer_rtx;
8865 max = cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD;
8866 if (max > X86_64_REGPARM_MAX)
8867 max = X86_64_REGPARM_MAX;
8869 bnd_reg = cum->bnd_regno + cum->force_bnd_pass;
8870 if (chkp_function_instrumented_p (current_function_decl))
8871 for (i = cum->regno; i < max; i++)
8873 rtx addr = plus_constant (Pmode, save_area, i * UNITS_PER_WORD);
8874 rtx reg = gen_rtx_REG (DImode,
8875 x86_64_int_parameter_registers[i]);
8879 if (bnd_reg <= LAST_BND_REG)
8880 bounds = gen_rtx_REG (BNDmode, bnd_reg);
8884 plus_constant (Pmode, arg_pointer_rtx,
8885 (LAST_BND_REG - bnd_reg) * GET_MODE_SIZE (Pmode));
8886 bounds = gen_reg_rtx (BNDmode);
8887 emit_insn (BNDmode == BND64mode
8888 ? gen_bnd64_ldx (bounds, ldx_addr, ptr)
8889 : gen_bnd32_ldx (bounds, ldx_addr, ptr));
8892 emit_insn (BNDmode == BND64mode
8893 ? gen_bnd64_stx (addr, ptr, bounds)
8894 : gen_bnd32_stx (addr, ptr, bounds));
8901 /* Checks if TYPE is of kind va_list char *. */
8904 is_va_list_char_pointer (tree type)
8908 /* For 32-bit it is always true. */
8911 canonic = ix86_canonical_va_list_type (type);
8912 return (canonic == ms_va_list_type_node
8913 || (ix86_abi == MS_ABI && canonic == va_list_type_node));
8916 /* Implement va_start. */
8919 ix86_va_start (tree valist, rtx nextarg)
8921 HOST_WIDE_INT words, n_gpr, n_fpr;
8922 tree f_gpr, f_fpr, f_ovf, f_sav;
8923 tree gpr, fpr, ovf, sav, t;
8927 if (flag_split_stack
8928 && cfun->machine->split_stack_varargs_pointer == NULL_RTX)
8930 unsigned int scratch_regno;
8932 /* When we are splitting the stack, we can't refer to the stack
8933 arguments using internal_arg_pointer, because they may be on
8934 the old stack. The split stack prologue will arrange to
8935 leave a pointer to the old stack arguments in a scratch
8936 register, which we here copy to a pseudo-register. The split
8937 stack prologue can't set the pseudo-register directly because
8938 it (the prologue) runs before any registers have been saved. */
8940 scratch_regno = split_stack_prologue_scratch_regno ();
8941 if (scratch_regno != INVALID_REGNUM)
8946 reg = gen_reg_rtx (Pmode);
8947 cfun->machine->split_stack_varargs_pointer = reg;
8950 emit_move_insn (reg, gen_rtx_REG (Pmode, scratch_regno));
8954 push_topmost_sequence ();
8955 emit_insn_after (seq, entry_of_function ());
8956 pop_topmost_sequence ();
8960 /* Only 64bit target needs something special. */
8961 if (!TARGET_64BIT || is_va_list_char_pointer (TREE_TYPE (valist)))
8963 if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
8964 std_expand_builtin_va_start (valist, nextarg);
8969 va_r = expand_expr (valist, NULL_RTX, VOIDmode, EXPAND_WRITE);
8970 next = expand_binop (ptr_mode, add_optab,
8971 cfun->machine->split_stack_varargs_pointer,
8972 crtl->args.arg_offset_rtx,
8973 NULL_RTX, 0, OPTAB_LIB_WIDEN);
8974 convert_move (va_r, next, 0);
8976 /* Store zero bounds for va_list. */
8977 if (chkp_function_instrumented_p (current_function_decl))
8978 chkp_expand_bounds_reset_for_mem (valist,
8979 make_tree (TREE_TYPE (valist),
8986 f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
8987 f_fpr = DECL_CHAIN (f_gpr);
8988 f_ovf = DECL_CHAIN (f_fpr);
8989 f_sav = DECL_CHAIN (f_ovf);
8991 valist = build_simple_mem_ref (valist);
8992 TREE_TYPE (valist) = TREE_TYPE (sysv_va_list_type_node);
8993 /* The following should be folded into the MEM_REF offset. */
8994 gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr), unshare_expr (valist),
8996 fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), unshare_expr (valist),
8998 ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), unshare_expr (valist),
9000 sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), unshare_expr (valist),
9003 /* Count number of gp and fp argument registers used. */
9004 words = crtl->args.info.words;
9005 n_gpr = crtl->args.info.regno;
9006 n_fpr = crtl->args.info.sse_regno;
9008 if (cfun->va_list_gpr_size)
9010 type = TREE_TYPE (gpr);
9011 t = build2 (MODIFY_EXPR, type,
9012 gpr, build_int_cst (type, n_gpr * 8));
9013 TREE_SIDE_EFFECTS (t) = 1;
9014 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
9017 if (TARGET_SSE && cfun->va_list_fpr_size)
9019 type = TREE_TYPE (fpr);
9020 t = build2 (MODIFY_EXPR, type, fpr,
9021 build_int_cst (type, n_fpr * 16 + 8*X86_64_REGPARM_MAX));
9022 TREE_SIDE_EFFECTS (t) = 1;
9023 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
9026 /* Find the overflow area. */
9027 type = TREE_TYPE (ovf);
9028 if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
9029 ovf_rtx = crtl->args.internal_arg_pointer;
9031 ovf_rtx = cfun->machine->split_stack_varargs_pointer;
9032 t = make_tree (type, ovf_rtx);
9034 t = fold_build_pointer_plus_hwi (t, words * UNITS_PER_WORD);
9036 /* Store zero bounds for overflow area pointer. */
9037 if (chkp_function_instrumented_p (current_function_decl))
9038 chkp_expand_bounds_reset_for_mem (ovf, t);
9040 t = build2 (MODIFY_EXPR, type, ovf, t);
9041 TREE_SIDE_EFFECTS (t) = 1;
9042 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
9044 if (ix86_varargs_gpr_size || ix86_varargs_fpr_size)
9046 /* Find the register save area.
9047 Prologue of the function save it right above stack frame. */
9048 type = TREE_TYPE (sav);
9049 t = make_tree (type, frame_pointer_rtx);
9050 if (!ix86_varargs_gpr_size)
9051 t = fold_build_pointer_plus_hwi (t, -8 * X86_64_REGPARM_MAX);
9053 /* Store zero bounds for save area pointer. */
9054 if (chkp_function_instrumented_p (current_function_decl))
9055 chkp_expand_bounds_reset_for_mem (sav, t);
9057 t = build2 (MODIFY_EXPR, type, sav, t);
9058 TREE_SIDE_EFFECTS (t) = 1;
9059 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
9063 /* Implement va_arg. */
9066 ix86_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
9069 static const int intreg[6] = { 0, 1, 2, 3, 4, 5 };
9070 tree f_gpr, f_fpr, f_ovf, f_sav;
9071 tree gpr, fpr, ovf, sav, t;
9073 tree lab_false, lab_over = NULL_TREE;
9078 machine_mode nat_mode;
9079 unsigned int arg_boundary;
9081 /* Only 64bit target needs something special. */
9082 if (!TARGET_64BIT || is_va_list_char_pointer (TREE_TYPE (valist)))
9083 return std_gimplify_va_arg_expr (valist, type, pre_p, post_p);
9085 f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
9086 f_fpr = DECL_CHAIN (f_gpr);
9087 f_ovf = DECL_CHAIN (f_fpr);
9088 f_sav = DECL_CHAIN (f_ovf);
9090 gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr),
9091 build_va_arg_indirect_ref (valist), f_gpr, NULL_TREE);
9092 valist = build_va_arg_indirect_ref (valist);
9093 fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), valist, f_fpr, NULL_TREE);
9094 ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE);
9095 sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE);
9097 indirect_p = pass_by_reference (NULL, TYPE_MODE (type), type, false);
9099 type = build_pointer_type (type);
9100 size = int_size_in_bytes (type);
9101 rsize = (size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
9103 nat_mode = type_natural_mode (type, NULL, false);
9118 /* Unnamed 256 and 512bit vector mode parameters are passed on stack. */
9119 if (!TARGET_64BIT_MS_ABI)
9126 container = construct_container (nat_mode, TYPE_MODE (type),
9127 type, 0, X86_64_REGPARM_MAX,
9128 X86_64_SSE_REGPARM_MAX, intreg,
9133 /* Pull the value out of the saved registers. */
9135 addr = create_tmp_var (ptr_type_node, "addr");
9139 int needed_intregs, needed_sseregs;
9141 tree int_addr, sse_addr;
9143 lab_false = create_artificial_label (UNKNOWN_LOCATION);
9144 lab_over = create_artificial_label (UNKNOWN_LOCATION);
9146 examine_argument (nat_mode, type, 0, &needed_intregs, &needed_sseregs);
9148 need_temp = (!REG_P (container)
9149 && ((needed_intregs && TYPE_ALIGN (type) > 64)
9150 || TYPE_ALIGN (type) > 128));
9152 /* In case we are passing structure, verify that it is consecutive block
9153 on the register save area. If not we need to do moves. */
9154 if (!need_temp && !REG_P (container))
9156 /* Verify that all registers are strictly consecutive */
9157 if (SSE_REGNO_P (REGNO (XEXP (XVECEXP (container, 0, 0), 0))))
9161 for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
9163 rtx slot = XVECEXP (container, 0, i);
9164 if (REGNO (XEXP (slot, 0)) != FIRST_SSE_REG + (unsigned int) i
9165 || INTVAL (XEXP (slot, 1)) != i * 16)
9173 for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
9175 rtx slot = XVECEXP (container, 0, i);
9176 if (REGNO (XEXP (slot, 0)) != (unsigned int) i
9177 || INTVAL (XEXP (slot, 1)) != i * 8)
9189 int_addr = create_tmp_var (ptr_type_node, "int_addr");
9190 sse_addr = create_tmp_var (ptr_type_node, "sse_addr");
9193 /* First ensure that we fit completely in registers. */
9196 t = build_int_cst (TREE_TYPE (gpr),
9197 (X86_64_REGPARM_MAX - needed_intregs + 1) * 8);
9198 t = build2 (GE_EXPR, boolean_type_node, gpr, t);
9199 t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
9200 t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
9201 gimplify_and_add (t, pre_p);
9205 t = build_int_cst (TREE_TYPE (fpr),
9206 (X86_64_SSE_REGPARM_MAX - needed_sseregs + 1) * 16
9207 + X86_64_REGPARM_MAX * 8);
9208 t = build2 (GE_EXPR, boolean_type_node, fpr, t);
9209 t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
9210 t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
9211 gimplify_and_add (t, pre_p);
9214 /* Compute index to start of area used for integer regs. */
9217 /* int_addr = gpr + sav; */
9218 t = fold_build_pointer_plus (sav, gpr);
9219 gimplify_assign (int_addr, t, pre_p);
9223 /* sse_addr = fpr + sav; */
9224 t = fold_build_pointer_plus (sav, fpr);
9225 gimplify_assign (sse_addr, t, pre_p);
9229 int i, prev_size = 0;
9230 tree temp = create_tmp_var (type, "va_arg_tmp");
9233 t = build1 (ADDR_EXPR, build_pointer_type (type), temp);
9234 gimplify_assign (addr, t, pre_p);
9236 for (i = 0; i < XVECLEN (container, 0); i++)
9238 rtx slot = XVECEXP (container, 0, i);
9239 rtx reg = XEXP (slot, 0);
9240 machine_mode mode = GET_MODE (reg);
9246 tree dest_addr, dest;
9247 int cur_size = GET_MODE_SIZE (mode);
9249 gcc_assert (prev_size <= INTVAL (XEXP (slot, 1)));
9250 prev_size = INTVAL (XEXP (slot, 1));
9251 if (prev_size + cur_size > size)
9253 cur_size = size - prev_size;
9254 mode = mode_for_size (cur_size * BITS_PER_UNIT, MODE_INT, 1);
9255 if (mode == BLKmode)
9258 piece_type = lang_hooks.types.type_for_mode (mode, 1);
9259 if (mode == GET_MODE (reg))
9260 addr_type = build_pointer_type (piece_type);
9262 addr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
9264 daddr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
9267 if (SSE_REGNO_P (REGNO (reg)))
9269 src_addr = sse_addr;
9270 src_offset = (REGNO (reg) - FIRST_SSE_REG) * 16;
9274 src_addr = int_addr;
9275 src_offset = REGNO (reg) * 8;
9277 src_addr = fold_convert (addr_type, src_addr);
9278 src_addr = fold_build_pointer_plus_hwi (src_addr, src_offset);
9280 dest_addr = fold_convert (daddr_type, addr);
9281 dest_addr = fold_build_pointer_plus_hwi (dest_addr, prev_size);
9282 if (cur_size == GET_MODE_SIZE (mode))
9284 src = build_va_arg_indirect_ref (src_addr);
9285 dest = build_va_arg_indirect_ref (dest_addr);
9287 gimplify_assign (dest, src, pre_p);
9292 = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMCPY),
9293 3, dest_addr, src_addr,
9294 size_int (cur_size));
9295 gimplify_and_add (copy, pre_p);
9297 prev_size += cur_size;
9303 t = build2 (PLUS_EXPR, TREE_TYPE (gpr), gpr,
9304 build_int_cst (TREE_TYPE (gpr), needed_intregs * 8));
9305 gimplify_assign (gpr, t, pre_p);
9310 t = build2 (PLUS_EXPR, TREE_TYPE (fpr), fpr,
9311 build_int_cst (TREE_TYPE (fpr), needed_sseregs * 16));
9312 gimplify_assign (fpr, t, pre_p);
9315 gimple_seq_add_stmt (pre_p, gimple_build_goto (lab_over));
9317 gimple_seq_add_stmt (pre_p, gimple_build_label (lab_false));
9320 /* ... otherwise out of the overflow area. */
9322 /* When we align parameter on stack for caller, if the parameter
9323 alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
9324 aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
9325 here with caller. */
9326 arg_boundary = ix86_function_arg_boundary (VOIDmode, type);
9327 if ((unsigned int) arg_boundary > MAX_SUPPORTED_STACK_ALIGNMENT)
9328 arg_boundary = MAX_SUPPORTED_STACK_ALIGNMENT;
9330 /* Care for on-stack alignment if needed. */
9331 if (arg_boundary <= 64 || size == 0)
9335 HOST_WIDE_INT align = arg_boundary / 8;
9336 t = fold_build_pointer_plus_hwi (ovf, align - 1);
9337 t = build2 (BIT_AND_EXPR, TREE_TYPE (t), t,
9338 build_int_cst (TREE_TYPE (t), -align));
9341 gimplify_expr (&t, pre_p, NULL, is_gimple_val, fb_rvalue);
9342 gimplify_assign (addr, t, pre_p);
9344 t = fold_build_pointer_plus_hwi (t, rsize * UNITS_PER_WORD);
9345 gimplify_assign (unshare_expr (ovf), t, pre_p);
9348 gimple_seq_add_stmt (pre_p, gimple_build_label (lab_over));
9350 ptrtype = build_pointer_type_for_mode (type, ptr_mode, true);
9351 addr = fold_convert (ptrtype, addr);
9354 addr = build_va_arg_indirect_ref (addr);
9355 return build_va_arg_indirect_ref (addr);
9358 /* Return true if OPNUM's MEM should be matched
9359 in movabs* patterns. */
9362 ix86_check_movabs (rtx insn, int opnum)
9366 set = PATTERN (insn);
9367 if (GET_CODE (set) == PARALLEL)
9368 set = XVECEXP (set, 0, 0);
9369 gcc_assert (GET_CODE (set) == SET);
9370 mem = XEXP (set, opnum);
9371 while (GET_CODE (mem) == SUBREG)
9372 mem = SUBREG_REG (mem);
9373 gcc_assert (MEM_P (mem));
9374 return volatile_ok || !MEM_VOLATILE_P (mem);
9377 /* Initialize the table of extra 80387 mathematical constants. */
9380 init_ext_80387_constants (void)
9382 static const char * cst[5] =
9384 "0.3010299956639811952256464283594894482", /* 0: fldlg2 */
9385 "0.6931471805599453094286904741849753009", /* 1: fldln2 */
9386 "1.4426950408889634073876517827983434472", /* 2: fldl2e */
9387 "3.3219280948873623478083405569094566090", /* 3: fldl2t */
9388 "3.1415926535897932385128089594061862044", /* 4: fldpi */
9392 for (i = 0; i < 5; i++)
9394 real_from_string (&ext_80387_constants_table[i], cst[i]);
9395 /* Ensure each constant is rounded to XFmode precision. */
9396 real_convert (&ext_80387_constants_table[i],
9397 XFmode, &ext_80387_constants_table[i]);
9400 ext_80387_constants_init = 1;
9403 /* Return non-zero if the constant is something that
9404 can be loaded with a special instruction. */
9407 standard_80387_constant_p (rtx x)
9409 machine_mode mode = GET_MODE (x);
9413 if (!(X87_FLOAT_MODE_P (mode) && (GET_CODE (x) == CONST_DOUBLE)))
9416 if (x == CONST0_RTX (mode))
9418 if (x == CONST1_RTX (mode))
9421 REAL_VALUE_FROM_CONST_DOUBLE (r, x);
9423 /* For XFmode constants, try to find a special 80387 instruction when
9424 optimizing for size or on those CPUs that benefit from them. */
9426 && (optimize_function_for_size_p (cfun) || TARGET_EXT_80387_CONSTANTS))
9430 if (! ext_80387_constants_init)
9431 init_ext_80387_constants ();
9433 for (i = 0; i < 5; i++)
9434 if (real_identical (&r, &ext_80387_constants_table[i]))
9438 /* Load of the constant -0.0 or -1.0 will be split as
9439 fldz;fchs or fld1;fchs sequence. */
9440 if (real_isnegzero (&r))
9442 if (real_identical (&r, &dconstm1))
9448 /* Return the opcode of the special instruction to be used to load
9452 standard_80387_constant_opcode (rtx x)
9454 switch (standard_80387_constant_p (x))
9478 /* Return the CONST_DOUBLE representing the 80387 constant that is
9479 loaded by the specified special instruction. The argument IDX
9480 matches the return value from standard_80387_constant_p. */
9483 standard_80387_constant_rtx (int idx)
9487 if (! ext_80387_constants_init)
9488 init_ext_80387_constants ();
9504 return CONST_DOUBLE_FROM_REAL_VALUE (ext_80387_constants_table[i],
9508 /* Return 1 if X is all 0s and 2 if x is all 1s
9509 in supported SSE/AVX vector mode. */
9512 standard_sse_constant_p (rtx x)
9514 machine_mode mode = GET_MODE (x);
9516 if (x == const0_rtx || x == CONST0_RTX (GET_MODE (x)))
9518 if (vector_all_ones_operand (x, mode))
9546 /* Return the opcode of the special instruction to be used to load
9550 standard_sse_constant_opcode (rtx_insn *insn, rtx x)
9552 switch (standard_sse_constant_p (x))
9555 switch (get_attr_mode (insn))
9558 return "vpxord\t%g0, %g0, %g0";
9560 return TARGET_AVX512DQ ? "vxorps\t%g0, %g0, %g0"
9561 : "vpxord\t%g0, %g0, %g0";
9563 return TARGET_AVX512DQ ? "vxorpd\t%g0, %g0, %g0"
9564 : "vpxorq\t%g0, %g0, %g0";
9566 return TARGET_AVX512VL ? "vpxord\t%t0, %t0, %t0"
9567 : "%vpxor\t%0, %d0";
9569 return "%vxorpd\t%0, %d0";
9571 return "%vxorps\t%0, %d0";
9574 return TARGET_AVX512VL ? "vpxord\t%x0, %x0, %x0"
9575 : "vpxor\t%x0, %x0, %x0";
9577 return "vxorpd\t%x0, %x0, %x0";
9579 return "vxorps\t%x0, %x0, %x0";
9587 || get_attr_mode (insn) == MODE_XI
9588 || get_attr_mode (insn) == MODE_V8DF
9589 || get_attr_mode (insn) == MODE_V16SF)
9590 return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";
9592 return "vpcmpeqd\t%0, %0, %0";
9594 return "pcmpeqd\t%0, %0";
9602 /* Returns true if OP contains a symbol reference */
9605 symbolic_reference_mentioned_p (rtx op)
9610 if (GET_CODE (op) == SYMBOL_REF || GET_CODE (op) == LABEL_REF)
9613 fmt = GET_RTX_FORMAT (GET_CODE (op));
9614 for (i = GET_RTX_LENGTH (GET_CODE (op)) - 1; i >= 0; i--)
9620 for (j = XVECLEN (op, i) - 1; j >= 0; j--)
9621 if (symbolic_reference_mentioned_p (XVECEXP (op, i, j)))
9625 else if (fmt[i] == 'e' && symbolic_reference_mentioned_p (XEXP (op, i)))
9632 /* Return true if it is appropriate to emit `ret' instructions in the
9633 body of a function. Do this only if the epilogue is simple, needing a
9634 couple of insns. Prior to reloading, we can't tell how many registers
9635 must be saved, so return false then. Return false if there is no frame
9636 marker to de-allocate. */
9639 ix86_can_use_return_insn_p (void)
9641 struct ix86_frame frame;
9643 if (! reload_completed || frame_pointer_needed)
9646 /* Don't allow more than 32k pop, since that's all we can do
9647 with one instruction. */
9648 if (crtl->args.pops_args && crtl->args.size >= 32768)
9651 ix86_compute_frame_layout (&frame);
9652 return (frame.stack_pointer_offset == UNITS_PER_WORD
9653 && (frame.nregs + frame.nsseregs) == 0);
9656 /* Value should be nonzero if functions must have frame pointers.
9657 Zero means the frame pointer need not be set up (and parms may
9658 be accessed via the stack pointer) in functions that seem suitable. */
9661 ix86_frame_pointer_required (void)
9663 /* If we accessed previous frames, then the generated code expects
9664 to be able to access the saved ebp value in our frame. */
9665 if (cfun->machine->accesses_prev_frame)
9668 /* Several x86 os'es need a frame pointer for other reasons,
9669 usually pertaining to setjmp. */
9670 if (SUBTARGET_FRAME_POINTER_REQUIRED)
9673 /* For older 32-bit runtimes setjmp requires valid frame-pointer. */
9674 if (TARGET_32BIT_MS_ABI && cfun->calls_setjmp)
9677 /* Win64 SEH, very large frames need a frame-pointer as maximum stack
9678 allocation is 4GB. */
9679 if (TARGET_64BIT_MS_ABI && get_frame_size () > SEH_MAX_FRAME_SIZE)
9682 /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
9683 turns off the frame pointer by default. Turn it back on now if
9684 we've not got a leaf function. */
9685 if (TARGET_OMIT_LEAF_FRAME_POINTER
9687 || ix86_current_function_calls_tls_descriptor))
9690 if (crtl->profile && !flag_fentry)
9696 /* Record that the current function accesses previous call frames. */
9699 ix86_setup_frame_addresses (void)
9701 cfun->machine->accesses_prev_frame = 1;
9704 #ifndef USE_HIDDEN_LINKONCE
9705 # if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
9706 # define USE_HIDDEN_LINKONCE 1
9708 # define USE_HIDDEN_LINKONCE 0
9712 static int pic_labels_used;
9714 /* Fills in the label name that should be used for a pc thunk for
9715 the given register. */
9718 get_pc_thunk_name (char name[32], unsigned int regno)
9720 gcc_assert (!TARGET_64BIT);
9722 if (USE_HIDDEN_LINKONCE)
9723 sprintf (name, "__x86.get_pc_thunk.%s", reg_names[regno]);
9725 ASM_GENERATE_INTERNAL_LABEL (name, "LPR", regno);
9729 /* This function generates code for -fpic that loads %ebx with
9730 the return address of the caller and then returns. */
9733 ix86_code_end (void)
9738 for (regno = AX_REG; regno <= SP_REG; regno++)
9743 if (!(pic_labels_used & (1 << regno)))
9746 get_pc_thunk_name (name, regno);
9748 decl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL,
9749 get_identifier (name),
9750 build_function_type_list (void_type_node, NULL_TREE));
9751 DECL_RESULT (decl) = build_decl (BUILTINS_LOCATION, RESULT_DECL,
9752 NULL_TREE, void_type_node);
9753 TREE_PUBLIC (decl) = 1;
9754 TREE_STATIC (decl) = 1;
9755 DECL_IGNORED_P (decl) = 1;
9760 switch_to_section (darwin_sections[text_coal_section]);
9761 fputs ("\t.weak_definition\t", asm_out_file);
9762 assemble_name (asm_out_file, name);
9763 fputs ("\n\t.private_extern\t", asm_out_file);
9764 assemble_name (asm_out_file, name);
9765 putc ('\n', asm_out_file);
9766 ASM_OUTPUT_LABEL (asm_out_file, name);
9767 DECL_WEAK (decl) = 1;
9771 if (USE_HIDDEN_LINKONCE)
9773 cgraph_node::create (decl)->set_comdat_group (DECL_ASSEMBLER_NAME (decl));
9775 targetm.asm_out.unique_section (decl, 0);
9776 switch_to_section (get_named_section (decl, NULL, 0));
9778 targetm.asm_out.globalize_label (asm_out_file, name);
9779 fputs ("\t.hidden\t", asm_out_file);
9780 assemble_name (asm_out_file, name);
9781 putc ('\n', asm_out_file);
9782 ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl);
9786 switch_to_section (text_section);
9787 ASM_OUTPUT_LABEL (asm_out_file, name);
9790 DECL_INITIAL (decl) = make_node (BLOCK);
9791 current_function_decl = decl;
9792 init_function_start (decl);
9793 first_function_block_is_cold = false;
9794 /* Make sure unwind info is emitted for the thunk if needed. */
9795 final_start_function (emit_barrier (), asm_out_file, 1);
9797 /* Pad stack IP move with 4 instructions (two NOPs count
9798 as one instruction). */
9799 if (TARGET_PAD_SHORT_FUNCTION)
9804 fputs ("\tnop\n", asm_out_file);
9807 xops[0] = gen_rtx_REG (Pmode, regno);
9808 xops[1] = gen_rtx_MEM (Pmode, stack_pointer_rtx);
9809 output_asm_insn ("mov%z0\t{%1, %0|%0, %1}", xops);
9810 output_asm_insn ("%!ret", NULL);
9811 final_end_function ();
9812 init_insn_lengths ();
9813 free_after_compilation (cfun);
9815 current_function_decl = NULL;
9818 if (flag_split_stack)
9819 file_end_indicate_split_stack ();
9822 /* Emit code for the SET_GOT patterns. */
9825 output_set_got (rtx dest, rtx label)
9831 if (TARGET_VXWORKS_RTP && flag_pic)
9833 /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
9834 xops[2] = gen_rtx_MEM (Pmode,
9835 gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_BASE));
9836 output_asm_insn ("mov{l}\t{%2, %0|%0, %2}", xops);
9838 /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
9839 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
9840 an unadorned address. */
9841 xops[2] = gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_INDEX);
9842 SYMBOL_REF_FLAGS (xops[2]) |= SYMBOL_FLAG_LOCAL;
9843 output_asm_insn ("mov{l}\t{%P2(%0), %0|%0, DWORD PTR %P2[%0]}", xops);
9847 xops[1] = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
9852 /* We don't need a pic base, we're not producing pic. */
9855 xops[2] = gen_rtx_LABEL_REF (Pmode, label ? label : gen_label_rtx ());
9856 output_asm_insn ("mov%z0\t{%2, %0|%0, %2}", xops);
9857 targetm.asm_out.internal_label (asm_out_file, "L",
9858 CODE_LABEL_NUMBER (XEXP (xops[2], 0)));
9863 get_pc_thunk_name (name, REGNO (dest));
9864 pic_labels_used |= 1 << REGNO (dest);
9866 xops[2] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (name));
9867 xops[2] = gen_rtx_MEM (QImode, xops[2]);
9868 output_asm_insn ("%!call\t%X2", xops);
9871 /* Output the Mach-O "canonical" pic base label name ("Lxx$pb") here.
9872 This is what will be referenced by the Mach-O PIC subsystem. */
9873 if (machopic_should_output_picbase_label () || !label)
9874 ASM_OUTPUT_LABEL (asm_out_file, MACHOPIC_FUNCTION_BASE_NAME);
9876 /* When we are restoring the pic base at the site of a nonlocal label,
9877 and we decided to emit the pic base above, we will still output a
9878 local label used for calculating the correction offset (even though
9879 the offset will be 0 in that case). */
9881 targetm.asm_out.internal_label (asm_out_file, "L",
9882 CODE_LABEL_NUMBER (label));
9887 output_asm_insn ("add%z0\t{%1, %0|%0, %1}", xops);
9892 /* Generate an "push" pattern for input ARG. */
9897 struct machine_function *m = cfun->machine;
9899 if (m->fs.cfa_reg == stack_pointer_rtx)
9900 m->fs.cfa_offset += UNITS_PER_WORD;
9901 m->fs.sp_offset += UNITS_PER_WORD;
9903 if (REG_P (arg) && GET_MODE (arg) != word_mode)
9904 arg = gen_rtx_REG (word_mode, REGNO (arg));
9906 return gen_rtx_SET (VOIDmode,
9907 gen_rtx_MEM (word_mode,
9908 gen_rtx_PRE_DEC (Pmode,
9909 stack_pointer_rtx)),
9913 /* Generate an "pop" pattern for input ARG. */
9918 if (REG_P (arg) && GET_MODE (arg) != word_mode)
9919 arg = gen_rtx_REG (word_mode, REGNO (arg));
9921 return gen_rtx_SET (VOIDmode,
9923 gen_rtx_MEM (word_mode,
9924 gen_rtx_POST_INC (Pmode,
9925 stack_pointer_rtx)));
9928 /* Return >= 0 if there is an unused call-clobbered register available
9929 for the entire function. */
9932 ix86_select_alt_pic_regnum (void)
9934 if (ix86_use_pseudo_pic_reg ())
9935 return INVALID_REGNUM;
9939 && !ix86_current_function_calls_tls_descriptor)
9942 /* Can't use the same register for both PIC and DRAP. */
9944 drap = REGNO (crtl->drap_reg);
9947 for (i = 2; i >= 0; --i)
9948 if (i != drap && !df_regs_ever_live_p (i))
9952 return INVALID_REGNUM;
9955 /* Return TRUE if we need to save REGNO. */
9958 ix86_save_reg (unsigned int regno, bool maybe_eh_return)
9960 if (regno == REAL_PIC_OFFSET_TABLE_REGNUM
9961 && pic_offset_table_rtx)
9963 if (ix86_use_pseudo_pic_reg ())
9965 /* REAL_PIC_OFFSET_TABLE_REGNUM used by call to
9966 _mcount in prologue. */
9967 if (!TARGET_64BIT && flag_pic && crtl->profile)
9970 else if (df_regs_ever_live_p (REAL_PIC_OFFSET_TABLE_REGNUM)
9972 || crtl->calls_eh_return
9973 || crtl->uses_const_pool
9974 || cfun->has_nonlocal_label)
9975 return ix86_select_alt_pic_regnum () == INVALID_REGNUM;
9978 if (crtl->calls_eh_return && maybe_eh_return)
9983 unsigned test = EH_RETURN_DATA_REGNO (i);
9984 if (test == INVALID_REGNUM)
9992 && regno == REGNO (crtl->drap_reg)
9993 && !cfun->machine->no_drap_save_restore)
9996 return (df_regs_ever_live_p (regno)
9997 && !call_used_regs[regno]
9998 && !fixed_regs[regno]
9999 && (regno != HARD_FRAME_POINTER_REGNUM || !frame_pointer_needed));
10002 /* Return number of saved general prupose registers. */
10005 ix86_nsaved_regs (void)
10010 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10011 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
10016 /* Return number of saved SSE registrers. */
10019 ix86_nsaved_sseregs (void)
10024 if (!TARGET_64BIT_MS_ABI)
10026 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10027 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
10032 /* Given FROM and TO register numbers, say whether this elimination is
10033 allowed. If stack alignment is needed, we can only replace argument
10034 pointer with hard frame pointer, or replace frame pointer with stack
10035 pointer. Otherwise, frame pointer elimination is automatically
10036 handled and all other eliminations are valid. */
10039 ix86_can_eliminate (const int from, const int to)
10041 if (stack_realign_fp)
10042 return ((from == ARG_POINTER_REGNUM
10043 && to == HARD_FRAME_POINTER_REGNUM)
10044 || (from == FRAME_POINTER_REGNUM
10045 && to == STACK_POINTER_REGNUM));
10047 return to == STACK_POINTER_REGNUM ? !frame_pointer_needed : true;
10050 /* Return the offset between two registers, one to be eliminated, and the other
10051 its replacement, at the start of a routine. */
10054 ix86_initial_elimination_offset (int from, int to)
10056 struct ix86_frame frame;
10057 ix86_compute_frame_layout (&frame);
10059 if (from == ARG_POINTER_REGNUM && to == HARD_FRAME_POINTER_REGNUM)
10060 return frame.hard_frame_pointer_offset;
10061 else if (from == FRAME_POINTER_REGNUM
10062 && to == HARD_FRAME_POINTER_REGNUM)
10063 return frame.hard_frame_pointer_offset - frame.frame_pointer_offset;
10066 gcc_assert (to == STACK_POINTER_REGNUM);
10068 if (from == ARG_POINTER_REGNUM)
10069 return frame.stack_pointer_offset;
10071 gcc_assert (from == FRAME_POINTER_REGNUM);
10072 return frame.stack_pointer_offset - frame.frame_pointer_offset;
10076 /* In a dynamically-aligned function, we can't know the offset from
10077 stack pointer to frame pointer, so we must ensure that setjmp
10078 eliminates fp against the hard fp (%ebp) rather than trying to
10079 index from %esp up to the top of the frame across a gap that is
10080 of unknown (at compile-time) size. */
10082 ix86_builtin_setjmp_frame_value (void)
10084 return stack_realign_fp ? hard_frame_pointer_rtx : virtual_stack_vars_rtx;
10087 /* When using -fsplit-stack, the allocation routines set a field in
10088 the TCB to the bottom of the stack plus this much space, measured
10091 #define SPLIT_STACK_AVAILABLE 256
10093 /* Fill structure ix86_frame about frame of currently computed function. */
10096 ix86_compute_frame_layout (struct ix86_frame *frame)
10098 unsigned HOST_WIDE_INT stack_alignment_needed;
10099 HOST_WIDE_INT offset;
10100 unsigned HOST_WIDE_INT preferred_alignment;
10101 HOST_WIDE_INT size = get_frame_size ();
10102 HOST_WIDE_INT to_allocate;
10104 frame->nregs = ix86_nsaved_regs ();
10105 frame->nsseregs = ix86_nsaved_sseregs ();
10107 /* 64-bit MS ABI seem to require stack alignment to be always 16 except for
10108 function prologues and leaf. */
10109 if ((TARGET_64BIT_MS_ABI && crtl->preferred_stack_boundary < 128)
10110 && (!crtl->is_leaf || cfun->calls_alloca != 0
10111 || ix86_current_function_calls_tls_descriptor))
10113 crtl->preferred_stack_boundary = 128;
10114 crtl->stack_alignment_needed = 128;
10116 /* preferred_stack_boundary is never updated for call
10117 expanded from tls descriptor. Update it here. We don't update it in
10118 expand stage because according to the comments before
10119 ix86_current_function_calls_tls_descriptor, tls calls may be optimized
10121 else if (ix86_current_function_calls_tls_descriptor
10122 && crtl->preferred_stack_boundary < PREFERRED_STACK_BOUNDARY)
10124 crtl->preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
10125 if (crtl->stack_alignment_needed < PREFERRED_STACK_BOUNDARY)
10126 crtl->stack_alignment_needed = PREFERRED_STACK_BOUNDARY;
10129 stack_alignment_needed = crtl->stack_alignment_needed / BITS_PER_UNIT;
10130 preferred_alignment = crtl->preferred_stack_boundary / BITS_PER_UNIT;
10132 gcc_assert (!size || stack_alignment_needed);
10133 gcc_assert (preferred_alignment >= STACK_BOUNDARY / BITS_PER_UNIT);
10134 gcc_assert (preferred_alignment <= stack_alignment_needed);
10136 /* For SEH we have to limit the amount of code movement into the prologue.
10137 At present we do this via a BLOCKAGE, at which point there's very little
10138 scheduling that can be done, which means that there's very little point
10139 in doing anything except PUSHs. */
10141 cfun->machine->use_fast_prologue_epilogue = false;
10143 /* During reload iteration the amount of registers saved can change.
10144 Recompute the value as needed. Do not recompute when amount of registers
10145 didn't change as reload does multiple calls to the function and does not
10146 expect the decision to change within single iteration. */
10147 else if (!optimize_bb_for_size_p (ENTRY_BLOCK_PTR_FOR_FN (cfun))
10148 && cfun->machine->use_fast_prologue_epilogue_nregs != frame->nregs)
10150 int count = frame->nregs;
10151 struct cgraph_node *node = cgraph_node::get (current_function_decl);
10153 cfun->machine->use_fast_prologue_epilogue_nregs = count;
10155 /* The fast prologue uses move instead of push to save registers. This
10156 is significantly longer, but also executes faster as modern hardware
10157 can execute the moves in parallel, but can't do that for push/pop.
10159 Be careful about choosing what prologue to emit: When function takes
10160 many instructions to execute we may use slow version as well as in
10161 case function is known to be outside hot spot (this is known with
10162 feedback only). Weight the size of function by number of registers
10163 to save as it is cheap to use one or two push instructions but very
10164 slow to use many of them. */
10166 count = (count - 1) * FAST_PROLOGUE_INSN_COUNT;
10167 if (node->frequency < NODE_FREQUENCY_NORMAL
10168 || (flag_branch_probabilities
10169 && node->frequency < NODE_FREQUENCY_HOT))
10170 cfun->machine->use_fast_prologue_epilogue = false;
10172 cfun->machine->use_fast_prologue_epilogue
10173 = !expensive_function_p (count);
10176 frame->save_regs_using_mov
10177 = (TARGET_PROLOGUE_USING_MOVE && cfun->machine->use_fast_prologue_epilogue
10178 /* If static stack checking is enabled and done with probes,
10179 the registers need to be saved before allocating the frame. */
10180 && flag_stack_check != STATIC_BUILTIN_STACK_CHECK);
10182 /* Skip return address. */
10183 offset = UNITS_PER_WORD;
10185 /* Skip pushed static chain. */
10186 if (ix86_static_chain_on_stack)
10187 offset += UNITS_PER_WORD;
10189 /* Skip saved base pointer. */
10190 if (frame_pointer_needed)
10191 offset += UNITS_PER_WORD;
10192 frame->hfp_save_offset = offset;
10194 /* The traditional frame pointer location is at the top of the frame. */
10195 frame->hard_frame_pointer_offset = offset;
10197 /* Register save area */
10198 offset += frame->nregs * UNITS_PER_WORD;
10199 frame->reg_save_offset = offset;
10201 /* On SEH target, registers are pushed just before the frame pointer
10204 frame->hard_frame_pointer_offset = offset;
10206 /* Align and set SSE register save area. */
10207 if (frame->nsseregs)
10209 /* The only ABI that has saved SSE registers (Win64) also has a
10210 16-byte aligned default stack, and thus we don't need to be
10211 within the re-aligned local stack frame to save them. */
10212 gcc_assert (INCOMING_STACK_BOUNDARY >= 128);
10213 offset = (offset + 16 - 1) & -16;
10214 offset += frame->nsseregs * 16;
10216 frame->sse_reg_save_offset = offset;
10218 /* The re-aligned stack starts here. Values before this point are not
10219 directly comparable with values below this point. In order to make
10220 sure that no value happens to be the same before and after, force
10221 the alignment computation below to add a non-zero value. */
10222 if (stack_realign_fp)
10223 offset = (offset + stack_alignment_needed) & -stack_alignment_needed;
10226 frame->va_arg_size = ix86_varargs_gpr_size + ix86_varargs_fpr_size;
10227 offset += frame->va_arg_size;
10229 /* Align start of frame for local function. */
10230 if (stack_realign_fp
10231 || offset != frame->sse_reg_save_offset
10234 || cfun->calls_alloca
10235 || ix86_current_function_calls_tls_descriptor)
10236 offset = (offset + stack_alignment_needed - 1) & -stack_alignment_needed;
10238 /* Frame pointer points here. */
10239 frame->frame_pointer_offset = offset;
10243 /* Add outgoing arguments area. Can be skipped if we eliminated
10244 all the function calls as dead code.
10245 Skipping is however impossible when function calls alloca. Alloca
10246 expander assumes that last crtl->outgoing_args_size
10247 of stack frame are unused. */
10248 if (ACCUMULATE_OUTGOING_ARGS
10249 && (!crtl->is_leaf || cfun->calls_alloca
10250 || ix86_current_function_calls_tls_descriptor))
10252 offset += crtl->outgoing_args_size;
10253 frame->outgoing_arguments_size = crtl->outgoing_args_size;
10256 frame->outgoing_arguments_size = 0;
10258 /* Align stack boundary. Only needed if we're calling another function
10259 or using alloca. */
10260 if (!crtl->is_leaf || cfun->calls_alloca
10261 || ix86_current_function_calls_tls_descriptor)
10262 offset = (offset + preferred_alignment - 1) & -preferred_alignment;
10264 /* We've reached end of stack frame. */
10265 frame->stack_pointer_offset = offset;
10267 /* Size prologue needs to allocate. */
10268 to_allocate = offset - frame->sse_reg_save_offset;
10270 if ((!to_allocate && frame->nregs <= 1)
10271 || (TARGET_64BIT && to_allocate >= (HOST_WIDE_INT) 0x80000000))
10272 frame->save_regs_using_mov = false;
10274 if (ix86_using_red_zone ()
10275 && crtl->sp_is_unchanging
10277 && !ix86_current_function_calls_tls_descriptor)
10279 frame->red_zone_size = to_allocate;
10280 if (frame->save_regs_using_mov)
10281 frame->red_zone_size += frame->nregs * UNITS_PER_WORD;
10282 if (frame->red_zone_size > RED_ZONE_SIZE - RED_ZONE_RESERVE)
10283 frame->red_zone_size = RED_ZONE_SIZE - RED_ZONE_RESERVE;
10286 frame->red_zone_size = 0;
10287 frame->stack_pointer_offset -= frame->red_zone_size;
10289 /* The SEH frame pointer location is near the bottom of the frame.
10290 This is enforced by the fact that the difference between the
10291 stack pointer and the frame pointer is limited to 240 bytes in
10292 the unwind data structure. */
10295 HOST_WIDE_INT diff;
10297 /* If we can leave the frame pointer where it is, do so. Also, returns
10298 the establisher frame for __builtin_frame_address (0). */
10299 diff = frame->stack_pointer_offset - frame->hard_frame_pointer_offset;
10300 if (diff <= SEH_MAX_FRAME_SIZE
10301 && (diff > 240 || (diff & 15) != 0)
10302 && !crtl->accesses_prior_frames)
10304 /* Ideally we'd determine what portion of the local stack frame
10305 (within the constraint of the lowest 240) is most heavily used.
10306 But without that complication, simply bias the frame pointer
10307 by 128 bytes so as to maximize the amount of the local stack
10308 frame that is addressable with 8-bit offsets. */
10309 frame->hard_frame_pointer_offset = frame->stack_pointer_offset - 128;
10314 /* This is semi-inlined memory_address_length, but simplified
10315 since we know that we're always dealing with reg+offset, and
10316 to avoid having to create and discard all that rtl. */
10319 choose_baseaddr_len (unsigned int regno, HOST_WIDE_INT offset)
10325 /* EBP and R13 cannot be encoded without an offset. */
10326 len = (regno == BP_REG || regno == R13_REG);
10328 else if (IN_RANGE (offset, -128, 127))
10331 /* ESP and R12 must be encoded with a SIB byte. */
10332 if (regno == SP_REG || regno == R12_REG)
10338 /* Return an RTX that points to CFA_OFFSET within the stack frame.
10339 The valid base registers are taken from CFUN->MACHINE->FS. */
10342 choose_baseaddr (HOST_WIDE_INT cfa_offset)
10344 const struct machine_function *m = cfun->machine;
10345 rtx base_reg = NULL;
10346 HOST_WIDE_INT base_offset = 0;
10348 if (m->use_fast_prologue_epilogue)
10350 /* Choose the base register most likely to allow the most scheduling
10351 opportunities. Generally FP is valid throughout the function,
10352 while DRAP must be reloaded within the epilogue. But choose either
10353 over the SP due to increased encoding size. */
10355 if (m->fs.fp_valid)
10357 base_reg = hard_frame_pointer_rtx;
10358 base_offset = m->fs.fp_offset - cfa_offset;
10360 else if (m->fs.drap_valid)
10362 base_reg = crtl->drap_reg;
10363 base_offset = 0 - cfa_offset;
10365 else if (m->fs.sp_valid)
10367 base_reg = stack_pointer_rtx;
10368 base_offset = m->fs.sp_offset - cfa_offset;
10373 HOST_WIDE_INT toffset;
10374 int len = 16, tlen;
10376 /* Choose the base register with the smallest address encoding.
10377 With a tie, choose FP > DRAP > SP. */
10378 if (m->fs.sp_valid)
10380 base_reg = stack_pointer_rtx;
10381 base_offset = m->fs.sp_offset - cfa_offset;
10382 len = choose_baseaddr_len (STACK_POINTER_REGNUM, base_offset);
10384 if (m->fs.drap_valid)
10386 toffset = 0 - cfa_offset;
10387 tlen = choose_baseaddr_len (REGNO (crtl->drap_reg), toffset);
10390 base_reg = crtl->drap_reg;
10391 base_offset = toffset;
10395 if (m->fs.fp_valid)
10397 toffset = m->fs.fp_offset - cfa_offset;
10398 tlen = choose_baseaddr_len (HARD_FRAME_POINTER_REGNUM, toffset);
10401 base_reg = hard_frame_pointer_rtx;
10402 base_offset = toffset;
10407 gcc_assert (base_reg != NULL);
10409 return plus_constant (Pmode, base_reg, base_offset);
10412 /* Emit code to save registers in the prologue. */
10415 ix86_emit_save_regs (void)
10417 unsigned int regno;
10420 for (regno = FIRST_PSEUDO_REGISTER - 1; regno-- > 0; )
10421 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
10423 insn = emit_insn (gen_push (gen_rtx_REG (word_mode, regno)));
10424 RTX_FRAME_RELATED_P (insn) = 1;
10428 /* Emit a single register save at CFA - CFA_OFFSET. */
10431 ix86_emit_save_reg_using_mov (machine_mode mode, unsigned int regno,
10432 HOST_WIDE_INT cfa_offset)
10434 struct machine_function *m = cfun->machine;
10435 rtx reg = gen_rtx_REG (mode, regno);
10436 rtx mem, addr, base, insn;
10438 addr = choose_baseaddr (cfa_offset);
10439 mem = gen_frame_mem (mode, addr);
10441 /* For SSE saves, we need to indicate the 128-bit alignment. */
10442 set_mem_align (mem, GET_MODE_ALIGNMENT (mode));
10444 insn = emit_move_insn (mem, reg);
10445 RTX_FRAME_RELATED_P (insn) = 1;
10448 if (GET_CODE (base) == PLUS)
10449 base = XEXP (base, 0);
10450 gcc_checking_assert (REG_P (base));
10452 /* When saving registers into a re-aligned local stack frame, avoid
10453 any tricky guessing by dwarf2out. */
10454 if (m->fs.realigned)
10456 gcc_checking_assert (stack_realign_drap);
10458 if (regno == REGNO (crtl->drap_reg))
10460 /* A bit of a hack. We force the DRAP register to be saved in
10461 the re-aligned stack frame, which provides us with a copy
10462 of the CFA that will last past the prologue. Install it. */
10463 gcc_checking_assert (cfun->machine->fs.fp_valid);
10464 addr = plus_constant (Pmode, hard_frame_pointer_rtx,
10465 cfun->machine->fs.fp_offset - cfa_offset);
10466 mem = gen_rtx_MEM (mode, addr);
10467 add_reg_note (insn, REG_CFA_DEF_CFA, mem);
10471 /* The frame pointer is a stable reference within the
10472 aligned frame. Use it. */
10473 gcc_checking_assert (cfun->machine->fs.fp_valid);
10474 addr = plus_constant (Pmode, hard_frame_pointer_rtx,
10475 cfun->machine->fs.fp_offset - cfa_offset);
10476 mem = gen_rtx_MEM (mode, addr);
10477 add_reg_note (insn, REG_CFA_EXPRESSION,
10478 gen_rtx_SET (VOIDmode, mem, reg));
10482 /* The memory may not be relative to the current CFA register,
10483 which means that we may need to generate a new pattern for
10484 use by the unwind info. */
10485 else if (base != m->fs.cfa_reg)
10487 addr = plus_constant (Pmode, m->fs.cfa_reg,
10488 m->fs.cfa_offset - cfa_offset);
10489 mem = gen_rtx_MEM (mode, addr);
10490 add_reg_note (insn, REG_CFA_OFFSET, gen_rtx_SET (VOIDmode, mem, reg));
10494 /* Emit code to save registers using MOV insns.
10495 First register is stored at CFA - CFA_OFFSET. */
10497 ix86_emit_save_regs_using_mov (HOST_WIDE_INT cfa_offset)
10499 unsigned int regno;
10501 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10502 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
10504 ix86_emit_save_reg_using_mov (word_mode, regno, cfa_offset);
10505 cfa_offset -= UNITS_PER_WORD;
10509 /* Emit code to save SSE registers using MOV insns.
10510 First register is stored at CFA - CFA_OFFSET. */
10512 ix86_emit_save_sse_regs_using_mov (HOST_WIDE_INT cfa_offset)
10514 unsigned int regno;
10516 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10517 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
10519 ix86_emit_save_reg_using_mov (V4SFmode, regno, cfa_offset);
10524 static GTY(()) rtx queued_cfa_restores;
10526 /* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
10527 manipulation insn. The value is on the stack at CFA - CFA_OFFSET.
10528 Don't add the note if the previously saved value will be left untouched
10529 within stack red-zone till return, as unwinders can find the same value
10530 in the register and on the stack. */
10533 ix86_add_cfa_restore_note (rtx insn, rtx reg, HOST_WIDE_INT cfa_offset)
10535 if (!crtl->shrink_wrapped
10536 && cfa_offset <= cfun->machine->fs.red_zone_offset)
10541 add_reg_note (insn, REG_CFA_RESTORE, reg);
10542 RTX_FRAME_RELATED_P (insn) = 1;
10545 queued_cfa_restores
10546 = alloc_reg_note (REG_CFA_RESTORE, reg, queued_cfa_restores);
10549 /* Add queued REG_CFA_RESTORE notes if any to INSN. */
10552 ix86_add_queued_cfa_restore_notes (rtx insn)
10555 if (!queued_cfa_restores)
10557 for (last = queued_cfa_restores; XEXP (last, 1); last = XEXP (last, 1))
10559 XEXP (last, 1) = REG_NOTES (insn);
10560 REG_NOTES (insn) = queued_cfa_restores;
10561 queued_cfa_restores = NULL_RTX;
10562 RTX_FRAME_RELATED_P (insn) = 1;
10565 /* Expand prologue or epilogue stack adjustment.
10566 The pattern exist to put a dependency on all ebp-based memory accesses.
10567 STYLE should be negative if instructions should be marked as frame related,
10568 zero if %r11 register is live and cannot be freely used and positive
10572 pro_epilogue_adjust_stack (rtx dest, rtx src, rtx offset,
10573 int style, bool set_cfa)
10575 struct machine_function *m = cfun->machine;
10577 bool add_frame_related_expr = false;
10579 if (Pmode == SImode)
10580 insn = gen_pro_epilogue_adjust_stack_si_add (dest, src, offset);
10581 else if (x86_64_immediate_operand (offset, DImode))
10582 insn = gen_pro_epilogue_adjust_stack_di_add (dest, src, offset);
10586 /* r11 is used by indirect sibcall return as well, set before the
10587 epilogue and used after the epilogue. */
10589 tmp = gen_rtx_REG (DImode, R11_REG);
10592 gcc_assert (src != hard_frame_pointer_rtx
10593 && dest != hard_frame_pointer_rtx);
10594 tmp = hard_frame_pointer_rtx;
10596 insn = emit_insn (gen_rtx_SET (DImode, tmp, offset));
10598 add_frame_related_expr = true;
10600 insn = gen_pro_epilogue_adjust_stack_di_add (dest, src, tmp);
10603 insn = emit_insn (insn);
10605 ix86_add_queued_cfa_restore_notes (insn);
10611 gcc_assert (m->fs.cfa_reg == src);
10612 m->fs.cfa_offset += INTVAL (offset);
10613 m->fs.cfa_reg = dest;
10615 r = gen_rtx_PLUS (Pmode, src, offset);
10616 r = gen_rtx_SET (VOIDmode, dest, r);
10617 add_reg_note (insn, REG_CFA_ADJUST_CFA, r);
10618 RTX_FRAME_RELATED_P (insn) = 1;
10620 else if (style < 0)
10622 RTX_FRAME_RELATED_P (insn) = 1;
10623 if (add_frame_related_expr)
10625 rtx r = gen_rtx_PLUS (Pmode, src, offset);
10626 r = gen_rtx_SET (VOIDmode, dest, r);
10627 add_reg_note (insn, REG_FRAME_RELATED_EXPR, r);
10631 if (dest == stack_pointer_rtx)
10633 HOST_WIDE_INT ooffset = m->fs.sp_offset;
10634 bool valid = m->fs.sp_valid;
10636 if (src == hard_frame_pointer_rtx)
10638 valid = m->fs.fp_valid;
10639 ooffset = m->fs.fp_offset;
10641 else if (src == crtl->drap_reg)
10643 valid = m->fs.drap_valid;
10648 /* Else there are two possibilities: SP itself, which we set
10649 up as the default above. Or EH_RETURN_STACKADJ_RTX, which is
10650 taken care of this by hand along the eh_return path. */
10651 gcc_checking_assert (src == stack_pointer_rtx
10652 || offset == const0_rtx);
10655 m->fs.sp_offset = ooffset - INTVAL (offset);
10656 m->fs.sp_valid = valid;
10660 /* Find an available register to be used as dynamic realign argument
10661 pointer regsiter. Such a register will be written in prologue and
10662 used in begin of body, so it must not be
10663 1. parameter passing register.
10665 We reuse static-chain register if it is available. Otherwise, we
10666 use DI for i386 and R13 for x86-64. We chose R13 since it has
10669 Return: the regno of chosen register. */
10671 static unsigned int
10672 find_drap_reg (void)
10674 tree decl = cfun->decl;
10678 /* Use R13 for nested function or function need static chain.
10679 Since function with tail call may use any caller-saved
10680 registers in epilogue, DRAP must not use caller-saved
10681 register in such case. */
10682 if (DECL_STATIC_CHAIN (decl) || crtl->tail_call_emit)
10689 /* Use DI for nested function or function need static chain.
10690 Since function with tail call may use any caller-saved
10691 registers in epilogue, DRAP must not use caller-saved
10692 register in such case. */
10693 if (DECL_STATIC_CHAIN (decl) || crtl->tail_call_emit)
10696 /* Reuse static chain register if it isn't used for parameter
10698 if (ix86_function_regparm (TREE_TYPE (decl), decl) <= 2)
10700 unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (decl));
10701 if ((ccvt & (IX86_CALLCVT_FASTCALL | IX86_CALLCVT_THISCALL)) == 0)
10708 /* Return minimum incoming stack alignment. */
10710 static unsigned int
10711 ix86_minimum_incoming_stack_boundary (bool sibcall)
10713 unsigned int incoming_stack_boundary;
10715 /* Prefer the one specified at command line. */
10716 if (ix86_user_incoming_stack_boundary)
10717 incoming_stack_boundary = ix86_user_incoming_stack_boundary;
10718 /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
10719 if -mstackrealign is used, it isn't used for sibcall check and
10720 estimated stack alignment is 128bit. */
10723 && ix86_force_align_arg_pointer
10724 && crtl->stack_alignment_estimated == 128)
10725 incoming_stack_boundary = MIN_STACK_BOUNDARY;
10727 incoming_stack_boundary = ix86_default_incoming_stack_boundary;
10729 /* Incoming stack alignment can be changed on individual functions
10730 via force_align_arg_pointer attribute. We use the smallest
10731 incoming stack boundary. */
10732 if (incoming_stack_boundary > MIN_STACK_BOUNDARY
10733 && lookup_attribute (ix86_force_align_arg_pointer_string,
10734 TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
10735 incoming_stack_boundary = MIN_STACK_BOUNDARY;
10737 /* The incoming stack frame has to be aligned at least at
10738 parm_stack_boundary. */
10739 if (incoming_stack_boundary < crtl->parm_stack_boundary)
10740 incoming_stack_boundary = crtl->parm_stack_boundary;
10742 /* Stack at entrance of main is aligned by runtime. We use the
10743 smallest incoming stack boundary. */
10744 if (incoming_stack_boundary > MAIN_STACK_BOUNDARY
10745 && DECL_NAME (current_function_decl)
10746 && MAIN_NAME_P (DECL_NAME (current_function_decl))
10747 && DECL_FILE_SCOPE_P (current_function_decl))
10748 incoming_stack_boundary = MAIN_STACK_BOUNDARY;
10750 return incoming_stack_boundary;
10753 /* Update incoming stack boundary and estimated stack alignment. */
10756 ix86_update_stack_boundary (void)
10758 ix86_incoming_stack_boundary
10759 = ix86_minimum_incoming_stack_boundary (false);
10761 /* x86_64 vararg needs 16byte stack alignment for register save
10765 && crtl->stack_alignment_estimated < 128)
10766 crtl->stack_alignment_estimated = 128;
10769 /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
10770 needed or an rtx for DRAP otherwise. */
10773 ix86_get_drap_rtx (void)
10775 if (ix86_force_drap || !ACCUMULATE_OUTGOING_ARGS)
10776 crtl->need_drap = true;
10778 if (stack_realign_drap)
10780 /* Assign DRAP to vDRAP and returns vDRAP */
10781 unsigned int regno = find_drap_reg ();
10784 rtx_insn *seq, *insn;
10786 arg_ptr = gen_rtx_REG (Pmode, regno);
10787 crtl->drap_reg = arg_ptr;
10790 drap_vreg = copy_to_reg (arg_ptr);
10791 seq = get_insns ();
10794 insn = emit_insn_before (seq, NEXT_INSN (entry_of_function ()));
10797 add_reg_note (insn, REG_CFA_SET_VDRAP, drap_vreg);
10798 RTX_FRAME_RELATED_P (insn) = 1;
10806 /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
10809 ix86_internal_arg_pointer (void)
10811 return virtual_incoming_args_rtx;
10814 struct scratch_reg {
10819 /* Return a short-lived scratch register for use on function entry.
10820 In 32-bit mode, it is valid only after the registers are saved
10821 in the prologue. This register must be released by means of
10822 release_scratch_register_on_entry once it is dead. */
10825 get_scratch_register_on_entry (struct scratch_reg *sr)
10833 /* We always use R11 in 64-bit mode. */
10838 tree decl = current_function_decl, fntype = TREE_TYPE (decl);
10840 = lookup_attribute ("fastcall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
10842 = lookup_attribute ("thiscall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
10843 bool static_chain_p = DECL_STATIC_CHAIN (decl);
10844 int regparm = ix86_function_regparm (fntype, decl);
10846 = crtl->drap_reg ? REGNO (crtl->drap_reg) : INVALID_REGNUM;
10848 /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
10849 for the static chain register. */
10850 if ((regparm < 1 || (fastcall_p && !static_chain_p))
10851 && drap_regno != AX_REG)
10853 /* 'thiscall' sets regparm to 1, uses ecx for arguments and edx
10854 for the static chain register. */
10855 else if (thiscall_p && !static_chain_p && drap_regno != AX_REG)
10857 else if (regparm < 2 && !thiscall_p && drap_regno != DX_REG)
10859 /* ecx is the static chain register. */
10860 else if (regparm < 3 && !fastcall_p && !thiscall_p
10862 && drap_regno != CX_REG)
10864 else if (ix86_save_reg (BX_REG, true))
10866 /* esi is the static chain register. */
10867 else if (!(regparm == 3 && static_chain_p)
10868 && ix86_save_reg (SI_REG, true))
10870 else if (ix86_save_reg (DI_REG, true))
10874 regno = (drap_regno == AX_REG ? DX_REG : AX_REG);
10879 sr->reg = gen_rtx_REG (Pmode, regno);
10882 rtx insn = emit_insn (gen_push (sr->reg));
10883 RTX_FRAME_RELATED_P (insn) = 1;
10887 /* Release a scratch register obtained from the preceding function. */
10890 release_scratch_register_on_entry (struct scratch_reg *sr)
10894 struct machine_function *m = cfun->machine;
10895 rtx x, insn = emit_insn (gen_pop (sr->reg));
10897 /* The RTX_FRAME_RELATED_P mechanism doesn't know about pop. */
10898 RTX_FRAME_RELATED_P (insn) = 1;
10899 x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (UNITS_PER_WORD));
10900 x = gen_rtx_SET (VOIDmode, stack_pointer_rtx, x);
10901 add_reg_note (insn, REG_FRAME_RELATED_EXPR, x);
10902 m->fs.sp_offset -= UNITS_PER_WORD;
10906 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
10908 /* Emit code to adjust the stack pointer by SIZE bytes while probing it. */
10911 ix86_adjust_stack_and_probe (const HOST_WIDE_INT size)
10913 /* We skip the probe for the first interval + a small dope of 4 words and
10914 probe that many bytes past the specified size to maintain a protection
10915 area at the botton of the stack. */
10916 const int dope = 4 * UNITS_PER_WORD;
10917 rtx size_rtx = GEN_INT (size), last;
10919 /* See if we have a constant small number of probes to generate. If so,
10920 that's the easy case. The run-time loop is made up of 11 insns in the
10921 generic case while the compile-time loop is made up of 3+2*(n-1) insns
10922 for n # of intervals. */
10923 if (size <= 5 * PROBE_INTERVAL)
10925 HOST_WIDE_INT i, adjust;
10926 bool first_probe = true;
10928 /* Adjust SP and probe at PROBE_INTERVAL + N * PROBE_INTERVAL for
10929 values of N from 1 until it exceeds SIZE. If only one probe is
10930 needed, this will not generate any code. Then adjust and probe
10931 to PROBE_INTERVAL + SIZE. */
10932 for (i = PROBE_INTERVAL; i < size; i += PROBE_INTERVAL)
10936 adjust = 2 * PROBE_INTERVAL + dope;
10937 first_probe = false;
10940 adjust = PROBE_INTERVAL;
10942 emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
10943 plus_constant (Pmode, stack_pointer_rtx,
10945 emit_stack_probe (stack_pointer_rtx);
10949 adjust = size + PROBE_INTERVAL + dope;
10951 adjust = size + PROBE_INTERVAL - i;
10953 emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
10954 plus_constant (Pmode, stack_pointer_rtx,
10956 emit_stack_probe (stack_pointer_rtx);
10958 /* Adjust back to account for the additional first interval. */
10959 last = emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
10960 plus_constant (Pmode, stack_pointer_rtx,
10961 PROBE_INTERVAL + dope)));
10964 /* Otherwise, do the same as above, but in a loop. Note that we must be
10965 extra careful with variables wrapping around because we might be at
10966 the very top (or the very bottom) of the address space and we have
10967 to be able to handle this case properly; in particular, we use an
10968 equality test for the loop condition. */
10971 HOST_WIDE_INT rounded_size;
10972 struct scratch_reg sr;
10974 get_scratch_register_on_entry (&sr);
10977 /* Step 1: round SIZE to the previous multiple of the interval. */
10979 rounded_size = size & -PROBE_INTERVAL;
10982 /* Step 2: compute initial and final value of the loop counter. */
10984 /* SP = SP_0 + PROBE_INTERVAL. */
10985 emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
10986 plus_constant (Pmode, stack_pointer_rtx,
10987 - (PROBE_INTERVAL + dope))));
10989 /* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE. */
10990 emit_move_insn (sr.reg, GEN_INT (-rounded_size));
10991 emit_insn (gen_rtx_SET (VOIDmode, sr.reg,
10992 gen_rtx_PLUS (Pmode, sr.reg,
10993 stack_pointer_rtx)));
10996 /* Step 3: the loop
10998 while (SP != LAST_ADDR)
11000 SP = SP + PROBE_INTERVAL
11004 adjusts SP and probes to PROBE_INTERVAL + N * PROBE_INTERVAL for
11005 values of N from 1 until it is equal to ROUNDED_SIZE. */
11007 emit_insn (ix86_gen_adjust_stack_and_probe (sr.reg, sr.reg, size_rtx));
11010 /* Step 4: adjust SP and probe at PROBE_INTERVAL + SIZE if we cannot
11011 assert at compile-time that SIZE is equal to ROUNDED_SIZE. */
11013 if (size != rounded_size)
11015 emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
11016 plus_constant (Pmode, stack_pointer_rtx,
11017 rounded_size - size)));
11018 emit_stack_probe (stack_pointer_rtx);
11021 /* Adjust back to account for the additional first interval. */
11022 last = emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
11023 plus_constant (Pmode, stack_pointer_rtx,
11024 PROBE_INTERVAL + dope)));
11026 release_scratch_register_on_entry (&sr);
11029 gcc_assert (cfun->machine->fs.cfa_reg != stack_pointer_rtx);
11031 /* Even if the stack pointer isn't the CFA register, we need to correctly
11032 describe the adjustments made to it, in particular differentiate the
11033 frame-related ones from the frame-unrelated ones. */
11036 rtx expr = gen_rtx_SEQUENCE (VOIDmode, rtvec_alloc (2));
11037 XVECEXP (expr, 0, 0)
11038 = gen_rtx_SET (VOIDmode, stack_pointer_rtx,
11039 plus_constant (Pmode, stack_pointer_rtx, -size));
11040 XVECEXP (expr, 0, 1)
11041 = gen_rtx_SET (VOIDmode, stack_pointer_rtx,
11042 plus_constant (Pmode, stack_pointer_rtx,
11043 PROBE_INTERVAL + dope + size));
11044 add_reg_note (last, REG_FRAME_RELATED_EXPR, expr);
11045 RTX_FRAME_RELATED_P (last) = 1;
11047 cfun->machine->fs.sp_offset += size;
11050 /* Make sure nothing is scheduled before we are done. */
11051 emit_insn (gen_blockage ());
11054 /* Adjust the stack pointer up to REG while probing it. */
11057 output_adjust_stack_and_probe (rtx reg)
11059 static int labelno = 0;
11060 char loop_lab[32], end_lab[32];
11063 ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno);
11064 ASM_GENERATE_INTERNAL_LABEL (end_lab, "LPSRE", labelno++);
11066 ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);
11068 /* Jump to END_LAB if SP == LAST_ADDR. */
11069 xops[0] = stack_pointer_rtx;
11071 output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);
11072 fputs ("\tje\t", asm_out_file);
11073 assemble_name_raw (asm_out_file, end_lab);
11074 fputc ('\n', asm_out_file);
11076 /* SP = SP + PROBE_INTERVAL. */
11077 xops[1] = GEN_INT (PROBE_INTERVAL);
11078 output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);
11081 xops[1] = const0_rtx;
11082 output_asm_insn ("or%z0\t{%1, (%0)|DWORD PTR [%0], %1}", xops);
11084 fprintf (asm_out_file, "\tjmp\t");
11085 assemble_name_raw (asm_out_file, loop_lab);
11086 fputc ('\n', asm_out_file);
11088 ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, end_lab);
11093 /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
11094 inclusive. These are offsets from the current stack pointer. */
11097 ix86_emit_probe_stack_range (HOST_WIDE_INT first, HOST_WIDE_INT size)
11099 /* See if we have a constant small number of probes to generate. If so,
11100 that's the easy case. The run-time loop is made up of 7 insns in the
11101 generic case while the compile-time loop is made up of n insns for n #
11103 if (size <= 7 * PROBE_INTERVAL)
11107 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
11108 it exceeds SIZE. If only one probe is needed, this will not
11109 generate any code. Then probe at FIRST + SIZE. */
11110 for (i = PROBE_INTERVAL; i < size; i += PROBE_INTERVAL)
11111 emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
11114 emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
11118 /* Otherwise, do the same as above, but in a loop. Note that we must be
11119 extra careful with variables wrapping around because we might be at
11120 the very top (or the very bottom) of the address space and we have
11121 to be able to handle this case properly; in particular, we use an
11122 equality test for the loop condition. */
11125 HOST_WIDE_INT rounded_size, last;
11126 struct scratch_reg sr;
11128 get_scratch_register_on_entry (&sr);
11131 /* Step 1: round SIZE to the previous multiple of the interval. */
11133 rounded_size = size & -PROBE_INTERVAL;
11136 /* Step 2: compute initial and final value of the loop counter. */
11138 /* TEST_OFFSET = FIRST. */
11139 emit_move_insn (sr.reg, GEN_INT (-first));
11141 /* LAST_OFFSET = FIRST + ROUNDED_SIZE. */
11142 last = first + rounded_size;
11145 /* Step 3: the loop
11147 while (TEST_ADDR != LAST_ADDR)
11149 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
11153 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
11154 until it is equal to ROUNDED_SIZE. */
11156 emit_insn (ix86_gen_probe_stack_range (sr.reg, sr.reg, GEN_INT (-last)));
11159 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
11160 that SIZE is equal to ROUNDED_SIZE. */
11162 if (size != rounded_size)
11163 emit_stack_probe (plus_constant (Pmode,
11164 gen_rtx_PLUS (Pmode,
11167 rounded_size - size));
11169 release_scratch_register_on_entry (&sr);
11172 /* Make sure nothing is scheduled before we are done. */
11173 emit_insn (gen_blockage ());
11176 /* Probe a range of stack addresses from REG to END, inclusive. These are
11177 offsets from the current stack pointer. */
11180 output_probe_stack_range (rtx reg, rtx end)
11182 static int labelno = 0;
11183 char loop_lab[32], end_lab[32];
11186 ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno);
11187 ASM_GENERATE_INTERNAL_LABEL (end_lab, "LPSRE", labelno++);
11189 ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);
11191 /* Jump to END_LAB if TEST_ADDR == LAST_ADDR. */
11194 output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);
11195 fputs ("\tje\t", asm_out_file);
11196 assemble_name_raw (asm_out_file, end_lab);
11197 fputc ('\n', asm_out_file);
11199 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
11200 xops[1] = GEN_INT (PROBE_INTERVAL);
11201 output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);
11203 /* Probe at TEST_ADDR. */
11204 xops[0] = stack_pointer_rtx;
11206 xops[2] = const0_rtx;
11207 output_asm_insn ("or%z0\t{%2, (%0,%1)|DWORD PTR [%0+%1], %2}", xops);
11209 fprintf (asm_out_file, "\tjmp\t");
11210 assemble_name_raw (asm_out_file, loop_lab);
11211 fputc ('\n', asm_out_file);
11213 ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, end_lab);
11218 /* Finalize stack_realign_needed flag, which will guide prologue/epilogue
11219 to be generated in correct form. */
11221 ix86_finalize_stack_realign_flags (void)
11223 /* Check if stack realign is really needed after reload, and
11224 stores result in cfun */
11225 unsigned int incoming_stack_boundary
11226 = (crtl->parm_stack_boundary > ix86_incoming_stack_boundary
11227 ? crtl->parm_stack_boundary : ix86_incoming_stack_boundary);
11228 unsigned int stack_realign = (incoming_stack_boundary
11230 ? crtl->max_used_stack_slot_alignment
11231 : crtl->stack_alignment_needed));
11233 if (crtl->stack_realign_finalized)
11235 /* After stack_realign_needed is finalized, we can't no longer
11237 gcc_assert (crtl->stack_realign_needed == stack_realign);
11241 /* If the only reason for frame_pointer_needed is that we conservatively
11242 assumed stack realignment might be needed, but in the end nothing that
11243 needed the stack alignment had been spilled, clear frame_pointer_needed
11244 and say we don't need stack realignment. */
11246 && frame_pointer_needed
11248 && flag_omit_frame_pointer
11249 && crtl->sp_is_unchanging
11250 && !ix86_current_function_calls_tls_descriptor
11251 && !crtl->accesses_prior_frames
11252 && !cfun->calls_alloca
11253 && !crtl->calls_eh_return
11254 && !(flag_stack_check && STACK_CHECK_MOVING_SP)
11255 && !ix86_frame_pointer_required ()
11256 && get_frame_size () == 0
11257 && ix86_nsaved_sseregs () == 0
11258 && ix86_varargs_gpr_size + ix86_varargs_fpr_size == 0)
11260 HARD_REG_SET set_up_by_prologue, prologue_used;
11263 CLEAR_HARD_REG_SET (prologue_used);
11264 CLEAR_HARD_REG_SET (set_up_by_prologue);
11265 add_to_hard_reg_set (&set_up_by_prologue, Pmode, STACK_POINTER_REGNUM);
11266 add_to_hard_reg_set (&set_up_by_prologue, Pmode, ARG_POINTER_REGNUM);
11267 add_to_hard_reg_set (&set_up_by_prologue, Pmode,
11268 HARD_FRAME_POINTER_REGNUM);
11269 FOR_EACH_BB_FN (bb, cfun)
11272 FOR_BB_INSNS (bb, insn)
11273 if (NONDEBUG_INSN_P (insn)
11274 && requires_stack_frame_p (insn, prologue_used,
11275 set_up_by_prologue))
11277 crtl->stack_realign_needed = stack_realign;
11278 crtl->stack_realign_finalized = true;
11283 /* If drap has been set, but it actually isn't live at the start
11284 of the function, there is no reason to set it up. */
11285 if (crtl->drap_reg)
11287 basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
11288 if (! REGNO_REG_SET_P (DF_LR_IN (bb), REGNO (crtl->drap_reg)))
11290 crtl->drap_reg = NULL_RTX;
11291 crtl->need_drap = false;
11295 cfun->machine->no_drap_save_restore = true;
11297 frame_pointer_needed = false;
11298 stack_realign = false;
11299 crtl->max_used_stack_slot_alignment = incoming_stack_boundary;
11300 crtl->stack_alignment_needed = incoming_stack_boundary;
11301 crtl->stack_alignment_estimated = incoming_stack_boundary;
11302 if (crtl->preferred_stack_boundary > incoming_stack_boundary)
11303 crtl->preferred_stack_boundary = incoming_stack_boundary;
11304 df_finish_pass (true);
11305 df_scan_alloc (NULL);
11307 df_compute_regs_ever_live (true);
11311 crtl->stack_realign_needed = stack_realign;
11312 crtl->stack_realign_finalized = true;
11315 /* Delete SET_GOT right after entry block if it is allocated to reg. */
11318 ix86_elim_entry_set_got (rtx reg)
11320 basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
11321 rtx_insn *c_insn = BB_HEAD (bb);
11322 if (!NONDEBUG_INSN_P (c_insn))
11323 c_insn = next_nonnote_nondebug_insn (c_insn);
11324 if (c_insn && NONJUMP_INSN_P (c_insn))
11326 rtx pat = PATTERN (c_insn);
11327 if (GET_CODE (pat) == PARALLEL)
11329 rtx vec = XVECEXP (pat, 0, 0);
11330 if (GET_CODE (vec) == SET
11331 && XINT (XEXP (vec, 1), 1) == UNSPEC_SET_GOT
11332 && REGNO (XEXP (vec, 0)) == REGNO (reg))
11333 delete_insn (c_insn);
11338 /* Expand the prologue into a bunch of separate insns. */
11341 ix86_expand_prologue (void)
11343 struct machine_function *m = cfun->machine;
11345 struct ix86_frame frame;
11346 HOST_WIDE_INT allocate;
11347 bool int_registers_saved;
11348 bool sse_registers_saved;
11350 ix86_finalize_stack_realign_flags ();
11352 /* DRAP should not coexist with stack_realign_fp */
11353 gcc_assert (!(crtl->drap_reg && stack_realign_fp));
11355 memset (&m->fs, 0, sizeof (m->fs));
11357 /* Initialize CFA state for before the prologue. */
11358 m->fs.cfa_reg = stack_pointer_rtx;
11359 m->fs.cfa_offset = INCOMING_FRAME_SP_OFFSET;
11361 /* Track SP offset to the CFA. We continue tracking this after we've
11362 swapped the CFA register away from SP. In the case of re-alignment
11363 this is fudged; we're interested to offsets within the local frame. */
11364 m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
11365 m->fs.sp_valid = true;
11367 ix86_compute_frame_layout (&frame);
11369 if (!TARGET_64BIT && ix86_function_ms_hook_prologue (current_function_decl))
11371 /* We should have already generated an error for any use of
11372 ms_hook on a nested function. */
11373 gcc_checking_assert (!ix86_static_chain_on_stack);
11375 /* Check if profiling is active and we shall use profiling before
11376 prologue variant. If so sorry. */
11377 if (crtl->profile && flag_fentry != 0)
11378 sorry ("ms_hook_prologue attribute isn%'t compatible "
11379 "with -mfentry for 32-bit");
11381 /* In ix86_asm_output_function_label we emitted:
11382 8b ff movl.s %edi,%edi
11384 8b ec movl.s %esp,%ebp
11386 This matches the hookable function prologue in Win32 API
11387 functions in Microsoft Windows XP Service Pack 2 and newer.
11388 Wine uses this to enable Windows apps to hook the Win32 API
11389 functions provided by Wine.
11391 What that means is that we've already set up the frame pointer. */
11393 if (frame_pointer_needed
11394 && !(crtl->drap_reg && crtl->stack_realign_needed))
11398 /* We've decided to use the frame pointer already set up.
11399 Describe this to the unwinder by pretending that both
11400 push and mov insns happen right here.
11402 Putting the unwind info here at the end of the ms_hook
11403 is done so that we can make absolutely certain we get
11404 the required byte sequence at the start of the function,
11405 rather than relying on an assembler that can produce
11406 the exact encoding required.
11408 However it does mean (in the unpatched case) that we have
11409 a 1 insn window where the asynchronous unwind info is
11410 incorrect. However, if we placed the unwind info at
11411 its correct location we would have incorrect unwind info
11412 in the patched case. Which is probably all moot since
11413 I don't expect Wine generates dwarf2 unwind info for the
11414 system libraries that use this feature. */
11416 insn = emit_insn (gen_blockage ());
11418 push = gen_push (hard_frame_pointer_rtx);
11419 mov = gen_rtx_SET (VOIDmode, hard_frame_pointer_rtx,
11420 stack_pointer_rtx);
11421 RTX_FRAME_RELATED_P (push) = 1;
11422 RTX_FRAME_RELATED_P (mov) = 1;
11424 RTX_FRAME_RELATED_P (insn) = 1;
11425 add_reg_note (insn, REG_FRAME_RELATED_EXPR,
11426 gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, push, mov)));
11428 /* Note that gen_push incremented m->fs.cfa_offset, even
11429 though we didn't emit the push insn here. */
11430 m->fs.cfa_reg = hard_frame_pointer_rtx;
11431 m->fs.fp_offset = m->fs.cfa_offset;
11432 m->fs.fp_valid = true;
11436 /* The frame pointer is not needed so pop %ebp again.
11437 This leaves us with a pristine state. */
11438 emit_insn (gen_pop (hard_frame_pointer_rtx));
11442 /* The first insn of a function that accepts its static chain on the
11443 stack is to push the register that would be filled in by a direct
11444 call. This insn will be skipped by the trampoline. */
11445 else if (ix86_static_chain_on_stack)
11447 insn = emit_insn (gen_push (ix86_static_chain (cfun->decl, false)));
11448 emit_insn (gen_blockage ());
11450 /* We don't want to interpret this push insn as a register save,
11451 only as a stack adjustment. The real copy of the register as
11452 a save will be done later, if needed. */
11453 t = plus_constant (Pmode, stack_pointer_rtx, -UNITS_PER_WORD);
11454 t = gen_rtx_SET (VOIDmode, stack_pointer_rtx, t);
11455 add_reg_note (insn, REG_CFA_ADJUST_CFA, t);
11456 RTX_FRAME_RELATED_P (insn) = 1;
11459 /* Emit prologue code to adjust stack alignment and setup DRAP, in case
11460 of DRAP is needed and stack realignment is really needed after reload */
11461 if (stack_realign_drap)
11463 int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
11465 /* Only need to push parameter pointer reg if it is caller saved. */
11466 if (!call_used_regs[REGNO (crtl->drap_reg)])
11468 /* Push arg pointer reg */
11469 insn = emit_insn (gen_push (crtl->drap_reg));
11470 RTX_FRAME_RELATED_P (insn) = 1;
11473 /* Grab the argument pointer. */
11474 t = plus_constant (Pmode, stack_pointer_rtx, m->fs.sp_offset);
11475 insn = emit_insn (gen_rtx_SET (VOIDmode, crtl->drap_reg, t));
11476 RTX_FRAME_RELATED_P (insn) = 1;
11477 m->fs.cfa_reg = crtl->drap_reg;
11478 m->fs.cfa_offset = 0;
11480 /* Align the stack. */
11481 insn = emit_insn (ix86_gen_andsp (stack_pointer_rtx,
11483 GEN_INT (-align_bytes)));
11484 RTX_FRAME_RELATED_P (insn) = 1;
11486 /* Replicate the return address on the stack so that return
11487 address can be reached via (argp - 1) slot. This is needed
11488 to implement macro RETURN_ADDR_RTX and intrinsic function
11489 expand_builtin_return_addr etc. */
11490 t = plus_constant (Pmode, crtl->drap_reg, -UNITS_PER_WORD);
11491 t = gen_frame_mem (word_mode, t);
11492 insn = emit_insn (gen_push (t));
11493 RTX_FRAME_RELATED_P (insn) = 1;
11495 /* For the purposes of frame and register save area addressing,
11496 we've started over with a new frame. */
11497 m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
11498 m->fs.realigned = true;
11501 int_registers_saved = (frame.nregs == 0);
11502 sse_registers_saved = (frame.nsseregs == 0);
11504 if (frame_pointer_needed && !m->fs.fp_valid)
11506 /* Note: AT&T enter does NOT have reversed args. Enter is probably
11507 slower on all targets. Also sdb doesn't like it. */
11508 insn = emit_insn (gen_push (hard_frame_pointer_rtx));
11509 RTX_FRAME_RELATED_P (insn) = 1;
11511 /* Push registers now, before setting the frame pointer
11513 if (!int_registers_saved
11515 && !frame.save_regs_using_mov)
11517 ix86_emit_save_regs ();
11518 int_registers_saved = true;
11519 gcc_assert (m->fs.sp_offset == frame.reg_save_offset);
11522 if (m->fs.sp_offset == frame.hard_frame_pointer_offset)
11524 insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
11525 RTX_FRAME_RELATED_P (insn) = 1;
11527 if (m->fs.cfa_reg == stack_pointer_rtx)
11528 m->fs.cfa_reg = hard_frame_pointer_rtx;
11529 m->fs.fp_offset = m->fs.sp_offset;
11530 m->fs.fp_valid = true;
11534 if (!int_registers_saved)
11536 /* If saving registers via PUSH, do so now. */
11537 if (!frame.save_regs_using_mov)
11539 ix86_emit_save_regs ();
11540 int_registers_saved = true;
11541 gcc_assert (m->fs.sp_offset == frame.reg_save_offset);
11544 /* When using red zone we may start register saving before allocating
11545 the stack frame saving one cycle of the prologue. However, avoid
11546 doing this if we have to probe the stack; at least on x86_64 the
11547 stack probe can turn into a call that clobbers a red zone location. */
11548 else if (ix86_using_red_zone ()
11549 && (! TARGET_STACK_PROBE
11550 || frame.stack_pointer_offset < CHECK_STACK_LIMIT))
11552 ix86_emit_save_regs_using_mov (frame.reg_save_offset);
11553 int_registers_saved = true;
11557 if (stack_realign_fp)
11559 int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
11560 gcc_assert (align_bytes > MIN_STACK_BOUNDARY / BITS_PER_UNIT);
11562 /* The computation of the size of the re-aligned stack frame means
11563 that we must allocate the size of the register save area before
11564 performing the actual alignment. Otherwise we cannot guarantee
11565 that there's enough storage above the realignment point. */
11566 if (m->fs.sp_offset != frame.sse_reg_save_offset)
11567 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
11568 GEN_INT (m->fs.sp_offset
11569 - frame.sse_reg_save_offset),
11572 /* Align the stack. */
11573 insn = emit_insn (ix86_gen_andsp (stack_pointer_rtx,
11575 GEN_INT (-align_bytes)));
11577 /* For the purposes of register save area addressing, the stack
11578 pointer is no longer valid. As for the value of sp_offset,
11579 see ix86_compute_frame_layout, which we need to match in order
11580 to pass verification of stack_pointer_offset at the end. */
11581 m->fs.sp_offset = (m->fs.sp_offset + align_bytes) & -align_bytes;
11582 m->fs.sp_valid = false;
11585 allocate = frame.stack_pointer_offset - m->fs.sp_offset;
11587 if (flag_stack_usage_info)
11589 /* We start to count from ARG_POINTER. */
11590 HOST_WIDE_INT stack_size = frame.stack_pointer_offset;
11592 /* If it was realigned, take into account the fake frame. */
11593 if (stack_realign_drap)
11595 if (ix86_static_chain_on_stack)
11596 stack_size += UNITS_PER_WORD;
11598 if (!call_used_regs[REGNO (crtl->drap_reg)])
11599 stack_size += UNITS_PER_WORD;
11601 /* This over-estimates by 1 minimal-stack-alignment-unit but
11602 mitigates that by counting in the new return address slot. */
11603 current_function_dynamic_stack_size
11604 += crtl->stack_alignment_needed / BITS_PER_UNIT;
11607 current_function_static_stack_size = stack_size;
11610 /* On SEH target with very large frame size, allocate an area to save
11611 SSE registers (as the very large allocation won't be described). */
11613 && frame.stack_pointer_offset > SEH_MAX_FRAME_SIZE
11614 && !sse_registers_saved)
11616 HOST_WIDE_INT sse_size =
11617 frame.sse_reg_save_offset - frame.reg_save_offset;
11619 gcc_assert (int_registers_saved);
11621 /* No need to do stack checking as the area will be immediately
11623 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
11624 GEN_INT (-sse_size), -1,
11625 m->fs.cfa_reg == stack_pointer_rtx);
11626 allocate -= sse_size;
11627 ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
11628 sse_registers_saved = true;
11631 /* The stack has already been decremented by the instruction calling us
11632 so probe if the size is non-negative to preserve the protection area. */
11633 if (allocate >= 0 && flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
11635 /* We expect the registers to be saved when probes are used. */
11636 gcc_assert (int_registers_saved);
11638 if (STACK_CHECK_MOVING_SP)
11640 if (!(crtl->is_leaf && !cfun->calls_alloca
11641 && allocate <= PROBE_INTERVAL))
11643 ix86_adjust_stack_and_probe (allocate);
11649 HOST_WIDE_INT size = allocate;
11651 if (TARGET_64BIT && size >= (HOST_WIDE_INT) 0x80000000)
11652 size = 0x80000000 - STACK_CHECK_PROTECT - 1;
11654 if (TARGET_STACK_PROBE)
11656 if (crtl->is_leaf && !cfun->calls_alloca)
11658 if (size > PROBE_INTERVAL)
11659 ix86_emit_probe_stack_range (0, size);
11662 ix86_emit_probe_stack_range (0, size + STACK_CHECK_PROTECT);
11666 if (crtl->is_leaf && !cfun->calls_alloca)
11668 if (size > PROBE_INTERVAL && size > STACK_CHECK_PROTECT)
11669 ix86_emit_probe_stack_range (STACK_CHECK_PROTECT,
11670 size - STACK_CHECK_PROTECT);
11673 ix86_emit_probe_stack_range (STACK_CHECK_PROTECT, size);
11680 else if (!ix86_target_stack_probe ()
11681 || frame.stack_pointer_offset < CHECK_STACK_LIMIT)
11683 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
11684 GEN_INT (-allocate), -1,
11685 m->fs.cfa_reg == stack_pointer_rtx);
11689 rtx eax = gen_rtx_REG (Pmode, AX_REG);
11691 rtx (*adjust_stack_insn)(rtx, rtx, rtx);
11692 const bool sp_is_cfa_reg = (m->fs.cfa_reg == stack_pointer_rtx);
11693 bool eax_live = ix86_eax_live_at_start_p ();
11694 bool r10_live = false;
11697 r10_live = (DECL_STATIC_CHAIN (current_function_decl) != 0);
11701 insn = emit_insn (gen_push (eax));
11702 allocate -= UNITS_PER_WORD;
11703 /* Note that SEH directives need to continue tracking the stack
11704 pointer even after the frame pointer has been set up. */
11705 if (sp_is_cfa_reg || TARGET_SEH)
11708 m->fs.cfa_offset += UNITS_PER_WORD;
11709 RTX_FRAME_RELATED_P (insn) = 1;
11710 add_reg_note (insn, REG_FRAME_RELATED_EXPR,
11711 gen_rtx_SET (VOIDmode, stack_pointer_rtx,
11712 plus_constant (Pmode, stack_pointer_rtx,
11713 -UNITS_PER_WORD)));
11719 r10 = gen_rtx_REG (Pmode, R10_REG);
11720 insn = emit_insn (gen_push (r10));
11721 allocate -= UNITS_PER_WORD;
11722 if (sp_is_cfa_reg || TARGET_SEH)
11725 m->fs.cfa_offset += UNITS_PER_WORD;
11726 RTX_FRAME_RELATED_P (insn) = 1;
11727 add_reg_note (insn, REG_FRAME_RELATED_EXPR,
11728 gen_rtx_SET (VOIDmode, stack_pointer_rtx,
11729 plus_constant (Pmode, stack_pointer_rtx,
11730 -UNITS_PER_WORD)));
11734 emit_move_insn (eax, GEN_INT (allocate));
11735 emit_insn (ix86_gen_allocate_stack_worker (eax, eax));
11737 /* Use the fact that AX still contains ALLOCATE. */
11738 adjust_stack_insn = (Pmode == DImode
11739 ? gen_pro_epilogue_adjust_stack_di_sub
11740 : gen_pro_epilogue_adjust_stack_si_sub);
11742 insn = emit_insn (adjust_stack_insn (stack_pointer_rtx,
11743 stack_pointer_rtx, eax));
11745 if (sp_is_cfa_reg || TARGET_SEH)
11748 m->fs.cfa_offset += allocate;
11749 RTX_FRAME_RELATED_P (insn) = 1;
11750 add_reg_note (insn, REG_FRAME_RELATED_EXPR,
11751 gen_rtx_SET (VOIDmode, stack_pointer_rtx,
11752 plus_constant (Pmode, stack_pointer_rtx,
11755 m->fs.sp_offset += allocate;
11757 /* Use stack_pointer_rtx for relative addressing so that code
11758 works for realigned stack, too. */
11759 if (r10_live && eax_live)
11761 t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
11762 emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
11763 gen_frame_mem (word_mode, t));
11764 t = plus_constant (Pmode, t, UNITS_PER_WORD);
11765 emit_move_insn (gen_rtx_REG (word_mode, AX_REG),
11766 gen_frame_mem (word_mode, t));
11768 else if (eax_live || r10_live)
11770 t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
11771 emit_move_insn (gen_rtx_REG (word_mode,
11772 (eax_live ? AX_REG : R10_REG)),
11773 gen_frame_mem (word_mode, t));
11776 gcc_assert (m->fs.sp_offset == frame.stack_pointer_offset);
11778 /* If we havn't already set up the frame pointer, do so now. */
11779 if (frame_pointer_needed && !m->fs.fp_valid)
11781 insn = ix86_gen_add3 (hard_frame_pointer_rtx, stack_pointer_rtx,
11782 GEN_INT (frame.stack_pointer_offset
11783 - frame.hard_frame_pointer_offset));
11784 insn = emit_insn (insn);
11785 RTX_FRAME_RELATED_P (insn) = 1;
11786 add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL);
11788 if (m->fs.cfa_reg == stack_pointer_rtx)
11789 m->fs.cfa_reg = hard_frame_pointer_rtx;
11790 m->fs.fp_offset = frame.hard_frame_pointer_offset;
11791 m->fs.fp_valid = true;
11794 if (!int_registers_saved)
11795 ix86_emit_save_regs_using_mov (frame.reg_save_offset);
11796 if (!sse_registers_saved)
11797 ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
11799 /* For the mcount profiling on 32 bit PIC mode we need to emit SET_GOT
11801 if (!TARGET_64BIT && pic_offset_table_rtx && crtl->profile && !flag_fentry)
11803 rtx pic = gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM);
11804 insn = emit_insn (gen_set_got (pic));
11805 RTX_FRAME_RELATED_P (insn) = 1;
11806 add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
11807 emit_insn (gen_prologue_use (pic));
11808 /* Deleting already emmitted SET_GOT if exist and allocated to
11809 REAL_PIC_OFFSET_TABLE_REGNUM. */
11810 ix86_elim_entry_set_got (pic);
11813 if (crtl->drap_reg && !crtl->stack_realign_needed)
11815 /* vDRAP is setup but after reload it turns out stack realign
11816 isn't necessary, here we will emit prologue to setup DRAP
11817 without stack realign adjustment */
11818 t = choose_baseaddr (0);
11819 emit_insn (gen_rtx_SET (VOIDmode, crtl->drap_reg, t));
11822 /* Prevent instructions from being scheduled into register save push
11823 sequence when access to the redzone area is done through frame pointer.
11824 The offset between the frame pointer and the stack pointer is calculated
11825 relative to the value of the stack pointer at the end of the function
11826 prologue, and moving instructions that access redzone area via frame
11827 pointer inside push sequence violates this assumption. */
11828 if (frame_pointer_needed && frame.red_zone_size)
11829 emit_insn (gen_memory_blockage ());
11831 /* Emit cld instruction if stringops are used in the function. */
11832 if (TARGET_CLD && ix86_current_function_needs_cld)
11833 emit_insn (gen_cld ());
11835 /* SEH requires that the prologue end within 256 bytes of the start of
11836 the function. Prevent instruction schedules that would extend that.
11837 Further, prevent alloca modifications to the stack pointer from being
11838 combined with prologue modifications. */
11840 emit_insn (gen_prologue_use (stack_pointer_rtx));
11843 /* Emit code to restore REG using a POP insn. */
11846 ix86_emit_restore_reg_using_pop (rtx reg)
11848 struct machine_function *m = cfun->machine;
11849 rtx insn = emit_insn (gen_pop (reg));
11851 ix86_add_cfa_restore_note (insn, reg, m->fs.sp_offset);
11852 m->fs.sp_offset -= UNITS_PER_WORD;
11854 if (m->fs.cfa_reg == crtl->drap_reg
11855 && REGNO (reg) == REGNO (crtl->drap_reg))
11857 /* Previously we'd represented the CFA as an expression
11858 like *(%ebp - 8). We've just popped that value from
11859 the stack, which means we need to reset the CFA to
11860 the drap register. This will remain until we restore
11861 the stack pointer. */
11862 add_reg_note (insn, REG_CFA_DEF_CFA, reg);
11863 RTX_FRAME_RELATED_P (insn) = 1;
11865 /* This means that the DRAP register is valid for addressing too. */
11866 m->fs.drap_valid = true;
11870 if (m->fs.cfa_reg == stack_pointer_rtx)
11872 rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
11873 x = gen_rtx_SET (VOIDmode, stack_pointer_rtx, x);
11874 add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
11875 RTX_FRAME_RELATED_P (insn) = 1;
11877 m->fs.cfa_offset -= UNITS_PER_WORD;
11880 /* When the frame pointer is the CFA, and we pop it, we are
11881 swapping back to the stack pointer as the CFA. This happens
11882 for stack frames that don't allocate other data, so we assume
11883 the stack pointer is now pointing at the return address, i.e.
11884 the function entry state, which makes the offset be 1 word. */
11885 if (reg == hard_frame_pointer_rtx)
11887 m->fs.fp_valid = false;
11888 if (m->fs.cfa_reg == hard_frame_pointer_rtx)
11890 m->fs.cfa_reg = stack_pointer_rtx;
11891 m->fs.cfa_offset -= UNITS_PER_WORD;
11893 add_reg_note (insn, REG_CFA_DEF_CFA,
11894 gen_rtx_PLUS (Pmode, stack_pointer_rtx,
11895 GEN_INT (m->fs.cfa_offset)));
11896 RTX_FRAME_RELATED_P (insn) = 1;
11901 /* Emit code to restore saved registers using POP insns. */
11904 ix86_emit_restore_regs_using_pop (void)
11906 unsigned int regno;
11908 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
11909 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, false))
11910 ix86_emit_restore_reg_using_pop (gen_rtx_REG (word_mode, regno));
11913 /* Emit code and notes for the LEAVE instruction. */
11916 ix86_emit_leave (void)
11918 struct machine_function *m = cfun->machine;
11919 rtx insn = emit_insn (ix86_gen_leave ());
11921 ix86_add_queued_cfa_restore_notes (insn);
11923 gcc_assert (m->fs.fp_valid);
11924 m->fs.sp_valid = true;
11925 m->fs.sp_offset = m->fs.fp_offset - UNITS_PER_WORD;
11926 m->fs.fp_valid = false;
11928 if (m->fs.cfa_reg == hard_frame_pointer_rtx)
11930 m->fs.cfa_reg = stack_pointer_rtx;
11931 m->fs.cfa_offset = m->fs.sp_offset;
11933 add_reg_note (insn, REG_CFA_DEF_CFA,
11934 plus_constant (Pmode, stack_pointer_rtx,
11936 RTX_FRAME_RELATED_P (insn) = 1;
11938 ix86_add_cfa_restore_note (insn, hard_frame_pointer_rtx,
11942 /* Emit code to restore saved registers using MOV insns.
11943 First register is restored from CFA - CFA_OFFSET. */
11945 ix86_emit_restore_regs_using_mov (HOST_WIDE_INT cfa_offset,
11946 bool maybe_eh_return)
11948 struct machine_function *m = cfun->machine;
11949 unsigned int regno;
11951 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
11952 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return))
11954 rtx reg = gen_rtx_REG (word_mode, regno);
11957 mem = choose_baseaddr (cfa_offset);
11958 mem = gen_frame_mem (word_mode, mem);
11959 insn = emit_move_insn (reg, mem);
11961 if (m->fs.cfa_reg == crtl->drap_reg && regno == REGNO (crtl->drap_reg))
11963 /* Previously we'd represented the CFA as an expression
11964 like *(%ebp - 8). We've just popped that value from
11965 the stack, which means we need to reset the CFA to
11966 the drap register. This will remain until we restore
11967 the stack pointer. */
11968 add_reg_note (insn, REG_CFA_DEF_CFA, reg);
11969 RTX_FRAME_RELATED_P (insn) = 1;
11971 /* This means that the DRAP register is valid for addressing. */
11972 m->fs.drap_valid = true;
11975 ix86_add_cfa_restore_note (NULL_RTX, reg, cfa_offset);
11977 cfa_offset -= UNITS_PER_WORD;
11981 /* Emit code to restore saved registers using MOV insns.
11982 First register is restored from CFA - CFA_OFFSET. */
11984 ix86_emit_restore_sse_regs_using_mov (HOST_WIDE_INT cfa_offset,
11985 bool maybe_eh_return)
11987 unsigned int regno;
11989 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
11990 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return))
11992 rtx reg = gen_rtx_REG (V4SFmode, regno);
11995 mem = choose_baseaddr (cfa_offset);
11996 mem = gen_rtx_MEM (V4SFmode, mem);
11997 set_mem_align (mem, 128);
11998 emit_move_insn (reg, mem);
12000 ix86_add_cfa_restore_note (NULL_RTX, reg, cfa_offset);
12006 /* Restore function stack, frame, and registers. */
12009 ix86_expand_epilogue (int style)
12011 struct machine_function *m = cfun->machine;
12012 struct machine_frame_state frame_state_save = m->fs;
12013 struct ix86_frame frame;
12014 bool restore_regs_via_mov;
12017 ix86_finalize_stack_realign_flags ();
12018 ix86_compute_frame_layout (&frame);
12020 m->fs.sp_valid = (!frame_pointer_needed
12021 || (crtl->sp_is_unchanging
12022 && !stack_realign_fp));
12023 gcc_assert (!m->fs.sp_valid
12024 || m->fs.sp_offset == frame.stack_pointer_offset);
12026 /* The FP must be valid if the frame pointer is present. */
12027 gcc_assert (frame_pointer_needed == m->fs.fp_valid);
12028 gcc_assert (!m->fs.fp_valid
12029 || m->fs.fp_offset == frame.hard_frame_pointer_offset);
12031 /* We must have *some* valid pointer to the stack frame. */
12032 gcc_assert (m->fs.sp_valid || m->fs.fp_valid);
12034 /* The DRAP is never valid at this point. */
12035 gcc_assert (!m->fs.drap_valid);
12037 /* See the comment about red zone and frame
12038 pointer usage in ix86_expand_prologue. */
12039 if (frame_pointer_needed && frame.red_zone_size)
12040 emit_insn (gen_memory_blockage ());
12042 using_drap = crtl->drap_reg && crtl->stack_realign_needed;
12043 gcc_assert (!using_drap || m->fs.cfa_reg == crtl->drap_reg);
12045 /* Determine the CFA offset of the end of the red-zone. */
12046 m->fs.red_zone_offset = 0;
12047 if (ix86_using_red_zone () && crtl->args.pops_args < 65536)
12049 /* The red-zone begins below the return address. */
12050 m->fs.red_zone_offset = RED_ZONE_SIZE + UNITS_PER_WORD;
12052 /* When the register save area is in the aligned portion of
12053 the stack, determine the maximum runtime displacement that
12054 matches up with the aligned frame. */
12055 if (stack_realign_drap)
12056 m->fs.red_zone_offset -= (crtl->stack_alignment_needed / BITS_PER_UNIT
12060 /* Special care must be taken for the normal return case of a function
12061 using eh_return: the eax and edx registers are marked as saved, but
12062 not restored along this path. Adjust the save location to match. */
12063 if (crtl->calls_eh_return && style != 2)
12064 frame.reg_save_offset -= 2 * UNITS_PER_WORD;
12066 /* EH_RETURN requires the use of moves to function properly. */
12067 if (crtl->calls_eh_return)
12068 restore_regs_via_mov = true;
12069 /* SEH requires the use of pops to identify the epilogue. */
12070 else if (TARGET_SEH)
12071 restore_regs_via_mov = false;
12072 /* If we're only restoring one register and sp is not valid then
12073 using a move instruction to restore the register since it's
12074 less work than reloading sp and popping the register. */
12075 else if (!m->fs.sp_valid && frame.nregs <= 1)
12076 restore_regs_via_mov = true;
12077 else if (TARGET_EPILOGUE_USING_MOVE
12078 && cfun->machine->use_fast_prologue_epilogue
12079 && (frame.nregs > 1
12080 || m->fs.sp_offset != frame.reg_save_offset))
12081 restore_regs_via_mov = true;
12082 else if (frame_pointer_needed
12084 && m->fs.sp_offset != frame.reg_save_offset)
12085 restore_regs_via_mov = true;
12086 else if (frame_pointer_needed
12087 && TARGET_USE_LEAVE
12088 && cfun->machine->use_fast_prologue_epilogue
12089 && frame.nregs == 1)
12090 restore_regs_via_mov = true;
12092 restore_regs_via_mov = false;
12094 if (restore_regs_via_mov || frame.nsseregs)
12096 /* Ensure that the entire register save area is addressable via
12097 the stack pointer, if we will restore via sp. */
12099 && m->fs.sp_offset > 0x7fffffff
12100 && !(m->fs.fp_valid || m->fs.drap_valid)
12101 && (frame.nsseregs + frame.nregs) != 0)
12103 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
12104 GEN_INT (m->fs.sp_offset
12105 - frame.sse_reg_save_offset),
12107 m->fs.cfa_reg == stack_pointer_rtx);
12111 /* If there are any SSE registers to restore, then we have to do it
12112 via moves, since there's obviously no pop for SSE regs. */
12113 if (frame.nsseregs)
12114 ix86_emit_restore_sse_regs_using_mov (frame.sse_reg_save_offset,
12117 if (restore_regs_via_mov)
12122 ix86_emit_restore_regs_using_mov (frame.reg_save_offset, style == 2);
12124 /* eh_return epilogues need %ecx added to the stack pointer. */
12127 rtx insn, sa = EH_RETURN_STACKADJ_RTX;
12129 /* Stack align doesn't work with eh_return. */
12130 gcc_assert (!stack_realign_drap);
12131 /* Neither does regparm nested functions. */
12132 gcc_assert (!ix86_static_chain_on_stack);
12134 if (frame_pointer_needed)
12136 t = gen_rtx_PLUS (Pmode, hard_frame_pointer_rtx, sa);
12137 t = plus_constant (Pmode, t, m->fs.fp_offset - UNITS_PER_WORD);
12138 emit_insn (gen_rtx_SET (VOIDmode, sa, t));
12140 t = gen_frame_mem (Pmode, hard_frame_pointer_rtx);
12141 insn = emit_move_insn (hard_frame_pointer_rtx, t);
12143 /* Note that we use SA as a temporary CFA, as the return
12144 address is at the proper place relative to it. We
12145 pretend this happens at the FP restore insn because
12146 prior to this insn the FP would be stored at the wrong
12147 offset relative to SA, and after this insn we have no
12148 other reasonable register to use for the CFA. We don't
12149 bother resetting the CFA to the SP for the duration of
12150 the return insn. */
12151 add_reg_note (insn, REG_CFA_DEF_CFA,
12152 plus_constant (Pmode, sa, UNITS_PER_WORD));
12153 ix86_add_queued_cfa_restore_notes (insn);
12154 add_reg_note (insn, REG_CFA_RESTORE, hard_frame_pointer_rtx);
12155 RTX_FRAME_RELATED_P (insn) = 1;
12157 m->fs.cfa_reg = sa;
12158 m->fs.cfa_offset = UNITS_PER_WORD;
12159 m->fs.fp_valid = false;
12161 pro_epilogue_adjust_stack (stack_pointer_rtx, sa,
12162 const0_rtx, style, false);
12166 t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, sa);
12167 t = plus_constant (Pmode, t, m->fs.sp_offset - UNITS_PER_WORD);
12168 insn = emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, t));
12169 ix86_add_queued_cfa_restore_notes (insn);
12171 gcc_assert (m->fs.cfa_reg == stack_pointer_rtx);
12172 if (m->fs.cfa_offset != UNITS_PER_WORD)
12174 m->fs.cfa_offset = UNITS_PER_WORD;
12175 add_reg_note (insn, REG_CFA_DEF_CFA,
12176 plus_constant (Pmode, stack_pointer_rtx,
12178 RTX_FRAME_RELATED_P (insn) = 1;
12181 m->fs.sp_offset = UNITS_PER_WORD;
12182 m->fs.sp_valid = true;
12187 /* SEH requires that the function end with (1) a stack adjustment
12188 if necessary, (2) a sequence of pops, and (3) a return or
12189 jump instruction. Prevent insns from the function body from
12190 being scheduled into this sequence. */
12193 /* Prevent a catch region from being adjacent to the standard
12194 epilogue sequence. Unfortuantely crtl->uses_eh_lsda nor
12195 several other flags that would be interesting to test are
12197 if (flag_non_call_exceptions)
12198 emit_insn (gen_nops (const1_rtx));
12200 emit_insn (gen_blockage ());
12203 /* First step is to deallocate the stack frame so that we can
12204 pop the registers. Also do it on SEH target for very large
12205 frame as the emitted instructions aren't allowed by the ABI in
12207 if (!m->fs.sp_valid
12209 && (m->fs.sp_offset - frame.reg_save_offset
12210 >= SEH_MAX_FRAME_SIZE)))
12212 pro_epilogue_adjust_stack (stack_pointer_rtx, hard_frame_pointer_rtx,
12213 GEN_INT (m->fs.fp_offset
12214 - frame.reg_save_offset),
12217 else if (m->fs.sp_offset != frame.reg_save_offset)
12219 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
12220 GEN_INT (m->fs.sp_offset
12221 - frame.reg_save_offset),
12223 m->fs.cfa_reg == stack_pointer_rtx);
12226 ix86_emit_restore_regs_using_pop ();
12229 /* If we used a stack pointer and haven't already got rid of it,
12231 if (m->fs.fp_valid)
12233 /* If the stack pointer is valid and pointing at the frame
12234 pointer store address, then we only need a pop. */
12235 if (m->fs.sp_valid && m->fs.sp_offset == frame.hfp_save_offset)
12236 ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
12237 /* Leave results in shorter dependency chains on CPUs that are
12238 able to grok it fast. */
12239 else if (TARGET_USE_LEAVE
12240 || optimize_bb_for_size_p (EXIT_BLOCK_PTR_FOR_FN (cfun))
12241 || !cfun->machine->use_fast_prologue_epilogue)
12242 ix86_emit_leave ();
12245 pro_epilogue_adjust_stack (stack_pointer_rtx,
12246 hard_frame_pointer_rtx,
12247 const0_rtx, style, !using_drap);
12248 ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
12254 int param_ptr_offset = UNITS_PER_WORD;
12257 gcc_assert (stack_realign_drap);
12259 if (ix86_static_chain_on_stack)
12260 param_ptr_offset += UNITS_PER_WORD;
12261 if (!call_used_regs[REGNO (crtl->drap_reg)])
12262 param_ptr_offset += UNITS_PER_WORD;
12264 insn = emit_insn (gen_rtx_SET
12265 (VOIDmode, stack_pointer_rtx,
12266 gen_rtx_PLUS (Pmode,
12268 GEN_INT (-param_ptr_offset))));
12269 m->fs.cfa_reg = stack_pointer_rtx;
12270 m->fs.cfa_offset = param_ptr_offset;
12271 m->fs.sp_offset = param_ptr_offset;
12272 m->fs.realigned = false;
12274 add_reg_note (insn, REG_CFA_DEF_CFA,
12275 gen_rtx_PLUS (Pmode, stack_pointer_rtx,
12276 GEN_INT (param_ptr_offset)));
12277 RTX_FRAME_RELATED_P (insn) = 1;
12279 if (!call_used_regs[REGNO (crtl->drap_reg)])
12280 ix86_emit_restore_reg_using_pop (crtl->drap_reg);
12283 /* At this point the stack pointer must be valid, and we must have
12284 restored all of the registers. We may not have deallocated the
12285 entire stack frame. We've delayed this until now because it may
12286 be possible to merge the local stack deallocation with the
12287 deallocation forced by ix86_static_chain_on_stack. */
12288 gcc_assert (m->fs.sp_valid);
12289 gcc_assert (!m->fs.fp_valid);
12290 gcc_assert (!m->fs.realigned);
12291 if (m->fs.sp_offset != UNITS_PER_WORD)
12293 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
12294 GEN_INT (m->fs.sp_offset - UNITS_PER_WORD),
12298 ix86_add_queued_cfa_restore_notes (get_last_insn ());
12300 /* Sibcall epilogues don't want a return instruction. */
12303 m->fs = frame_state_save;
12307 if (crtl->args.pops_args && crtl->args.size)
12309 rtx popc = GEN_INT (crtl->args.pops_args);
12311 /* i386 can only pop 64K bytes. If asked to pop more, pop return
12312 address, do explicit add, and jump indirectly to the caller. */
12314 if (crtl->args.pops_args >= 65536)
12316 rtx ecx = gen_rtx_REG (SImode, CX_REG);
12319 /* There is no "pascal" calling convention in any 64bit ABI. */
12320 gcc_assert (!TARGET_64BIT);
12322 insn = emit_insn (gen_pop (ecx));
12323 m->fs.cfa_offset -= UNITS_PER_WORD;
12324 m->fs.sp_offset -= UNITS_PER_WORD;
12326 rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
12327 x = gen_rtx_SET (VOIDmode, stack_pointer_rtx, x);
12328 add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
12329 add_reg_note (insn, REG_CFA_REGISTER,
12330 gen_rtx_SET (VOIDmode, ecx, pc_rtx));
12331 RTX_FRAME_RELATED_P (insn) = 1;
12333 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
12335 emit_jump_insn (gen_simple_return_indirect_internal (ecx));
12338 emit_jump_insn (gen_simple_return_pop_internal (popc));
12341 emit_jump_insn (gen_simple_return_internal ());
12343 /* Restore the state back to the state from the prologue,
12344 so that it's correct for the next epilogue. */
12345 m->fs = frame_state_save;
12348 /* Reset from the function's potential modifications. */
12351 ix86_output_function_epilogue (FILE *file ATTRIBUTE_UNUSED, HOST_WIDE_INT)
12353 if (pic_offset_table_rtx
12354 && !ix86_use_pseudo_pic_reg ())
12355 SET_REGNO (pic_offset_table_rtx, REAL_PIC_OFFSET_TABLE_REGNUM);
12357 /* Mach-O doesn't support labels at the end of objects, so if
12358 it looks like we might want one, insert a NOP. */
12360 rtx_insn *insn = get_last_insn ();
12361 rtx_insn *deleted_debug_label = NULL;
12364 && NOTE_KIND (insn) != NOTE_INSN_DELETED_LABEL)
12366 /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
12367 notes only, instead set their CODE_LABEL_NUMBER to -1,
12368 otherwise there would be code generation differences
12369 in between -g and -g0. */
12370 if (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_DELETED_DEBUG_LABEL)
12371 deleted_debug_label = insn;
12372 insn = PREV_INSN (insn);
12377 && NOTE_KIND (insn) == NOTE_INSN_DELETED_LABEL)))
12378 fputs ("\tnop\n", file);
12379 else if (deleted_debug_label)
12380 for (insn = deleted_debug_label; insn; insn = NEXT_INSN (insn))
12381 if (NOTE_KIND (insn) == NOTE_INSN_DELETED_DEBUG_LABEL)
12382 CODE_LABEL_NUMBER (insn) = -1;
12388 /* Return a scratch register to use in the split stack prologue. The
12389 split stack prologue is used for -fsplit-stack. It is the first
12390 instructions in the function, even before the regular prologue.
12391 The scratch register can be any caller-saved register which is not
12392 used for parameters or for the static chain. */
12394 static unsigned int
12395 split_stack_prologue_scratch_regno (void)
12401 bool is_fastcall, is_thiscall;
12404 is_fastcall = (lookup_attribute ("fastcall",
12405 TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
12407 is_thiscall = (lookup_attribute ("thiscall",
12408 TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
12410 regparm = ix86_function_regparm (TREE_TYPE (cfun->decl), cfun->decl);
12414 if (DECL_STATIC_CHAIN (cfun->decl))
12416 sorry ("-fsplit-stack does not support fastcall with "
12417 "nested function");
12418 return INVALID_REGNUM;
12422 else if (is_thiscall)
12424 if (!DECL_STATIC_CHAIN (cfun->decl))
12428 else if (regparm < 3)
12430 if (!DECL_STATIC_CHAIN (cfun->decl))
12436 sorry ("-fsplit-stack does not support 2 register "
12437 "parameters for a nested function");
12438 return INVALID_REGNUM;
12445 /* FIXME: We could make this work by pushing a register
12446 around the addition and comparison. */
12447 sorry ("-fsplit-stack does not support 3 register parameters");
12448 return INVALID_REGNUM;
12453 /* A SYMBOL_REF for the function which allocates new stackspace for
12456 static GTY(()) rtx split_stack_fn;
12458 /* A SYMBOL_REF for the more stack function when using the large
12461 static GTY(()) rtx split_stack_fn_large;
12463 /* Handle -fsplit-stack. These are the first instructions in the
12464 function, even before the regular prologue. */
12467 ix86_expand_split_stack_prologue (void)
12469 struct ix86_frame frame;
12470 HOST_WIDE_INT allocate;
12471 unsigned HOST_WIDE_INT args_size;
12472 rtx_code_label *label;
12473 rtx limit, current, jump_insn, allocate_rtx, call_insn, call_fusage;
12474 rtx scratch_reg = NULL_RTX;
12475 rtx_code_label *varargs_label = NULL;
12478 gcc_assert (flag_split_stack && reload_completed);
12480 ix86_finalize_stack_realign_flags ();
12481 ix86_compute_frame_layout (&frame);
12482 allocate = frame.stack_pointer_offset - INCOMING_FRAME_SP_OFFSET;
12484 /* This is the label we will branch to if we have enough stack
12485 space. We expect the basic block reordering pass to reverse this
12486 branch if optimizing, so that we branch in the unlikely case. */
12487 label = gen_label_rtx ();
12489 /* We need to compare the stack pointer minus the frame size with
12490 the stack boundary in the TCB. The stack boundary always gives
12491 us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
12492 can compare directly. Otherwise we need to do an addition. */
12494 limit = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
12495 UNSPEC_STACK_CHECK);
12496 limit = gen_rtx_CONST (Pmode, limit);
12497 limit = gen_rtx_MEM (Pmode, limit);
12498 if (allocate < SPLIT_STACK_AVAILABLE)
12499 current = stack_pointer_rtx;
12502 unsigned int scratch_regno;
12505 /* We need a scratch register to hold the stack pointer minus
12506 the required frame size. Since this is the very start of the
12507 function, the scratch register can be any caller-saved
12508 register which is not used for parameters. */
12509 offset = GEN_INT (- allocate);
12510 scratch_regno = split_stack_prologue_scratch_regno ();
12511 if (scratch_regno == INVALID_REGNUM)
12513 scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
12514 if (!TARGET_64BIT || x86_64_immediate_operand (offset, Pmode))
12516 /* We don't use ix86_gen_add3 in this case because it will
12517 want to split to lea, but when not optimizing the insn
12518 will not be split after this point. */
12519 emit_insn (gen_rtx_SET (VOIDmode, scratch_reg,
12520 gen_rtx_PLUS (Pmode, stack_pointer_rtx,
12525 emit_move_insn (scratch_reg, offset);
12526 emit_insn (ix86_gen_add3 (scratch_reg, scratch_reg,
12527 stack_pointer_rtx));
12529 current = scratch_reg;
12532 ix86_expand_branch (GEU, current, limit, label);
12533 jump_insn = get_last_insn ();
12534 JUMP_LABEL (jump_insn) = label;
12536 /* Mark the jump as very likely to be taken. */
12537 add_int_reg_note (jump_insn, REG_BR_PROB,
12538 REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100);
12540 if (split_stack_fn == NULL_RTX)
12542 split_stack_fn = gen_rtx_SYMBOL_REF (Pmode, "__morestack");
12543 SYMBOL_REF_FLAGS (split_stack_fn) |= SYMBOL_FLAG_LOCAL;
12545 fn = split_stack_fn;
12547 /* Get more stack space. We pass in the desired stack space and the
12548 size of the arguments to copy to the new stack. In 32-bit mode
12549 we push the parameters; __morestack will return on a new stack
12550 anyhow. In 64-bit mode we pass the parameters in r10 and
12552 allocate_rtx = GEN_INT (allocate);
12553 args_size = crtl->args.size >= 0 ? crtl->args.size : 0;
12554 call_fusage = NULL_RTX;
12559 reg10 = gen_rtx_REG (Pmode, R10_REG);
12560 reg11 = gen_rtx_REG (Pmode, R11_REG);
12562 /* If this function uses a static chain, it will be in %r10.
12563 Preserve it across the call to __morestack. */
12564 if (DECL_STATIC_CHAIN (cfun->decl))
12568 rax = gen_rtx_REG (word_mode, AX_REG);
12569 emit_move_insn (rax, gen_rtx_REG (word_mode, R10_REG));
12570 use_reg (&call_fusage, rax);
12573 if ((ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
12576 HOST_WIDE_INT argval;
12578 gcc_assert (Pmode == DImode);
12579 /* When using the large model we need to load the address
12580 into a register, and we've run out of registers. So we
12581 switch to a different calling convention, and we call a
12582 different function: __morestack_large. We pass the
12583 argument size in the upper 32 bits of r10 and pass the
12584 frame size in the lower 32 bits. */
12585 gcc_assert ((allocate & (HOST_WIDE_INT) 0xffffffff) == allocate);
12586 gcc_assert ((args_size & 0xffffffff) == args_size);
12588 if (split_stack_fn_large == NULL_RTX)
12590 split_stack_fn_large =
12591 gen_rtx_SYMBOL_REF (Pmode, "__morestack_large_model");
12592 SYMBOL_REF_FLAGS (split_stack_fn_large) |= SYMBOL_FLAG_LOCAL;
12594 if (ix86_cmodel == CM_LARGE_PIC)
12596 rtx_code_label *label;
12599 label = gen_label_rtx ();
12600 emit_label (label);
12601 LABEL_PRESERVE_P (label) = 1;
12602 emit_insn (gen_set_rip_rex64 (reg10, label));
12603 emit_insn (gen_set_got_offset_rex64 (reg11, label));
12604 emit_insn (ix86_gen_add3 (reg10, reg10, reg11));
12605 x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, split_stack_fn_large),
12607 x = gen_rtx_CONST (Pmode, x);
12608 emit_move_insn (reg11, x);
12609 x = gen_rtx_PLUS (Pmode, reg10, reg11);
12610 x = gen_const_mem (Pmode, x);
12611 emit_move_insn (reg11, x);
12614 emit_move_insn (reg11, split_stack_fn_large);
12618 argval = ((args_size << 16) << 16) + allocate;
12619 emit_move_insn (reg10, GEN_INT (argval));
12623 emit_move_insn (reg10, allocate_rtx);
12624 emit_move_insn (reg11, GEN_INT (args_size));
12625 use_reg (&call_fusage, reg11);
12628 use_reg (&call_fusage, reg10);
12632 emit_insn (gen_push (GEN_INT (args_size)));
12633 emit_insn (gen_push (allocate_rtx));
12635 call_insn = ix86_expand_call (NULL_RTX, gen_rtx_MEM (QImode, fn),
12636 GEN_INT (UNITS_PER_WORD), constm1_rtx,
12638 add_function_usage_to (call_insn, call_fusage);
12640 /* In order to make call/return prediction work right, we now need
12641 to execute a return instruction. See
12642 libgcc/config/i386/morestack.S for the details on how this works.
12644 For flow purposes gcc must not see this as a return
12645 instruction--we need control flow to continue at the subsequent
12646 label. Therefore, we use an unspec. */
12647 gcc_assert (crtl->args.pops_args < 65536);
12648 emit_insn (gen_split_stack_return (GEN_INT (crtl->args.pops_args)));
12650 /* If we are in 64-bit mode and this function uses a static chain,
12651 we saved %r10 in %rax before calling _morestack. */
12652 if (TARGET_64BIT && DECL_STATIC_CHAIN (cfun->decl))
12653 emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
12654 gen_rtx_REG (word_mode, AX_REG));
12656 /* If this function calls va_start, we need to store a pointer to
12657 the arguments on the old stack, because they may not have been
12658 all copied to the new stack. At this point the old stack can be
12659 found at the frame pointer value used by __morestack, because
12660 __morestack has set that up before calling back to us. Here we
12661 store that pointer in a scratch register, and in
12662 ix86_expand_prologue we store the scratch register in a stack
12664 if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
12666 unsigned int scratch_regno;
12670 scratch_regno = split_stack_prologue_scratch_regno ();
12671 scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
12672 frame_reg = gen_rtx_REG (Pmode, BP_REG);
12676 return address within this function
12677 return address of caller of this function
12679 So we add three words to get to the stack arguments.
12683 return address within this function
12684 first argument to __morestack
12685 second argument to __morestack
12686 return address of caller of this function
12688 So we add five words to get to the stack arguments.
12690 words = TARGET_64BIT ? 3 : 5;
12691 emit_insn (gen_rtx_SET (VOIDmode, scratch_reg,
12692 gen_rtx_PLUS (Pmode, frame_reg,
12693 GEN_INT (words * UNITS_PER_WORD))));
12695 varargs_label = gen_label_rtx ();
12696 emit_jump_insn (gen_jump (varargs_label));
12697 JUMP_LABEL (get_last_insn ()) = varargs_label;
12702 emit_label (label);
12703 LABEL_NUSES (label) = 1;
12705 /* If this function calls va_start, we now have to set the scratch
12706 register for the case where we do not call __morestack. In this
12707 case we need to set it based on the stack pointer. */
12708 if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
12710 emit_insn (gen_rtx_SET (VOIDmode, scratch_reg,
12711 gen_rtx_PLUS (Pmode, stack_pointer_rtx,
12712 GEN_INT (UNITS_PER_WORD))));
12714 emit_label (varargs_label);
12715 LABEL_NUSES (varargs_label) = 1;
12719 /* We may have to tell the dataflow pass that the split stack prologue
12720 is initializing a scratch register. */
12723 ix86_live_on_entry (bitmap regs)
12725 if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
12727 gcc_assert (flag_split_stack);
12728 bitmap_set_bit (regs, split_stack_prologue_scratch_regno ());
12732 /* Extract the parts of an RTL expression that is a valid memory address
12733 for an instruction. Return 0 if the structure of the address is
12734 grossly off. Return -1 if the address contains ASHIFT, so it is not
12735 strictly valid, but still used for computing length of lea instruction. */
12738 ix86_decompose_address (rtx addr, struct ix86_address *out)
12740 rtx base = NULL_RTX, index = NULL_RTX, disp = NULL_RTX;
12741 rtx base_reg, index_reg;
12742 HOST_WIDE_INT scale = 1;
12743 rtx scale_rtx = NULL_RTX;
12746 enum ix86_address_seg seg = SEG_DEFAULT;
12748 /* Allow zero-extended SImode addresses,
12749 they will be emitted with addr32 prefix. */
12750 if (TARGET_64BIT && GET_MODE (addr) == DImode)
12752 if (GET_CODE (addr) == ZERO_EXTEND
12753 && GET_MODE (XEXP (addr, 0)) == SImode)
12755 addr = XEXP (addr, 0);
12756 if (CONST_INT_P (addr))
12759 else if (GET_CODE (addr) == AND
12760 && const_32bit_mask (XEXP (addr, 1), DImode))
12762 addr = simplify_gen_subreg (SImode, XEXP (addr, 0), DImode, 0);
12763 if (addr == NULL_RTX)
12766 if (CONST_INT_P (addr))
12771 /* Allow SImode subregs of DImode addresses,
12772 they will be emitted with addr32 prefix. */
12773 if (TARGET_64BIT && GET_MODE (addr) == SImode)
12775 if (GET_CODE (addr) == SUBREG
12776 && GET_MODE (SUBREG_REG (addr)) == DImode)
12778 addr = SUBREG_REG (addr);
12779 if (CONST_INT_P (addr))
12786 else if (GET_CODE (addr) == SUBREG)
12788 if (REG_P (SUBREG_REG (addr)))
12793 else if (GET_CODE (addr) == PLUS)
12795 rtx addends[4], op;
12803 addends[n++] = XEXP (op, 1);
12806 while (GET_CODE (op) == PLUS);
12811 for (i = n; i >= 0; --i)
12814 switch (GET_CODE (op))
12819 index = XEXP (op, 0);
12820 scale_rtx = XEXP (op, 1);
12826 index = XEXP (op, 0);
12827 tmp = XEXP (op, 1);
12828 if (!CONST_INT_P (tmp))
12830 scale = INTVAL (tmp);
12831 if ((unsigned HOST_WIDE_INT) scale > 3)
12833 scale = 1 << scale;
12838 if (GET_CODE (op) != UNSPEC)
12843 if (XINT (op, 1) == UNSPEC_TP
12844 && TARGET_TLS_DIRECT_SEG_REFS
12845 && seg == SEG_DEFAULT)
12846 seg = DEFAULT_TLS_SEG_REG;
12852 if (!REG_P (SUBREG_REG (op)))
12879 else if (GET_CODE (addr) == MULT)
12881 index = XEXP (addr, 0); /* index*scale */
12882 scale_rtx = XEXP (addr, 1);
12884 else if (GET_CODE (addr) == ASHIFT)
12886 /* We're called for lea too, which implements ashift on occasion. */
12887 index = XEXP (addr, 0);
12888 tmp = XEXP (addr, 1);
12889 if (!CONST_INT_P (tmp))
12891 scale = INTVAL (tmp);
12892 if ((unsigned HOST_WIDE_INT) scale > 3)
12894 scale = 1 << scale;
12898 disp = addr; /* displacement */
12904 else if (GET_CODE (index) == SUBREG
12905 && REG_P (SUBREG_REG (index)))
12911 /* Extract the integral value of scale. */
12914 if (!CONST_INT_P (scale_rtx))
12916 scale = INTVAL (scale_rtx);
12919 base_reg = base && GET_CODE (base) == SUBREG ? SUBREG_REG (base) : base;
12920 index_reg = index && GET_CODE (index) == SUBREG ? SUBREG_REG (index) : index;
12922 /* Avoid useless 0 displacement. */
12923 if (disp == const0_rtx && (base || index))
12926 /* Allow arg pointer and stack pointer as index if there is not scaling. */
12927 if (base_reg && index_reg && scale == 1
12928 && (index_reg == arg_pointer_rtx
12929 || index_reg == frame_pointer_rtx
12930 || (REG_P (index_reg) && REGNO (index_reg) == STACK_POINTER_REGNUM)))
12932 std::swap (base, index);
12933 std::swap (base_reg, index_reg);
12936 /* Special case: %ebp cannot be encoded as a base without a displacement.
12940 && (base_reg == hard_frame_pointer_rtx
12941 || base_reg == frame_pointer_rtx
12942 || base_reg == arg_pointer_rtx
12943 || (REG_P (base_reg)
12944 && (REGNO (base_reg) == HARD_FRAME_POINTER_REGNUM
12945 || REGNO (base_reg) == R13_REG))))
12948 /* Special case: on K6, [%esi] makes the instruction vector decoded.
12949 Avoid this by transforming to [%esi+0].
12950 Reload calls address legitimization without cfun defined, so we need
12951 to test cfun for being non-NULL. */
12952 if (TARGET_K6 && cfun && optimize_function_for_speed_p (cfun)
12953 && base_reg && !index_reg && !disp
12954 && REG_P (base_reg) && REGNO (base_reg) == SI_REG)
12957 /* Special case: encode reg+reg instead of reg*2. */
12958 if (!base && index && scale == 2)
12959 base = index, base_reg = index_reg, scale = 1;
12961 /* Special case: scaling cannot be encoded without base or displacement. */
12962 if (!base && !disp && index && scale != 1)
12966 out->index = index;
12968 out->scale = scale;
12974 /* Return cost of the memory address x.
12975 For i386, it is better to use a complex address than let gcc copy
12976 the address into a reg and make a new pseudo. But not if the address
12977 requires to two regs - that would mean more pseudos with longer
12980 ix86_address_cost (rtx x, machine_mode, addr_space_t, bool)
12982 struct ix86_address parts;
12984 int ok = ix86_decompose_address (x, &parts);
12988 if (parts.base && GET_CODE (parts.base) == SUBREG)
12989 parts.base = SUBREG_REG (parts.base);
12990 if (parts.index && GET_CODE (parts.index) == SUBREG)
12991 parts.index = SUBREG_REG (parts.index);
12993 /* Attempt to minimize number of registers in the address by increasing
12994 address cost for each used register. We don't increase address cost
12995 for "pic_offset_table_rtx". When a memopt with "pic_offset_table_rtx"
12996 is not invariant itself it most likely means that base or index is not
12997 invariant. Therefore only "pic_offset_table_rtx" could be hoisted out,
12998 which is not profitable for x86. */
13000 && (!REG_P (parts.base) || REGNO (parts.base) >= FIRST_PSEUDO_REGISTER)
13001 && (current_pass->type == GIMPLE_PASS
13002 || !pic_offset_table_rtx
13003 || !REG_P (parts.base)
13004 || REGNO (pic_offset_table_rtx) != REGNO (parts.base)))
13008 && (!REG_P (parts.index) || REGNO (parts.index) >= FIRST_PSEUDO_REGISTER)
13009 && (current_pass->type == GIMPLE_PASS
13010 || !pic_offset_table_rtx
13011 || !REG_P (parts.index)
13012 || REGNO (pic_offset_table_rtx) != REGNO (parts.index)))
13015 /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
13016 since it's predecode logic can't detect the length of instructions
13017 and it degenerates to vector decoded. Increase cost of such
13018 addresses here. The penalty is minimally 2 cycles. It may be worthwhile
13019 to split such addresses or even refuse such addresses at all.
13021 Following addressing modes are affected:
13026 The first and last case may be avoidable by explicitly coding the zero in
13027 memory address, but I don't have AMD-K6 machine handy to check this
13031 && ((!parts.disp && parts.base && parts.index && parts.scale != 1)
13032 || (parts.disp && !parts.base && parts.index && parts.scale != 1)
13033 || (!parts.disp && parts.base && parts.index && parts.scale == 1)))
13039 /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
13040 this is used for to form addresses to local data when -fPIC is in
13044 darwin_local_data_pic (rtx disp)
13046 return (GET_CODE (disp) == UNSPEC
13047 && XINT (disp, 1) == UNSPEC_MACHOPIC_OFFSET);
13050 /* Determine if a given RTX is a valid constant. We already know this
13051 satisfies CONSTANT_P. */
13054 ix86_legitimate_constant_p (machine_mode, rtx x)
13056 /* Pointer bounds constants are not valid. */
13057 if (POINTER_BOUNDS_MODE_P (GET_MODE (x)))
13060 switch (GET_CODE (x))
13065 if (GET_CODE (x) == PLUS)
13067 if (!CONST_INT_P (XEXP (x, 1)))
13072 if (TARGET_MACHO && darwin_local_data_pic (x))
13075 /* Only some unspecs are valid as "constants". */
13076 if (GET_CODE (x) == UNSPEC)
13077 switch (XINT (x, 1))
13080 case UNSPEC_GOTOFF:
13081 case UNSPEC_PLTOFF:
13082 return TARGET_64BIT;
13084 case UNSPEC_NTPOFF:
13085 x = XVECEXP (x, 0, 0);
13086 return (GET_CODE (x) == SYMBOL_REF
13087 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
13088 case UNSPEC_DTPOFF:
13089 x = XVECEXP (x, 0, 0);
13090 return (GET_CODE (x) == SYMBOL_REF
13091 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC);
13096 /* We must have drilled down to a symbol. */
13097 if (GET_CODE (x) == LABEL_REF)
13099 if (GET_CODE (x) != SYMBOL_REF)
13104 /* TLS symbols are never valid. */
13105 if (SYMBOL_REF_TLS_MODEL (x))
13108 /* DLLIMPORT symbols are never valid. */
13109 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
13110 && SYMBOL_REF_DLLIMPORT_P (x))
13114 /* mdynamic-no-pic */
13115 if (MACHO_DYNAMIC_NO_PIC_P)
13116 return machopic_symbol_defined_p (x);
13121 if (GET_MODE (x) == TImode
13122 && x != CONST0_RTX (TImode)
13128 if (!standard_sse_constant_p (x))
13135 /* Otherwise we handle everything else in the move patterns. */
13139 /* Determine if it's legal to put X into the constant pool. This
13140 is not possible for the address of thread-local symbols, which
13141 is checked above. */
13144 ix86_cannot_force_const_mem (machine_mode mode, rtx x)
13146 /* We can always put integral constants and vectors in memory. */
13147 switch (GET_CODE (x))
13157 return !ix86_legitimate_constant_p (mode, x);
13160 /* Nonzero if the symbol is marked as dllimport, or as stub-variable,
13164 is_imported_p (rtx x)
13166 if (!TARGET_DLLIMPORT_DECL_ATTRIBUTES
13167 || GET_CODE (x) != SYMBOL_REF)
13170 return SYMBOL_REF_DLLIMPORT_P (x) || SYMBOL_REF_STUBVAR_P (x);
13174 /* Nonzero if the constant value X is a legitimate general operand
13175 when generating PIC code. It is given that flag_pic is on and
13176 that X satisfies CONSTANT_P or is a CONST_DOUBLE. */
13179 legitimate_pic_operand_p (rtx x)
13183 switch (GET_CODE (x))
13186 inner = XEXP (x, 0);
13187 if (GET_CODE (inner) == PLUS
13188 && CONST_INT_P (XEXP (inner, 1)))
13189 inner = XEXP (inner, 0);
13191 /* Only some unspecs are valid as "constants". */
13192 if (GET_CODE (inner) == UNSPEC)
13193 switch (XINT (inner, 1))
13196 case UNSPEC_GOTOFF:
13197 case UNSPEC_PLTOFF:
13198 return TARGET_64BIT;
13200 x = XVECEXP (inner, 0, 0);
13201 return (GET_CODE (x) == SYMBOL_REF
13202 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
13203 case UNSPEC_MACHOPIC_OFFSET:
13204 return legitimate_pic_address_disp_p (x);
13212 return legitimate_pic_address_disp_p (x);
13219 /* Determine if a given CONST RTX is a valid memory displacement
13223 legitimate_pic_address_disp_p (rtx disp)
13227 /* In 64bit mode we can allow direct addresses of symbols and labels
13228 when they are not dynamic symbols. */
13231 rtx op0 = disp, op1;
13233 switch (GET_CODE (disp))
13239 if (GET_CODE (XEXP (disp, 0)) != PLUS)
13241 op0 = XEXP (XEXP (disp, 0), 0);
13242 op1 = XEXP (XEXP (disp, 0), 1);
13243 if (!CONST_INT_P (op1)
13244 || INTVAL (op1) >= 16*1024*1024
13245 || INTVAL (op1) < -16*1024*1024)
13247 if (GET_CODE (op0) == LABEL_REF)
13249 if (GET_CODE (op0) == CONST
13250 && GET_CODE (XEXP (op0, 0)) == UNSPEC
13251 && XINT (XEXP (op0, 0), 1) == UNSPEC_PCREL)
13253 if (GET_CODE (op0) == UNSPEC
13254 && XINT (op0, 1) == UNSPEC_PCREL)
13256 if (GET_CODE (op0) != SYMBOL_REF)
13261 /* TLS references should always be enclosed in UNSPEC.
13262 The dllimported symbol needs always to be resolved. */
13263 if (SYMBOL_REF_TLS_MODEL (op0)
13264 || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && SYMBOL_REF_DLLIMPORT_P (op0)))
13269 if (is_imported_p (op0))
13272 if (SYMBOL_REF_FAR_ADDR_P (op0)
13273 || !SYMBOL_REF_LOCAL_P (op0))
13276 /* Function-symbols need to be resolved only for
13278 For the small-model we don't need to resolve anything
13280 if ((ix86_cmodel != CM_LARGE_PIC
13281 && SYMBOL_REF_FUNCTION_P (op0))
13282 || ix86_cmodel == CM_SMALL_PIC)
13284 /* Non-external symbols don't need to be resolved for
13285 large, and medium-model. */
13286 if ((ix86_cmodel == CM_LARGE_PIC
13287 || ix86_cmodel == CM_MEDIUM_PIC)
13288 && !SYMBOL_REF_EXTERNAL_P (op0))
13291 else if (!SYMBOL_REF_FAR_ADDR_P (op0)
13292 && (SYMBOL_REF_LOCAL_P (op0)
13293 || (HAVE_LD_PIE_COPYRELOC
13295 && !SYMBOL_REF_WEAK (op0)
13296 && !SYMBOL_REF_FUNCTION_P (op0)))
13297 && ix86_cmodel != CM_LARGE_PIC)
13305 if (GET_CODE (disp) != CONST)
13307 disp = XEXP (disp, 0);
13311 /* We are unsafe to allow PLUS expressions. This limit allowed distance
13312 of GOT tables. We should not need these anyway. */
13313 if (GET_CODE (disp) != UNSPEC
13314 || (XINT (disp, 1) != UNSPEC_GOTPCREL
13315 && XINT (disp, 1) != UNSPEC_GOTOFF
13316 && XINT (disp, 1) != UNSPEC_PCREL
13317 && XINT (disp, 1) != UNSPEC_PLTOFF))
13320 if (GET_CODE (XVECEXP (disp, 0, 0)) != SYMBOL_REF
13321 && GET_CODE (XVECEXP (disp, 0, 0)) != LABEL_REF)
13327 if (GET_CODE (disp) == PLUS)
13329 if (!CONST_INT_P (XEXP (disp, 1)))
13331 disp = XEXP (disp, 0);
13335 if (TARGET_MACHO && darwin_local_data_pic (disp))
13338 if (GET_CODE (disp) != UNSPEC)
13341 switch (XINT (disp, 1))
13346 /* We need to check for both symbols and labels because VxWorks loads
13347 text labels with @GOT rather than @GOTOFF. See gotoff_operand for
13349 return (GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
13350 || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF);
13351 case UNSPEC_GOTOFF:
13352 /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
13353 While ABI specify also 32bit relocation but we don't produce it in
13354 small PIC model at all. */
13355 if ((GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
13356 || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF)
13358 return !TARGET_PECOFF && gotoff_operand (XVECEXP (disp, 0, 0), Pmode);
13360 case UNSPEC_GOTTPOFF:
13361 case UNSPEC_GOTNTPOFF:
13362 case UNSPEC_INDNTPOFF:
13365 disp = XVECEXP (disp, 0, 0);
13366 return (GET_CODE (disp) == SYMBOL_REF
13367 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_INITIAL_EXEC);
13368 case UNSPEC_NTPOFF:
13369 disp = XVECEXP (disp, 0, 0);
13370 return (GET_CODE (disp) == SYMBOL_REF
13371 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_EXEC);
13372 case UNSPEC_DTPOFF:
13373 disp = XVECEXP (disp, 0, 0);
13374 return (GET_CODE (disp) == SYMBOL_REF
13375 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_DYNAMIC);
13381 /* Our implementation of LEGITIMIZE_RELOAD_ADDRESS. Returns a value to
13382 replace the input X, or the original X if no replacement is called for.
13383 The output parameter *WIN is 1 if the calling macro should goto WIN,
13384 0 if it should not. */
13387 ix86_legitimize_reload_address (rtx x, machine_mode, int opnum, int type,
13390 /* Reload can generate:
13392 (plus:DI (plus:DI (unspec:DI [(const_int 0 [0])] UNSPEC_TP)
13396 This RTX is rejected from ix86_legitimate_address_p due to
13397 non-strictness of base register 97. Following this rejection,
13398 reload pushes all three components into separate registers,
13399 creating invalid memory address RTX.
13401 Following code reloads only the invalid part of the
13402 memory address RTX. */
13404 if (GET_CODE (x) == PLUS
13405 && REG_P (XEXP (x, 1))
13406 && GET_CODE (XEXP (x, 0)) == PLUS
13407 && REG_P (XEXP (XEXP (x, 0), 1)))
13410 bool something_reloaded = false;
13412 base = XEXP (XEXP (x, 0), 1);
13413 if (!REG_OK_FOR_BASE_STRICT_P (base))
13415 push_reload (base, NULL_RTX, &XEXP (XEXP (x, 0), 1), NULL,
13416 BASE_REG_CLASS, GET_MODE (x), VOIDmode, 0, 0,
13417 opnum, (enum reload_type) type);
13418 something_reloaded = true;
13421 index = XEXP (x, 1);
13422 if (!REG_OK_FOR_INDEX_STRICT_P (index))
13424 push_reload (index, NULL_RTX, &XEXP (x, 1), NULL,
13425 INDEX_REG_CLASS, GET_MODE (x), VOIDmode, 0, 0,
13426 opnum, (enum reload_type) type);
13427 something_reloaded = true;
13430 gcc_assert (something_reloaded);
13437 /* Determine if op is suitable RTX for an address register.
13438 Return naked register if a register or a register subreg is
13439 found, otherwise return NULL_RTX. */
13442 ix86_validate_address_register (rtx op)
13444 machine_mode mode = GET_MODE (op);
13446 /* Only SImode or DImode registers can form the address. */
13447 if (mode != SImode && mode != DImode)
13452 else if (GET_CODE (op) == SUBREG)
13454 rtx reg = SUBREG_REG (op);
13459 mode = GET_MODE (reg);
13461 /* Don't allow SUBREGs that span more than a word. It can
13462 lead to spill failures when the register is one word out
13463 of a two word structure. */
13464 if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
13467 /* Allow only SUBREGs of non-eliminable hard registers. */
13468 if (register_no_elim_operand (reg, mode))
13472 /* Op is not a register. */
13476 /* Recognizes RTL expressions that are valid memory addresses for an
13477 instruction. The MODE argument is the machine mode for the MEM
13478 expression that wants to use this address.
13480 It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
13481 convert common non-canonical forms to canonical form so that they will
13485 ix86_legitimate_address_p (machine_mode, rtx addr, bool strict)
13487 struct ix86_address parts;
13488 rtx base, index, disp;
13489 HOST_WIDE_INT scale;
13490 enum ix86_address_seg seg;
13492 if (ix86_decompose_address (addr, &parts) <= 0)
13493 /* Decomposition failed. */
13497 index = parts.index;
13499 scale = parts.scale;
13502 /* Validate base register. */
13505 rtx reg = ix86_validate_address_register (base);
13507 if (reg == NULL_RTX)
13510 if ((strict && ! REG_OK_FOR_BASE_STRICT_P (reg))
13511 || (! strict && ! REG_OK_FOR_BASE_NONSTRICT_P (reg)))
13512 /* Base is not valid. */
13516 /* Validate index register. */
13519 rtx reg = ix86_validate_address_register (index);
13521 if (reg == NULL_RTX)
13524 if ((strict && ! REG_OK_FOR_INDEX_STRICT_P (reg))
13525 || (! strict && ! REG_OK_FOR_INDEX_NONSTRICT_P (reg)))
13526 /* Index is not valid. */
13530 /* Index and base should have the same mode. */
13532 && GET_MODE (base) != GET_MODE (index))
13535 /* Address override works only on the (%reg) part of %fs:(%reg). */
13536 if (seg != SEG_DEFAULT
13537 && ((base && GET_MODE (base) != word_mode)
13538 || (index && GET_MODE (index) != word_mode)))
13541 /* Validate scale factor. */
13545 /* Scale without index. */
13548 if (scale != 2 && scale != 4 && scale != 8)
13549 /* Scale is not a valid multiplier. */
13553 /* Validate displacement. */
13556 if (GET_CODE (disp) == CONST
13557 && GET_CODE (XEXP (disp, 0)) == UNSPEC
13558 && XINT (XEXP (disp, 0), 1) != UNSPEC_MACHOPIC_OFFSET)
13559 switch (XINT (XEXP (disp, 0), 1))
13561 /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit when
13562 used. While ABI specify also 32bit relocations, we don't produce
13563 them at all and use IP relative instead. */
13565 case UNSPEC_GOTOFF:
13566 gcc_assert (flag_pic);
13568 goto is_legitimate_pic;
13570 /* 64bit address unspec. */
13573 case UNSPEC_GOTPCREL:
13575 gcc_assert (flag_pic);
13576 goto is_legitimate_pic;
13578 case UNSPEC_GOTTPOFF:
13579 case UNSPEC_GOTNTPOFF:
13580 case UNSPEC_INDNTPOFF:
13581 case UNSPEC_NTPOFF:
13582 case UNSPEC_DTPOFF:
13585 case UNSPEC_STACK_CHECK:
13586 gcc_assert (flag_split_stack);
13590 /* Invalid address unspec. */
13594 else if (SYMBOLIC_CONST (disp)
13598 && MACHOPIC_INDIRECT
13599 && !machopic_operand_p (disp)
13605 if (TARGET_64BIT && (index || base))
13607 /* foo@dtpoff(%rX) is ok. */
13608 if (GET_CODE (disp) != CONST
13609 || GET_CODE (XEXP (disp, 0)) != PLUS
13610 || GET_CODE (XEXP (XEXP (disp, 0), 0)) != UNSPEC
13611 || !CONST_INT_P (XEXP (XEXP (disp, 0), 1))
13612 || (XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_DTPOFF
13613 && XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_NTPOFF))
13614 /* Non-constant pic memory reference. */
13617 else if ((!TARGET_MACHO || flag_pic)
13618 && ! legitimate_pic_address_disp_p (disp))
13619 /* Displacement is an invalid pic construct. */
13622 else if (MACHO_DYNAMIC_NO_PIC_P
13623 && !ix86_legitimate_constant_p (Pmode, disp))
13624 /* displacment must be referenced via non_lazy_pointer */
13628 /* This code used to verify that a symbolic pic displacement
13629 includes the pic_offset_table_rtx register.
13631 While this is good idea, unfortunately these constructs may
13632 be created by "adds using lea" optimization for incorrect
13641 This code is nonsensical, but results in addressing
13642 GOT table with pic_offset_table_rtx base. We can't
13643 just refuse it easily, since it gets matched by
13644 "addsi3" pattern, that later gets split to lea in the
13645 case output register differs from input. While this
13646 can be handled by separate addsi pattern for this case
13647 that never results in lea, this seems to be easier and
13648 correct fix for crash to disable this test. */
13650 else if (GET_CODE (disp) != LABEL_REF
13651 && !CONST_INT_P (disp)
13652 && (GET_CODE (disp) != CONST
13653 || !ix86_legitimate_constant_p (Pmode, disp))
13654 && (GET_CODE (disp) != SYMBOL_REF
13655 || !ix86_legitimate_constant_p (Pmode, disp)))
13656 /* Displacement is not constant. */
13658 else if (TARGET_64BIT
13659 && !x86_64_immediate_operand (disp, VOIDmode))
13660 /* Displacement is out of range. */
13662 /* In x32 mode, constant addresses are sign extended to 64bit, so
13663 we have to prevent addresses from 0x80000000 to 0xffffffff. */
13664 else if (TARGET_X32 && !(index || base)
13665 && CONST_INT_P (disp)
13666 && val_signbit_known_set_p (SImode, INTVAL (disp)))
13670 /* Everything looks valid. */
13674 /* Determine if a given RTX is a valid constant address. */
13677 constant_address_p (rtx x)
13679 return CONSTANT_P (x) && ix86_legitimate_address_p (Pmode, x, 1);
13682 /* Return a unique alias set for the GOT. */
13684 static alias_set_type
13685 ix86_GOT_alias_set (void)
13687 static alias_set_type set = -1;
13689 set = new_alias_set ();
13693 /* Set regs_ever_live for PIC base address register
13694 to true if required. */
13696 set_pic_reg_ever_live ()
13698 if (reload_in_progress)
13699 df_set_regs_ever_live (REGNO (pic_offset_table_rtx), true);
13702 /* Return a legitimate reference for ORIG (an address) using the
13703 register REG. If REG is 0, a new pseudo is generated.
13705 There are two types of references that must be handled:
13707 1. Global data references must load the address from the GOT, via
13708 the PIC reg. An insn is emitted to do this load, and the reg is
13711 2. Static data references, constant pool addresses, and code labels
13712 compute the address as an offset from the GOT, whose base is in
13713 the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
13714 differentiate them from global data objects. The returned
13715 address is the PIC reg + an unspec constant.
13717 TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
13718 reg also appears in the address. */
13721 legitimize_pic_address (rtx orig, rtx reg)
13724 rtx new_rtx = orig;
13727 if (TARGET_MACHO && !TARGET_64BIT)
13730 reg = gen_reg_rtx (Pmode);
13731 /* Use the generic Mach-O PIC machinery. */
13732 return machopic_legitimize_pic_address (orig, GET_MODE (orig), reg);
13736 if (TARGET_64BIT && TARGET_DLLIMPORT_DECL_ATTRIBUTES)
13738 rtx tmp = legitimize_pe_coff_symbol (addr, true);
13743 if (TARGET_64BIT && legitimate_pic_address_disp_p (addr))
13745 else if (TARGET_64BIT && !TARGET_PECOFF
13746 && ix86_cmodel != CM_SMALL_PIC && gotoff_operand (addr, Pmode))
13749 /* This symbol may be referenced via a displacement from the PIC
13750 base address (@GOTOFF). */
13752 set_pic_reg_ever_live ();
13753 if (GET_CODE (addr) == CONST)
13754 addr = XEXP (addr, 0);
13755 if (GET_CODE (addr) == PLUS)
13757 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
13759 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
13762 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);
13763 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13765 tmpreg = gen_reg_rtx (Pmode);
13768 emit_move_insn (tmpreg, new_rtx);
13772 new_rtx = expand_simple_binop (Pmode, PLUS, reg, pic_offset_table_rtx,
13773 tmpreg, 1, OPTAB_DIRECT);
13777 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, tmpreg);
13779 else if (!TARGET_64BIT && !TARGET_PECOFF && gotoff_operand (addr, Pmode))
13781 /* This symbol may be referenced via a displacement from the PIC
13782 base address (@GOTOFF). */
13784 set_pic_reg_ever_live ();
13785 if (GET_CODE (addr) == CONST)
13786 addr = XEXP (addr, 0);
13787 if (GET_CODE (addr) == PLUS)
13789 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
13791 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
13794 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);
13795 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13796 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
13800 emit_move_insn (reg, new_rtx);
13804 else if ((GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_TLS_MODEL (addr) == 0)
13805 /* We can't use @GOTOFF for text labels on VxWorks;
13806 see gotoff_operand. */
13807 || (TARGET_VXWORKS_RTP && GET_CODE (addr) == LABEL_REF))
13809 rtx tmp = legitimize_pe_coff_symbol (addr, true);
13813 /* For x64 PE-COFF there is no GOT table. So we use address
13815 if (TARGET_64BIT && TARGET_PECOFF)
13817 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_PCREL);
13818 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13821 reg = gen_reg_rtx (Pmode);
13822 emit_move_insn (reg, new_rtx);
13825 else if (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC)
13827 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTPCREL);
13828 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13829 new_rtx = gen_const_mem (Pmode, new_rtx);
13830 set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
13833 reg = gen_reg_rtx (Pmode);
13834 /* Use directly gen_movsi, otherwise the address is loaded
13835 into register for CSE. We don't want to CSE this addresses,
13836 instead we CSE addresses from the GOT table, so skip this. */
13837 emit_insn (gen_movsi (reg, new_rtx));
13842 /* This symbol must be referenced via a load from the
13843 Global Offset Table (@GOT). */
13845 set_pic_reg_ever_live ();
13846 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOT);
13847 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13849 new_rtx = force_reg (Pmode, new_rtx);
13850 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
13851 new_rtx = gen_const_mem (Pmode, new_rtx);
13852 set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
13855 reg = gen_reg_rtx (Pmode);
13856 emit_move_insn (reg, new_rtx);
13862 if (CONST_INT_P (addr)
13863 && !x86_64_immediate_operand (addr, VOIDmode))
13867 emit_move_insn (reg, addr);
13871 new_rtx = force_reg (Pmode, addr);
13873 else if (GET_CODE (addr) == CONST)
13875 addr = XEXP (addr, 0);
13877 /* We must match stuff we generate before. Assume the only
13878 unspecs that can get here are ours. Not that we could do
13879 anything with them anyway.... */
13880 if (GET_CODE (addr) == UNSPEC
13881 || (GET_CODE (addr) == PLUS
13882 && GET_CODE (XEXP (addr, 0)) == UNSPEC))
13884 gcc_assert (GET_CODE (addr) == PLUS);
13886 if (GET_CODE (addr) == PLUS)
13888 rtx op0 = XEXP (addr, 0), op1 = XEXP (addr, 1);
13890 /* Check first to see if this is a constant offset from a @GOTOFF
13891 symbol reference. */
13892 if (!TARGET_PECOFF && gotoff_operand (op0, Pmode)
13893 && CONST_INT_P (op1))
13897 set_pic_reg_ever_live ();
13898 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op0),
13900 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, op1);
13901 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13902 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
13906 emit_move_insn (reg, new_rtx);
13912 if (INTVAL (op1) < -16*1024*1024
13913 || INTVAL (op1) >= 16*1024*1024)
13915 if (!x86_64_immediate_operand (op1, Pmode))
13916 op1 = force_reg (Pmode, op1);
13917 new_rtx = gen_rtx_PLUS (Pmode, force_reg (Pmode, op0), op1);
13923 rtx base = legitimize_pic_address (op0, reg);
13924 machine_mode mode = GET_MODE (base);
13926 = legitimize_pic_address (op1, base == reg ? NULL_RTX : reg);
13928 if (CONST_INT_P (new_rtx))
13930 if (INTVAL (new_rtx) < -16*1024*1024
13931 || INTVAL (new_rtx) >= 16*1024*1024)
13933 if (!x86_64_immediate_operand (new_rtx, mode))
13934 new_rtx = force_reg (mode, new_rtx);
13936 = gen_rtx_PLUS (mode, force_reg (mode, base), new_rtx);
13939 new_rtx = plus_constant (mode, base, INTVAL (new_rtx));
13943 /* For %rip addressing, we have to use just disp32, not
13946 && (GET_CODE (base) == SYMBOL_REF
13947 || GET_CODE (base) == LABEL_REF))
13948 base = force_reg (mode, base);
13949 if (GET_CODE (new_rtx) == PLUS
13950 && CONSTANT_P (XEXP (new_rtx, 1)))
13952 base = gen_rtx_PLUS (mode, base, XEXP (new_rtx, 0));
13953 new_rtx = XEXP (new_rtx, 1);
13955 new_rtx = gen_rtx_PLUS (mode, base, new_rtx);
13963 /* Load the thread pointer. If TO_REG is true, force it into a register. */
13966 get_thread_pointer (machine_mode tp_mode, bool to_reg)
13968 rtx tp = gen_rtx_UNSPEC (ptr_mode, gen_rtvec (1, const0_rtx), UNSPEC_TP);
13970 if (GET_MODE (tp) != tp_mode)
13972 gcc_assert (GET_MODE (tp) == SImode);
13973 gcc_assert (tp_mode == DImode);
13975 tp = gen_rtx_ZERO_EXTEND (tp_mode, tp);
13979 tp = copy_to_mode_reg (tp_mode, tp);
13984 /* Construct the SYMBOL_REF for the tls_get_addr function. */
13986 static GTY(()) rtx ix86_tls_symbol;
13989 ix86_tls_get_addr (void)
13991 if (!ix86_tls_symbol)
13994 = ((TARGET_ANY_GNU_TLS && !TARGET_64BIT)
13995 ? "___tls_get_addr" : "__tls_get_addr");
13997 ix86_tls_symbol = gen_rtx_SYMBOL_REF (Pmode, sym);
14000 if (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF)
14002 rtx unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, ix86_tls_symbol),
14004 return gen_rtx_PLUS (Pmode, pic_offset_table_rtx,
14005 gen_rtx_CONST (Pmode, unspec));
14008 return ix86_tls_symbol;
14011 /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
14013 static GTY(()) rtx ix86_tls_module_base_symbol;
14016 ix86_tls_module_base (void)
14018 if (!ix86_tls_module_base_symbol)
14020 ix86_tls_module_base_symbol
14021 = gen_rtx_SYMBOL_REF (Pmode, "_TLS_MODULE_BASE_");
14023 SYMBOL_REF_FLAGS (ix86_tls_module_base_symbol)
14024 |= TLS_MODEL_GLOBAL_DYNAMIC << SYMBOL_FLAG_TLS_SHIFT;
14027 return ix86_tls_module_base_symbol;
14030 /* A subroutine of ix86_legitimize_address and ix86_expand_move. FOR_MOV is
14031 false if we expect this to be used for a memory address and true if
14032 we expect to load the address into a register. */
14035 legitimize_tls_address (rtx x, enum tls_model model, bool for_mov)
14037 rtx dest, base, off;
14038 rtx pic = NULL_RTX, tp = NULL_RTX;
14039 machine_mode tp_mode = Pmode;
14042 /* Fall back to global dynamic model if tool chain cannot support local
14044 if (TARGET_SUN_TLS && !TARGET_64BIT
14045 && !HAVE_AS_IX86_TLSLDMPLT && !HAVE_AS_IX86_TLSLDM
14046 && model == TLS_MODEL_LOCAL_DYNAMIC)
14047 model = TLS_MODEL_GLOBAL_DYNAMIC;
14051 case TLS_MODEL_GLOBAL_DYNAMIC:
14052 dest = gen_reg_rtx (Pmode);
14056 if (flag_pic && !TARGET_PECOFF)
14057 pic = pic_offset_table_rtx;
14060 pic = gen_reg_rtx (Pmode);
14061 emit_insn (gen_set_got (pic));
14065 if (TARGET_GNU2_TLS)
14068 emit_insn (gen_tls_dynamic_gnu2_64 (dest, x));
14070 emit_insn (gen_tls_dynamic_gnu2_32 (dest, x, pic));
14072 tp = get_thread_pointer (Pmode, true);
14073 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, tp, dest));
14075 if (GET_MODE (x) != Pmode)
14076 x = gen_rtx_ZERO_EXTEND (Pmode, x);
14078 set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
14082 rtx caddr = ix86_tls_get_addr ();
14086 rtx rax = gen_rtx_REG (Pmode, AX_REG);
14091 (ix86_gen_tls_global_dynamic_64 (rax, x, caddr));
14092 insns = get_insns ();
14095 if (GET_MODE (x) != Pmode)
14096 x = gen_rtx_ZERO_EXTEND (Pmode, x);
14098 RTL_CONST_CALL_P (insns) = 1;
14099 emit_libcall_block (insns, dest, rax, x);
14102 emit_insn (gen_tls_global_dynamic_32 (dest, x, pic, caddr));
14106 case TLS_MODEL_LOCAL_DYNAMIC:
14107 base = gen_reg_rtx (Pmode);
14112 pic = pic_offset_table_rtx;
14115 pic = gen_reg_rtx (Pmode);
14116 emit_insn (gen_set_got (pic));
14120 if (TARGET_GNU2_TLS)
14122 rtx tmp = ix86_tls_module_base ();
14125 emit_insn (gen_tls_dynamic_gnu2_64 (base, tmp));
14127 emit_insn (gen_tls_dynamic_gnu2_32 (base, tmp, pic));
14129 tp = get_thread_pointer (Pmode, true);
14130 set_unique_reg_note (get_last_insn (), REG_EQUAL,
14131 gen_rtx_MINUS (Pmode, tmp, tp));
14135 rtx caddr = ix86_tls_get_addr ();
14139 rtx rax = gen_rtx_REG (Pmode, AX_REG);
14145 (ix86_gen_tls_local_dynamic_base_64 (rax, caddr));
14146 insns = get_insns ();
14149 /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
14150 share the LD_BASE result with other LD model accesses. */
14151 eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
14152 UNSPEC_TLS_LD_BASE);
14154 RTL_CONST_CALL_P (insns) = 1;
14155 emit_libcall_block (insns, base, rax, eqv);
14158 emit_insn (gen_tls_local_dynamic_base_32 (base, pic, caddr));
14161 off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x), UNSPEC_DTPOFF);
14162 off = gen_rtx_CONST (Pmode, off);
14164 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, base, off));
14166 if (TARGET_GNU2_TLS)
14168 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, dest, tp));
14170 if (GET_MODE (x) != Pmode)
14171 x = gen_rtx_ZERO_EXTEND (Pmode, x);
14173 set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
14177 case TLS_MODEL_INITIAL_EXEC:
14180 if (TARGET_SUN_TLS && !TARGET_X32)
14182 /* The Sun linker took the AMD64 TLS spec literally
14183 and can only handle %rax as destination of the
14184 initial executable code sequence. */
14186 dest = gen_reg_rtx (DImode);
14187 emit_insn (gen_tls_initial_exec_64_sun (dest, x));
14191 /* Generate DImode references to avoid %fs:(%reg32)
14192 problems and linker IE->LE relaxation bug. */
14195 type = UNSPEC_GOTNTPOFF;
14199 set_pic_reg_ever_live ();
14200 pic = pic_offset_table_rtx;
14201 type = TARGET_ANY_GNU_TLS ? UNSPEC_GOTNTPOFF : UNSPEC_GOTTPOFF;
14203 else if (!TARGET_ANY_GNU_TLS)
14205 pic = gen_reg_rtx (Pmode);
14206 emit_insn (gen_set_got (pic));
14207 type = UNSPEC_GOTTPOFF;
14212 type = UNSPEC_INDNTPOFF;
14215 off = gen_rtx_UNSPEC (tp_mode, gen_rtvec (1, x), type);
14216 off = gen_rtx_CONST (tp_mode, off);
14218 off = gen_rtx_PLUS (tp_mode, pic, off);
14219 off = gen_const_mem (tp_mode, off);
14220 set_mem_alias_set (off, ix86_GOT_alias_set ());
14222 if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
14224 base = get_thread_pointer (tp_mode,
14225 for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
14226 off = force_reg (tp_mode, off);
14227 return gen_rtx_PLUS (tp_mode, base, off);
14231 base = get_thread_pointer (Pmode, true);
14232 dest = gen_reg_rtx (Pmode);
14233 emit_insn (ix86_gen_sub3 (dest, base, off));
14237 case TLS_MODEL_LOCAL_EXEC:
14238 off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x),
14239 (TARGET_64BIT || TARGET_ANY_GNU_TLS)
14240 ? UNSPEC_NTPOFF : UNSPEC_TPOFF);
14241 off = gen_rtx_CONST (Pmode, off);
14243 if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
14245 base = get_thread_pointer (Pmode,
14246 for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
14247 return gen_rtx_PLUS (Pmode, base, off);
14251 base = get_thread_pointer (Pmode, true);
14252 dest = gen_reg_rtx (Pmode);
14253 emit_insn (ix86_gen_sub3 (dest, base, off));
14258 gcc_unreachable ();
14264 /* Create or return the unique __imp_DECL dllimport symbol corresponding
14265 to symbol DECL if BEIMPORT is true. Otherwise create or return the
14266 unique refptr-DECL symbol corresponding to symbol DECL. */
14268 struct dllimport_hasher : ggc_cache_hasher<tree_map *>
14270 static inline hashval_t hash (tree_map *m) { return m->hash; }
14272 equal (tree_map *a, tree_map *b)
14274 return a->base.from == b->base.from;
14278 handle_cache_entry (tree_map *&m)
14280 extern void gt_ggc_mx (tree_map *&);
14281 if (m == HTAB_EMPTY_ENTRY || m == HTAB_DELETED_ENTRY)
14283 else if (ggc_marked_p (m->base.from))
14286 m = static_cast<tree_map *> (HTAB_DELETED_ENTRY);
14290 static GTY((cache)) hash_table<dllimport_hasher> *dllimport_map;
14293 get_dllimport_decl (tree decl, bool beimport)
14295 struct tree_map *h, in;
14297 const char *prefix;
14298 size_t namelen, prefixlen;
14303 if (!dllimport_map)
14304 dllimport_map = hash_table<dllimport_hasher>::create_ggc (512);
14306 in.hash = htab_hash_pointer (decl);
14307 in.base.from = decl;
14308 tree_map **loc = dllimport_map->find_slot_with_hash (&in, in.hash, INSERT);
14313 *loc = h = ggc_alloc<tree_map> ();
14315 h->base.from = decl;
14316 h->to = to = build_decl (DECL_SOURCE_LOCATION (decl),
14317 VAR_DECL, NULL, ptr_type_node);
14318 DECL_ARTIFICIAL (to) = 1;
14319 DECL_IGNORED_P (to) = 1;
14320 DECL_EXTERNAL (to) = 1;
14321 TREE_READONLY (to) = 1;
14323 name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
14324 name = targetm.strip_name_encoding (name);
14326 prefix = name[0] == FASTCALL_PREFIX || user_label_prefix[0] == 0
14327 ? "*__imp_" : "*__imp__";
14329 prefix = user_label_prefix[0] == 0 ? "*.refptr." : "*refptr.";
14330 namelen = strlen (name);
14331 prefixlen = strlen (prefix);
14332 imp_name = (char *) alloca (namelen + prefixlen + 1);
14333 memcpy (imp_name, prefix, prefixlen);
14334 memcpy (imp_name + prefixlen, name, namelen + 1);
14336 name = ggc_alloc_string (imp_name, namelen + prefixlen);
14337 rtl = gen_rtx_SYMBOL_REF (Pmode, name);
14338 SET_SYMBOL_REF_DECL (rtl, to);
14339 SYMBOL_REF_FLAGS (rtl) = SYMBOL_FLAG_LOCAL | SYMBOL_FLAG_STUBVAR;
14342 SYMBOL_REF_FLAGS (rtl) |= SYMBOL_FLAG_EXTERNAL;
14343 #ifdef SUB_TARGET_RECORD_STUB
14344 SUB_TARGET_RECORD_STUB (name);
14348 rtl = gen_const_mem (Pmode, rtl);
14349 set_mem_alias_set (rtl, ix86_GOT_alias_set ());
14351 SET_DECL_RTL (to, rtl);
14352 SET_DECL_ASSEMBLER_NAME (to, get_identifier (name));
14357 /* Expand SYMBOL into its corresponding far-addresse symbol.
14358 WANT_REG is true if we require the result be a register. */
14361 legitimize_pe_coff_extern_decl (rtx symbol, bool want_reg)
14366 gcc_assert (SYMBOL_REF_DECL (symbol));
14367 imp_decl = get_dllimport_decl (SYMBOL_REF_DECL (symbol), false);
14369 x = DECL_RTL (imp_decl);
14371 x = force_reg (Pmode, x);
14375 /* Expand SYMBOL into its corresponding dllimport symbol. WANT_REG is
14376 true if we require the result be a register. */
14379 legitimize_dllimport_symbol (rtx symbol, bool want_reg)
14384 gcc_assert (SYMBOL_REF_DECL (symbol));
14385 imp_decl = get_dllimport_decl (SYMBOL_REF_DECL (symbol), true);
14387 x = DECL_RTL (imp_decl);
14389 x = force_reg (Pmode, x);
14393 /* Expand SYMBOL into its corresponding dllimport or refptr symbol. WANT_REG
14394 is true if we require the result be a register. */
14397 legitimize_pe_coff_symbol (rtx addr, bool inreg)
14399 if (!TARGET_PECOFF)
14402 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
14404 if (GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_DLLIMPORT_P (addr))
14405 return legitimize_dllimport_symbol (addr, inreg);
14406 if (GET_CODE (addr) == CONST
14407 && GET_CODE (XEXP (addr, 0)) == PLUS
14408 && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF
14409 && SYMBOL_REF_DLLIMPORT_P (XEXP (XEXP (addr, 0), 0)))
14411 rtx t = legitimize_dllimport_symbol (XEXP (XEXP (addr, 0), 0), inreg);
14412 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (addr, 0), 1));
14416 if (ix86_cmodel != CM_LARGE_PIC && ix86_cmodel != CM_MEDIUM_PIC)
14418 if (GET_CODE (addr) == SYMBOL_REF
14419 && !is_imported_p (addr)
14420 && SYMBOL_REF_EXTERNAL_P (addr)
14421 && SYMBOL_REF_DECL (addr))
14422 return legitimize_pe_coff_extern_decl (addr, inreg);
14424 if (GET_CODE (addr) == CONST
14425 && GET_CODE (XEXP (addr, 0)) == PLUS
14426 && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF
14427 && !is_imported_p (XEXP (XEXP (addr, 0), 0))
14428 && SYMBOL_REF_EXTERNAL_P (XEXP (XEXP (addr, 0), 0))
14429 && SYMBOL_REF_DECL (XEXP (XEXP (addr, 0), 0)))
14431 rtx t = legitimize_pe_coff_extern_decl (XEXP (XEXP (addr, 0), 0), inreg);
14432 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (addr, 0), 1));
14437 /* Try machine-dependent ways of modifying an illegitimate address
14438 to be legitimate. If we find one, return the new, valid address.
14439 This macro is used in only one place: `memory_address' in explow.c.
14441 OLDX is the address as it was before break_out_memory_refs was called.
14442 In some cases it is useful to look at this to decide what needs to be done.
14444 It is always safe for this macro to do nothing. It exists to recognize
14445 opportunities to optimize the output.
14447 For the 80386, we handle X+REG by loading X into a register R and
14448 using R+REG. R will go in a general reg and indexing will be used.
14449 However, if REG is a broken-out memory address or multiplication,
14450 nothing needs to be done because REG can certainly go in a general reg.
14452 When -fpic is used, special handling is needed for symbolic references.
14453 See comments by legitimize_pic_address in i386.c for details. */
14456 ix86_legitimize_address (rtx x, rtx, machine_mode mode)
14458 bool changed = false;
14461 log = GET_CODE (x) == SYMBOL_REF ? SYMBOL_REF_TLS_MODEL (x) : 0;
14463 return legitimize_tls_address (x, (enum tls_model) log, false);
14464 if (GET_CODE (x) == CONST
14465 && GET_CODE (XEXP (x, 0)) == PLUS
14466 && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
14467 && (log = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (x, 0), 0))))
14469 rtx t = legitimize_tls_address (XEXP (XEXP (x, 0), 0),
14470 (enum tls_model) log, false);
14471 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (x, 0), 1));
14474 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
14476 rtx tmp = legitimize_pe_coff_symbol (x, true);
14481 if (flag_pic && SYMBOLIC_CONST (x))
14482 return legitimize_pic_address (x, 0);
14485 if (MACHO_DYNAMIC_NO_PIC_P && SYMBOLIC_CONST (x))
14486 return machopic_indirect_data_reference (x, 0);
14489 /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
14490 if (GET_CODE (x) == ASHIFT
14491 && CONST_INT_P (XEXP (x, 1))
14492 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (x, 1)) < 4)
14495 log = INTVAL (XEXP (x, 1));
14496 x = gen_rtx_MULT (Pmode, force_reg (Pmode, XEXP (x, 0)),
14497 GEN_INT (1 << log));
14500 if (GET_CODE (x) == PLUS)
14502 /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
14504 if (GET_CODE (XEXP (x, 0)) == ASHIFT
14505 && CONST_INT_P (XEXP (XEXP (x, 0), 1))
14506 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 0), 1)) < 4)
14509 log = INTVAL (XEXP (XEXP (x, 0), 1));
14510 XEXP (x, 0) = gen_rtx_MULT (Pmode,
14511 force_reg (Pmode, XEXP (XEXP (x, 0), 0)),
14512 GEN_INT (1 << log));
14515 if (GET_CODE (XEXP (x, 1)) == ASHIFT
14516 && CONST_INT_P (XEXP (XEXP (x, 1), 1))
14517 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 1), 1)) < 4)
14520 log = INTVAL (XEXP (XEXP (x, 1), 1));
14521 XEXP (x, 1) = gen_rtx_MULT (Pmode,
14522 force_reg (Pmode, XEXP (XEXP (x, 1), 0)),
14523 GEN_INT (1 << log));
14526 /* Put multiply first if it isn't already. */
14527 if (GET_CODE (XEXP (x, 1)) == MULT)
14529 std::swap (XEXP (x, 0), XEXP (x, 1));
14533 /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
14534 into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
14535 created by virtual register instantiation, register elimination, and
14536 similar optimizations. */
14537 if (GET_CODE (XEXP (x, 0)) == MULT && GET_CODE (XEXP (x, 1)) == PLUS)
14540 x = gen_rtx_PLUS (Pmode,
14541 gen_rtx_PLUS (Pmode, XEXP (x, 0),
14542 XEXP (XEXP (x, 1), 0)),
14543 XEXP (XEXP (x, 1), 1));
14547 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
14548 into (plus (plus (mult (reg) (const)) (reg)) (const)). */
14549 else if (GET_CODE (x) == PLUS && GET_CODE (XEXP (x, 0)) == PLUS
14550 && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
14551 && GET_CODE (XEXP (XEXP (x, 0), 1)) == PLUS
14552 && CONSTANT_P (XEXP (x, 1)))
14555 rtx other = NULL_RTX;
14557 if (CONST_INT_P (XEXP (x, 1)))
14559 constant = XEXP (x, 1);
14560 other = XEXP (XEXP (XEXP (x, 0), 1), 1);
14562 else if (CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 1), 1)))
14564 constant = XEXP (XEXP (XEXP (x, 0), 1), 1);
14565 other = XEXP (x, 1);
14573 x = gen_rtx_PLUS (Pmode,
14574 gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 0),
14575 XEXP (XEXP (XEXP (x, 0), 1), 0)),
14576 plus_constant (Pmode, other,
14577 INTVAL (constant)));
14581 if (changed && ix86_legitimate_address_p (mode, x, false))
14584 if (GET_CODE (XEXP (x, 0)) == MULT)
14587 XEXP (x, 0) = copy_addr_to_reg (XEXP (x, 0));
14590 if (GET_CODE (XEXP (x, 1)) == MULT)
14593 XEXP (x, 1) = copy_addr_to_reg (XEXP (x, 1));
14597 && REG_P (XEXP (x, 1))
14598 && REG_P (XEXP (x, 0)))
14601 if (flag_pic && SYMBOLIC_CONST (XEXP (x, 1)))
14604 x = legitimize_pic_address (x, 0);
14607 if (changed && ix86_legitimate_address_p (mode, x, false))
14610 if (REG_P (XEXP (x, 0)))
14612 rtx temp = gen_reg_rtx (Pmode);
14613 rtx val = force_operand (XEXP (x, 1), temp);
14616 val = convert_to_mode (Pmode, val, 1);
14617 emit_move_insn (temp, val);
14620 XEXP (x, 1) = temp;
14624 else if (REG_P (XEXP (x, 1)))
14626 rtx temp = gen_reg_rtx (Pmode);
14627 rtx val = force_operand (XEXP (x, 0), temp);
14630 val = convert_to_mode (Pmode, val, 1);
14631 emit_move_insn (temp, val);
14634 XEXP (x, 0) = temp;
14642 /* Print an integer constant expression in assembler syntax. Addition
14643 and subtraction are the only arithmetic that may appear in these
14644 expressions. FILE is the stdio stream to write to, X is the rtx, and
14645 CODE is the operand print code from the output string. */
14648 output_pic_addr_const (FILE *file, rtx x, int code)
14652 switch (GET_CODE (x))
14655 gcc_assert (flag_pic);
14660 if (TARGET_64BIT || ! TARGET_MACHO_BRANCH_ISLANDS)
14661 output_addr_const (file, x);
14664 const char *name = XSTR (x, 0);
14666 /* Mark the decl as referenced so that cgraph will
14667 output the function. */
14668 if (SYMBOL_REF_DECL (x))
14669 mark_decl_referenced (SYMBOL_REF_DECL (x));
14672 if (MACHOPIC_INDIRECT
14673 && machopic_classify_symbol (x) == MACHOPIC_UNDEFINED_FUNCTION)
14674 name = machopic_indirection_name (x, /*stub_p=*/true);
14676 assemble_name (file, name);
14678 if (!TARGET_MACHO && !(TARGET_64BIT && TARGET_PECOFF)
14679 && code == 'P' && ! SYMBOL_REF_LOCAL_P (x))
14680 fputs ("@PLT", file);
14687 ASM_GENERATE_INTERNAL_LABEL (buf, "L", CODE_LABEL_NUMBER (x));
14688 assemble_name (asm_out_file, buf);
14692 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
14696 /* This used to output parentheses around the expression,
14697 but that does not work on the 386 (either ATT or BSD assembler). */
14698 output_pic_addr_const (file, XEXP (x, 0), code);
14702 if (GET_MODE (x) == VOIDmode)
14704 /* We can use %d if the number is <32 bits and positive. */
14705 if (CONST_DOUBLE_HIGH (x) || CONST_DOUBLE_LOW (x) < 0)
14706 fprintf (file, "0x%lx%08lx",
14707 (unsigned long) CONST_DOUBLE_HIGH (x),
14708 (unsigned long) CONST_DOUBLE_LOW (x));
14710 fprintf (file, HOST_WIDE_INT_PRINT_DEC, CONST_DOUBLE_LOW (x));
14713 /* We can't handle floating point constants;
14714 TARGET_PRINT_OPERAND must handle them. */
14715 output_operand_lossage ("floating constant misused");
14719 /* Some assemblers need integer constants to appear first. */
14720 if (CONST_INT_P (XEXP (x, 0)))
14722 output_pic_addr_const (file, XEXP (x, 0), code);
14724 output_pic_addr_const (file, XEXP (x, 1), code);
14728 gcc_assert (CONST_INT_P (XEXP (x, 1)));
14729 output_pic_addr_const (file, XEXP (x, 1), code);
14731 output_pic_addr_const (file, XEXP (x, 0), code);
14737 putc (ASSEMBLER_DIALECT == ASM_INTEL ? '(' : '[', file);
14738 output_pic_addr_const (file, XEXP (x, 0), code);
14740 output_pic_addr_const (file, XEXP (x, 1), code);
14742 putc (ASSEMBLER_DIALECT == ASM_INTEL ? ')' : ']', file);
14746 if (XINT (x, 1) == UNSPEC_STACK_CHECK)
14748 bool f = i386_asm_output_addr_const_extra (file, x);
14753 gcc_assert (XVECLEN (x, 0) == 1);
14754 output_pic_addr_const (file, XVECEXP (x, 0, 0), code);
14755 switch (XINT (x, 1))
14758 fputs ("@GOT", file);
14760 case UNSPEC_GOTOFF:
14761 fputs ("@GOTOFF", file);
14763 case UNSPEC_PLTOFF:
14764 fputs ("@PLTOFF", file);
14767 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
14768 "(%rip)" : "[rip]", file);
14770 case UNSPEC_GOTPCREL:
14771 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
14772 "@GOTPCREL(%rip)" : "@GOTPCREL[rip]", file);
14774 case UNSPEC_GOTTPOFF:
14775 /* FIXME: This might be @TPOFF in Sun ld too. */
14776 fputs ("@gottpoff", file);
14779 fputs ("@tpoff", file);
14781 case UNSPEC_NTPOFF:
14783 fputs ("@tpoff", file);
14785 fputs ("@ntpoff", file);
14787 case UNSPEC_DTPOFF:
14788 fputs ("@dtpoff", file);
14790 case UNSPEC_GOTNTPOFF:
14792 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
14793 "@gottpoff(%rip)": "@gottpoff[rip]", file);
14795 fputs ("@gotntpoff", file);
14797 case UNSPEC_INDNTPOFF:
14798 fputs ("@indntpoff", file);
14801 case UNSPEC_MACHOPIC_OFFSET:
14803 machopic_output_function_base_name (file);
14807 output_operand_lossage ("invalid UNSPEC as operand");
14813 output_operand_lossage ("invalid expression as operand");
14817 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
14818 We need to emit DTP-relative relocations. */
14820 static void ATTRIBUTE_UNUSED
14821 i386_output_dwarf_dtprel (FILE *file, int size, rtx x)
14823 fputs (ASM_LONG, file);
14824 output_addr_const (file, x);
14825 fputs ("@dtpoff", file);
14831 fputs (", 0", file);
14834 gcc_unreachable ();
14838 /* Return true if X is a representation of the PIC register. This copes
14839 with calls from ix86_find_base_term, where the register might have
14840 been replaced by a cselib value. */
14843 ix86_pic_register_p (rtx x)
14845 if (GET_CODE (x) == VALUE && CSELIB_VAL_PTR (x))
14846 return (pic_offset_table_rtx
14847 && rtx_equal_for_cselib_p (x, pic_offset_table_rtx));
14848 else if (!REG_P (x))
14850 else if (pic_offset_table_rtx)
14852 if (REGNO (x) == REGNO (pic_offset_table_rtx))
14854 if (HARD_REGISTER_P (x)
14855 && !HARD_REGISTER_P (pic_offset_table_rtx)
14856 && ORIGINAL_REGNO (x) == REGNO (pic_offset_table_rtx))
14861 return REGNO (x) == PIC_OFFSET_TABLE_REGNUM;
14864 /* Helper function for ix86_delegitimize_address.
14865 Attempt to delegitimize TLS local-exec accesses. */
14868 ix86_delegitimize_tls_address (rtx orig_x)
14870 rtx x = orig_x, unspec;
14871 struct ix86_address addr;
14873 if (!TARGET_TLS_DIRECT_SEG_REFS)
14877 if (GET_CODE (x) != PLUS || GET_MODE (x) != Pmode)
14879 if (ix86_decompose_address (x, &addr) == 0
14880 || addr.seg != DEFAULT_TLS_SEG_REG
14881 || addr.disp == NULL_RTX
14882 || GET_CODE (addr.disp) != CONST)
14884 unspec = XEXP (addr.disp, 0);
14885 if (GET_CODE (unspec) == PLUS && CONST_INT_P (XEXP (unspec, 1)))
14886 unspec = XEXP (unspec, 0);
14887 if (GET_CODE (unspec) != UNSPEC || XINT (unspec, 1) != UNSPEC_NTPOFF)
14889 x = XVECEXP (unspec, 0, 0);
14890 gcc_assert (GET_CODE (x) == SYMBOL_REF);
14891 if (unspec != XEXP (addr.disp, 0))
14892 x = gen_rtx_PLUS (Pmode, x, XEXP (XEXP (addr.disp, 0), 1));
14895 rtx idx = addr.index;
14896 if (addr.scale != 1)
14897 idx = gen_rtx_MULT (Pmode, idx, GEN_INT (addr.scale));
14898 x = gen_rtx_PLUS (Pmode, idx, x);
14901 x = gen_rtx_PLUS (Pmode, addr.base, x);
14902 if (MEM_P (orig_x))
14903 x = replace_equiv_address_nv (orig_x, x);
14907 /* In the name of slightly smaller debug output, and to cater to
14908 general assembler lossage, recognize PIC+GOTOFF and turn it back
14909 into a direct symbol reference.
14911 On Darwin, this is necessary to avoid a crash, because Darwin
14912 has a different PIC label for each routine but the DWARF debugging
14913 information is not associated with any particular routine, so it's
14914 necessary to remove references to the PIC label from RTL stored by
14915 the DWARF output code. */
14918 ix86_delegitimize_address (rtx x)
14920 rtx orig_x = delegitimize_mem_from_attrs (x);
14921 /* addend is NULL or some rtx if x is something+GOTOFF where
14922 something doesn't include the PIC register. */
14923 rtx addend = NULL_RTX;
14924 /* reg_addend is NULL or a multiple of some register. */
14925 rtx reg_addend = NULL_RTX;
14926 /* const_addend is NULL or a const_int. */
14927 rtx const_addend = NULL_RTX;
14928 /* This is the result, or NULL. */
14929 rtx result = NULL_RTX;
14938 if (GET_CODE (x) == CONST
14939 && GET_CODE (XEXP (x, 0)) == PLUS
14940 && GET_MODE (XEXP (x, 0)) == Pmode
14941 && CONST_INT_P (XEXP (XEXP (x, 0), 1))
14942 && GET_CODE (XEXP (XEXP (x, 0), 0)) == UNSPEC
14943 && XINT (XEXP (XEXP (x, 0), 0), 1) == UNSPEC_PCREL)
14945 rtx x2 = XVECEXP (XEXP (XEXP (x, 0), 0), 0, 0);
14946 x = gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 1), x2);
14947 if (MEM_P (orig_x))
14948 x = replace_equiv_address_nv (orig_x, x);
14952 if (GET_CODE (x) == CONST
14953 && GET_CODE (XEXP (x, 0)) == UNSPEC
14954 && (XINT (XEXP (x, 0), 1) == UNSPEC_GOTPCREL
14955 || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL)
14956 && (MEM_P (orig_x) || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL))
14958 x = XVECEXP (XEXP (x, 0), 0, 0);
14959 if (GET_MODE (orig_x) != GET_MODE (x) && MEM_P (orig_x))
14961 x = simplify_gen_subreg (GET_MODE (orig_x), x,
14969 if (ix86_cmodel != CM_MEDIUM_PIC && ix86_cmodel != CM_LARGE_PIC)
14970 return ix86_delegitimize_tls_address (orig_x);
14972 /* Fall thru into the code shared with -m32 for -mcmodel=large -fpic
14973 and -mcmodel=medium -fpic. */
14976 if (GET_CODE (x) != PLUS
14977 || GET_CODE (XEXP (x, 1)) != CONST)
14978 return ix86_delegitimize_tls_address (orig_x);
14980 if (ix86_pic_register_p (XEXP (x, 0)))
14981 /* %ebx + GOT/GOTOFF */
14983 else if (GET_CODE (XEXP (x, 0)) == PLUS)
14985 /* %ebx + %reg * scale + GOT/GOTOFF */
14986 reg_addend = XEXP (x, 0);
14987 if (ix86_pic_register_p (XEXP (reg_addend, 0)))
14988 reg_addend = XEXP (reg_addend, 1);
14989 else if (ix86_pic_register_p (XEXP (reg_addend, 1)))
14990 reg_addend = XEXP (reg_addend, 0);
14993 reg_addend = NULL_RTX;
14994 addend = XEXP (x, 0);
14998 addend = XEXP (x, 0);
15000 x = XEXP (XEXP (x, 1), 0);
15001 if (GET_CODE (x) == PLUS
15002 && CONST_INT_P (XEXP (x, 1)))
15004 const_addend = XEXP (x, 1);
15008 if (GET_CODE (x) == UNSPEC
15009 && ((XINT (x, 1) == UNSPEC_GOT && MEM_P (orig_x) && !addend)
15010 || (XINT (x, 1) == UNSPEC_GOTOFF && !MEM_P (orig_x))
15011 || (XINT (x, 1) == UNSPEC_PLTOFF && ix86_cmodel == CM_LARGE_PIC
15012 && !MEM_P (orig_x) && !addend)))
15013 result = XVECEXP (x, 0, 0);
15015 if (!TARGET_64BIT && TARGET_MACHO && darwin_local_data_pic (x)
15016 && !MEM_P (orig_x))
15017 result = XVECEXP (x, 0, 0);
15020 return ix86_delegitimize_tls_address (orig_x);
15023 result = gen_rtx_CONST (Pmode, gen_rtx_PLUS (Pmode, result, const_addend));
15025 result = gen_rtx_PLUS (Pmode, reg_addend, result);
15028 /* If the rest of original X doesn't involve the PIC register, add
15029 addend and subtract pic_offset_table_rtx. This can happen e.g.
15031 leal (%ebx, %ecx, 4), %ecx
15033 movl foo@GOTOFF(%ecx), %edx
15034 in which case we return (%ecx - %ebx) + foo
15035 or (%ecx - _GLOBAL_OFFSET_TABLE_) + foo if pseudo_pic_reg
15036 and reload has completed. */
15037 if (pic_offset_table_rtx
15038 && (!reload_completed || !ix86_use_pseudo_pic_reg ()))
15039 result = gen_rtx_PLUS (Pmode, gen_rtx_MINUS (Pmode, copy_rtx (addend),
15040 pic_offset_table_rtx),
15042 else if (pic_offset_table_rtx && !TARGET_MACHO && !TARGET_VXWORKS_RTP)
15044 rtx tmp = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
15045 tmp = gen_rtx_MINUS (Pmode, copy_rtx (addend), tmp);
15046 result = gen_rtx_PLUS (Pmode, tmp, result);
15051 if (GET_MODE (orig_x) != Pmode && MEM_P (orig_x))
15053 result = simplify_gen_subreg (GET_MODE (orig_x), result, Pmode, 0);
15054 if (result == NULL_RTX)
15060 /* If X is a machine specific address (i.e. a symbol or label being
15061 referenced as a displacement from the GOT implemented using an
15062 UNSPEC), then return the base term. Otherwise return X. */
15065 ix86_find_base_term (rtx x)
15071 if (GET_CODE (x) != CONST)
15073 term = XEXP (x, 0);
15074 if (GET_CODE (term) == PLUS
15075 && (CONST_INT_P (XEXP (term, 1))
15076 || GET_CODE (XEXP (term, 1)) == CONST_DOUBLE))
15077 term = XEXP (term, 0);
15078 if (GET_CODE (term) != UNSPEC
15079 || (XINT (term, 1) != UNSPEC_GOTPCREL
15080 && XINT (term, 1) != UNSPEC_PCREL))
15083 return XVECEXP (term, 0, 0);
15086 return ix86_delegitimize_address (x);
15090 put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
15091 bool fp, FILE *file)
15093 const char *suffix;
15095 if (mode == CCFPmode || mode == CCFPUmode)
15097 code = ix86_fp_compare_code_to_integer (code);
15101 code = reverse_condition (code);
15152 gcc_assert (mode == CCmode || mode == CCNOmode || mode == CCGCmode);
15156 /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
15157 Those same assemblers have the same but opposite lossage on cmov. */
15158 if (mode == CCmode)
15159 suffix = fp ? "nbe" : "a";
15161 gcc_unreachable ();
15177 gcc_unreachable ();
15181 if (mode == CCmode)
15183 else if (mode == CCCmode)
15184 suffix = fp ? "b" : "c";
15186 gcc_unreachable ();
15202 gcc_unreachable ();
15206 if (mode == CCmode)
15208 else if (mode == CCCmode)
15209 suffix = fp ? "nb" : "nc";
15211 gcc_unreachable ();
15214 gcc_assert (mode == CCmode || mode == CCGCmode || mode == CCNOmode);
15218 if (mode == CCmode)
15221 gcc_unreachable ();
15224 suffix = fp ? "u" : "p";
15227 suffix = fp ? "nu" : "np";
15230 gcc_unreachable ();
15232 fputs (suffix, file);
15235 /* Print the name of register X to FILE based on its machine mode and number.
15236 If CODE is 'w', pretend the mode is HImode.
15237 If CODE is 'b', pretend the mode is QImode.
15238 If CODE is 'k', pretend the mode is SImode.
15239 If CODE is 'q', pretend the mode is DImode.
15240 If CODE is 'x', pretend the mode is V4SFmode.
15241 If CODE is 't', pretend the mode is V8SFmode.
15242 If CODE is 'g', pretend the mode is V16SFmode.
15243 If CODE is 'h', pretend the reg is the 'high' byte register.
15244 If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
15245 If CODE is 'd', duplicate the operand for AVX instruction.
15249 print_reg (rtx x, int code, FILE *file)
15252 unsigned int regno;
15253 bool duplicated = code == 'd' && TARGET_AVX;
15255 if (ASSEMBLER_DIALECT == ASM_ATT)
15260 gcc_assert (TARGET_64BIT);
15261 fputs ("rip", file);
15265 regno = true_regnum (x);
15266 gcc_assert (regno != ARG_POINTER_REGNUM
15267 && regno != FRAME_POINTER_REGNUM
15268 && regno != FLAGS_REG
15269 && regno != FPSR_REG
15270 && regno != FPCR_REG);
15272 if (code == 'w' || MMX_REG_P (x))
15274 else if (code == 'b')
15276 else if (code == 'k')
15278 else if (code == 'q')
15280 else if (code == 'y')
15282 else if (code == 'h')
15284 else if (code == 'x')
15286 else if (code == 't')
15288 else if (code == 'g')
15291 code = GET_MODE_SIZE (GET_MODE (x));
15293 /* Irritatingly, AMD extended registers use different naming convention
15294 from the normal registers: "r%d[bwd]" */
15295 if (REX_INT_REGNO_P (regno))
15297 gcc_assert (TARGET_64BIT);
15299 fprint_ul (file, regno - FIRST_REX_INT_REG + 8);
15303 error ("extended registers have no high halves");
15318 error ("unsupported operand size for extended register");
15328 if (STACK_TOP_P (x))
15337 if (! ANY_FP_REG_P (x) && ! ANY_MASK_REG_P (x) && ! ANY_BND_REG_P (x))
15338 putc (code == 8 && TARGET_64BIT ? 'r' : 'e', file);
15343 reg = hi_reg_name[regno];
15346 if (regno >= ARRAY_SIZE (qi_reg_name))
15348 reg = qi_reg_name[regno];
15351 if (regno >= ARRAY_SIZE (qi_high_reg_name))
15353 reg = qi_high_reg_name[regno];
15358 gcc_assert (!duplicated);
15360 fputs (hi_reg_name[regno] + 1, file);
15366 gcc_assert (!duplicated);
15368 fputs (hi_reg_name[REGNO (x)] + 1, file);
15373 gcc_unreachable ();
15379 if (ASSEMBLER_DIALECT == ASM_ATT)
15380 fprintf (file, ", %%%s", reg);
15382 fprintf (file, ", %s", reg);
15386 /* Meaning of CODE:
15387 L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
15388 C -- print opcode suffix for set/cmov insn.
15389 c -- like C, but print reversed condition
15390 F,f -- likewise, but for floating-point.
15391 O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
15393 R -- print embeded rounding and sae.
15394 r -- print only sae.
15395 z -- print the opcode suffix for the size of the current operand.
15396 Z -- likewise, with special suffixes for x87 instructions.
15397 * -- print a star (in certain assembler syntax)
15398 A -- print an absolute memory reference.
15399 E -- print address with DImode register names if TARGET_64BIT.
15400 w -- print the operand as if it's a "word" (HImode) even if it isn't.
15401 s -- print a shift double count, followed by the assemblers argument
15403 b -- print the QImode name of the register for the indicated operand.
15404 %b0 would print %al if operands[0] is reg 0.
15405 w -- likewise, print the HImode name of the register.
15406 k -- likewise, print the SImode name of the register.
15407 q -- likewise, print the DImode name of the register.
15408 x -- likewise, print the V4SFmode name of the register.
15409 t -- likewise, print the V8SFmode name of the register.
15410 g -- likewise, print the V16SFmode name of the register.
15411 h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
15412 y -- print "st(0)" instead of "st" as a register.
15413 d -- print duplicated register operand for AVX instruction.
15414 D -- print condition for SSE cmp instruction.
15415 P -- if PIC, print an @PLT suffix.
15416 p -- print raw symbol name.
15417 X -- don't print any sort of PIC '@' suffix for a symbol.
15418 & -- print some in-use local-dynamic symbol name.
15419 H -- print a memory address offset by 8; used for sse high-parts
15420 Y -- print condition for XOP pcom* instruction.
15421 + -- print a branch hint as 'cs' or 'ds' prefix
15422 ; -- print a semicolon (after prefixes due to bug in older gas).
15423 ~ -- print "i" if TARGET_AVX2, "f" otherwise.
15424 @ -- print a segment register of thread base pointer load
15425 ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
15426 ! -- print MPX prefix for jxx/call/ret instructions if required.
15430 ix86_print_operand (FILE *file, rtx x, int code)
15437 switch (ASSEMBLER_DIALECT)
15444 /* Intel syntax. For absolute addresses, registers should not
15445 be surrounded by braces. */
15449 ix86_print_operand (file, x, 0);
15456 gcc_unreachable ();
15459 ix86_print_operand (file, x, 0);
15463 /* Wrap address in an UNSPEC to declare special handling. */
15465 x = gen_rtx_UNSPEC (DImode, gen_rtvec (1, x), UNSPEC_LEA_ADDR);
15467 output_address (x);
15471 if (ASSEMBLER_DIALECT == ASM_ATT)
15476 if (ASSEMBLER_DIALECT == ASM_ATT)
15481 if (ASSEMBLER_DIALECT == ASM_ATT)
15486 if (ASSEMBLER_DIALECT == ASM_ATT)
15491 if (ASSEMBLER_DIALECT == ASM_ATT)
15496 if (ASSEMBLER_DIALECT == ASM_ATT)
15501 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
15502 if (ASSEMBLER_DIALECT != ASM_ATT)
15505 switch (GET_MODE_SIZE (GET_MODE (x)))
15520 output_operand_lossage
15521 ("invalid operand size for operand code 'O'");
15530 if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
15532 /* Opcodes don't get size suffixes if using Intel opcodes. */
15533 if (ASSEMBLER_DIALECT == ASM_INTEL)
15536 switch (GET_MODE_SIZE (GET_MODE (x)))
15555 output_operand_lossage
15556 ("invalid operand size for operand code 'z'");
15561 if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
15563 (0, "non-integer operand used with operand code 'z'");
15567 /* 387 opcodes don't get size suffixes if using Intel opcodes. */
15568 if (ASSEMBLER_DIALECT == ASM_INTEL)
15571 if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
15573 switch (GET_MODE_SIZE (GET_MODE (x)))
15576 #ifdef HAVE_AS_IX86_FILDS
15586 #ifdef HAVE_AS_IX86_FILDQ
15589 fputs ("ll", file);
15597 else if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
15599 /* 387 opcodes don't get size suffixes
15600 if the operands are registers. */
15601 if (STACK_REG_P (x))
15604 switch (GET_MODE_SIZE (GET_MODE (x)))
15625 output_operand_lossage
15626 ("invalid operand type used with operand code 'Z'");
15630 output_operand_lossage
15631 ("invalid operand size for operand code 'Z'");
15650 if (CONST_INT_P (x) || ! SHIFT_DOUBLE_OMITS_COUNT)
15652 ix86_print_operand (file, x, 0);
15653 fputs (", ", file);
15658 switch (GET_CODE (x))
15661 fputs ("neq", file);
15664 fputs ("eq", file);
15668 fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "ge" : "unlt", file);
15672 fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "gt" : "unle", file);
15676 fputs ("le", file);
15680 fputs ("lt", file);
15683 fputs ("unord", file);
15686 fputs ("ord", file);
15689 fputs ("ueq", file);
15692 fputs ("nlt", file);
15695 fputs ("nle", file);
15698 fputs ("ule", file);
15701 fputs ("ult", file);
15704 fputs ("une", file);
15707 output_operand_lossage ("operand is not a condition code, "
15708 "invalid operand code 'Y'");
15714 /* Little bit of braindamage here. The SSE compare instructions
15715 does use completely different names for the comparisons that the
15716 fp conditional moves. */
15717 switch (GET_CODE (x))
15722 fputs ("eq_us", file);
15726 fputs ("eq", file);
15731 fputs ("nge", file);
15735 fputs ("lt", file);
15740 fputs ("ngt", file);
15744 fputs ("le", file);
15747 fputs ("unord", file);
15752 fputs ("neq_oq", file);
15756 fputs ("neq", file);
15761 fputs ("ge", file);
15765 fputs ("nlt", file);
15770 fputs ("gt", file);
15774 fputs ("nle", file);
15777 fputs ("ord", file);
15780 output_operand_lossage ("operand is not a condition code, "
15781 "invalid operand code 'D'");
15788 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
15789 if (ASSEMBLER_DIALECT == ASM_ATT)
15795 if (!COMPARISON_P (x))
15797 output_operand_lossage ("operand is not a condition code, "
15798 "invalid operand code '%c'", code);
15801 put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)),
15802 code == 'c' || code == 'f',
15803 code == 'F' || code == 'f',
15808 if (!offsettable_memref_p (x))
15810 output_operand_lossage ("operand is not an offsettable memory "
15811 "reference, invalid operand code 'H'");
15814 /* It doesn't actually matter what mode we use here, as we're
15815 only going to use this for printing. */
15816 x = adjust_address_nv (x, DImode, 8);
15817 /* Output 'qword ptr' for intel assembler dialect. */
15818 if (ASSEMBLER_DIALECT == ASM_INTEL)
15823 gcc_assert (CONST_INT_P (x));
15825 if (INTVAL (x) & IX86_HLE_ACQUIRE)
15826 #ifdef HAVE_AS_IX86_HLE
15827 fputs ("xacquire ", file);
15829 fputs ("\n" ASM_BYTE "0xf2\n\t", file);
15831 else if (INTVAL (x) & IX86_HLE_RELEASE)
15832 #ifdef HAVE_AS_IX86_HLE
15833 fputs ("xrelease ", file);
15835 fputs ("\n" ASM_BYTE "0xf3\n\t", file);
15837 /* We do not want to print value of the operand. */
15841 if (x == const0_rtx || x == CONST0_RTX (GET_MODE (x)))
15842 fputs ("{z}", file);
15846 gcc_assert (CONST_INT_P (x));
15847 gcc_assert (INTVAL (x) == ROUND_SAE);
15849 if (ASSEMBLER_DIALECT == ASM_INTEL)
15850 fputs (", ", file);
15852 fputs ("{sae}", file);
15854 if (ASSEMBLER_DIALECT == ASM_ATT)
15855 fputs (", ", file);
15860 gcc_assert (CONST_INT_P (x));
15862 if (ASSEMBLER_DIALECT == ASM_INTEL)
15863 fputs (", ", file);
15865 switch (INTVAL (x))
15867 case ROUND_NEAREST_INT | ROUND_SAE:
15868 fputs ("{rn-sae}", file);
15870 case ROUND_NEG_INF | ROUND_SAE:
15871 fputs ("{rd-sae}", file);
15873 case ROUND_POS_INF | ROUND_SAE:
15874 fputs ("{ru-sae}", file);
15876 case ROUND_ZERO | ROUND_SAE:
15877 fputs ("{rz-sae}", file);
15880 gcc_unreachable ();
15883 if (ASSEMBLER_DIALECT == ASM_ATT)
15884 fputs (", ", file);
15889 if (ASSEMBLER_DIALECT == ASM_ATT)
15895 const char *name = get_some_local_dynamic_name ();
15897 output_operand_lossage ("'%%&' used without any "
15898 "local dynamic TLS references");
15900 assemble_name (file, name);
15909 || optimize_function_for_size_p (cfun)
15910 || !TARGET_BRANCH_PREDICTION_HINTS)
15913 x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
15916 int pred_val = XINT (x, 0);
15918 if (pred_val < REG_BR_PROB_BASE * 45 / 100
15919 || pred_val > REG_BR_PROB_BASE * 55 / 100)
15921 bool taken = pred_val > REG_BR_PROB_BASE / 2;
15923 = final_forward_branch_p (current_output_insn) == 0;
15925 /* Emit hints only in the case default branch prediction
15926 heuristics would fail. */
15927 if (taken != cputaken)
15929 /* We use 3e (DS) prefix for taken branches and
15930 2e (CS) prefix for not taken branches. */
15932 fputs ("ds ; ", file);
15934 fputs ("cs ; ", file);
15942 #ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
15948 if (ASSEMBLER_DIALECT == ASM_ATT)
15951 /* The kernel uses a different segment register for performance
15952 reasons; a system call would not have to trash the userspace
15953 segment register, which would be expensive. */
15954 if (TARGET_64BIT && ix86_cmodel != CM_KERNEL)
15955 fputs ("fs", file);
15957 fputs ("gs", file);
15961 putc (TARGET_AVX2 ? 'i' : 'f', file);
15965 if (TARGET_64BIT && Pmode != word_mode)
15966 fputs ("addr32 ", file);
15970 if (ix86_bnd_prefixed_insn_p (current_output_insn))
15971 fputs ("bnd ", file);
15975 output_operand_lossage ("invalid operand code '%c'", code);
15980 print_reg (x, code, file);
15982 else if (MEM_P (x))
15984 /* No `byte ptr' prefix for call instructions or BLKmode operands. */
15985 if (ASSEMBLER_DIALECT == ASM_INTEL && code != 'X' && code != 'P'
15986 && GET_MODE (x) != BLKmode)
15989 switch (GET_MODE_SIZE (GET_MODE (x)))
15991 case 1: size = "BYTE"; break;
15992 case 2: size = "WORD"; break;
15993 case 4: size = "DWORD"; break;
15994 case 8: size = "QWORD"; break;
15995 case 12: size = "TBYTE"; break;
15997 if (GET_MODE (x) == XFmode)
16002 case 32: size = "YMMWORD"; break;
16003 case 64: size = "ZMMWORD"; break;
16005 gcc_unreachable ();
16008 /* Check for explicit size override (codes 'b', 'w', 'k',
16012 else if (code == 'w')
16014 else if (code == 'k')
16016 else if (code == 'q')
16018 else if (code == 'x')
16021 fputs (size, file);
16022 fputs (" PTR ", file);
16026 /* Avoid (%rip) for call operands. */
16027 if (CONSTANT_ADDRESS_P (x) && code == 'P'
16028 && !CONST_INT_P (x))
16029 output_addr_const (file, x);
16030 else if (this_is_asm_operands && ! address_operand (x, VOIDmode))
16031 output_operand_lossage ("invalid constraints for operand");
16033 output_address (x);
16036 else if (GET_CODE (x) == CONST_DOUBLE && GET_MODE (x) == SFmode)
16041 REAL_VALUE_FROM_CONST_DOUBLE (r, x);
16042 REAL_VALUE_TO_TARGET_SINGLE (r, l);
16044 if (ASSEMBLER_DIALECT == ASM_ATT)
16046 /* Sign extend 32bit SFmode immediate to 8 bytes. */
16048 fprintf (file, "0x%08" HOST_LONG_LONG_FORMAT "x",
16049 (unsigned long long) (int) l);
16051 fprintf (file, "0x%08x", (unsigned int) l);
16054 else if (GET_CODE (x) == CONST_DOUBLE && GET_MODE (x) == DFmode)
16059 REAL_VALUE_FROM_CONST_DOUBLE (r, x);
16060 REAL_VALUE_TO_TARGET_DOUBLE (r, l);
16062 if (ASSEMBLER_DIALECT == ASM_ATT)
16064 fprintf (file, "0x%lx%08lx", l[1] & 0xffffffff, l[0] & 0xffffffff);
16067 /* These float cases don't actually occur as immediate operands. */
16068 else if (GET_CODE (x) == CONST_DOUBLE && GET_MODE (x) == XFmode)
16072 real_to_decimal (dstr, CONST_DOUBLE_REAL_VALUE (x), sizeof (dstr), 0, 1);
16073 fputs (dstr, file);
16078 /* We have patterns that allow zero sets of memory, for instance.
16079 In 64-bit mode, we should probably support all 8-byte vectors,
16080 since we can in fact encode that into an immediate. */
16081 if (GET_CODE (x) == CONST_VECTOR)
16083 gcc_assert (x == CONST0_RTX (GET_MODE (x)));
16087 if (code != 'P' && code != 'p')
16089 if (CONST_INT_P (x) || GET_CODE (x) == CONST_DOUBLE)
16091 if (ASSEMBLER_DIALECT == ASM_ATT)
16094 else if (GET_CODE (x) == CONST || GET_CODE (x) == SYMBOL_REF
16095 || GET_CODE (x) == LABEL_REF)
16097 if (ASSEMBLER_DIALECT == ASM_ATT)
16100 fputs ("OFFSET FLAT:", file);
16103 if (CONST_INT_P (x))
16104 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
16105 else if (flag_pic || MACHOPIC_INDIRECT)
16106 output_pic_addr_const (file, x, code);
16108 output_addr_const (file, x);
16113 ix86_print_operand_punct_valid_p (unsigned char code)
16115 return (code == '@' || code == '*' || code == '+' || code == '&'
16116 || code == ';' || code == '~' || code == '^' || code == '!');
16119 /* Print a memory operand whose address is ADDR. */
16122 ix86_print_operand_address (FILE *file, rtx addr)
16124 struct ix86_address parts;
16125 rtx base, index, disp;
16131 if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_VSIBADDR)
16133 ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
16134 gcc_assert (parts.index == NULL_RTX);
16135 parts.index = XVECEXP (addr, 0, 1);
16136 parts.scale = INTVAL (XVECEXP (addr, 0, 2));
16137 addr = XVECEXP (addr, 0, 0);
16140 else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_LEA_ADDR)
16142 gcc_assert (TARGET_64BIT);
16143 ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
16146 else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_BNDMK_ADDR)
16148 ok = ix86_decompose_address (XVECEXP (addr, 0, 1), &parts);
16149 gcc_assert (parts.base == NULL_RTX || parts.index == NULL_RTX);
16150 if (parts.base != NULL_RTX)
16152 parts.index = parts.base;
16155 parts.base = XVECEXP (addr, 0, 0);
16156 addr = XVECEXP (addr, 0, 0);
16158 else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_BNDLDX_ADDR)
16160 ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
16161 gcc_assert (parts.index == NULL_RTX);
16162 parts.index = XVECEXP (addr, 0, 1);
16163 addr = XVECEXP (addr, 0, 0);
16166 ok = ix86_decompose_address (addr, &parts);
16171 index = parts.index;
16173 scale = parts.scale;
16181 if (ASSEMBLER_DIALECT == ASM_ATT)
16183 fputs ((parts.seg == SEG_FS ? "fs:" : "gs:"), file);
16186 gcc_unreachable ();
16189 /* Use one byte shorter RIP relative addressing for 64bit mode. */
16190 if (TARGET_64BIT && !base && !index)
16194 if (GET_CODE (disp) == CONST
16195 && GET_CODE (XEXP (disp, 0)) == PLUS
16196 && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
16197 symbol = XEXP (XEXP (disp, 0), 0);
16199 if (GET_CODE (symbol) == LABEL_REF
16200 || (GET_CODE (symbol) == SYMBOL_REF
16201 && SYMBOL_REF_TLS_MODEL (symbol) == 0))
16204 if (!base && !index)
16206 /* Displacement only requires special attention. */
16208 if (CONST_INT_P (disp))
16210 if (ASSEMBLER_DIALECT == ASM_INTEL && parts.seg == SEG_DEFAULT)
16211 fputs ("ds:", file);
16212 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (disp));
16215 output_pic_addr_const (file, disp, 0);
16217 output_addr_const (file, disp);
16221 /* Print SImode register names to force addr32 prefix. */
16222 if (SImode_address_operand (addr, VOIDmode))
16224 #ifdef ENABLE_CHECKING
16225 gcc_assert (TARGET_64BIT);
16226 switch (GET_CODE (addr))
16229 gcc_assert (GET_MODE (addr) == SImode);
16230 gcc_assert (GET_MODE (SUBREG_REG (addr)) == DImode);
16234 gcc_assert (GET_MODE (addr) == DImode);
16237 gcc_unreachable ();
16240 gcc_assert (!code);
16246 && CONST_INT_P (disp)
16247 && INTVAL (disp) < -16*1024*1024)
16249 /* X32 runs in 64-bit mode, where displacement, DISP, in
16250 address DISP(%r64), is encoded as 32-bit immediate sign-
16251 extended from 32-bit to 64-bit. For -0x40000300(%r64),
16252 address is %r64 + 0xffffffffbffffd00. When %r64 <
16253 0x40000300, like 0x37ffe064, address is 0xfffffffff7ffdd64,
16254 which is invalid for x32. The correct address is %r64
16255 - 0x40000300 == 0xf7ffdd64. To properly encode
16256 -0x40000300(%r64) for x32, we zero-extend negative
16257 displacement by forcing addr32 prefix which truncates
16258 0xfffffffff7ffdd64 to 0xf7ffdd64. In theory, we should
16259 zero-extend all negative displacements, including -1(%rsp).
16260 However, for small negative displacements, sign-extension
16261 won't cause overflow. We only zero-extend negative
16262 displacements if they < -16*1024*1024, which is also used
16263 to check legitimate address displacements for PIC. */
16267 if (ASSEMBLER_DIALECT == ASM_ATT)
16272 output_pic_addr_const (file, disp, 0);
16273 else if (GET_CODE (disp) == LABEL_REF)
16274 output_asm_label (disp);
16276 output_addr_const (file, disp);
16281 print_reg (base, code, file);
16285 print_reg (index, vsib ? 0 : code, file);
16286 if (scale != 1 || vsib)
16287 fprintf (file, ",%d", scale);
16293 rtx offset = NULL_RTX;
16297 /* Pull out the offset of a symbol; print any symbol itself. */
16298 if (GET_CODE (disp) == CONST
16299 && GET_CODE (XEXP (disp, 0)) == PLUS
16300 && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
16302 offset = XEXP (XEXP (disp, 0), 1);
16303 disp = gen_rtx_CONST (VOIDmode,
16304 XEXP (XEXP (disp, 0), 0));
16308 output_pic_addr_const (file, disp, 0);
16309 else if (GET_CODE (disp) == LABEL_REF)
16310 output_asm_label (disp);
16311 else if (CONST_INT_P (disp))
16314 output_addr_const (file, disp);
16320 print_reg (base, code, file);
16323 if (INTVAL (offset) >= 0)
16325 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
16329 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
16336 print_reg (index, vsib ? 0 : code, file);
16337 if (scale != 1 || vsib)
16338 fprintf (file, "*%d", scale);
16345 /* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */
16348 i386_asm_output_addr_const_extra (FILE *file, rtx x)
16352 if (GET_CODE (x) != UNSPEC)
16355 op = XVECEXP (x, 0, 0);
16356 switch (XINT (x, 1))
16358 case UNSPEC_GOTTPOFF:
16359 output_addr_const (file, op);
16360 /* FIXME: This might be @TPOFF in Sun ld. */
16361 fputs ("@gottpoff", file);
16364 output_addr_const (file, op);
16365 fputs ("@tpoff", file);
16367 case UNSPEC_NTPOFF:
16368 output_addr_const (file, op);
16370 fputs ("@tpoff", file);
16372 fputs ("@ntpoff", file);
16374 case UNSPEC_DTPOFF:
16375 output_addr_const (file, op);
16376 fputs ("@dtpoff", file);
16378 case UNSPEC_GOTNTPOFF:
16379 output_addr_const (file, op);
16381 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
16382 "@gottpoff(%rip)" : "@gottpoff[rip]", file);
16384 fputs ("@gotntpoff", file);
16386 case UNSPEC_INDNTPOFF:
16387 output_addr_const (file, op);
16388 fputs ("@indntpoff", file);
16391 case UNSPEC_MACHOPIC_OFFSET:
16392 output_addr_const (file, op);
16394 machopic_output_function_base_name (file);
16398 case UNSPEC_STACK_CHECK:
16402 gcc_assert (flag_split_stack);
16404 #ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
16405 offset = TARGET_THREAD_SPLIT_STACK_OFFSET;
16407 gcc_unreachable ();
16410 fprintf (file, "%s:%d", TARGET_64BIT ? "%fs" : "%gs", offset);
16421 /* Split one or more double-mode RTL references into pairs of half-mode
16422 references. The RTL can be REG, offsettable MEM, integer constant, or
16423 CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
16424 split and "num" is its length. lo_half and hi_half are output arrays
16425 that parallel "operands". */
16428 split_double_mode (machine_mode mode, rtx operands[],
16429 int num, rtx lo_half[], rtx hi_half[])
16431 machine_mode half_mode;
16437 half_mode = DImode;
16440 half_mode = SImode;
16443 gcc_unreachable ();
16446 byte = GET_MODE_SIZE (half_mode);
16450 rtx op = operands[num];
16452 /* simplify_subreg refuse to split volatile memory addresses,
16453 but we still have to handle it. */
16456 lo_half[num] = adjust_address (op, half_mode, 0);
16457 hi_half[num] = adjust_address (op, half_mode, byte);
16461 lo_half[num] = simplify_gen_subreg (half_mode, op,
16462 GET_MODE (op) == VOIDmode
16463 ? mode : GET_MODE (op), 0);
16464 hi_half[num] = simplify_gen_subreg (half_mode, op,
16465 GET_MODE (op) == VOIDmode
16466 ? mode : GET_MODE (op), byte);
16471 /* Output code to perform a 387 binary operation in INSN, one of PLUS,
16472 MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
16473 is the expression of the binary operation. The output may either be
16474 emitted here, or returned to the caller, like all output_* functions.
16476 There is no guarantee that the operands are the same mode, as they
16477 might be within FLOAT or FLOAT_EXTEND expressions. */
16479 #ifndef SYSV386_COMPAT
16480 /* Set to 1 for compatibility with brain-damaged assemblers. No-one
16481 wants to fix the assemblers because that causes incompatibility
16482 with gcc. No-one wants to fix gcc because that causes
16483 incompatibility with assemblers... You can use the option of
16484 -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
16485 #define SYSV386_COMPAT 1
16489 output_387_binary_op (rtx insn, rtx *operands)
16491 static char buf[40];
16494 int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]) || SSE_REG_P (operands[2]);
16496 #ifdef ENABLE_CHECKING
16497 /* Even if we do not want to check the inputs, this documents input
16498 constraints. Which helps in understanding the following code. */
16499 if (STACK_REG_P (operands[0])
16500 && ((REG_P (operands[1])
16501 && REGNO (operands[0]) == REGNO (operands[1])
16502 && (STACK_REG_P (operands[2]) || MEM_P (operands[2])))
16503 || (REG_P (operands[2])
16504 && REGNO (operands[0]) == REGNO (operands[2])
16505 && (STACK_REG_P (operands[1]) || MEM_P (operands[1]))))
16506 && (STACK_TOP_P (operands[1]) || STACK_TOP_P (operands[2])))
16509 gcc_assert (is_sse);
16512 switch (GET_CODE (operands[3]))
16515 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
16516 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
16524 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
16525 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
16533 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
16534 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
16542 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
16543 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
16551 gcc_unreachable ();
16558 strcpy (buf, ssep);
16559 if (GET_MODE (operands[0]) == SFmode)
16560 strcat (buf, "ss\t{%2, %1, %0|%0, %1, %2}");
16562 strcat (buf, "sd\t{%2, %1, %0|%0, %1, %2}");
16566 strcpy (buf, ssep + 1);
16567 if (GET_MODE (operands[0]) == SFmode)
16568 strcat (buf, "ss\t{%2, %0|%0, %2}");
16570 strcat (buf, "sd\t{%2, %0|%0, %2}");
16576 switch (GET_CODE (operands[3]))
16580 if (REG_P (operands[2]) && REGNO (operands[0]) == REGNO (operands[2]))
16581 std::swap (operands[1], operands[2]);
16583 /* know operands[0] == operands[1]. */
16585 if (MEM_P (operands[2]))
16591 if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
16593 if (STACK_TOP_P (operands[0]))
16594 /* How is it that we are storing to a dead operand[2]?
16595 Well, presumably operands[1] is dead too. We can't
16596 store the result to st(0) as st(0) gets popped on this
16597 instruction. Instead store to operands[2] (which I
16598 think has to be st(1)). st(1) will be popped later.
16599 gcc <= 2.8.1 didn't have this check and generated
16600 assembly code that the Unixware assembler rejected. */
16601 p = "p\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
16603 p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
16607 if (STACK_TOP_P (operands[0]))
16608 p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
16610 p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
16615 if (MEM_P (operands[1]))
16621 if (MEM_P (operands[2]))
16627 if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
16630 /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
16631 derived assemblers, confusingly reverse the direction of
16632 the operation for fsub{r} and fdiv{r} when the
16633 destination register is not st(0). The Intel assembler
16634 doesn't have this brain damage. Read !SYSV386_COMPAT to
16635 figure out what the hardware really does. */
16636 if (STACK_TOP_P (operands[0]))
16637 p = "{p\t%0, %2|rp\t%2, %0}";
16639 p = "{rp\t%2, %0|p\t%0, %2}";
16641 if (STACK_TOP_P (operands[0]))
16642 /* As above for fmul/fadd, we can't store to st(0). */
16643 p = "rp\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
16645 p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
16650 if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
16653 if (STACK_TOP_P (operands[0]))
16654 p = "{rp\t%0, %1|p\t%1, %0}";
16656 p = "{p\t%1, %0|rp\t%0, %1}";
16658 if (STACK_TOP_P (operands[0]))
16659 p = "p\t{%0, %1|%1, %0}"; /* st(1) = st(1) op st(0); pop */
16661 p = "rp\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2); pop */
16666 if (STACK_TOP_P (operands[0]))
16668 if (STACK_TOP_P (operands[1]))
16669 p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
16671 p = "r\t{%y1, %0|%0, %y1}"; /* st(0) = st(r1) op st(0) */
16674 else if (STACK_TOP_P (operands[1]))
16677 p = "{\t%1, %0|r\t%0, %1}";
16679 p = "r\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2) */
16685 p = "{r\t%2, %0|\t%0, %2}";
16687 p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
16693 gcc_unreachable ();
16700 /* Check if a 256bit AVX register is referenced inside of EXP. */
16703 ix86_check_avx256_register (const_rtx exp)
16705 if (GET_CODE (exp) == SUBREG)
16706 exp = SUBREG_REG (exp);
16708 return (REG_P (exp)
16709 && VALID_AVX256_REG_OR_OI_MODE (GET_MODE (exp)));
16712 /* Return needed mode for entity in optimize_mode_switching pass. */
16715 ix86_avx_u128_mode_needed (rtx_insn *insn)
16721 /* Needed mode is set to AVX_U128_CLEAN if there are
16722 no 256bit modes used in function arguments. */
16723 for (link = CALL_INSN_FUNCTION_USAGE (insn);
16725 link = XEXP (link, 1))
16727 if (GET_CODE (XEXP (link, 0)) == USE)
16729 rtx arg = XEXP (XEXP (link, 0), 0);
16731 if (ix86_check_avx256_register (arg))
16732 return AVX_U128_DIRTY;
16736 return AVX_U128_CLEAN;
16739 /* Require DIRTY mode if a 256bit AVX register is referenced. Hardware
16740 changes state only when a 256bit register is written to, but we need
16741 to prevent the compiler from moving optimal insertion point above
16742 eventual read from 256bit register. */
16743 subrtx_iterator::array_type array;
16744 FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
16745 if (ix86_check_avx256_register (*iter))
16746 return AVX_U128_DIRTY;
16748 return AVX_U128_ANY;
16751 /* Return mode that i387 must be switched into
16752 prior to the execution of insn. */
16755 ix86_i387_mode_needed (int entity, rtx_insn *insn)
16757 enum attr_i387_cw mode;
16759 /* The mode UNINITIALIZED is used to store control word after a
16760 function call or ASM pattern. The mode ANY specify that function
16761 has no requirements on the control word and make no changes in the
16762 bits we are interested in. */
16765 || (NONJUMP_INSN_P (insn)
16766 && (asm_noperands (PATTERN (insn)) >= 0
16767 || GET_CODE (PATTERN (insn)) == ASM_INPUT)))
16768 return I387_CW_UNINITIALIZED;
16770 if (recog_memoized (insn) < 0)
16771 return I387_CW_ANY;
16773 mode = get_attr_i387_cw (insn);
16778 if (mode == I387_CW_TRUNC)
16783 if (mode == I387_CW_FLOOR)
16788 if (mode == I387_CW_CEIL)
16793 if (mode == I387_CW_MASK_PM)
16798 gcc_unreachable ();
16801 return I387_CW_ANY;
16804 /* Return mode that entity must be switched into
16805 prior to the execution of insn. */
16808 ix86_mode_needed (int entity, rtx_insn *insn)
16813 return ix86_avx_u128_mode_needed (insn);
16818 return ix86_i387_mode_needed (entity, insn);
16820 gcc_unreachable ();
16825 /* Check if a 256bit AVX register is referenced in stores. */
16828 ix86_check_avx256_stores (rtx dest, const_rtx, void *data)
16830 if (ix86_check_avx256_register (dest))
16832 bool *used = (bool *) data;
16837 /* Calculate mode of upper 128bit AVX registers after the insn. */
16840 ix86_avx_u128_mode_after (int mode, rtx_insn *insn)
16842 rtx pat = PATTERN (insn);
16844 if (vzeroupper_operation (pat, VOIDmode)
16845 || vzeroall_operation (pat, VOIDmode))
16846 return AVX_U128_CLEAN;
16848 /* We know that state is clean after CALL insn if there are no
16849 256bit registers used in the function return register. */
16852 bool avx_reg256_found = false;
16853 note_stores (pat, ix86_check_avx256_stores, &avx_reg256_found);
16855 return avx_reg256_found ? AVX_U128_DIRTY : AVX_U128_CLEAN;
16858 /* Otherwise, return current mode. Remember that if insn
16859 references AVX 256bit registers, the mode was already changed
16860 to DIRTY from MODE_NEEDED. */
16864 /* Return the mode that an insn results in. */
16867 ix86_mode_after (int entity, int mode, rtx_insn *insn)
16872 return ix86_avx_u128_mode_after (mode, insn);
16879 gcc_unreachable ();
16884 ix86_avx_u128_mode_entry (void)
16888 /* Entry mode is set to AVX_U128_DIRTY if there are
16889 256bit modes used in function arguments. */
16890 for (arg = DECL_ARGUMENTS (current_function_decl); arg;
16891 arg = TREE_CHAIN (arg))
16893 rtx incoming = DECL_INCOMING_RTL (arg);
16895 if (incoming && ix86_check_avx256_register (incoming))
16896 return AVX_U128_DIRTY;
16899 return AVX_U128_CLEAN;
16902 /* Return a mode that ENTITY is assumed to be
16903 switched to at function entry. */
16906 ix86_mode_entry (int entity)
16911 return ix86_avx_u128_mode_entry ();
16916 return I387_CW_ANY;
16918 gcc_unreachable ();
16923 ix86_avx_u128_mode_exit (void)
16925 rtx reg = crtl->return_rtx;
16927 /* Exit mode is set to AVX_U128_DIRTY if there are
16928 256bit modes used in the function return register. */
16929 if (reg && ix86_check_avx256_register (reg))
16930 return AVX_U128_DIRTY;
16932 return AVX_U128_CLEAN;
16935 /* Return a mode that ENTITY is assumed to be
16936 switched to at function exit. */
16939 ix86_mode_exit (int entity)
16944 return ix86_avx_u128_mode_exit ();
16949 return I387_CW_ANY;
16951 gcc_unreachable ();
16956 ix86_mode_priority (int, int n)
16961 /* Output code to initialize control word copies used by trunc?f?i and
16962 rounding patterns. CURRENT_MODE is set to current control word,
16963 while NEW_MODE is set to new control word. */
16966 emit_i387_cw_initialization (int mode)
16968 rtx stored_mode = assign_386_stack_local (HImode, SLOT_CW_STORED);
16971 enum ix86_stack_slot slot;
16973 rtx reg = gen_reg_rtx (HImode);
16975 emit_insn (gen_x86_fnstcw_1 (stored_mode));
16976 emit_move_insn (reg, copy_rtx (stored_mode));
16978 if (TARGET_64BIT || TARGET_PARTIAL_REG_STALL
16979 || optimize_insn_for_size_p ())
16983 case I387_CW_TRUNC:
16984 /* round toward zero (truncate) */
16985 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0c00)));
16986 slot = SLOT_CW_TRUNC;
16989 case I387_CW_FLOOR:
16990 /* round down toward -oo */
16991 emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
16992 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0400)));
16993 slot = SLOT_CW_FLOOR;
16997 /* round up toward +oo */
16998 emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
16999 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0800)));
17000 slot = SLOT_CW_CEIL;
17003 case I387_CW_MASK_PM:
17004 /* mask precision exception for nearbyint() */
17005 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020)));
17006 slot = SLOT_CW_MASK_PM;
17010 gcc_unreachable ();
17017 case I387_CW_TRUNC:
17018 /* round toward zero (truncate) */
17019 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0xc)));
17020 slot = SLOT_CW_TRUNC;
17023 case I387_CW_FLOOR:
17024 /* round down toward -oo */
17025 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0x4)));
17026 slot = SLOT_CW_FLOOR;
17030 /* round up toward +oo */
17031 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0x8)));
17032 slot = SLOT_CW_CEIL;
17035 case I387_CW_MASK_PM:
17036 /* mask precision exception for nearbyint() */
17037 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020)));
17038 slot = SLOT_CW_MASK_PM;
17042 gcc_unreachable ();
17046 gcc_assert (slot < MAX_386_STACK_LOCALS);
17048 new_mode = assign_386_stack_local (HImode, slot);
17049 emit_move_insn (new_mode, reg);
17052 /* Emit vzeroupper. */
17055 ix86_avx_emit_vzeroupper (HARD_REG_SET regs_live)
17059 /* Cancel automatic vzeroupper insertion if there are
17060 live call-saved SSE registers at the insertion point. */
17062 for (i = FIRST_SSE_REG; i <= LAST_SSE_REG; i++)
17063 if (TEST_HARD_REG_BIT (regs_live, i) && !call_used_regs[i])
17067 for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
17068 if (TEST_HARD_REG_BIT (regs_live, i) && !call_used_regs[i])
17071 emit_insn (gen_avx_vzeroupper ());
17074 /* Generate one or more insns to set ENTITY to MODE. */
17076 /* Generate one or more insns to set ENTITY to MODE. HARD_REG_LIVE
17077 is the set of hard registers live at the point where the insn(s)
17078 are to be inserted. */
17081 ix86_emit_mode_set (int entity, int mode, int prev_mode ATTRIBUTE_UNUSED,
17082 HARD_REG_SET regs_live)
17087 if (mode == AVX_U128_CLEAN)
17088 ix86_avx_emit_vzeroupper (regs_live);
17094 if (mode != I387_CW_ANY
17095 && mode != I387_CW_UNINITIALIZED)
17096 emit_i387_cw_initialization (mode);
17099 gcc_unreachable ();
17103 /* Output code for INSN to convert a float to a signed int. OPERANDS
17104 are the insn operands. The output may be [HSD]Imode and the input
17105 operand may be [SDX]Fmode. */
17108 output_fix_trunc (rtx_insn *insn, rtx *operands, bool fisttp)
17110 int stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;
17111 int dimode_p = GET_MODE (operands[0]) == DImode;
17112 int round_mode = get_attr_i387_cw (insn);
17114 /* Jump through a hoop or two for DImode, since the hardware has no
17115 non-popping instruction. We used to do this a different way, but
17116 that was somewhat fragile and broke with post-reload splitters. */
17117 if ((dimode_p || fisttp) && !stack_top_dies)
17118 output_asm_insn ("fld\t%y1", operands);
17120 gcc_assert (STACK_TOP_P (operands[1]));
17121 gcc_assert (MEM_P (operands[0]));
17122 gcc_assert (GET_MODE (operands[1]) != TFmode);
17125 output_asm_insn ("fisttp%Z0\t%0", operands);
17128 if (round_mode != I387_CW_ANY)
17129 output_asm_insn ("fldcw\t%3", operands);
17130 if (stack_top_dies || dimode_p)
17131 output_asm_insn ("fistp%Z0\t%0", operands);
17133 output_asm_insn ("fist%Z0\t%0", operands);
17134 if (round_mode != I387_CW_ANY)
17135 output_asm_insn ("fldcw\t%2", operands);
17141 /* Output code for x87 ffreep insn. The OPNO argument, which may only
17142 have the values zero or one, indicates the ffreep insn's operand
17143 from the OPERANDS array. */
17145 static const char *
17146 output_387_ffreep (rtx *operands ATTRIBUTE_UNUSED, int opno)
17148 if (TARGET_USE_FFREEP)
17149 #ifdef HAVE_AS_IX86_FFREEP
17150 return opno ? "ffreep\t%y1" : "ffreep\t%y0";
17153 static char retval[32];
17154 int regno = REGNO (operands[opno]);
17156 gcc_assert (STACK_REGNO_P (regno));
17158 regno -= FIRST_STACK_REG;
17160 snprintf (retval, sizeof (retval), ASM_SHORT "0xc%ddf", regno);
17165 return opno ? "fstp\t%y1" : "fstp\t%y0";
17169 /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
17170 should be used. UNORDERED_P is true when fucom should be used. */
17173 output_fp_compare (rtx insn, rtx *operands, bool eflags_p, bool unordered_p)
17175 int stack_top_dies;
17176 rtx cmp_op0, cmp_op1;
17177 int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]);
17181 cmp_op0 = operands[0];
17182 cmp_op1 = operands[1];
17186 cmp_op0 = operands[1];
17187 cmp_op1 = operands[2];
17192 if (GET_MODE (operands[0]) == SFmode)
17194 return "%vucomiss\t{%1, %0|%0, %1}";
17196 return "%vcomiss\t{%1, %0|%0, %1}";
17199 return "%vucomisd\t{%1, %0|%0, %1}";
17201 return "%vcomisd\t{%1, %0|%0, %1}";
17204 gcc_assert (STACK_TOP_P (cmp_op0));
17206 stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;
17208 if (cmp_op1 == CONST0_RTX (GET_MODE (cmp_op1)))
17210 if (stack_top_dies)
17212 output_asm_insn ("ftst\n\tfnstsw\t%0", operands);
17213 return output_387_ffreep (operands, 1);
17216 return "ftst\n\tfnstsw\t%0";
17219 if (STACK_REG_P (cmp_op1)
17221 && find_regno_note (insn, REG_DEAD, REGNO (cmp_op1))
17222 && REGNO (cmp_op1) != FIRST_STACK_REG)
17224 /* If both the top of the 387 stack dies, and the other operand
17225 is also a stack register that dies, then this must be a
17226 `fcompp' float compare */
17230 /* There is no double popping fcomi variant. Fortunately,
17231 eflags is immune from the fstp's cc clobbering. */
17233 output_asm_insn ("fucomip\t{%y1, %0|%0, %y1}", operands);
17235 output_asm_insn ("fcomip\t{%y1, %0|%0, %y1}", operands);
17236 return output_387_ffreep (operands, 0);
17241 return "fucompp\n\tfnstsw\t%0";
17243 return "fcompp\n\tfnstsw\t%0";
17248 /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies. */
17250 static const char * const alt[16] =
17252 "fcom%Z2\t%y2\n\tfnstsw\t%0",
17253 "fcomp%Z2\t%y2\n\tfnstsw\t%0",
17254 "fucom%Z2\t%y2\n\tfnstsw\t%0",
17255 "fucomp%Z2\t%y2\n\tfnstsw\t%0",
17257 "ficom%Z2\t%y2\n\tfnstsw\t%0",
17258 "ficomp%Z2\t%y2\n\tfnstsw\t%0",
17262 "fcomi\t{%y1, %0|%0, %y1}",
17263 "fcomip\t{%y1, %0|%0, %y1}",
17264 "fucomi\t{%y1, %0|%0, %y1}",
17265 "fucomip\t{%y1, %0|%0, %y1}",
17276 mask = eflags_p << 3;
17277 mask |= (GET_MODE_CLASS (GET_MODE (cmp_op1)) == MODE_INT) << 2;
17278 mask |= unordered_p << 1;
17279 mask |= stack_top_dies;
17281 gcc_assert (mask < 16);
17290 ix86_output_addr_vec_elt (FILE *file, int value)
17292 const char *directive = ASM_LONG;
17296 directive = ASM_QUAD;
17298 gcc_assert (!TARGET_64BIT);
17301 fprintf (file, "%s%s%d\n", directive, LPREFIX, value);
17305 ix86_output_addr_diff_elt (FILE *file, int value, int rel)
17307 const char *directive = ASM_LONG;
17310 if (TARGET_64BIT && CASE_VECTOR_MODE == DImode)
17311 directive = ASM_QUAD;
17313 gcc_assert (!TARGET_64BIT);
17315 /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
17316 if (TARGET_64BIT || TARGET_VXWORKS_RTP)
17317 fprintf (file, "%s%s%d-%s%d\n",
17318 directive, LPREFIX, value, LPREFIX, rel);
17319 else if (HAVE_AS_GOTOFF_IN_DATA)
17320 fprintf (file, ASM_LONG "%s%d@GOTOFF\n", LPREFIX, value);
17322 else if (TARGET_MACHO)
17324 fprintf (file, ASM_LONG "%s%d-", LPREFIX, value);
17325 machopic_output_function_base_name (file);
17330 asm_fprintf (file, ASM_LONG "%U%s+[.-%s%d]\n",
17331 GOT_SYMBOL_NAME, LPREFIX, value);
17334 /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
17338 ix86_expand_clear (rtx dest)
17342 /* We play register width games, which are only valid after reload. */
17343 gcc_assert (reload_completed);
17345 /* Avoid HImode and its attendant prefix byte. */
17346 if (GET_MODE_SIZE (GET_MODE (dest)) < 4)
17347 dest = gen_rtx_REG (SImode, REGNO (dest));
17348 tmp = gen_rtx_SET (VOIDmode, dest, const0_rtx);
17350 if (!TARGET_USE_MOV0 || optimize_insn_for_size_p ())
17352 rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
17353 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
17359 /* X is an unchanging MEM. If it is a constant pool reference, return
17360 the constant pool rtx, else NULL. */
17363 maybe_get_pool_constant (rtx x)
17365 x = ix86_delegitimize_address (XEXP (x, 0));
17367 if (GET_CODE (x) == SYMBOL_REF && CONSTANT_POOL_ADDRESS_P (x))
17368 return get_pool_constant (x);
17374 ix86_expand_move (machine_mode mode, rtx operands[])
17377 enum tls_model model;
17382 if (GET_CODE (op1) == SYMBOL_REF)
17386 model = SYMBOL_REF_TLS_MODEL (op1);
17389 op1 = legitimize_tls_address (op1, model, true);
17390 op1 = force_operand (op1, op0);
17393 op1 = convert_to_mode (mode, op1, 1);
17395 else if ((tmp = legitimize_pe_coff_symbol (op1, false)) != NULL_RTX)
17398 else if (GET_CODE (op1) == CONST
17399 && GET_CODE (XEXP (op1, 0)) == PLUS
17400 && GET_CODE (XEXP (XEXP (op1, 0), 0)) == SYMBOL_REF)
17402 rtx addend = XEXP (XEXP (op1, 0), 1);
17403 rtx symbol = XEXP (XEXP (op1, 0), 0);
17406 model = SYMBOL_REF_TLS_MODEL (symbol);
17408 tmp = legitimize_tls_address (symbol, model, true);
17410 tmp = legitimize_pe_coff_symbol (symbol, true);
17414 tmp = force_operand (tmp, NULL);
17415 tmp = expand_simple_binop (Pmode, PLUS, tmp, addend,
17416 op0, 1, OPTAB_DIRECT);
17419 op1 = convert_to_mode (mode, tmp, 1);
17423 if ((flag_pic || MACHOPIC_INDIRECT)
17424 && symbolic_operand (op1, mode))
17426 if (TARGET_MACHO && !TARGET_64BIT)
17429 /* dynamic-no-pic */
17430 if (MACHOPIC_INDIRECT)
17432 rtx temp = ((reload_in_progress
17433 || ((op0 && REG_P (op0))
17435 ? op0 : gen_reg_rtx (Pmode));
17436 op1 = machopic_indirect_data_reference (op1, temp);
17438 op1 = machopic_legitimize_pic_address (op1, mode,
17439 temp == op1 ? 0 : temp);
17441 if (op0 != op1 && GET_CODE (op0) != MEM)
17443 rtx insn = gen_rtx_SET (VOIDmode, op0, op1);
17447 if (GET_CODE (op0) == MEM)
17448 op1 = force_reg (Pmode, op1);
17452 if (GET_CODE (temp) != REG)
17453 temp = gen_reg_rtx (Pmode);
17454 temp = legitimize_pic_address (op1, temp);
17459 /* dynamic-no-pic */
17465 op1 = force_reg (mode, op1);
17466 else if (!(TARGET_64BIT && x86_64_movabs_operand (op1, DImode)))
17468 rtx reg = can_create_pseudo_p () ? NULL_RTX : op0;
17469 op1 = legitimize_pic_address (op1, reg);
17472 op1 = convert_to_mode (mode, op1, 1);
17479 && (PUSH_ROUNDING (GET_MODE_SIZE (mode)) != GET_MODE_SIZE (mode)
17480 || !push_operand (op0, mode))
17482 op1 = force_reg (mode, op1);
17484 if (push_operand (op0, mode)
17485 && ! general_no_elim_operand (op1, mode))
17486 op1 = copy_to_mode_reg (mode, op1);
17488 /* Force large constants in 64bit compilation into register
17489 to get them CSEed. */
17490 if (can_create_pseudo_p ()
17491 && (mode == DImode) && TARGET_64BIT
17492 && immediate_operand (op1, mode)
17493 && !x86_64_zext_immediate_operand (op1, VOIDmode)
17494 && !register_operand (op0, mode)
17496 op1 = copy_to_mode_reg (mode, op1);
17498 if (can_create_pseudo_p ()
17499 && FLOAT_MODE_P (mode)
17500 && GET_CODE (op1) == CONST_DOUBLE)
17502 /* If we are loading a floating point constant to a register,
17503 force the value to memory now, since we'll get better code
17504 out the back end. */
17506 op1 = validize_mem (force_const_mem (mode, op1));
17507 if (!register_operand (op0, mode))
17509 rtx temp = gen_reg_rtx (mode);
17510 emit_insn (gen_rtx_SET (VOIDmode, temp, op1));
17511 emit_move_insn (op0, temp);
17517 emit_insn (gen_rtx_SET (VOIDmode, op0, op1));
17521 ix86_expand_vector_move (machine_mode mode, rtx operands[])
17523 rtx op0 = operands[0], op1 = operands[1];
17524 unsigned int align = GET_MODE_ALIGNMENT (mode);
17526 if (push_operand (op0, VOIDmode))
17527 op0 = emit_move_resolve_push (mode, op0);
17529 /* Force constants other than zero into memory. We do not know how
17530 the instructions used to build constants modify the upper 64 bits
17531 of the register, once we have that information we may be able
17532 to handle some of them more efficiently. */
17533 if (can_create_pseudo_p ()
17534 && register_operand (op0, mode)
17535 && (CONSTANT_P (op1)
17536 || (GET_CODE (op1) == SUBREG
17537 && CONSTANT_P (SUBREG_REG (op1))))
17538 && !standard_sse_constant_p (op1))
17539 op1 = validize_mem (force_const_mem (mode, op1));
17541 /* We need to check memory alignment for SSE mode since attribute
17542 can make operands unaligned. */
17543 if (can_create_pseudo_p ()
17544 && SSE_REG_MODE_P (mode)
17545 && ((MEM_P (op0) && (MEM_ALIGN (op0) < align))
17546 || (MEM_P (op1) && (MEM_ALIGN (op1) < align))))
17550 /* ix86_expand_vector_move_misalign() does not like constants ... */
17551 if (CONSTANT_P (op1)
17552 || (GET_CODE (op1) == SUBREG
17553 && CONSTANT_P (SUBREG_REG (op1))))
17554 op1 = validize_mem (force_const_mem (mode, op1));
17556 /* ... nor both arguments in memory. */
17557 if (!register_operand (op0, mode)
17558 && !register_operand (op1, mode))
17559 op1 = force_reg (mode, op1);
17561 tmp[0] = op0; tmp[1] = op1;
17562 ix86_expand_vector_move_misalign (mode, tmp);
17566 /* Make operand1 a register if it isn't already. */
17567 if (can_create_pseudo_p ()
17568 && !register_operand (op0, mode)
17569 && !register_operand (op1, mode))
17571 emit_move_insn (op0, force_reg (GET_MODE (op0), op1));
17575 emit_insn (gen_rtx_SET (VOIDmode, op0, op1));
17578 /* Split 32-byte AVX unaligned load and store if needed. */
17581 ix86_avx256_split_vector_move_misalign (rtx op0, rtx op1)
17584 rtx (*extract) (rtx, rtx, rtx);
17585 rtx (*load_unaligned) (rtx, rtx);
17586 rtx (*store_unaligned) (rtx, rtx);
17589 switch (GET_MODE (op0))
17592 gcc_unreachable ();
17594 extract = gen_avx_vextractf128v32qi;
17595 load_unaligned = gen_avx_loaddquv32qi;
17596 store_unaligned = gen_avx_storedquv32qi;
17600 extract = gen_avx_vextractf128v8sf;
17601 load_unaligned = gen_avx_loadups256;
17602 store_unaligned = gen_avx_storeups256;
17606 extract = gen_avx_vextractf128v4df;
17607 load_unaligned = gen_avx_loadupd256;
17608 store_unaligned = gen_avx_storeupd256;
17615 if (TARGET_AVX256_SPLIT_UNALIGNED_LOAD
17616 && optimize_insn_for_speed_p ())
17618 rtx r = gen_reg_rtx (mode);
17619 m = adjust_address (op1, mode, 0);
17620 emit_move_insn (r, m);
17621 m = adjust_address (op1, mode, 16);
17622 r = gen_rtx_VEC_CONCAT (GET_MODE (op0), r, m);
17623 emit_move_insn (op0, r);
17625 /* Normal *mov<mode>_internal pattern will handle
17626 unaligned loads just fine if misaligned_operand
17627 is true, and without the UNSPEC it can be combined
17628 with arithmetic instructions. */
17629 else if (misaligned_operand (op1, GET_MODE (op1)))
17630 emit_insn (gen_rtx_SET (VOIDmode, op0, op1));
17632 emit_insn (load_unaligned (op0, op1));
17634 else if (MEM_P (op0))
17636 if (TARGET_AVX256_SPLIT_UNALIGNED_STORE
17637 && optimize_insn_for_speed_p ())
17639 m = adjust_address (op0, mode, 0);
17640 emit_insn (extract (m, op1, const0_rtx));
17641 m = adjust_address (op0, mode, 16);
17642 emit_insn (extract (m, op1, const1_rtx));
17645 emit_insn (store_unaligned (op0, op1));
17648 gcc_unreachable ();
17651 /* Implement the movmisalign patterns for SSE. Non-SSE modes go
17652 straight to ix86_expand_vector_move. */
17653 /* Code generation for scalar reg-reg moves of single and double precision data:
17654 if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
17658 if (x86_sse_partial_reg_dependency == true)
17663 Code generation for scalar loads of double precision data:
17664 if (x86_sse_split_regs == true)
17665 movlpd mem, reg (gas syntax)
17669 Code generation for unaligned packed loads of single precision data
17670 (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
17671 if (x86_sse_unaligned_move_optimal)
17674 if (x86_sse_partial_reg_dependency == true)
17686 Code generation for unaligned packed loads of double precision data
17687 (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
17688 if (x86_sse_unaligned_move_optimal)
17691 if (x86_sse_split_regs == true)
17704 ix86_expand_vector_move_misalign (machine_mode mode, rtx operands[])
17706 rtx op0, op1, orig_op0 = NULL_RTX, m;
17707 rtx (*load_unaligned) (rtx, rtx);
17708 rtx (*store_unaligned) (rtx, rtx);
17713 if (GET_MODE_SIZE (mode) == 64)
17715 switch (GET_MODE_CLASS (mode))
17717 case MODE_VECTOR_INT:
17719 if (GET_MODE (op0) != V16SImode)
17724 op0 = gen_reg_rtx (V16SImode);
17727 op0 = gen_lowpart (V16SImode, op0);
17729 op1 = gen_lowpart (V16SImode, op1);
17732 case MODE_VECTOR_FLOAT:
17733 switch (GET_MODE (op0))
17736 gcc_unreachable ();
17738 load_unaligned = gen_avx512f_loaddquv16si;
17739 store_unaligned = gen_avx512f_storedquv16si;
17742 load_unaligned = gen_avx512f_loadups512;
17743 store_unaligned = gen_avx512f_storeups512;
17746 load_unaligned = gen_avx512f_loadupd512;
17747 store_unaligned = gen_avx512f_storeupd512;
17752 emit_insn (load_unaligned (op0, op1));
17753 else if (MEM_P (op0))
17754 emit_insn (store_unaligned (op0, op1));
17756 gcc_unreachable ();
17758 emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
17762 gcc_unreachable ();
17769 && GET_MODE_SIZE (mode) == 32)
17771 switch (GET_MODE_CLASS (mode))
17773 case MODE_VECTOR_INT:
17775 if (GET_MODE (op0) != V32QImode)
17780 op0 = gen_reg_rtx (V32QImode);
17783 op0 = gen_lowpart (V32QImode, op0);
17785 op1 = gen_lowpart (V32QImode, op1);
17788 case MODE_VECTOR_FLOAT:
17789 ix86_avx256_split_vector_move_misalign (op0, op1);
17791 emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
17795 gcc_unreachable ();
17803 /* Normal *mov<mode>_internal pattern will handle
17804 unaligned loads just fine if misaligned_operand
17805 is true, and without the UNSPEC it can be combined
17806 with arithmetic instructions. */
17808 && (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
17809 || GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT)
17810 && misaligned_operand (op1, GET_MODE (op1)))
17811 emit_insn (gen_rtx_SET (VOIDmode, op0, op1));
17812 /* ??? If we have typed data, then it would appear that using
17813 movdqu is the only way to get unaligned data loaded with
17815 else if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
17817 if (GET_MODE (op0) != V16QImode)
17820 op0 = gen_reg_rtx (V16QImode);
17822 op1 = gen_lowpart (V16QImode, op1);
17823 /* We will eventually emit movups based on insn attributes. */
17824 emit_insn (gen_sse2_loaddquv16qi (op0, op1));
17826 emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
17828 else if (TARGET_SSE2 && mode == V2DFmode)
17833 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
17834 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17835 || optimize_insn_for_size_p ())
17837 /* We will eventually emit movups based on insn attributes. */
17838 emit_insn (gen_sse2_loadupd (op0, op1));
17842 /* When SSE registers are split into halves, we can avoid
17843 writing to the top half twice. */
17844 if (TARGET_SSE_SPLIT_REGS)
17846 emit_clobber (op0);
17851 /* ??? Not sure about the best option for the Intel chips.
17852 The following would seem to satisfy; the register is
17853 entirely cleared, breaking the dependency chain. We
17854 then store to the upper half, with a dependency depth
17855 of one. A rumor has it that Intel recommends two movsd
17856 followed by an unpacklpd, but this is unconfirmed. And
17857 given that the dependency depth of the unpacklpd would
17858 still be one, I'm not sure why this would be better. */
17859 zero = CONST0_RTX (V2DFmode);
17862 m = adjust_address (op1, DFmode, 0);
17863 emit_insn (gen_sse2_loadlpd (op0, zero, m));
17864 m = adjust_address (op1, DFmode, 8);
17865 emit_insn (gen_sse2_loadhpd (op0, op0, m));
17872 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
17873 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17874 || optimize_insn_for_size_p ())
17876 if (GET_MODE (op0) != V4SFmode)
17879 op0 = gen_reg_rtx (V4SFmode);
17881 op1 = gen_lowpart (V4SFmode, op1);
17882 emit_insn (gen_sse_loadups (op0, op1));
17884 emit_move_insn (orig_op0,
17885 gen_lowpart (GET_MODE (orig_op0), op0));
17889 if (mode != V4SFmode)
17890 t = gen_reg_rtx (V4SFmode);
17894 if (TARGET_SSE_PARTIAL_REG_DEPENDENCY)
17895 emit_move_insn (t, CONST0_RTX (V4SFmode));
17899 m = adjust_address (op1, V2SFmode, 0);
17900 emit_insn (gen_sse_loadlps (t, t, m));
17901 m = adjust_address (op1, V2SFmode, 8);
17902 emit_insn (gen_sse_loadhps (t, t, m));
17903 if (mode != V4SFmode)
17904 emit_move_insn (op0, gen_lowpart (mode, t));
17907 else if (MEM_P (op0))
17909 if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
17911 op0 = gen_lowpart (V16QImode, op0);
17912 op1 = gen_lowpart (V16QImode, op1);
17913 /* We will eventually emit movups based on insn attributes. */
17914 emit_insn (gen_sse2_storedquv16qi (op0, op1));
17916 else if (TARGET_SSE2 && mode == V2DFmode)
17919 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
17920 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17921 || optimize_insn_for_size_p ())
17922 /* We will eventually emit movups based on insn attributes. */
17923 emit_insn (gen_sse2_storeupd (op0, op1));
17926 m = adjust_address (op0, DFmode, 0);
17927 emit_insn (gen_sse2_storelpd (m, op1));
17928 m = adjust_address (op0, DFmode, 8);
17929 emit_insn (gen_sse2_storehpd (m, op1));
17934 if (mode != V4SFmode)
17935 op1 = gen_lowpart (V4SFmode, op1);
17938 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
17939 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17940 || optimize_insn_for_size_p ())
17942 op0 = gen_lowpart (V4SFmode, op0);
17943 emit_insn (gen_sse_storeups (op0, op1));
17947 m = adjust_address (op0, V2SFmode, 0);
17948 emit_insn (gen_sse_storelps (m, op1));
17949 m = adjust_address (op0, V2SFmode, 8);
17950 emit_insn (gen_sse_storehps (m, op1));
17955 gcc_unreachable ();
17958 /* Helper function of ix86_fixup_binary_operands to canonicalize
17959 operand order. Returns true if the operands should be swapped. */
17962 ix86_swap_binary_operands_p (enum rtx_code code, machine_mode mode,
17965 rtx dst = operands[0];
17966 rtx src1 = operands[1];
17967 rtx src2 = operands[2];
17969 /* If the operation is not commutative, we can't do anything. */
17970 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH)
17973 /* Highest priority is that src1 should match dst. */
17974 if (rtx_equal_p (dst, src1))
17976 if (rtx_equal_p (dst, src2))
17979 /* Next highest priority is that immediate constants come second. */
17980 if (immediate_operand (src2, mode))
17982 if (immediate_operand (src1, mode))
17985 /* Lowest priority is that memory references should come second. */
17995 /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
17996 destination to use for the operation. If different from the true
17997 destination in operands[0], a copy operation will be required. */
18000 ix86_fixup_binary_operands (enum rtx_code code, machine_mode mode,
18003 rtx dst = operands[0];
18004 rtx src1 = operands[1];
18005 rtx src2 = operands[2];
18007 /* Canonicalize operand order. */
18008 if (ix86_swap_binary_operands_p (code, mode, operands))
18010 /* It is invalid to swap operands of different modes. */
18011 gcc_assert (GET_MODE (src1) == GET_MODE (src2));
18013 std::swap (src1, src2);
18016 /* Both source operands cannot be in memory. */
18017 if (MEM_P (src1) && MEM_P (src2))
18019 /* Optimization: Only read from memory once. */
18020 if (rtx_equal_p (src1, src2))
18022 src2 = force_reg (mode, src2);
18025 else if (rtx_equal_p (dst, src1))
18026 src2 = force_reg (mode, src2);
18028 src1 = force_reg (mode, src1);
18031 /* If the destination is memory, and we do not have matching source
18032 operands, do things in registers. */
18033 if (MEM_P (dst) && !rtx_equal_p (dst, src1))
18034 dst = gen_reg_rtx (mode);
18036 /* Source 1 cannot be a constant. */
18037 if (CONSTANT_P (src1))
18038 src1 = force_reg (mode, src1);
18040 /* Source 1 cannot be a non-matching memory. */
18041 if (MEM_P (src1) && !rtx_equal_p (dst, src1))
18042 src1 = force_reg (mode, src1);
18044 /* Improve address combine. */
18046 && GET_MODE_CLASS (mode) == MODE_INT
18048 src2 = force_reg (mode, src2);
18050 operands[1] = src1;
18051 operands[2] = src2;
18055 /* Similarly, but assume that the destination has already been
18056 set up properly. */
18059 ix86_fixup_binary_operands_no_copy (enum rtx_code code,
18060 machine_mode mode, rtx operands[])
18062 rtx dst = ix86_fixup_binary_operands (code, mode, operands);
18063 gcc_assert (dst == operands[0]);
18066 /* Attempt to expand a binary operator. Make the expansion closer to the
18067 actual machine, then just general_operand, which will allow 3 separate
18068 memory references (one output, two input) in a single insn. */
18071 ix86_expand_binary_operator (enum rtx_code code, machine_mode mode,
18074 rtx src1, src2, dst, op, clob;
18076 dst = ix86_fixup_binary_operands (code, mode, operands);
18077 src1 = operands[1];
18078 src2 = operands[2];
18080 /* Emit the instruction. */
18082 op = gen_rtx_SET (VOIDmode, dst, gen_rtx_fmt_ee (code, mode, src1, src2));
18083 if (reload_in_progress)
18085 /* Reload doesn't know about the flags register, and doesn't know that
18086 it doesn't want to clobber it. We can only do this with PLUS. */
18087 gcc_assert (code == PLUS);
18090 else if (reload_completed
18092 && !rtx_equal_p (dst, src1))
18094 /* This is going to be an LEA; avoid splitting it later. */
18099 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
18100 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
18103 /* Fix up the destination if needed. */
18104 if (dst != operands[0])
18105 emit_move_insn (operands[0], dst);
18108 /* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
18109 the given OPERANDS. */
18112 ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
18115 rtx op1 = NULL_RTX, op2 = NULL_RTX;
18116 if (GET_CODE (operands[1]) == SUBREG)
18121 else if (GET_CODE (operands[2]) == SUBREG)
18126 /* Optimize (__m128i) d | (__m128i) e and similar code
18127 when d and e are float vectors into float vector logical
18128 insn. In C/C++ without using intrinsics there is no other way
18129 to express vector logical operation on float vectors than
18130 to cast them temporarily to integer vectors. */
18132 && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
18133 && ((GET_CODE (op2) == SUBREG || GET_CODE (op2) == CONST_VECTOR))
18134 && GET_MODE_CLASS (GET_MODE (SUBREG_REG (op1))) == MODE_VECTOR_FLOAT
18135 && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op1))) == GET_MODE_SIZE (mode)
18136 && SUBREG_BYTE (op1) == 0
18137 && (GET_CODE (op2) == CONST_VECTOR
18138 || (GET_MODE (SUBREG_REG (op1)) == GET_MODE (SUBREG_REG (op2))
18139 && SUBREG_BYTE (op2) == 0))
18140 && can_create_pseudo_p ())
18143 switch (GET_MODE (SUBREG_REG (op1)))
18151 dst = gen_reg_rtx (GET_MODE (SUBREG_REG (op1)));
18152 if (GET_CODE (op2) == CONST_VECTOR)
18154 op2 = gen_lowpart (GET_MODE (dst), op2);
18155 op2 = force_reg (GET_MODE (dst), op2);
18160 op2 = SUBREG_REG (operands[2]);
18161 if (!nonimmediate_operand (op2, GET_MODE (dst)))
18162 op2 = force_reg (GET_MODE (dst), op2);
18164 op1 = SUBREG_REG (op1);
18165 if (!nonimmediate_operand (op1, GET_MODE (dst)))
18166 op1 = force_reg (GET_MODE (dst), op1);
18167 emit_insn (gen_rtx_SET (VOIDmode, dst,
18168 gen_rtx_fmt_ee (code, GET_MODE (dst),
18170 emit_move_insn (operands[0], gen_lowpart (mode, dst));
18176 if (!nonimmediate_operand (operands[1], mode))
18177 operands[1] = force_reg (mode, operands[1]);
18178 if (!nonimmediate_operand (operands[2], mode))
18179 operands[2] = force_reg (mode, operands[2]);
18180 ix86_fixup_binary_operands_no_copy (code, mode, operands);
18181 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
18182 gen_rtx_fmt_ee (code, mode, operands[1],
18186 /* Return TRUE or FALSE depending on whether the binary operator meets the
18187 appropriate constraints. */
18190 ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
18193 rtx dst = operands[0];
18194 rtx src1 = operands[1];
18195 rtx src2 = operands[2];
18197 /* Both source operands cannot be in memory. */
18198 if (MEM_P (src1) && MEM_P (src2))
18201 /* Canonicalize operand order for commutative operators. */
18202 if (ix86_swap_binary_operands_p (code, mode, operands))
18203 std::swap (src1, src2);
18205 /* If the destination is memory, we must have a matching source operand. */
18206 if (MEM_P (dst) && !rtx_equal_p (dst, src1))
18209 /* Source 1 cannot be a constant. */
18210 if (CONSTANT_P (src1))
18213 /* Source 1 cannot be a non-matching memory. */
18214 if (MEM_P (src1) && !rtx_equal_p (dst, src1))
18215 /* Support "andhi/andsi/anddi" as a zero-extending move. */
18216 return (code == AND
18219 || (TARGET_64BIT && mode == DImode))
18220 && satisfies_constraint_L (src2));
18225 /* Attempt to expand a unary operator. Make the expansion closer to the
18226 actual machine, then just general_operand, which will allow 2 separate
18227 memory references (one output, one input) in a single insn. */
18230 ix86_expand_unary_operator (enum rtx_code code, machine_mode mode,
18233 bool matching_memory = false;
18234 rtx src, dst, op, clob;
18239 /* If the destination is memory, and we do not have matching source
18240 operands, do things in registers. */
18243 if (rtx_equal_p (dst, src))
18244 matching_memory = true;
18246 dst = gen_reg_rtx (mode);
18249 /* When source operand is memory, destination must match. */
18250 if (MEM_P (src) && !matching_memory)
18251 src = force_reg (mode, src);
18253 /* Emit the instruction. */
18255 op = gen_rtx_SET (VOIDmode, dst, gen_rtx_fmt_e (code, mode, src));
18256 if (reload_in_progress || code == NOT)
18258 /* Reload doesn't know about the flags register, and doesn't know that
18259 it doesn't want to clobber it. */
18260 gcc_assert (code == NOT);
18265 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
18266 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
18269 /* Fix up the destination if needed. */
18270 if (dst != operands[0])
18271 emit_move_insn (operands[0], dst);
18274 /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
18275 divisor are within the range [0-255]. */
18278 ix86_split_idivmod (machine_mode mode, rtx operands[],
18281 rtx_code_label *end_label, *qimode_label;
18282 rtx insn, div, mod;
18283 rtx scratch, tmp0, tmp1, tmp2;
18284 rtx (*gen_divmod4_1) (rtx, rtx, rtx, rtx);
18285 rtx (*gen_zero_extend) (rtx, rtx);
18286 rtx (*gen_test_ccno_1) (rtx, rtx);
18291 gen_divmod4_1 = signed_p ? gen_divmodsi4_1 : gen_udivmodsi4_1;
18292 gen_test_ccno_1 = gen_testsi_ccno_1;
18293 gen_zero_extend = gen_zero_extendqisi2;
18296 gen_divmod4_1 = signed_p ? gen_divmoddi4_1 : gen_udivmoddi4_1;
18297 gen_test_ccno_1 = gen_testdi_ccno_1;
18298 gen_zero_extend = gen_zero_extendqidi2;
18301 gcc_unreachable ();
18304 end_label = gen_label_rtx ();
18305 qimode_label = gen_label_rtx ();
18307 scratch = gen_reg_rtx (mode);
18309 /* Use 8bit unsigned divimod if dividend and divisor are within
18310 the range [0-255]. */
18311 emit_move_insn (scratch, operands[2]);
18312 scratch = expand_simple_binop (mode, IOR, scratch, operands[3],
18313 scratch, 1, OPTAB_DIRECT);
18314 emit_insn (gen_test_ccno_1 (scratch, GEN_INT (-0x100)));
18315 tmp0 = gen_rtx_REG (CCNOmode, FLAGS_REG);
18316 tmp0 = gen_rtx_EQ (VOIDmode, tmp0, const0_rtx);
18317 tmp0 = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp0,
18318 gen_rtx_LABEL_REF (VOIDmode, qimode_label),
18320 insn = emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp0));
18321 predict_jump (REG_BR_PROB_BASE * 50 / 100);
18322 JUMP_LABEL (insn) = qimode_label;
18324 /* Generate original signed/unsigned divimod. */
18325 div = gen_divmod4_1 (operands[0], operands[1],
18326 operands[2], operands[3]);
18329 /* Branch to the end. */
18330 emit_jump_insn (gen_jump (end_label));
18333 /* Generate 8bit unsigned divide. */
18334 emit_label (qimode_label);
18335 /* Don't use operands[0] for result of 8bit divide since not all
18336 registers support QImode ZERO_EXTRACT. */
18337 tmp0 = simplify_gen_subreg (HImode, scratch, mode, 0);
18338 tmp1 = simplify_gen_subreg (HImode, operands[2], mode, 0);
18339 tmp2 = simplify_gen_subreg (QImode, operands[3], mode, 0);
18340 emit_insn (gen_udivmodhiqi3 (tmp0, tmp1, tmp2));
18344 div = gen_rtx_DIV (SImode, operands[2], operands[3]);
18345 mod = gen_rtx_MOD (SImode, operands[2], operands[3]);
18349 div = gen_rtx_UDIV (SImode, operands[2], operands[3]);
18350 mod = gen_rtx_UMOD (SImode, operands[2], operands[3]);
18353 /* Extract remainder from AH. */
18354 tmp1 = gen_rtx_ZERO_EXTRACT (mode, tmp0, GEN_INT (8), GEN_INT (8));
18355 if (REG_P (operands[1]))
18356 insn = emit_move_insn (operands[1], tmp1);
18359 /* Need a new scratch register since the old one has result
18361 scratch = gen_reg_rtx (mode);
18362 emit_move_insn (scratch, tmp1);
18363 insn = emit_move_insn (operands[1], scratch);
18365 set_unique_reg_note (insn, REG_EQUAL, mod);
18367 /* Zero extend quotient from AL. */
18368 tmp1 = gen_lowpart (QImode, tmp0);
18369 insn = emit_insn (gen_zero_extend (operands[0], tmp1));
18370 set_unique_reg_note (insn, REG_EQUAL, div);
18372 emit_label (end_label);
18375 #define LEA_MAX_STALL (3)
18376 #define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)
18378 /* Increase given DISTANCE in half-cycles according to
18379 dependencies between PREV and NEXT instructions.
18380 Add 1 half-cycle if there is no dependency and
18381 go to next cycle if there is some dependecy. */
18383 static unsigned int
18384 increase_distance (rtx_insn *prev, rtx_insn *next, unsigned int distance)
18388 if (!prev || !next)
18389 return distance + (distance & 1) + 2;
18391 if (!DF_INSN_USES (next) || !DF_INSN_DEFS (prev))
18392 return distance + 1;
18394 FOR_EACH_INSN_USE (use, next)
18395 FOR_EACH_INSN_DEF (def, prev)
18396 if (!DF_REF_IS_ARTIFICIAL (def)
18397 && DF_REF_REGNO (use) == DF_REF_REGNO (def))
18398 return distance + (distance & 1) + 2;
18400 return distance + 1;
18403 /* Function checks if instruction INSN defines register number
18404 REGNO1 or REGNO2. */
18407 insn_defines_reg (unsigned int regno1, unsigned int regno2,
18412 FOR_EACH_INSN_DEF (def, insn)
18413 if (DF_REF_REG_DEF_P (def)
18414 && !DF_REF_IS_ARTIFICIAL (def)
18415 && (regno1 == DF_REF_REGNO (def)
18416 || regno2 == DF_REF_REGNO (def)))
18422 /* Function checks if instruction INSN uses register number
18423 REGNO as a part of address expression. */
18426 insn_uses_reg_mem (unsigned int regno, rtx insn)
18430 FOR_EACH_INSN_USE (use, insn)
18431 if (DF_REF_REG_MEM_P (use) && regno == DF_REF_REGNO (use))
18437 /* Search backward for non-agu definition of register number REGNO1
18438 or register number REGNO2 in basic block starting from instruction
18439 START up to head of basic block or instruction INSN.
18441 Function puts true value into *FOUND var if definition was found
18442 and false otherwise.
18444 Distance in half-cycles between START and found instruction or head
18445 of BB is added to DISTANCE and returned. */
18448 distance_non_agu_define_in_bb (unsigned int regno1, unsigned int regno2,
18449 rtx_insn *insn, int distance,
18450 rtx_insn *start, bool *found)
18452 basic_block bb = start ? BLOCK_FOR_INSN (start) : NULL;
18453 rtx_insn *prev = start;
18454 rtx_insn *next = NULL;
18460 && distance < LEA_SEARCH_THRESHOLD)
18462 if (NONDEBUG_INSN_P (prev) && NONJUMP_INSN_P (prev))
18464 distance = increase_distance (prev, next, distance);
18465 if (insn_defines_reg (regno1, regno2, prev))
18467 if (recog_memoized (prev) < 0
18468 || get_attr_type (prev) != TYPE_LEA)
18477 if (prev == BB_HEAD (bb))
18480 prev = PREV_INSN (prev);
18486 /* Search backward for non-agu definition of register number REGNO1
18487 or register number REGNO2 in INSN's basic block until
18488 1. Pass LEA_SEARCH_THRESHOLD instructions, or
18489 2. Reach neighbour BBs boundary, or
18490 3. Reach agu definition.
18491 Returns the distance between the non-agu definition point and INSN.
18492 If no definition point, returns -1. */
18495 distance_non_agu_define (unsigned int regno1, unsigned int regno2,
18498 basic_block bb = BLOCK_FOR_INSN (insn);
18500 bool found = false;
18502 if (insn != BB_HEAD (bb))
18503 distance = distance_non_agu_define_in_bb (regno1, regno2, insn,
18504 distance, PREV_INSN (insn),
18507 if (!found && distance < LEA_SEARCH_THRESHOLD)
18511 bool simple_loop = false;
18513 FOR_EACH_EDGE (e, ei, bb->preds)
18516 simple_loop = true;
18521 distance = distance_non_agu_define_in_bb (regno1, regno2,
18523 BB_END (bb), &found);
18526 int shortest_dist = -1;
18527 bool found_in_bb = false;
18529 FOR_EACH_EDGE (e, ei, bb->preds)
18532 = distance_non_agu_define_in_bb (regno1, regno2,
18538 if (shortest_dist < 0)
18539 shortest_dist = bb_dist;
18540 else if (bb_dist > 0)
18541 shortest_dist = MIN (bb_dist, shortest_dist);
18547 distance = shortest_dist;
18551 /* get_attr_type may modify recog data. We want to make sure
18552 that recog data is valid for instruction INSN, on which
18553 distance_non_agu_define is called. INSN is unchanged here. */
18554 extract_insn_cached (insn);
18559 return distance >> 1;
18562 /* Return the distance in half-cycles between INSN and the next
18563 insn that uses register number REGNO in memory address added
18564 to DISTANCE. Return -1 if REGNO0 is set.
18566 Put true value into *FOUND if register usage was found and
18568 Put true value into *REDEFINED if register redefinition was
18569 found and false otherwise. */
18572 distance_agu_use_in_bb (unsigned int regno,
18573 rtx_insn *insn, int distance, rtx_insn *start,
18574 bool *found, bool *redefined)
18576 basic_block bb = NULL;
18577 rtx_insn *next = start;
18578 rtx_insn *prev = NULL;
18581 *redefined = false;
18583 if (start != NULL_RTX)
18585 bb = BLOCK_FOR_INSN (start);
18586 if (start != BB_HEAD (bb))
18587 /* If insn and start belong to the same bb, set prev to insn,
18588 so the call to increase_distance will increase the distance
18589 between insns by 1. */
18595 && distance < LEA_SEARCH_THRESHOLD)
18597 if (NONDEBUG_INSN_P (next) && NONJUMP_INSN_P (next))
18599 distance = increase_distance(prev, next, distance);
18600 if (insn_uses_reg_mem (regno, next))
18602 /* Return DISTANCE if OP0 is used in memory
18603 address in NEXT. */
18608 if (insn_defines_reg (regno, INVALID_REGNUM, next))
18610 /* Return -1 if OP0 is set in NEXT. */
18618 if (next == BB_END (bb))
18621 next = NEXT_INSN (next);
18627 /* Return the distance between INSN and the next insn that uses
18628 register number REGNO0 in memory address. Return -1 if no such
18629 a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */
18632 distance_agu_use (unsigned int regno0, rtx_insn *insn)
18634 basic_block bb = BLOCK_FOR_INSN (insn);
18636 bool found = false;
18637 bool redefined = false;
18639 if (insn != BB_END (bb))
18640 distance = distance_agu_use_in_bb (regno0, insn, distance,
18642 &found, &redefined);
18644 if (!found && !redefined && distance < LEA_SEARCH_THRESHOLD)
18648 bool simple_loop = false;
18650 FOR_EACH_EDGE (e, ei, bb->succs)
18653 simple_loop = true;
18658 distance = distance_agu_use_in_bb (regno0, insn,
18659 distance, BB_HEAD (bb),
18660 &found, &redefined);
18663 int shortest_dist = -1;
18664 bool found_in_bb = false;
18665 bool redefined_in_bb = false;
18667 FOR_EACH_EDGE (e, ei, bb->succs)
18670 = distance_agu_use_in_bb (regno0, insn,
18671 distance, BB_HEAD (e->dest),
18672 &found_in_bb, &redefined_in_bb);
18675 if (shortest_dist < 0)
18676 shortest_dist = bb_dist;
18677 else if (bb_dist > 0)
18678 shortest_dist = MIN (bb_dist, shortest_dist);
18684 distance = shortest_dist;
18688 if (!found || redefined)
18691 return distance >> 1;
18694 /* Define this macro to tune LEA priority vs ADD, it take effect when
18695 there is a dilemma of choicing LEA or ADD
18696 Negative value: ADD is more preferred than LEA
18698 Positive value: LEA is more preferred than ADD*/
18699 #define IX86_LEA_PRIORITY 0
18701 /* Return true if usage of lea INSN has performance advantage
18702 over a sequence of instructions. Instructions sequence has
18703 SPLIT_COST cycles higher latency than lea latency. */
18706 ix86_lea_outperforms (rtx_insn *insn, unsigned int regno0, unsigned int regno1,
18707 unsigned int regno2, int split_cost, bool has_scale)
18709 int dist_define, dist_use;
18711 /* For Silvermont if using a 2-source or 3-source LEA for
18712 non-destructive destination purposes, or due to wanting
18713 ability to use SCALE, the use of LEA is justified. */
18714 if (TARGET_SILVERMONT || TARGET_INTEL)
18718 if (split_cost < 1)
18720 if (regno0 == regno1 || regno0 == regno2)
18725 dist_define = distance_non_agu_define (regno1, regno2, insn);
18726 dist_use = distance_agu_use (regno0, insn);
18728 if (dist_define < 0 || dist_define >= LEA_MAX_STALL)
18730 /* If there is no non AGU operand definition, no AGU
18731 operand usage and split cost is 0 then both lea
18732 and non lea variants have same priority. Currently
18733 we prefer lea for 64 bit code and non lea on 32 bit
18735 if (dist_use < 0 && split_cost == 0)
18736 return TARGET_64BIT || IX86_LEA_PRIORITY;
18741 /* With longer definitions distance lea is more preferable.
18742 Here we change it to take into account splitting cost and
18744 dist_define += split_cost + IX86_LEA_PRIORITY;
18746 /* If there is no use in memory addess then we just check
18747 that split cost exceeds AGU stall. */
18749 return dist_define > LEA_MAX_STALL;
18751 /* If this insn has both backward non-agu dependence and forward
18752 agu dependence, the one with short distance takes effect. */
18753 return dist_define >= dist_use;
18756 /* Return true if it is legal to clobber flags by INSN and
18757 false otherwise. */
18760 ix86_ok_to_clobber_flags (rtx_insn *insn)
18762 basic_block bb = BLOCK_FOR_INSN (insn);
18768 if (NONDEBUG_INSN_P (insn))
18770 FOR_EACH_INSN_USE (use, insn)
18771 if (DF_REF_REG_USE_P (use) && DF_REF_REGNO (use) == FLAGS_REG)
18774 if (insn_defines_reg (FLAGS_REG, INVALID_REGNUM, insn))
18778 if (insn == BB_END (bb))
18781 insn = NEXT_INSN (insn);
18784 live = df_get_live_out(bb);
18785 return !REGNO_REG_SET_P (live, FLAGS_REG);
18788 /* Return true if we need to split op0 = op1 + op2 into a sequence of
18789 move and add to avoid AGU stalls. */
18792 ix86_avoid_lea_for_add (rtx_insn *insn, rtx operands[])
18794 unsigned int regno0, regno1, regno2;
18796 /* Check if we need to optimize. */
18797 if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
18800 /* Check it is correct to split here. */
18801 if (!ix86_ok_to_clobber_flags(insn))
18804 regno0 = true_regnum (operands[0]);
18805 regno1 = true_regnum (operands[1]);
18806 regno2 = true_regnum (operands[2]);
18808 /* We need to split only adds with non destructive
18809 destination operand. */
18810 if (regno0 == regno1 || regno0 == regno2)
18813 return !ix86_lea_outperforms (insn, regno0, regno1, regno2, 1, false);
18816 /* Return true if we should emit lea instruction instead of mov
18820 ix86_use_lea_for_mov (rtx_insn *insn, rtx operands[])
18822 unsigned int regno0, regno1;
18824 /* Check if we need to optimize. */
18825 if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
18828 /* Use lea for reg to reg moves only. */
18829 if (!REG_P (operands[0]) || !REG_P (operands[1]))
18832 regno0 = true_regnum (operands[0]);
18833 regno1 = true_regnum (operands[1]);
18835 return ix86_lea_outperforms (insn, regno0, regno1, INVALID_REGNUM, 0, false);
18838 /* Return true if we need to split lea into a sequence of
18839 instructions to avoid AGU stalls. */
18842 ix86_avoid_lea_for_addr (rtx_insn *insn, rtx operands[])
18844 unsigned int regno0, regno1, regno2;
18846 struct ix86_address parts;
18849 /* Check we need to optimize. */
18850 if (!TARGET_AVOID_LEA_FOR_ADDR || optimize_function_for_size_p (cfun))
18853 /* The "at least two components" test below might not catch simple
18854 move or zero extension insns if parts.base is non-NULL and parts.disp
18855 is const0_rtx as the only components in the address, e.g. if the
18856 register is %rbp or %r13. As this test is much cheaper and moves or
18857 zero extensions are the common case, do this check first. */
18858 if (REG_P (operands[1])
18859 || (SImode_address_operand (operands[1], VOIDmode)
18860 && REG_P (XEXP (operands[1], 0))))
18863 /* Check if it is OK to split here. */
18864 if (!ix86_ok_to_clobber_flags (insn))
18867 ok = ix86_decompose_address (operands[1], &parts);
18870 /* There should be at least two components in the address. */
18871 if ((parts.base != NULL_RTX) + (parts.index != NULL_RTX)
18872 + (parts.disp != NULL_RTX) + (parts.scale > 1) < 2)
18875 /* We should not split into add if non legitimate pic
18876 operand is used as displacement. */
18877 if (parts.disp && flag_pic && !LEGITIMATE_PIC_OPERAND_P (parts.disp))
18880 regno0 = true_regnum (operands[0]) ;
18881 regno1 = INVALID_REGNUM;
18882 regno2 = INVALID_REGNUM;
18885 regno1 = true_regnum (parts.base);
18887 regno2 = true_regnum (parts.index);
18891 /* Compute how many cycles we will add to execution time
18892 if split lea into a sequence of instructions. */
18893 if (parts.base || parts.index)
18895 /* Have to use mov instruction if non desctructive
18896 destination form is used. */
18897 if (regno1 != regno0 && regno2 != regno0)
18900 /* Have to add index to base if both exist. */
18901 if (parts.base && parts.index)
18904 /* Have to use shift and adds if scale is 2 or greater. */
18905 if (parts.scale > 1)
18907 if (regno0 != regno1)
18909 else if (regno2 == regno0)
18912 split_cost += parts.scale;
18915 /* Have to use add instruction with immediate if
18916 disp is non zero. */
18917 if (parts.disp && parts.disp != const0_rtx)
18920 /* Subtract the price of lea. */
18924 return !ix86_lea_outperforms (insn, regno0, regno1, regno2, split_cost,
18928 /* Emit x86 binary operand CODE in mode MODE, where the first operand
18929 matches destination. RTX includes clobber of FLAGS_REG. */
18932 ix86_emit_binop (enum rtx_code code, machine_mode mode,
18937 op = gen_rtx_SET (VOIDmode, dst, gen_rtx_fmt_ee (code, mode, dst, src));
18938 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
18940 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
18943 /* Return true if regno1 def is nearest to the insn. */
18946 find_nearest_reg_def (rtx_insn *insn, int regno1, int regno2)
18948 rtx_insn *prev = insn;
18949 rtx_insn *start = BB_HEAD (BLOCK_FOR_INSN (insn));
18953 while (prev && prev != start)
18955 if (!INSN_P (prev) || !NONDEBUG_INSN_P (prev))
18957 prev = PREV_INSN (prev);
18960 if (insn_defines_reg (regno1, INVALID_REGNUM, prev))
18962 else if (insn_defines_reg (regno2, INVALID_REGNUM, prev))
18964 prev = PREV_INSN (prev);
18967 /* None of the regs is defined in the bb. */
18971 /* Split lea instructions into a sequence of instructions
18972 which are executed on ALU to avoid AGU stalls.
18973 It is assumed that it is allowed to clobber flags register
18974 at lea position. */
18977 ix86_split_lea_for_addr (rtx_insn *insn, rtx operands[], machine_mode mode)
18979 unsigned int regno0, regno1, regno2;
18980 struct ix86_address parts;
18984 ok = ix86_decompose_address (operands[1], &parts);
18987 target = gen_lowpart (mode, operands[0]);
18989 regno0 = true_regnum (target);
18990 regno1 = INVALID_REGNUM;
18991 regno2 = INVALID_REGNUM;
18995 parts.base = gen_lowpart (mode, parts.base);
18996 regno1 = true_regnum (parts.base);
19001 parts.index = gen_lowpart (mode, parts.index);
19002 regno2 = true_regnum (parts.index);
19006 parts.disp = gen_lowpart (mode, parts.disp);
19008 if (parts.scale > 1)
19010 /* Case r1 = r1 + ... */
19011 if (regno1 == regno0)
19013 /* If we have a case r1 = r1 + C * r2 then we
19014 should use multiplication which is very
19015 expensive. Assume cost model is wrong if we
19016 have such case here. */
19017 gcc_assert (regno2 != regno0);
19019 for (adds = parts.scale; adds > 0; adds--)
19020 ix86_emit_binop (PLUS, mode, target, parts.index);
19024 /* r1 = r2 + r3 * C case. Need to move r3 into r1. */
19025 if (regno0 != regno2)
19026 emit_insn (gen_rtx_SET (VOIDmode, target, parts.index));
19028 /* Use shift for scaling. */
19029 ix86_emit_binop (ASHIFT, mode, target,
19030 GEN_INT (exact_log2 (parts.scale)));
19033 ix86_emit_binop (PLUS, mode, target, parts.base);
19035 if (parts.disp && parts.disp != const0_rtx)
19036 ix86_emit_binop (PLUS, mode, target, parts.disp);
19039 else if (!parts.base && !parts.index)
19041 gcc_assert(parts.disp);
19042 emit_insn (gen_rtx_SET (VOIDmode, target, parts.disp));
19048 if (regno0 != regno2)
19049 emit_insn (gen_rtx_SET (VOIDmode, target, parts.index));
19051 else if (!parts.index)
19053 if (regno0 != regno1)
19054 emit_insn (gen_rtx_SET (VOIDmode, target, parts.base));
19058 if (regno0 == regno1)
19060 else if (regno0 == regno2)
19066 /* Find better operand for SET instruction, depending
19067 on which definition is farther from the insn. */
19068 if (find_nearest_reg_def (insn, regno1, regno2))
19069 tmp = parts.index, tmp1 = parts.base;
19071 tmp = parts.base, tmp1 = parts.index;
19073 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
19075 if (parts.disp && parts.disp != const0_rtx)
19076 ix86_emit_binop (PLUS, mode, target, parts.disp);
19078 ix86_emit_binop (PLUS, mode, target, tmp1);
19082 ix86_emit_binop (PLUS, mode, target, tmp);
19085 if (parts.disp && parts.disp != const0_rtx)
19086 ix86_emit_binop (PLUS, mode, target, parts.disp);
19090 /* Return true if it is ok to optimize an ADD operation to LEA
19091 operation to avoid flag register consumation. For most processors,
19092 ADD is faster than LEA. For the processors like BONNELL, if the
19093 destination register of LEA holds an actual address which will be
19094 used soon, LEA is better and otherwise ADD is better. */
19097 ix86_lea_for_add_ok (rtx_insn *insn, rtx operands[])
19099 unsigned int regno0 = true_regnum (operands[0]);
19100 unsigned int regno1 = true_regnum (operands[1]);
19101 unsigned int regno2 = true_regnum (operands[2]);
19103 /* If a = b + c, (a!=b && a!=c), must use lea form. */
19104 if (regno0 != regno1 && regno0 != regno2)
19107 if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
19110 return ix86_lea_outperforms (insn, regno0, regno1, regno2, 0, false);
19113 /* Return true if destination reg of SET_BODY is shift count of
19117 ix86_dep_by_shift_count_body (const_rtx set_body, const_rtx use_body)
19123 /* Retrieve destination of SET_BODY. */
19124 switch (GET_CODE (set_body))
19127 set_dest = SET_DEST (set_body);
19128 if (!set_dest || !REG_P (set_dest))
19132 for (i = XVECLEN (set_body, 0) - 1; i >= 0; i--)
19133 if (ix86_dep_by_shift_count_body (XVECEXP (set_body, 0, i),
19141 /* Retrieve shift count of USE_BODY. */
19142 switch (GET_CODE (use_body))
19145 shift_rtx = XEXP (use_body, 1);
19148 for (i = XVECLEN (use_body, 0) - 1; i >= 0; i--)
19149 if (ix86_dep_by_shift_count_body (set_body,
19150 XVECEXP (use_body, 0, i)))
19158 && (GET_CODE (shift_rtx) == ASHIFT
19159 || GET_CODE (shift_rtx) == LSHIFTRT
19160 || GET_CODE (shift_rtx) == ASHIFTRT
19161 || GET_CODE (shift_rtx) == ROTATE
19162 || GET_CODE (shift_rtx) == ROTATERT))
19164 rtx shift_count = XEXP (shift_rtx, 1);
19166 /* Return true if shift count is dest of SET_BODY. */
19167 if (REG_P (shift_count))
19169 /* Add check since it can be invoked before register
19170 allocation in pre-reload schedule. */
19171 if (reload_completed
19172 && true_regnum (set_dest) == true_regnum (shift_count))
19174 else if (REGNO(set_dest) == REGNO(shift_count))
19182 /* Return true if destination reg of SET_INSN is shift count of
19186 ix86_dep_by_shift_count (const_rtx set_insn, const_rtx use_insn)
19188 return ix86_dep_by_shift_count_body (PATTERN (set_insn),
19189 PATTERN (use_insn));
19192 /* Return TRUE or FALSE depending on whether the unary operator meets the
19193 appropriate constraints. */
19196 ix86_unary_operator_ok (enum rtx_code,
19200 /* If one of operands is memory, source and destination must match. */
19201 if ((MEM_P (operands[0])
19202 || MEM_P (operands[1]))
19203 && ! rtx_equal_p (operands[0], operands[1]))
19208 /* Return TRUE if the operands to a vec_interleave_{high,low}v2df
19209 are ok, keeping in mind the possible movddup alternative. */
19212 ix86_vec_interleave_v2df_operator_ok (rtx operands[3], bool high)
19214 if (MEM_P (operands[0]))
19215 return rtx_equal_p (operands[0], operands[1 + high]);
19216 if (MEM_P (operands[1]) && MEM_P (operands[2]))
19217 return TARGET_SSE3 && rtx_equal_p (operands[1], operands[2]);
19221 /* Post-reload splitter for converting an SF or DFmode value in an
19222 SSE register into an unsigned SImode. */
19225 ix86_split_convert_uns_si_sse (rtx operands[])
19227 machine_mode vecmode;
19228 rtx value, large, zero_or_two31, input, two31, x;
19230 large = operands[1];
19231 zero_or_two31 = operands[2];
19232 input = operands[3];
19233 two31 = operands[4];
19234 vecmode = GET_MODE (large);
19235 value = gen_rtx_REG (vecmode, REGNO (operands[0]));
19237 /* Load up the value into the low element. We must ensure that the other
19238 elements are valid floats -- zero is the easiest such value. */
19241 if (vecmode == V4SFmode)
19242 emit_insn (gen_vec_setv4sf_0 (value, CONST0_RTX (V4SFmode), input));
19244 emit_insn (gen_sse2_loadlpd (value, CONST0_RTX (V2DFmode), input));
19248 input = gen_rtx_REG (vecmode, REGNO (input));
19249 emit_move_insn (value, CONST0_RTX (vecmode));
19250 if (vecmode == V4SFmode)
19251 emit_insn (gen_sse_movss (value, value, input));
19253 emit_insn (gen_sse2_movsd (value, value, input));
19256 emit_move_insn (large, two31);
19257 emit_move_insn (zero_or_two31, MEM_P (two31) ? large : two31);
19259 x = gen_rtx_fmt_ee (LE, vecmode, large, value);
19260 emit_insn (gen_rtx_SET (VOIDmode, large, x));
19262 x = gen_rtx_AND (vecmode, zero_or_two31, large);
19263 emit_insn (gen_rtx_SET (VOIDmode, zero_or_two31, x));
19265 x = gen_rtx_MINUS (vecmode, value, zero_or_two31);
19266 emit_insn (gen_rtx_SET (VOIDmode, value, x));
19268 large = gen_rtx_REG (V4SImode, REGNO (large));
19269 emit_insn (gen_ashlv4si3 (large, large, GEN_INT (31)));
19271 x = gen_rtx_REG (V4SImode, REGNO (value));
19272 if (vecmode == V4SFmode)
19273 emit_insn (gen_fix_truncv4sfv4si2 (x, value));
19275 emit_insn (gen_sse2_cvttpd2dq (x, value));
19278 emit_insn (gen_xorv4si3 (value, value, large));
19281 /* Convert an unsigned DImode value into a DFmode, using only SSE.
19282 Expects the 64-bit DImode to be supplied in a pair of integral
19283 registers. Requires SSE2; will use SSE3 if available. For x86_32,
19284 -mfpmath=sse, !optimize_size only. */
19287 ix86_expand_convert_uns_didf_sse (rtx target, rtx input)
19289 REAL_VALUE_TYPE bias_lo_rvt, bias_hi_rvt;
19290 rtx int_xmm, fp_xmm;
19291 rtx biases, exponents;
19294 int_xmm = gen_reg_rtx (V4SImode);
19295 if (TARGET_INTER_UNIT_MOVES_TO_VEC)
19296 emit_insn (gen_movdi_to_sse (int_xmm, input));
19297 else if (TARGET_SSE_SPLIT_REGS)
19299 emit_clobber (int_xmm);
19300 emit_move_insn (gen_lowpart (DImode, int_xmm), input);
19304 x = gen_reg_rtx (V2DImode);
19305 ix86_expand_vector_init_one_nonzero (false, V2DImode, x, input, 0);
19306 emit_move_insn (int_xmm, gen_lowpart (V4SImode, x));
19309 x = gen_rtx_CONST_VECTOR (V4SImode,
19310 gen_rtvec (4, GEN_INT (0x43300000UL),
19311 GEN_INT (0x45300000UL),
19312 const0_rtx, const0_rtx));
19313 exponents = validize_mem (force_const_mem (V4SImode, x));
19315 /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
19316 emit_insn (gen_vec_interleave_lowv4si (int_xmm, int_xmm, exponents));
19318 /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
19319 yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
19320 Similarly (0x45300000UL ## fp_value_hi_xmm) yields
19321 (0x1.0p84 + double(fp_value_hi_xmm)).
19322 Note these exponents differ by 32. */
19324 fp_xmm = copy_to_mode_reg (V2DFmode, gen_lowpart (V2DFmode, int_xmm));
19326 /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
19327 in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
19328 real_ldexp (&bias_lo_rvt, &dconst1, 52);
19329 real_ldexp (&bias_hi_rvt, &dconst1, 84);
19330 biases = const_double_from_real_value (bias_lo_rvt, DFmode);
19331 x = const_double_from_real_value (bias_hi_rvt, DFmode);
19332 biases = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec (2, biases, x));
19333 biases = validize_mem (force_const_mem (V2DFmode, biases));
19334 emit_insn (gen_subv2df3 (fp_xmm, fp_xmm, biases));
19336 /* Add the upper and lower DFmode values together. */
19338 emit_insn (gen_sse3_haddv2df3 (fp_xmm, fp_xmm, fp_xmm));
19341 x = copy_to_mode_reg (V2DFmode, fp_xmm);
19342 emit_insn (gen_vec_interleave_highv2df (fp_xmm, fp_xmm, fp_xmm));
19343 emit_insn (gen_addv2df3 (fp_xmm, fp_xmm, x));
19346 ix86_expand_vector_extract (false, target, fp_xmm, 0);
19349 /* Not used, but eases macroization of patterns. */
19351 ix86_expand_convert_uns_sixf_sse (rtx, rtx)
19353 gcc_unreachable ();
19356 /* Convert an unsigned SImode value into a DFmode. Only currently used
19357 for SSE, but applicable anywhere. */
19360 ix86_expand_convert_uns_sidf_sse (rtx target, rtx input)
19362 REAL_VALUE_TYPE TWO31r;
19365 x = expand_simple_binop (SImode, PLUS, input, GEN_INT (-2147483647 - 1),
19366 NULL, 1, OPTAB_DIRECT);
19368 fp = gen_reg_rtx (DFmode);
19369 emit_insn (gen_floatsidf2 (fp, x));
19371 real_ldexp (&TWO31r, &dconst1, 31);
19372 x = const_double_from_real_value (TWO31r, DFmode);
19374 x = expand_simple_binop (DFmode, PLUS, fp, x, target, 0, OPTAB_DIRECT);
19376 emit_move_insn (target, x);
19379 /* Convert a signed DImode value into a DFmode. Only used for SSE in
19380 32-bit mode; otherwise we have a direct convert instruction. */
19383 ix86_expand_convert_sign_didf_sse (rtx target, rtx input)
19385 REAL_VALUE_TYPE TWO32r;
19386 rtx fp_lo, fp_hi, x;
19388 fp_lo = gen_reg_rtx (DFmode);
19389 fp_hi = gen_reg_rtx (DFmode);
19391 emit_insn (gen_floatsidf2 (fp_hi, gen_highpart (SImode, input)));
19393 real_ldexp (&TWO32r, &dconst1, 32);
19394 x = const_double_from_real_value (TWO32r, DFmode);
19395 fp_hi = expand_simple_binop (DFmode, MULT, fp_hi, x, fp_hi, 0, OPTAB_DIRECT);
19397 ix86_expand_convert_uns_sidf_sse (fp_lo, gen_lowpart (SImode, input));
19399 x = expand_simple_binop (DFmode, PLUS, fp_hi, fp_lo, target,
19402 emit_move_insn (target, x);
19405 /* Convert an unsigned SImode value into a SFmode, using only SSE.
19406 For x86_32, -mfpmath=sse, !optimize_size only. */
19408 ix86_expand_convert_uns_sisf_sse (rtx target, rtx input)
19410 REAL_VALUE_TYPE ONE16r;
19411 rtx fp_hi, fp_lo, int_hi, int_lo, x;
19413 real_ldexp (&ONE16r, &dconst1, 16);
19414 x = const_double_from_real_value (ONE16r, SFmode);
19415 int_lo = expand_simple_binop (SImode, AND, input, GEN_INT(0xffff),
19416 NULL, 0, OPTAB_DIRECT);
19417 int_hi = expand_simple_binop (SImode, LSHIFTRT, input, GEN_INT(16),
19418 NULL, 0, OPTAB_DIRECT);
19419 fp_hi = gen_reg_rtx (SFmode);
19420 fp_lo = gen_reg_rtx (SFmode);
19421 emit_insn (gen_floatsisf2 (fp_hi, int_hi));
19422 emit_insn (gen_floatsisf2 (fp_lo, int_lo));
19423 fp_hi = expand_simple_binop (SFmode, MULT, fp_hi, x, fp_hi,
19425 fp_hi = expand_simple_binop (SFmode, PLUS, fp_hi, fp_lo, target,
19427 if (!rtx_equal_p (target, fp_hi))
19428 emit_move_insn (target, fp_hi);
19431 /* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
19432 a vector of unsigned ints VAL to vector of floats TARGET. */
19435 ix86_expand_vector_convert_uns_vsivsf (rtx target, rtx val)
19438 REAL_VALUE_TYPE TWO16r;
19439 machine_mode intmode = GET_MODE (val);
19440 machine_mode fltmode = GET_MODE (target);
19441 rtx (*cvt) (rtx, rtx);
19443 if (intmode == V4SImode)
19444 cvt = gen_floatv4siv4sf2;
19446 cvt = gen_floatv8siv8sf2;
19447 tmp[0] = ix86_build_const_vector (intmode, 1, GEN_INT (0xffff));
19448 tmp[0] = force_reg (intmode, tmp[0]);
19449 tmp[1] = expand_simple_binop (intmode, AND, val, tmp[0], NULL_RTX, 1,
19451 tmp[2] = expand_simple_binop (intmode, LSHIFTRT, val, GEN_INT (16),
19452 NULL_RTX, 1, OPTAB_DIRECT);
19453 tmp[3] = gen_reg_rtx (fltmode);
19454 emit_insn (cvt (tmp[3], tmp[1]));
19455 tmp[4] = gen_reg_rtx (fltmode);
19456 emit_insn (cvt (tmp[4], tmp[2]));
19457 real_ldexp (&TWO16r, &dconst1, 16);
19458 tmp[5] = const_double_from_real_value (TWO16r, SFmode);
19459 tmp[5] = force_reg (fltmode, ix86_build_const_vector (fltmode, 1, tmp[5]));
19460 tmp[6] = expand_simple_binop (fltmode, MULT, tmp[4], tmp[5], NULL_RTX, 1,
19462 tmp[7] = expand_simple_binop (fltmode, PLUS, tmp[3], tmp[6], target, 1,
19464 if (tmp[7] != target)
19465 emit_move_insn (target, tmp[7]);
19468 /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
19469 pattern can be used on it instead of *ufix_trunc* resp. fixuns_trunc*.
19470 This is done by doing just signed conversion if < 0x1p31, and otherwise by
19471 subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
19474 ix86_expand_adjust_ufix_to_sfix_si (rtx val, rtx *xorp)
19476 REAL_VALUE_TYPE TWO31r;
19477 rtx two31r, tmp[4];
19478 machine_mode mode = GET_MODE (val);
19479 machine_mode scalarmode = GET_MODE_INNER (mode);
19480 machine_mode intmode = GET_MODE_SIZE (mode) == 32 ? V8SImode : V4SImode;
19481 rtx (*cmp) (rtx, rtx, rtx, rtx);
19484 for (i = 0; i < 3; i++)
19485 tmp[i] = gen_reg_rtx (mode);
19486 real_ldexp (&TWO31r, &dconst1, 31);
19487 two31r = const_double_from_real_value (TWO31r, scalarmode);
19488 two31r = ix86_build_const_vector (mode, 1, two31r);
19489 two31r = force_reg (mode, two31r);
19492 case V8SFmode: cmp = gen_avx_maskcmpv8sf3; break;
19493 case V4SFmode: cmp = gen_sse_maskcmpv4sf3; break;
19494 case V4DFmode: cmp = gen_avx_maskcmpv4df3; break;
19495 case V2DFmode: cmp = gen_sse2_maskcmpv2df3; break;
19496 default: gcc_unreachable ();
19498 tmp[3] = gen_rtx_LE (mode, two31r, val);
19499 emit_insn (cmp (tmp[0], two31r, val, tmp[3]));
19500 tmp[1] = expand_simple_binop (mode, AND, tmp[0], two31r, tmp[1],
19502 if (intmode == V4SImode || TARGET_AVX2)
19503 *xorp = expand_simple_binop (intmode, ASHIFT,
19504 gen_lowpart (intmode, tmp[0]),
19505 GEN_INT (31), NULL_RTX, 0,
19509 rtx two31 = GEN_INT ((unsigned HOST_WIDE_INT) 1 << 31);
19510 two31 = ix86_build_const_vector (intmode, 1, two31);
19511 *xorp = expand_simple_binop (intmode, AND,
19512 gen_lowpart (intmode, tmp[0]),
19513 two31, NULL_RTX, 0,
19516 return expand_simple_binop (mode, MINUS, val, tmp[1], tmp[2],
19520 /* A subroutine of ix86_build_signbit_mask. If VECT is true,
19521 then replicate the value for all elements of the vector
19525 ix86_build_const_vector (machine_mode mode, bool vect, rtx value)
19529 machine_mode scalar_mode;
19552 n_elt = GET_MODE_NUNITS (mode);
19553 v = rtvec_alloc (n_elt);
19554 scalar_mode = GET_MODE_INNER (mode);
19556 RTVEC_ELT (v, 0) = value;
19558 for (i = 1; i < n_elt; ++i)
19559 RTVEC_ELT (v, i) = vect ? value : CONST0_RTX (scalar_mode);
19561 return gen_rtx_CONST_VECTOR (mode, v);
19564 gcc_unreachable ();
19568 /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
19569 and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
19570 for an SSE register. If VECT is true, then replicate the mask for
19571 all elements of the vector register. If INVERT is true, then create
19572 a mask excluding the sign bit. */
19575 ix86_build_signbit_mask (machine_mode mode, bool vect, bool invert)
19577 machine_mode vec_mode, imode;
19578 HOST_WIDE_INT hi, lo;
19583 /* Find the sign bit, sign extended to 2*HWI. */
19593 mode = GET_MODE_INNER (mode);
19595 lo = 0x80000000, hi = lo < 0;
19605 mode = GET_MODE_INNER (mode);
19607 if (HOST_BITS_PER_WIDE_INT >= 64)
19608 lo = (HOST_WIDE_INT)1 << shift, hi = -1;
19610 lo = 0, hi = (HOST_WIDE_INT)1 << (shift - HOST_BITS_PER_WIDE_INT);
19615 vec_mode = VOIDmode;
19616 if (HOST_BITS_PER_WIDE_INT >= 64)
19619 lo = 0, hi = (HOST_WIDE_INT)1 << shift;
19626 lo = 0, hi = (HOST_WIDE_INT)1 << (shift - HOST_BITS_PER_WIDE_INT);
19630 lo = ~lo, hi = ~hi;
19636 mask = immed_double_const (lo, hi, imode);
19638 vec = gen_rtvec (2, v, mask);
19639 v = gen_rtx_CONST_VECTOR (V2DImode, vec);
19640 v = copy_to_mode_reg (mode, gen_lowpart (mode, v));
19647 gcc_unreachable ();
19651 lo = ~lo, hi = ~hi;
19653 /* Force this value into the low part of a fp vector constant. */
19654 mask = immed_double_const (lo, hi, imode);
19655 mask = gen_lowpart (mode, mask);
19657 if (vec_mode == VOIDmode)
19658 return force_reg (mode, mask);
19660 v = ix86_build_const_vector (vec_mode, vect, mask);
19661 return force_reg (vec_mode, v);
19664 /* Generate code for floating point ABS or NEG. */
19667 ix86_expand_fp_absneg_operator (enum rtx_code code, machine_mode mode,
19670 rtx mask, set, dst, src;
19671 bool use_sse = false;
19672 bool vector_mode = VECTOR_MODE_P (mode);
19673 machine_mode vmode = mode;
19677 else if (mode == TFmode)
19679 else if (TARGET_SSE_MATH)
19681 use_sse = SSE_FLOAT_MODE_P (mode);
19682 if (mode == SFmode)
19684 else if (mode == DFmode)
19688 /* NEG and ABS performed with SSE use bitwise mask operations.
19689 Create the appropriate mask now. */
19691 mask = ix86_build_signbit_mask (vmode, vector_mode, code == ABS);
19698 set = gen_rtx_fmt_e (code, mode, src);
19699 set = gen_rtx_SET (VOIDmode, dst, set);
19706 use = gen_rtx_USE (VOIDmode, mask);
19708 par = gen_rtvec (2, set, use);
19711 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
19712 par = gen_rtvec (3, set, use, clob);
19714 emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
19720 /* Expand a copysign operation. Special case operand 0 being a constant. */
19723 ix86_expand_copysign (rtx operands[])
19725 machine_mode mode, vmode;
19726 rtx dest, op0, op1, mask, nmask;
19728 dest = operands[0];
19732 mode = GET_MODE (dest);
19734 if (mode == SFmode)
19736 else if (mode == DFmode)
19741 if (GET_CODE (op0) == CONST_DOUBLE)
19743 rtx (*copysign_insn)(rtx, rtx, rtx, rtx);
19745 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
19746 op0 = simplify_unary_operation (ABS, mode, op0, mode);
19748 if (mode == SFmode || mode == DFmode)
19750 if (op0 == CONST0_RTX (mode))
19751 op0 = CONST0_RTX (vmode);
19754 rtx v = ix86_build_const_vector (vmode, false, op0);
19756 op0 = force_reg (vmode, v);
19759 else if (op0 != CONST0_RTX (mode))
19760 op0 = force_reg (mode, op0);
19762 mask = ix86_build_signbit_mask (vmode, 0, 0);
19764 if (mode == SFmode)
19765 copysign_insn = gen_copysignsf3_const;
19766 else if (mode == DFmode)
19767 copysign_insn = gen_copysigndf3_const;
19769 copysign_insn = gen_copysigntf3_const;
19771 emit_insn (copysign_insn (dest, op0, op1, mask));
19775 rtx (*copysign_insn)(rtx, rtx, rtx, rtx, rtx, rtx);
19777 nmask = ix86_build_signbit_mask (vmode, 0, 1);
19778 mask = ix86_build_signbit_mask (vmode, 0, 0);
19780 if (mode == SFmode)
19781 copysign_insn = gen_copysignsf3_var;
19782 else if (mode == DFmode)
19783 copysign_insn = gen_copysigndf3_var;
19785 copysign_insn = gen_copysigntf3_var;
19787 emit_insn (copysign_insn (dest, NULL_RTX, op0, op1, nmask, mask));
19791 /* Deconstruct a copysign operation into bit masks. Operand 0 is known to
19792 be a constant, and so has already been expanded into a vector constant. */
19795 ix86_split_copysign_const (rtx operands[])
19797 machine_mode mode, vmode;
19798 rtx dest, op0, mask, x;
19800 dest = operands[0];
19802 mask = operands[3];
19804 mode = GET_MODE (dest);
19805 vmode = GET_MODE (mask);
19807 dest = simplify_gen_subreg (vmode, dest, mode, 0);
19808 x = gen_rtx_AND (vmode, dest, mask);
19809 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
19811 if (op0 != CONST0_RTX (vmode))
19813 x = gen_rtx_IOR (vmode, dest, op0);
19814 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
19818 /* Deconstruct a copysign operation into bit masks. Operand 0 is variable,
19819 so we have to do two masks. */
19822 ix86_split_copysign_var (rtx operands[])
19824 machine_mode mode, vmode;
19825 rtx dest, scratch, op0, op1, mask, nmask, x;
19827 dest = operands[0];
19828 scratch = operands[1];
19831 nmask = operands[4];
19832 mask = operands[5];
19834 mode = GET_MODE (dest);
19835 vmode = GET_MODE (mask);
19837 if (rtx_equal_p (op0, op1))
19839 /* Shouldn't happen often (it's useless, obviously), but when it does
19840 we'd generate incorrect code if we continue below. */
19841 emit_move_insn (dest, op0);
19845 if (REG_P (mask) && REGNO (dest) == REGNO (mask)) /* alternative 0 */
19847 gcc_assert (REGNO (op1) == REGNO (scratch));
19849 x = gen_rtx_AND (vmode, scratch, mask);
19850 emit_insn (gen_rtx_SET (VOIDmode, scratch, x));
19853 op0 = simplify_gen_subreg (vmode, op0, mode, 0);
19854 x = gen_rtx_NOT (vmode, dest);
19855 x = gen_rtx_AND (vmode, x, op0);
19856 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
19860 if (REGNO (op1) == REGNO (scratch)) /* alternative 1,3 */
19862 x = gen_rtx_AND (vmode, scratch, mask);
19864 else /* alternative 2,4 */
19866 gcc_assert (REGNO (mask) == REGNO (scratch));
19867 op1 = simplify_gen_subreg (vmode, op1, mode, 0);
19868 x = gen_rtx_AND (vmode, scratch, op1);
19870 emit_insn (gen_rtx_SET (VOIDmode, scratch, x));
19872 if (REGNO (op0) == REGNO (dest)) /* alternative 1,2 */
19874 dest = simplify_gen_subreg (vmode, op0, mode, 0);
19875 x = gen_rtx_AND (vmode, dest, nmask);
19877 else /* alternative 3,4 */
19879 gcc_assert (REGNO (nmask) == REGNO (dest));
19881 op0 = simplify_gen_subreg (vmode, op0, mode, 0);
19882 x = gen_rtx_AND (vmode, dest, op0);
19884 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
19887 x = gen_rtx_IOR (vmode, dest, scratch);
19888 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
19891 /* Return TRUE or FALSE depending on whether the first SET in INSN
19892 has source and destination with matching CC modes, and that the
19893 CC mode is at least as constrained as REQ_MODE. */
19896 ix86_match_ccmode (rtx insn, machine_mode req_mode)
19899 machine_mode set_mode;
19901 set = PATTERN (insn);
19902 if (GET_CODE (set) == PARALLEL)
19903 set = XVECEXP (set, 0, 0);
19904 gcc_assert (GET_CODE (set) == SET);
19905 gcc_assert (GET_CODE (SET_SRC (set)) == COMPARE);
19907 set_mode = GET_MODE (SET_DEST (set));
19911 if (req_mode != CCNOmode
19912 && (req_mode != CCmode
19913 || XEXP (SET_SRC (set), 1) != const0_rtx))
19917 if (req_mode == CCGCmode)
19921 if (req_mode == CCGOCmode || req_mode == CCNOmode)
19925 if (req_mode == CCZmode)
19935 if (set_mode != req_mode)
19940 gcc_unreachable ();
19943 return GET_MODE (SET_SRC (set)) == set_mode;
19946 /* Generate insn patterns to do an integer compare of OPERANDS. */
19949 ix86_expand_int_compare (enum rtx_code code, rtx op0, rtx op1)
19951 machine_mode cmpmode;
19954 cmpmode = SELECT_CC_MODE (code, op0, op1);
19955 flags = gen_rtx_REG (cmpmode, FLAGS_REG);
19957 /* This is very simple, but making the interface the same as in the
19958 FP case makes the rest of the code easier. */
19959 tmp = gen_rtx_COMPARE (cmpmode, op0, op1);
19960 emit_insn (gen_rtx_SET (VOIDmode, flags, tmp));
19962 /* Return the test that should be put into the flags user, i.e.
19963 the bcc, scc, or cmov instruction. */
19964 return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
19967 /* Figure out whether to use ordered or unordered fp comparisons.
19968 Return the appropriate mode to use. */
19971 ix86_fp_compare_mode (enum rtx_code)
19973 /* ??? In order to make all comparisons reversible, we do all comparisons
19974 non-trapping when compiling for IEEE. Once gcc is able to distinguish
19975 all forms trapping and nontrapping comparisons, we can make inequality
19976 comparisons trapping again, since it results in better code when using
19977 FCOM based compares. */
19978 return TARGET_IEEE_FP ? CCFPUmode : CCFPmode;
19982 ix86_cc_mode (enum rtx_code code, rtx op0, rtx op1)
19984 machine_mode mode = GET_MODE (op0);
19986 if (SCALAR_FLOAT_MODE_P (mode))
19988 gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
19989 return ix86_fp_compare_mode (code);
19994 /* Only zero flag is needed. */
19995 case EQ: /* ZF=0 */
19996 case NE: /* ZF!=0 */
19998 /* Codes needing carry flag. */
19999 case GEU: /* CF=0 */
20000 case LTU: /* CF=1 */
20001 /* Detect overflow checks. They need just the carry flag. */
20002 if (GET_CODE (op0) == PLUS
20003 && rtx_equal_p (op1, XEXP (op0, 0)))
20007 case GTU: /* CF=0 & ZF=0 */
20008 case LEU: /* CF=1 | ZF=1 */
20010 /* Codes possibly doable only with sign flag when
20011 comparing against zero. */
20012 case GE: /* SF=OF or SF=0 */
20013 case LT: /* SF<>OF or SF=1 */
20014 if (op1 == const0_rtx)
20017 /* For other cases Carry flag is not required. */
20019 /* Codes doable only with sign flag when comparing
20020 against zero, but we miss jump instruction for it
20021 so we need to use relational tests against overflow
20022 that thus needs to be zero. */
20023 case GT: /* ZF=0 & SF=OF */
20024 case LE: /* ZF=1 | SF<>OF */
20025 if (op1 == const0_rtx)
20029 /* strcmp pattern do (use flags) and combine may ask us for proper
20034 gcc_unreachable ();
20038 /* Return the fixed registers used for condition codes. */
20041 ix86_fixed_condition_code_regs (unsigned int *p1, unsigned int *p2)
20048 /* If two condition code modes are compatible, return a condition code
20049 mode which is compatible with both. Otherwise, return
20052 static machine_mode
20053 ix86_cc_modes_compatible (machine_mode m1, machine_mode m2)
20058 if (GET_MODE_CLASS (m1) != MODE_CC || GET_MODE_CLASS (m2) != MODE_CC)
20061 if ((m1 == CCGCmode && m2 == CCGOCmode)
20062 || (m1 == CCGOCmode && m2 == CCGCmode))
20065 if (m1 == CCZmode && (m2 == CCGCmode || m2 == CCGOCmode))
20067 else if (m2 == CCZmode && (m1 == CCGCmode || m1 == CCGOCmode))
20073 gcc_unreachable ();
20103 /* These are only compatible with themselves, which we already
20110 /* Return a comparison we can do and that it is equivalent to
20111 swap_condition (code) apart possibly from orderedness.
20112 But, never change orderedness if TARGET_IEEE_FP, returning
20113 UNKNOWN in that case if necessary. */
20115 static enum rtx_code
20116 ix86_fp_swap_condition (enum rtx_code code)
20120 case GT: /* GTU - CF=0 & ZF=0 */
20121 return TARGET_IEEE_FP ? UNKNOWN : UNLT;
20122 case GE: /* GEU - CF=0 */
20123 return TARGET_IEEE_FP ? UNKNOWN : UNLE;
20124 case UNLT: /* LTU - CF=1 */
20125 return TARGET_IEEE_FP ? UNKNOWN : GT;
20126 case UNLE: /* LEU - CF=1 | ZF=1 */
20127 return TARGET_IEEE_FP ? UNKNOWN : GE;
20129 return swap_condition (code);
20133 /* Return cost of comparison CODE using the best strategy for performance.
20134 All following functions do use number of instructions as a cost metrics.
20135 In future this should be tweaked to compute bytes for optimize_size and
20136 take into account performance of various instructions on various CPUs. */
20139 ix86_fp_comparison_cost (enum rtx_code code)
20143 /* The cost of code using bit-twiddling on %ah. */
20160 arith_cost = TARGET_IEEE_FP ? 5 : 4;
20164 arith_cost = TARGET_IEEE_FP ? 6 : 4;
20167 gcc_unreachable ();
20170 switch (ix86_fp_comparison_strategy (code))
20172 case IX86_FPCMP_COMI:
20173 return arith_cost > 4 ? 3 : 2;
20174 case IX86_FPCMP_SAHF:
20175 return arith_cost > 4 ? 4 : 3;
20181 /* Return strategy to use for floating-point. We assume that fcomi is always
20182 preferrable where available, since that is also true when looking at size
20183 (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test). */
20185 enum ix86_fpcmp_strategy
20186 ix86_fp_comparison_strategy (enum rtx_code)
20188 /* Do fcomi/sahf based test when profitable. */
20191 return IX86_FPCMP_COMI;
20193 if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
20194 return IX86_FPCMP_SAHF;
20196 return IX86_FPCMP_ARITH;
20199 /* Swap, force into registers, or otherwise massage the two operands
20200 to a fp comparison. The operands are updated in place; the new
20201 comparison code is returned. */
20203 static enum rtx_code
20204 ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
20206 machine_mode fpcmp_mode = ix86_fp_compare_mode (code);
20207 rtx op0 = *pop0, op1 = *pop1;
20208 machine_mode op_mode = GET_MODE (op0);
20209 int is_sse = TARGET_SSE_MATH && SSE_FLOAT_MODE_P (op_mode);
20211 /* All of the unordered compare instructions only work on registers.
20212 The same is true of the fcomi compare instructions. The XFmode
20213 compare instructions require registers except when comparing
20214 against zero or when converting operand 1 from fixed point to
20218 && (fpcmp_mode == CCFPUmode
20219 || (op_mode == XFmode
20220 && ! (standard_80387_constant_p (op0) == 1
20221 || standard_80387_constant_p (op1) == 1)
20222 && GET_CODE (op1) != FLOAT)
20223 || ix86_fp_comparison_strategy (code) == IX86_FPCMP_COMI))
20225 op0 = force_reg (op_mode, op0);
20226 op1 = force_reg (op_mode, op1);
20230 /* %%% We only allow op1 in memory; op0 must be st(0). So swap
20231 things around if they appear profitable, otherwise force op0
20232 into a register. */
20234 if (standard_80387_constant_p (op0) == 0
20236 && ! (standard_80387_constant_p (op1) == 0
20239 enum rtx_code new_code = ix86_fp_swap_condition (code);
20240 if (new_code != UNKNOWN)
20242 std::swap (op0, op1);
20248 op0 = force_reg (op_mode, op0);
20250 if (CONSTANT_P (op1))
20252 int tmp = standard_80387_constant_p (op1);
20254 op1 = validize_mem (force_const_mem (op_mode, op1));
20258 op1 = force_reg (op_mode, op1);
20261 op1 = force_reg (op_mode, op1);
20265 /* Try to rearrange the comparison to make it cheaper. */
20266 if (ix86_fp_comparison_cost (code)
20267 > ix86_fp_comparison_cost (swap_condition (code))
20268 && (REG_P (op1) || can_create_pseudo_p ()))
20270 std::swap (op0, op1);
20271 code = swap_condition (code);
20273 op0 = force_reg (op_mode, op0);
20281 /* Convert comparison codes we use to represent FP comparison to integer
20282 code that will result in proper branch. Return UNKNOWN if no such code
20286 ix86_fp_compare_code_to_integer (enum rtx_code code)
20315 /* Generate insn patterns to do a floating point compare of OPERANDS. */
20318 ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1, rtx scratch)
20320 machine_mode fpcmp_mode, intcmp_mode;
20323 fpcmp_mode = ix86_fp_compare_mode (code);
20324 code = ix86_prepare_fp_compare_args (code, &op0, &op1);
20326 /* Do fcomi/sahf based test when profitable. */
20327 switch (ix86_fp_comparison_strategy (code))
20329 case IX86_FPCMP_COMI:
20330 intcmp_mode = fpcmp_mode;
20331 tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
20332 tmp = gen_rtx_SET (VOIDmode, gen_rtx_REG (fpcmp_mode, FLAGS_REG),
20337 case IX86_FPCMP_SAHF:
20338 intcmp_mode = fpcmp_mode;
20339 tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
20340 tmp = gen_rtx_SET (VOIDmode, gen_rtx_REG (fpcmp_mode, FLAGS_REG),
20344 scratch = gen_reg_rtx (HImode);
20345 tmp2 = gen_rtx_CLOBBER (VOIDmode, scratch);
20346 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, tmp2)));
20349 case IX86_FPCMP_ARITH:
20350 /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first. */
20351 tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
20352 tmp2 = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
20354 scratch = gen_reg_rtx (HImode);
20355 emit_insn (gen_rtx_SET (VOIDmode, scratch, tmp2));
20357 /* In the unordered case, we have to check C2 for NaN's, which
20358 doesn't happen to work out to anything nice combination-wise.
20359 So do some bit twiddling on the value we've got in AH to come
20360 up with an appropriate set of condition codes. */
20362 intcmp_mode = CCNOmode;
20367 if (code == GT || !TARGET_IEEE_FP)
20369 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x45)));
20374 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20375 emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
20376 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x44)));
20377 intcmp_mode = CCmode;
20383 if (code == LT && TARGET_IEEE_FP)
20385 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20386 emit_insn (gen_cmpqi_ext_3 (scratch, const1_rtx));
20387 intcmp_mode = CCmode;
20392 emit_insn (gen_testqi_ext_ccno_0 (scratch, const1_rtx));
20398 if (code == GE || !TARGET_IEEE_FP)
20400 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x05)));
20405 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20406 emit_insn (gen_xorqi_cc_ext_1 (scratch, scratch, const1_rtx));
20412 if (code == LE && TARGET_IEEE_FP)
20414 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20415 emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
20416 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
20417 intcmp_mode = CCmode;
20422 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x45)));
20428 if (code == EQ && TARGET_IEEE_FP)
20430 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20431 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
20432 intcmp_mode = CCmode;
20437 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x40)));
20443 if (code == NE && TARGET_IEEE_FP)
20445 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20446 emit_insn (gen_xorqi_cc_ext_1 (scratch, scratch,
20452 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x40)));
20458 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x04)));
20462 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x04)));
20467 gcc_unreachable ();
20475 /* Return the test that should be put into the flags user, i.e.
20476 the bcc, scc, or cmov instruction. */
20477 return gen_rtx_fmt_ee (code, VOIDmode,
20478 gen_rtx_REG (intcmp_mode, FLAGS_REG),
20483 ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1)
20487 if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC)
20488 ret = gen_rtx_fmt_ee (code, VOIDmode, op0, op1);
20490 else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0)))
20492 gcc_assert (!DECIMAL_FLOAT_MODE_P (GET_MODE (op0)));
20493 ret = ix86_expand_fp_compare (code, op0, op1, NULL_RTX);
20496 ret = ix86_expand_int_compare (code, op0, op1);
20502 ix86_expand_branch (enum rtx_code code, rtx op0, rtx op1, rtx label)
20504 machine_mode mode = GET_MODE (op0);
20516 tmp = ix86_expand_compare (code, op0, op1);
20517 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
20518 gen_rtx_LABEL_REF (VOIDmode, label),
20520 emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp));
20527 /* Expand DImode branch into multiple compare+branch. */
20530 rtx_code_label *label2;
20531 enum rtx_code code1, code2, code3;
20532 machine_mode submode;
20534 if (CONSTANT_P (op0) && !CONSTANT_P (op1))
20536 std::swap (op0, op1);
20537 code = swap_condition (code);
20540 split_double_mode (mode, &op0, 1, lo+0, hi+0);
20541 split_double_mode (mode, &op1, 1, lo+1, hi+1);
20543 submode = mode == DImode ? SImode : DImode;
20545 /* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
20546 avoid two branches. This costs one extra insn, so disable when
20547 optimizing for size. */
20549 if ((code == EQ || code == NE)
20550 && (!optimize_insn_for_size_p ()
20551 || hi[1] == const0_rtx || lo[1] == const0_rtx))
20556 if (hi[1] != const0_rtx)
20557 xor1 = expand_binop (submode, xor_optab, xor1, hi[1],
20558 NULL_RTX, 0, OPTAB_WIDEN);
20561 if (lo[1] != const0_rtx)
20562 xor0 = expand_binop (submode, xor_optab, xor0, lo[1],
20563 NULL_RTX, 0, OPTAB_WIDEN);
20565 tmp = expand_binop (submode, ior_optab, xor1, xor0,
20566 NULL_RTX, 0, OPTAB_WIDEN);
20568 ix86_expand_branch (code, tmp, const0_rtx, label);
20572 /* Otherwise, if we are doing less-than or greater-or-equal-than,
20573 op1 is a constant and the low word is zero, then we can just
20574 examine the high word. Similarly for low word -1 and
20575 less-or-equal-than or greater-than. */
20577 if (CONST_INT_P (hi[1]))
20580 case LT: case LTU: case GE: case GEU:
20581 if (lo[1] == const0_rtx)
20583 ix86_expand_branch (code, hi[0], hi[1], label);
20587 case LE: case LEU: case GT: case GTU:
20588 if (lo[1] == constm1_rtx)
20590 ix86_expand_branch (code, hi[0], hi[1], label);
20598 /* Otherwise, we need two or three jumps. */
20600 label2 = gen_label_rtx ();
20603 code2 = swap_condition (code);
20604 code3 = unsigned_condition (code);
20608 case LT: case GT: case LTU: case GTU:
20611 case LE: code1 = LT; code2 = GT; break;
20612 case GE: code1 = GT; code2 = LT; break;
20613 case LEU: code1 = LTU; code2 = GTU; break;
20614 case GEU: code1 = GTU; code2 = LTU; break;
20616 case EQ: code1 = UNKNOWN; code2 = NE; break;
20617 case NE: code2 = UNKNOWN; break;
20620 gcc_unreachable ();
20625 * if (hi(a) < hi(b)) goto true;
20626 * if (hi(a) > hi(b)) goto false;
20627 * if (lo(a) < lo(b)) goto true;
20631 if (code1 != UNKNOWN)
20632 ix86_expand_branch (code1, hi[0], hi[1], label);
20633 if (code2 != UNKNOWN)
20634 ix86_expand_branch (code2, hi[0], hi[1], label2);
20636 ix86_expand_branch (code3, lo[0], lo[1], label);
20638 if (code2 != UNKNOWN)
20639 emit_label (label2);
20644 gcc_assert (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC);
20649 /* Split branch based on floating point condition. */
20651 ix86_split_fp_branch (enum rtx_code code, rtx op1, rtx op2,
20652 rtx target1, rtx target2, rtx tmp)
20657 if (target2 != pc_rtx)
20659 std::swap (target1, target2);
20660 code = reverse_condition_maybe_unordered (code);
20663 condition = ix86_expand_fp_compare (code, op1, op2,
20666 i = emit_jump_insn (gen_rtx_SET
20668 gen_rtx_IF_THEN_ELSE (VOIDmode,
20669 condition, target1, target2)));
20670 if (split_branch_probability >= 0)
20671 add_int_reg_note (i, REG_BR_PROB, split_branch_probability);
20675 ix86_expand_setcc (rtx dest, enum rtx_code code, rtx op0, rtx op1)
20679 gcc_assert (GET_MODE (dest) == QImode);
20681 ret = ix86_expand_compare (code, op0, op1);
20682 PUT_MODE (ret, QImode);
20683 emit_insn (gen_rtx_SET (VOIDmode, dest, ret));
20686 /* Expand comparison setting or clearing carry flag. Return true when
20687 successful and set pop for the operation. */
20689 ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
20691 machine_mode mode =
20692 GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1);
20694 /* Do not handle double-mode compares that go through special path. */
20695 if (mode == (TARGET_64BIT ? TImode : DImode))
20698 if (SCALAR_FLOAT_MODE_P (mode))
20701 rtx_insn *compare_seq;
20703 gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
20705 /* Shortcut: following common codes never translate
20706 into carry flag compares. */
20707 if (code == EQ || code == NE || code == UNEQ || code == LTGT
20708 || code == ORDERED || code == UNORDERED)
20711 /* These comparisons require zero flag; swap operands so they won't. */
20712 if ((code == GT || code == UNLE || code == LE || code == UNGT)
20713 && !TARGET_IEEE_FP)
20715 std::swap (op0, op1);
20716 code = swap_condition (code);
20719 /* Try to expand the comparison and verify that we end up with
20720 carry flag based comparison. This fails to be true only when
20721 we decide to expand comparison using arithmetic that is not
20722 too common scenario. */
20724 compare_op = ix86_expand_fp_compare (code, op0, op1, NULL_RTX);
20725 compare_seq = get_insns ();
20728 if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode
20729 || GET_MODE (XEXP (compare_op, 0)) == CCFPUmode)
20730 code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op));
20732 code = GET_CODE (compare_op);
20734 if (code != LTU && code != GEU)
20737 emit_insn (compare_seq);
20742 if (!INTEGRAL_MODE_P (mode))
20751 /* Convert a==0 into (unsigned)a<1. */
20754 if (op1 != const0_rtx)
20757 code = (code == EQ ? LTU : GEU);
20760 /* Convert a>b into b<a or a>=b-1. */
20763 if (CONST_INT_P (op1))
20765 op1 = gen_int_mode (INTVAL (op1) + 1, GET_MODE (op0));
20766 /* Bail out on overflow. We still can swap operands but that
20767 would force loading of the constant into register. */
20768 if (op1 == const0_rtx
20769 || !x86_64_immediate_operand (op1, GET_MODE (op1)))
20771 code = (code == GTU ? GEU : LTU);
20775 std::swap (op0, op1);
20776 code = (code == GTU ? LTU : GEU);
20780 /* Convert a>=0 into (unsigned)a<0x80000000. */
20783 if (mode == DImode || op1 != const0_rtx)
20785 op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
20786 code = (code == LT ? GEU : LTU);
20790 if (mode == DImode || op1 != constm1_rtx)
20792 op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
20793 code = (code == LE ? GEU : LTU);
20799 /* Swapping operands may cause constant to appear as first operand. */
20800 if (!nonimmediate_operand (op0, VOIDmode))
20802 if (!can_create_pseudo_p ())
20804 op0 = force_reg (mode, op0);
20806 *pop = ix86_expand_compare (code, op0, op1);
20807 gcc_assert (GET_CODE (*pop) == LTU || GET_CODE (*pop) == GEU);
20812 ix86_expand_int_movcc (rtx operands[])
20814 enum rtx_code code = GET_CODE (operands[1]), compare_code;
20815 rtx_insn *compare_seq;
20817 machine_mode mode = GET_MODE (operands[0]);
20818 bool sign_bit_compare_p = false;
20819 rtx op0 = XEXP (operands[1], 0);
20820 rtx op1 = XEXP (operands[1], 1);
20822 if (GET_MODE (op0) == TImode
20823 || (GET_MODE (op0) == DImode
20828 compare_op = ix86_expand_compare (code, op0, op1);
20829 compare_seq = get_insns ();
20832 compare_code = GET_CODE (compare_op);
20834 if ((op1 == const0_rtx && (code == GE || code == LT))
20835 || (op1 == constm1_rtx && (code == GT || code == LE)))
20836 sign_bit_compare_p = true;
20838 /* Don't attempt mode expansion here -- if we had to expand 5 or 6
20839 HImode insns, we'd be swallowed in word prefix ops. */
20841 if ((mode != HImode || TARGET_FAST_PREFIX)
20842 && (mode != (TARGET_64BIT ? TImode : DImode))
20843 && CONST_INT_P (operands[2])
20844 && CONST_INT_P (operands[3]))
20846 rtx out = operands[0];
20847 HOST_WIDE_INT ct = INTVAL (operands[2]);
20848 HOST_WIDE_INT cf = INTVAL (operands[3]);
20849 HOST_WIDE_INT diff;
20852 /* Sign bit compares are better done using shifts than we do by using
20854 if (sign_bit_compare_p
20855 || ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
20857 /* Detect overlap between destination and compare sources. */
20860 if (!sign_bit_compare_p)
20863 bool fpcmp = false;
20865 compare_code = GET_CODE (compare_op);
20867 flags = XEXP (compare_op, 0);
20869 if (GET_MODE (flags) == CCFPmode
20870 || GET_MODE (flags) == CCFPUmode)
20874 = ix86_fp_compare_code_to_integer (compare_code);
20877 /* To simplify rest of code, restrict to the GEU case. */
20878 if (compare_code == LTU)
20880 std::swap (ct, cf);
20881 compare_code = reverse_condition (compare_code);
20882 code = reverse_condition (code);
20887 PUT_CODE (compare_op,
20888 reverse_condition_maybe_unordered
20889 (GET_CODE (compare_op)));
20891 PUT_CODE (compare_op,
20892 reverse_condition (GET_CODE (compare_op)));
20896 if (reg_overlap_mentioned_p (out, op0)
20897 || reg_overlap_mentioned_p (out, op1))
20898 tmp = gen_reg_rtx (mode);
20900 if (mode == DImode)
20901 emit_insn (gen_x86_movdicc_0_m1 (tmp, flags, compare_op));
20903 emit_insn (gen_x86_movsicc_0_m1 (gen_lowpart (SImode, tmp),
20904 flags, compare_op));
20908 if (code == GT || code == GE)
20909 code = reverse_condition (code);
20912 std::swap (ct, cf);
20915 tmp = emit_store_flag (tmp, code, op0, op1, VOIDmode, 0, -1);
20928 tmp = expand_simple_binop (mode, PLUS,
20930 copy_rtx (tmp), 1, OPTAB_DIRECT);
20941 tmp = expand_simple_binop (mode, IOR,
20943 copy_rtx (tmp), 1, OPTAB_DIRECT);
20945 else if (diff == -1 && ct)
20955 tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
20957 tmp = expand_simple_binop (mode, PLUS,
20958 copy_rtx (tmp), GEN_INT (cf),
20959 copy_rtx (tmp), 1, OPTAB_DIRECT);
20967 * andl cf - ct, dest
20977 tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
20980 tmp = expand_simple_binop (mode, AND,
20982 gen_int_mode (cf - ct, mode),
20983 copy_rtx (tmp), 1, OPTAB_DIRECT);
20985 tmp = expand_simple_binop (mode, PLUS,
20986 copy_rtx (tmp), GEN_INT (ct),
20987 copy_rtx (tmp), 1, OPTAB_DIRECT);
20990 if (!rtx_equal_p (tmp, out))
20991 emit_move_insn (copy_rtx (out), copy_rtx (tmp));
20998 machine_mode cmp_mode = GET_MODE (op0);
20999 enum rtx_code new_code;
21001 if (SCALAR_FLOAT_MODE_P (cmp_mode))
21003 gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
21005 /* We may be reversing unordered compare to normal compare, that
21006 is not valid in general (we may convert non-trapping condition
21007 to trapping one), however on i386 we currently emit all
21008 comparisons unordered. */
21009 new_code = reverse_condition_maybe_unordered (code);
21012 new_code = ix86_reverse_condition (code, cmp_mode);
21013 if (new_code != UNKNOWN)
21015 std::swap (ct, cf);
21021 compare_code = UNKNOWN;
21022 if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_INT
21023 && CONST_INT_P (op1))
21025 if (op1 == const0_rtx
21026 && (code == LT || code == GE))
21027 compare_code = code;
21028 else if (op1 == constm1_rtx)
21032 else if (code == GT)
21037 /* Optimize dest = (op0 < 0) ? -1 : cf. */
21038 if (compare_code != UNKNOWN
21039 && GET_MODE (op0) == GET_MODE (out)
21040 && (cf == -1 || ct == -1))
21042 /* If lea code below could be used, only optimize
21043 if it results in a 2 insn sequence. */
21045 if (! (diff == 1 || diff == 2 || diff == 4 || diff == 8
21046 || diff == 3 || diff == 5 || diff == 9)
21047 || (compare_code == LT && ct == -1)
21048 || (compare_code == GE && cf == -1))
21051 * notl op1 (if necessary)
21059 code = reverse_condition (code);
21062 out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
21064 out = expand_simple_binop (mode, IOR,
21066 out, 1, OPTAB_DIRECT);
21067 if (out != operands[0])
21068 emit_move_insn (operands[0], out);
21075 if ((diff == 1 || diff == 2 || diff == 4 || diff == 8
21076 || diff == 3 || diff == 5 || diff == 9)
21077 && ((mode != QImode && mode != HImode) || !TARGET_PARTIAL_REG_STALL)
21079 || x86_64_immediate_operand (GEN_INT (cf), VOIDmode)))
21085 * lea cf(dest*(ct-cf)),dest
21089 * This also catches the degenerate setcc-only case.
21095 out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
21098 /* On x86_64 the lea instruction operates on Pmode, so we need
21099 to get arithmetics done in proper mode to match. */
21101 tmp = copy_rtx (out);
21105 out1 = copy_rtx (out);
21106 tmp = gen_rtx_MULT (mode, out1, GEN_INT (diff & ~1));
21110 tmp = gen_rtx_PLUS (mode, tmp, out1);
21116 tmp = gen_rtx_PLUS (mode, tmp, GEN_INT (cf));
21119 if (!rtx_equal_p (tmp, out))
21122 out = force_operand (tmp, copy_rtx (out));
21124 emit_insn (gen_rtx_SET (VOIDmode, copy_rtx (out), copy_rtx (tmp)));
21126 if (!rtx_equal_p (out, operands[0]))
21127 emit_move_insn (operands[0], copy_rtx (out));
21133 * General case: Jumpful:
21134 * xorl dest,dest cmpl op1, op2
21135 * cmpl op1, op2 movl ct, dest
21136 * setcc dest jcc 1f
21137 * decl dest movl cf, dest
21138 * andl (cf-ct),dest 1:
21141 * Size 20. Size 14.
21143 * This is reasonably steep, but branch mispredict costs are
21144 * high on modern cpus, so consider failing only if optimizing
21148 if ((!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
21149 && BRANCH_COST (optimize_insn_for_speed_p (),
21154 machine_mode cmp_mode = GET_MODE (op0);
21155 enum rtx_code new_code;
21157 if (SCALAR_FLOAT_MODE_P (cmp_mode))
21159 gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
21161 /* We may be reversing unordered compare to normal compare,
21162 that is not valid in general (we may convert non-trapping
21163 condition to trapping one), however on i386 we currently
21164 emit all comparisons unordered. */
21165 new_code = reverse_condition_maybe_unordered (code);
21169 new_code = ix86_reverse_condition (code, cmp_mode);
21170 if (compare_code != UNKNOWN && new_code != UNKNOWN)
21171 compare_code = reverse_condition (compare_code);
21174 if (new_code != UNKNOWN)
21182 if (compare_code != UNKNOWN)
21184 /* notl op1 (if needed)
21189 For x < 0 (resp. x <= -1) there will be no notl,
21190 so if possible swap the constants to get rid of the
21192 True/false will be -1/0 while code below (store flag
21193 followed by decrement) is 0/-1, so the constants need
21194 to be exchanged once more. */
21196 if (compare_code == GE || !cf)
21198 code = reverse_condition (code);
21202 std::swap (ct, cf);
21204 out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
21208 out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
21210 out = expand_simple_binop (mode, PLUS, copy_rtx (out),
21212 copy_rtx (out), 1, OPTAB_DIRECT);
21215 out = expand_simple_binop (mode, AND, copy_rtx (out),
21216 gen_int_mode (cf - ct, mode),
21217 copy_rtx (out), 1, OPTAB_DIRECT);
21219 out = expand_simple_binop (mode, PLUS, copy_rtx (out), GEN_INT (ct),
21220 copy_rtx (out), 1, OPTAB_DIRECT);
21221 if (!rtx_equal_p (out, operands[0]))
21222 emit_move_insn (operands[0], copy_rtx (out));
21228 if (!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
21230 /* Try a few things more with specific constants and a variable. */
21233 rtx var, orig_out, out, tmp;
21235 if (BRANCH_COST (optimize_insn_for_speed_p (), false) <= 2)
21238 /* If one of the two operands is an interesting constant, load a
21239 constant with the above and mask it in with a logical operation. */
21241 if (CONST_INT_P (operands[2]))
21244 if (INTVAL (operands[2]) == 0 && operands[3] != constm1_rtx)
21245 operands[3] = constm1_rtx, op = and_optab;
21246 else if (INTVAL (operands[2]) == -1 && operands[3] != const0_rtx)
21247 operands[3] = const0_rtx, op = ior_optab;
21251 else if (CONST_INT_P (operands[3]))
21254 if (INTVAL (operands[3]) == 0 && operands[2] != constm1_rtx)
21255 operands[2] = constm1_rtx, op = and_optab;
21256 else if (INTVAL (operands[3]) == -1 && operands[3] != const0_rtx)
21257 operands[2] = const0_rtx, op = ior_optab;
21264 orig_out = operands[0];
21265 tmp = gen_reg_rtx (mode);
21268 /* Recurse to get the constant loaded. */
21269 if (ix86_expand_int_movcc (operands) == 0)
21272 /* Mask in the interesting variable. */
21273 out = expand_binop (mode, op, var, tmp, orig_out, 0,
21275 if (!rtx_equal_p (out, orig_out))
21276 emit_move_insn (copy_rtx (orig_out), copy_rtx (out));
21282 * For comparison with above,
21292 if (! nonimmediate_operand (operands[2], mode))
21293 operands[2] = force_reg (mode, operands[2]);
21294 if (! nonimmediate_operand (operands[3], mode))
21295 operands[3] = force_reg (mode, operands[3]);
21297 if (! register_operand (operands[2], VOIDmode)
21299 || ! register_operand (operands[3], VOIDmode)))
21300 operands[2] = force_reg (mode, operands[2]);
21303 && ! register_operand (operands[3], VOIDmode))
21304 operands[3] = force_reg (mode, operands[3]);
21306 emit_insn (compare_seq);
21307 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
21308 gen_rtx_IF_THEN_ELSE (mode,
21309 compare_op, operands[2],
21314 /* Swap, force into registers, or otherwise massage the two operands
21315 to an sse comparison with a mask result. Thus we differ a bit from
21316 ix86_prepare_fp_compare_args which expects to produce a flags result.
21318 The DEST operand exists to help determine whether to commute commutative
21319 operators. The POP0/POP1 operands are updated in place. The new
21320 comparison code is returned, or UNKNOWN if not implementable. */
21322 static enum rtx_code
21323 ix86_prepare_sse_fp_compare_args (rtx dest, enum rtx_code code,
21324 rtx *pop0, rtx *pop1)
21330 /* AVX supports all the needed comparisons. */
21333 /* We have no LTGT as an operator. We could implement it with
21334 NE & ORDERED, but this requires an extra temporary. It's
21335 not clear that it's worth it. */
21342 /* These are supported directly. */
21349 /* AVX has 3 operand comparisons, no need to swap anything. */
21352 /* For commutative operators, try to canonicalize the destination
21353 operand to be first in the comparison - this helps reload to
21354 avoid extra moves. */
21355 if (!dest || !rtx_equal_p (dest, *pop1))
21363 /* These are not supported directly before AVX, and furthermore
21364 ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
21365 comparison operands to transform into something that is
21367 std::swap (*pop0, *pop1);
21368 code = swap_condition (code);
21372 gcc_unreachable ();
21378 /* Detect conditional moves that exactly match min/max operational
21379 semantics. Note that this is IEEE safe, as long as we don't
21380 interchange the operands.
21382 Returns FALSE if this conditional move doesn't match a MIN/MAX,
21383 and TRUE if the operation is successful and instructions are emitted. */
21386 ix86_expand_sse_fp_minmax (rtx dest, enum rtx_code code, rtx cmp_op0,
21387 rtx cmp_op1, rtx if_true, rtx if_false)
21395 else if (code == UNGE)
21396 std::swap (if_true, if_false);
21400 if (rtx_equal_p (cmp_op0, if_true) && rtx_equal_p (cmp_op1, if_false))
21402 else if (rtx_equal_p (cmp_op1, if_true) && rtx_equal_p (cmp_op0, if_false))
21407 mode = GET_MODE (dest);
21409 /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
21410 but MODE may be a vector mode and thus not appropriate. */
21411 if (!flag_finite_math_only || !flag_unsafe_math_optimizations)
21413 int u = is_min ? UNSPEC_IEEE_MIN : UNSPEC_IEEE_MAX;
21416 if_true = force_reg (mode, if_true);
21417 v = gen_rtvec (2, if_true, if_false);
21418 tmp = gen_rtx_UNSPEC (mode, v, u);
21422 code = is_min ? SMIN : SMAX;
21423 tmp = gen_rtx_fmt_ee (code, mode, if_true, if_false);
21426 emit_insn (gen_rtx_SET (VOIDmode, dest, tmp));
21430 /* Expand an sse vector comparison. Return the register with the result. */
21433 ix86_expand_sse_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1,
21434 rtx op_true, rtx op_false)
21436 machine_mode mode = GET_MODE (dest);
21437 machine_mode cmp_ops_mode = GET_MODE (cmp_op0);
21439 /* In general case result of comparison can differ from operands' type. */
21440 machine_mode cmp_mode;
21442 /* In AVX512F the result of comparison is an integer mask. */
21443 bool maskcmp = false;
21446 if (GET_MODE_SIZE (cmp_ops_mode) == 64)
21448 cmp_mode = mode_for_size (GET_MODE_NUNITS (cmp_ops_mode), MODE_INT, 0);
21449 gcc_assert (cmp_mode != BLKmode);
21454 cmp_mode = cmp_ops_mode;
21457 cmp_op0 = force_reg (cmp_ops_mode, cmp_op0);
21458 if (!nonimmediate_operand (cmp_op1, cmp_ops_mode))
21459 cmp_op1 = force_reg (cmp_ops_mode, cmp_op1);
21462 || reg_overlap_mentioned_p (dest, op_true)
21463 || reg_overlap_mentioned_p (dest, op_false))
21464 dest = gen_reg_rtx (maskcmp ? cmp_mode : mode);
21466 /* Compare patterns for int modes are unspec in AVX512F only. */
21467 if (maskcmp && (code == GT || code == EQ))
21469 rtx (*gen)(rtx, rtx, rtx);
21471 switch (cmp_ops_mode)
21474 gcc_assert (TARGET_AVX512BW);
21475 gen = code == GT ? gen_avx512bw_gtv64qi3 : gen_avx512bw_eqv64qi3_1;
21478 gcc_assert (TARGET_AVX512BW);
21479 gen = code == GT ? gen_avx512bw_gtv32hi3 : gen_avx512bw_eqv32hi3_1;
21482 gen = code == GT ? gen_avx512f_gtv16si3 : gen_avx512f_eqv16si3_1;
21485 gen = code == GT ? gen_avx512f_gtv8di3 : gen_avx512f_eqv8di3_1;
21493 emit_insn (gen (dest, cmp_op0, cmp_op1));
21497 x = gen_rtx_fmt_ee (code, cmp_mode, cmp_op0, cmp_op1);
21499 if (cmp_mode != mode && !maskcmp)
21501 x = force_reg (cmp_ops_mode, x);
21502 convert_move (dest, x, false);
21505 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
21510 /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
21511 operations. This is used for both scalar and vector conditional moves. */
21514 ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
21516 machine_mode mode = GET_MODE (dest);
21517 machine_mode cmpmode = GET_MODE (cmp);
21519 /* In AVX512F the result of comparison is an integer mask. */
21520 bool maskcmp = (mode != cmpmode && TARGET_AVX512F);
21524 if (vector_all_ones_operand (op_true, mode)
21525 && rtx_equal_p (op_false, CONST0_RTX (mode))
21528 emit_insn (gen_rtx_SET (VOIDmode, dest, cmp));
21530 else if (op_false == CONST0_RTX (mode)
21533 op_true = force_reg (mode, op_true);
21534 x = gen_rtx_AND (mode, cmp, op_true);
21535 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
21537 else if (op_true == CONST0_RTX (mode)
21540 op_false = force_reg (mode, op_false);
21541 x = gen_rtx_NOT (mode, cmp);
21542 x = gen_rtx_AND (mode, x, op_false);
21543 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
21545 else if (INTEGRAL_MODE_P (mode) && op_true == CONSTM1_RTX (mode)
21548 op_false = force_reg (mode, op_false);
21549 x = gen_rtx_IOR (mode, cmp, op_false);
21550 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
21552 else if (TARGET_XOP
21555 op_true = force_reg (mode, op_true);
21557 if (!nonimmediate_operand (op_false, mode))
21558 op_false = force_reg (mode, op_false);
21560 emit_insn (gen_rtx_SET (mode, dest,
21561 gen_rtx_IF_THEN_ELSE (mode, cmp,
21567 rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
21570 if (!nonimmediate_operand (op_true, mode))
21571 op_true = force_reg (mode, op_true);
21573 op_false = force_reg (mode, op_false);
21579 gen = gen_sse4_1_blendvps;
21583 gen = gen_sse4_1_blendvpd;
21591 gen = gen_sse4_1_pblendvb;
21592 if (mode != V16QImode)
21593 d = gen_reg_rtx (V16QImode);
21594 op_false = gen_lowpart (V16QImode, op_false);
21595 op_true = gen_lowpart (V16QImode, op_true);
21596 cmp = gen_lowpart (V16QImode, cmp);
21601 gen = gen_avx_blendvps256;
21605 gen = gen_avx_blendvpd256;
21613 gen = gen_avx2_pblendvb;
21614 if (mode != V32QImode)
21615 d = gen_reg_rtx (V32QImode);
21616 op_false = gen_lowpart (V32QImode, op_false);
21617 op_true = gen_lowpart (V32QImode, op_true);
21618 cmp = gen_lowpart (V32QImode, cmp);
21623 gen = gen_avx512bw_blendmv64qi;
21626 gen = gen_avx512bw_blendmv32hi;
21629 gen = gen_avx512f_blendmv16si;
21632 gen = gen_avx512f_blendmv8di;
21635 gen = gen_avx512f_blendmv8df;
21638 gen = gen_avx512f_blendmv16sf;
21647 emit_insn (gen (d, op_false, op_true, cmp));
21649 emit_move_insn (dest, gen_lowpart (GET_MODE (dest), d));
21653 op_true = force_reg (mode, op_true);
21655 t2 = gen_reg_rtx (mode);
21657 t3 = gen_reg_rtx (mode);
21661 x = gen_rtx_AND (mode, op_true, cmp);
21662 emit_insn (gen_rtx_SET (VOIDmode, t2, x));
21664 x = gen_rtx_NOT (mode, cmp);
21665 x = gen_rtx_AND (mode, x, op_false);
21666 emit_insn (gen_rtx_SET (VOIDmode, t3, x));
21668 x = gen_rtx_IOR (mode, t3, t2);
21669 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
21674 /* Expand a floating-point conditional move. Return true if successful. */
21677 ix86_expand_fp_movcc (rtx operands[])
21679 machine_mode mode = GET_MODE (operands[0]);
21680 enum rtx_code code = GET_CODE (operands[1]);
21681 rtx tmp, compare_op;
21682 rtx op0 = XEXP (operands[1], 0);
21683 rtx op1 = XEXP (operands[1], 1);
21685 if (TARGET_SSE_MATH && SSE_FLOAT_MODE_P (mode))
21687 machine_mode cmode;
21689 /* Since we've no cmove for sse registers, don't force bad register
21690 allocation just to gain access to it. Deny movcc when the
21691 comparison mode doesn't match the move mode. */
21692 cmode = GET_MODE (op0);
21693 if (cmode == VOIDmode)
21694 cmode = GET_MODE (op1);
21698 code = ix86_prepare_sse_fp_compare_args (operands[0], code, &op0, &op1);
21699 if (code == UNKNOWN)
21702 if (ix86_expand_sse_fp_minmax (operands[0], code, op0, op1,
21703 operands[2], operands[3]))
21706 tmp = ix86_expand_sse_cmp (operands[0], code, op0, op1,
21707 operands[2], operands[3]);
21708 ix86_expand_sse_movcc (operands[0], tmp, operands[2], operands[3]);
21712 if (GET_MODE (op0) == TImode
21713 || (GET_MODE (op0) == DImode
21717 /* The floating point conditional move instructions don't directly
21718 support conditions resulting from a signed integer comparison. */
21720 compare_op = ix86_expand_compare (code, op0, op1);
21721 if (!fcmov_comparison_operator (compare_op, VOIDmode))
21723 tmp = gen_reg_rtx (QImode);
21724 ix86_expand_setcc (tmp, code, op0, op1);
21726 compare_op = ix86_expand_compare (NE, tmp, const0_rtx);
21729 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
21730 gen_rtx_IF_THEN_ELSE (mode, compare_op,
21731 operands[2], operands[3])));
21736 /* Expand a floating-point vector conditional move; a vcond operation
21737 rather than a movcc operation. */
21740 ix86_expand_fp_vcond (rtx operands[])
21742 enum rtx_code code = GET_CODE (operands[3]);
21745 code = ix86_prepare_sse_fp_compare_args (operands[0], code,
21746 &operands[4], &operands[5]);
21747 if (code == UNKNOWN)
21750 switch (GET_CODE (operands[3]))
21753 temp = ix86_expand_sse_cmp (operands[0], ORDERED, operands[4],
21754 operands[5], operands[0], operands[0]);
21755 cmp = ix86_expand_sse_cmp (operands[0], NE, operands[4],
21756 operands[5], operands[1], operands[2]);
21760 temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[4],
21761 operands[5], operands[0], operands[0]);
21762 cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[4],
21763 operands[5], operands[1], operands[2]);
21767 gcc_unreachable ();
21769 cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
21771 ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
21775 if (ix86_expand_sse_fp_minmax (operands[0], code, operands[4],
21776 operands[5], operands[1], operands[2]))
21779 cmp = ix86_expand_sse_cmp (operands[0], code, operands[4], operands[5],
21780 operands[1], operands[2]);
21781 ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
21785 /* Expand a signed/unsigned integral vector conditional move. */
21788 ix86_expand_int_vcond (rtx operands[])
21790 machine_mode data_mode = GET_MODE (operands[0]);
21791 machine_mode mode = GET_MODE (operands[4]);
21792 enum rtx_code code = GET_CODE (operands[3]);
21793 bool negate = false;
21796 cop0 = operands[4];
21797 cop1 = operands[5];
21799 /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
21800 and x < 0 ? 1 : 0 into (unsigned) x >> 31. */
21801 if ((code == LT || code == GE)
21802 && data_mode == mode
21803 && cop1 == CONST0_RTX (mode)
21804 && operands[1 + (code == LT)] == CONST0_RTX (data_mode)
21805 && GET_MODE_SIZE (GET_MODE_INNER (data_mode)) > 1
21806 && GET_MODE_SIZE (GET_MODE_INNER (data_mode)) <= 8
21807 && (GET_MODE_SIZE (data_mode) == 16
21808 || (TARGET_AVX2 && GET_MODE_SIZE (data_mode) == 32)))
21810 rtx negop = operands[2 - (code == LT)];
21811 int shift = GET_MODE_BITSIZE (GET_MODE_INNER (data_mode)) - 1;
21812 if (negop == CONST1_RTX (data_mode))
21814 rtx res = expand_simple_binop (mode, LSHIFTRT, cop0, GEN_INT (shift),
21815 operands[0], 1, OPTAB_DIRECT);
21816 if (res != operands[0])
21817 emit_move_insn (operands[0], res);
21820 else if (GET_MODE_INNER (data_mode) != DImode
21821 && vector_all_ones_operand (negop, data_mode))
21823 rtx res = expand_simple_binop (mode, ASHIFTRT, cop0, GEN_INT (shift),
21824 operands[0], 0, OPTAB_DIRECT);
21825 if (res != operands[0])
21826 emit_move_insn (operands[0], res);
21831 if (!nonimmediate_operand (cop1, mode))
21832 cop1 = force_reg (mode, cop1);
21833 if (!general_operand (operands[1], data_mode))
21834 operands[1] = force_reg (data_mode, operands[1]);
21835 if (!general_operand (operands[2], data_mode))
21836 operands[2] = force_reg (data_mode, operands[2]);
21838 /* XOP supports all of the comparisons on all 128-bit vector int types. */
21840 && (mode == V16QImode || mode == V8HImode
21841 || mode == V4SImode || mode == V2DImode))
21845 /* Canonicalize the comparison to EQ, GT, GTU. */
21856 code = reverse_condition (code);
21862 code = reverse_condition (code);
21868 std::swap (cop0, cop1);
21869 code = swap_condition (code);
21873 gcc_unreachable ();
21876 /* Only SSE4.1/SSE4.2 supports V2DImode. */
21877 if (mode == V2DImode)
21882 /* SSE4.1 supports EQ. */
21883 if (!TARGET_SSE4_1)
21889 /* SSE4.2 supports GT/GTU. */
21890 if (!TARGET_SSE4_2)
21895 gcc_unreachable ();
21899 /* Unsigned parallel compare is not supported by the hardware.
21900 Play some tricks to turn this into a signed comparison
21904 cop0 = force_reg (mode, cop0);
21916 rtx (*gen_sub3) (rtx, rtx, rtx);
21920 case V16SImode: gen_sub3 = gen_subv16si3; break;
21921 case V8DImode: gen_sub3 = gen_subv8di3; break;
21922 case V8SImode: gen_sub3 = gen_subv8si3; break;
21923 case V4DImode: gen_sub3 = gen_subv4di3; break;
21924 case V4SImode: gen_sub3 = gen_subv4si3; break;
21925 case V2DImode: gen_sub3 = gen_subv2di3; break;
21927 gcc_unreachable ();
21929 /* Subtract (-(INT MAX) - 1) from both operands to make
21931 mask = ix86_build_signbit_mask (mode, true, false);
21932 t1 = gen_reg_rtx (mode);
21933 emit_insn (gen_sub3 (t1, cop0, mask));
21935 t2 = gen_reg_rtx (mode);
21936 emit_insn (gen_sub3 (t2, cop1, mask));
21950 /* Perform a parallel unsigned saturating subtraction. */
21951 x = gen_reg_rtx (mode);
21952 emit_insn (gen_rtx_SET (VOIDmode, x,
21953 gen_rtx_US_MINUS (mode, cop0, cop1)));
21956 cop1 = CONST0_RTX (mode);
21962 gcc_unreachable ();
21967 /* Allow the comparison to be done in one mode, but the movcc to
21968 happen in another mode. */
21969 if (data_mode == mode)
21971 x = ix86_expand_sse_cmp (operands[0], code, cop0, cop1,
21972 operands[1+negate], operands[2-negate]);
21976 gcc_assert (GET_MODE_SIZE (data_mode) == GET_MODE_SIZE (mode));
21977 x = ix86_expand_sse_cmp (gen_reg_rtx (mode), code, cop0, cop1,
21978 operands[1+negate], operands[2-negate]);
21979 if (GET_MODE (x) == mode)
21980 x = gen_lowpart (data_mode, x);
21983 ix86_expand_sse_movcc (operands[0], x, operands[1+negate],
21984 operands[2-negate]);
21988 /* AVX512F does support 64-byte integer vector operations,
21989 thus the longest vector we are faced with is V64QImode. */
21990 #define MAX_VECT_LEN 64
21992 struct expand_vec_perm_d
21994 rtx target, op0, op1;
21995 unsigned char perm[MAX_VECT_LEN];
21996 machine_mode vmode;
21997 unsigned char nelt;
21998 bool one_operand_p;
22003 ix86_expand_vec_perm_vpermi2 (rtx target, rtx op0, rtx mask, rtx op1,
22004 struct expand_vec_perm_d *d)
22006 /* ix86_expand_vec_perm_vpermi2 is called from both const and non-const
22007 expander, so args are either in d, or in op0, op1 etc. */
22008 machine_mode mode = GET_MODE (d ? d->op0 : op0);
22009 machine_mode maskmode = mode;
22010 rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
22015 if (TARGET_AVX512VL && TARGET_AVX512BW)
22016 gen = gen_avx512vl_vpermi2varv8hi3;
22019 if (TARGET_AVX512VL && TARGET_AVX512BW)
22020 gen = gen_avx512vl_vpermi2varv16hi3;
22023 if (TARGET_AVX512VBMI)
22024 gen = gen_avx512bw_vpermi2varv64qi3;
22027 if (TARGET_AVX512BW)
22028 gen = gen_avx512bw_vpermi2varv32hi3;
22031 if (TARGET_AVX512VL)
22032 gen = gen_avx512vl_vpermi2varv4si3;
22035 if (TARGET_AVX512VL)
22036 gen = gen_avx512vl_vpermi2varv8si3;
22039 if (TARGET_AVX512F)
22040 gen = gen_avx512f_vpermi2varv16si3;
22043 if (TARGET_AVX512VL)
22045 gen = gen_avx512vl_vpermi2varv4sf3;
22046 maskmode = V4SImode;
22050 if (TARGET_AVX512VL)
22052 gen = gen_avx512vl_vpermi2varv8sf3;
22053 maskmode = V8SImode;
22057 if (TARGET_AVX512F)
22059 gen = gen_avx512f_vpermi2varv16sf3;
22060 maskmode = V16SImode;
22064 if (TARGET_AVX512VL)
22065 gen = gen_avx512vl_vpermi2varv2di3;
22068 if (TARGET_AVX512VL)
22069 gen = gen_avx512vl_vpermi2varv4di3;
22072 if (TARGET_AVX512F)
22073 gen = gen_avx512f_vpermi2varv8di3;
22076 if (TARGET_AVX512VL)
22078 gen = gen_avx512vl_vpermi2varv2df3;
22079 maskmode = V2DImode;
22083 if (TARGET_AVX512VL)
22085 gen = gen_avx512vl_vpermi2varv4df3;
22086 maskmode = V4DImode;
22090 if (TARGET_AVX512F)
22092 gen = gen_avx512f_vpermi2varv8df3;
22093 maskmode = V8DImode;
22103 /* ix86_expand_vec_perm_vpermi2 is called from both const and non-const
22104 expander, so args are either in d, or in op0, op1 etc. */
22108 target = d->target;
22111 for (int i = 0; i < d->nelt; ++i)
22112 vec[i] = GEN_INT (d->perm[i]);
22113 mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
22116 emit_insn (gen (target, op0, force_reg (maskmode, mask), op1));
22120 /* Expand a variable vector permutation. */
22123 ix86_expand_vec_perm (rtx operands[])
22125 rtx target = operands[0];
22126 rtx op0 = operands[1];
22127 rtx op1 = operands[2];
22128 rtx mask = operands[3];
22129 rtx t1, t2, t3, t4, t5, t6, t7, t8, vt, vt2, vec[32];
22130 machine_mode mode = GET_MODE (op0);
22131 machine_mode maskmode = GET_MODE (mask);
22133 bool one_operand_shuffle = rtx_equal_p (op0, op1);
22135 /* Number of elements in the vector. */
22136 w = GET_MODE_NUNITS (mode);
22137 e = GET_MODE_UNIT_SIZE (mode);
22138 gcc_assert (w <= 64);
22140 if (ix86_expand_vec_perm_vpermi2 (target, op0, mask, op1, NULL))
22145 if (mode == V4DImode || mode == V4DFmode || mode == V16HImode)
22147 /* Unfortunately, the VPERMQ and VPERMPD instructions only support
22148 an constant shuffle operand. With a tiny bit of effort we can
22149 use VPERMD instead. A re-interpretation stall for V4DFmode is
22150 unfortunate but there's no avoiding it.
22151 Similarly for V16HImode we don't have instructions for variable
22152 shuffling, while for V32QImode we can use after preparing suitable
22153 masks vpshufb; vpshufb; vpermq; vpor. */
22155 if (mode == V16HImode)
22157 maskmode = mode = V32QImode;
22163 maskmode = mode = V8SImode;
22167 t1 = gen_reg_rtx (maskmode);
22169 /* Replicate the low bits of the V4DImode mask into V8SImode:
22171 t1 = { A A B B C C D D }. */
22172 for (i = 0; i < w / 2; ++i)
22173 vec[i*2 + 1] = vec[i*2] = GEN_INT (i * 2);
22174 vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
22175 vt = force_reg (maskmode, vt);
22176 mask = gen_lowpart (maskmode, mask);
22177 if (maskmode == V8SImode)
22178 emit_insn (gen_avx2_permvarv8si (t1, mask, vt));
22180 emit_insn (gen_avx2_pshufbv32qi3 (t1, mask, vt));
22182 /* Multiply the shuffle indicies by two. */
22183 t1 = expand_simple_binop (maskmode, PLUS, t1, t1, t1, 1,
22186 /* Add one to the odd shuffle indicies:
22187 t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
22188 for (i = 0; i < w / 2; ++i)
22190 vec[i * 2] = const0_rtx;
22191 vec[i * 2 + 1] = const1_rtx;
22193 vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
22194 vt = validize_mem (force_const_mem (maskmode, vt));
22195 t1 = expand_simple_binop (maskmode, PLUS, t1, vt, t1, 1,
22198 /* Continue as if V8SImode (resp. V32QImode) was used initially. */
22199 operands[3] = mask = t1;
22200 target = gen_reg_rtx (mode);
22201 op0 = gen_lowpart (mode, op0);
22202 op1 = gen_lowpart (mode, op1);
22208 /* The VPERMD and VPERMPS instructions already properly ignore
22209 the high bits of the shuffle elements. No need for us to
22210 perform an AND ourselves. */
22211 if (one_operand_shuffle)
22213 emit_insn (gen_avx2_permvarv8si (target, op0, mask));
22214 if (target != operands[0])
22215 emit_move_insn (operands[0],
22216 gen_lowpart (GET_MODE (operands[0]), target));
22220 t1 = gen_reg_rtx (V8SImode);
22221 t2 = gen_reg_rtx (V8SImode);
22222 emit_insn (gen_avx2_permvarv8si (t1, op0, mask));
22223 emit_insn (gen_avx2_permvarv8si (t2, op1, mask));
22229 mask = gen_lowpart (V8SImode, mask);
22230 if (one_operand_shuffle)
22231 emit_insn (gen_avx2_permvarv8sf (target, op0, mask));
22234 t1 = gen_reg_rtx (V8SFmode);
22235 t2 = gen_reg_rtx (V8SFmode);
22236 emit_insn (gen_avx2_permvarv8sf (t1, op0, mask));
22237 emit_insn (gen_avx2_permvarv8sf (t2, op1, mask));
22243 /* By combining the two 128-bit input vectors into one 256-bit
22244 input vector, we can use VPERMD and VPERMPS for the full
22245 two-operand shuffle. */
22246 t1 = gen_reg_rtx (V8SImode);
22247 t2 = gen_reg_rtx (V8SImode);
22248 emit_insn (gen_avx_vec_concatv8si (t1, op0, op1));
22249 emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
22250 emit_insn (gen_avx2_permvarv8si (t1, t1, t2));
22251 emit_insn (gen_avx_vextractf128v8si (target, t1, const0_rtx));
22255 t1 = gen_reg_rtx (V8SFmode);
22256 t2 = gen_reg_rtx (V8SImode);
22257 mask = gen_lowpart (V4SImode, mask);
22258 emit_insn (gen_avx_vec_concatv8sf (t1, op0, op1));
22259 emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
22260 emit_insn (gen_avx2_permvarv8sf (t1, t1, t2));
22261 emit_insn (gen_avx_vextractf128v8sf (target, t1, const0_rtx));
22265 t1 = gen_reg_rtx (V32QImode);
22266 t2 = gen_reg_rtx (V32QImode);
22267 t3 = gen_reg_rtx (V32QImode);
22268 vt2 = GEN_INT (-128);
22269 for (i = 0; i < 32; i++)
22271 vt = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
22272 vt = force_reg (V32QImode, vt);
22273 for (i = 0; i < 32; i++)
22274 vec[i] = i < 16 ? vt2 : const0_rtx;
22275 vt2 = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
22276 vt2 = force_reg (V32QImode, vt2);
22277 /* From mask create two adjusted masks, which contain the same
22278 bits as mask in the low 7 bits of each vector element.
22279 The first mask will have the most significant bit clear
22280 if it requests element from the same 128-bit lane
22281 and MSB set if it requests element from the other 128-bit lane.
22282 The second mask will have the opposite values of the MSB,
22283 and additionally will have its 128-bit lanes swapped.
22284 E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
22285 t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
22286 t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
22287 stands for other 12 bytes. */
22288 /* The bit whether element is from the same lane or the other
22289 lane is bit 4, so shift it up by 3 to the MSB position. */
22290 t5 = gen_reg_rtx (V4DImode);
22291 emit_insn (gen_ashlv4di3 (t5, gen_lowpart (V4DImode, mask),
22293 /* Clear MSB bits from the mask just in case it had them set. */
22294 emit_insn (gen_avx2_andnotv32qi3 (t2, vt, mask));
22295 /* After this t1 will have MSB set for elements from other lane. */
22296 emit_insn (gen_xorv32qi3 (t1, gen_lowpart (V32QImode, t5), vt2));
22297 /* Clear bits other than MSB. */
22298 emit_insn (gen_andv32qi3 (t1, t1, vt));
22299 /* Or in the lower bits from mask into t3. */
22300 emit_insn (gen_iorv32qi3 (t3, t1, t2));
22301 /* And invert MSB bits in t1, so MSB is set for elements from the same
22303 emit_insn (gen_xorv32qi3 (t1, t1, vt));
22304 /* Swap 128-bit lanes in t3. */
22305 t6 = gen_reg_rtx (V4DImode);
22306 emit_insn (gen_avx2_permv4di_1 (t6, gen_lowpart (V4DImode, t3),
22307 const2_rtx, GEN_INT (3),
22308 const0_rtx, const1_rtx));
22309 /* And or in the lower bits from mask into t1. */
22310 emit_insn (gen_iorv32qi3 (t1, t1, t2));
22311 if (one_operand_shuffle)
22313 /* Each of these shuffles will put 0s in places where
22314 element from the other 128-bit lane is needed, otherwise
22315 will shuffle in the requested value. */
22316 emit_insn (gen_avx2_pshufbv32qi3 (t3, op0,
22317 gen_lowpart (V32QImode, t6)));
22318 emit_insn (gen_avx2_pshufbv32qi3 (t1, op0, t1));
22319 /* For t3 the 128-bit lanes are swapped again. */
22320 t7 = gen_reg_rtx (V4DImode);
22321 emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t3),
22322 const2_rtx, GEN_INT (3),
22323 const0_rtx, const1_rtx));
22324 /* And oring both together leads to the result. */
22325 emit_insn (gen_iorv32qi3 (target, t1,
22326 gen_lowpart (V32QImode, t7)));
22327 if (target != operands[0])
22328 emit_move_insn (operands[0],
22329 gen_lowpart (GET_MODE (operands[0]), target));
22333 t4 = gen_reg_rtx (V32QImode);
22334 /* Similarly to the above one_operand_shuffle code,
22335 just for repeated twice for each operand. merge_two:
22336 code will merge the two results together. */
22337 emit_insn (gen_avx2_pshufbv32qi3 (t4, op0,
22338 gen_lowpart (V32QImode, t6)));
22339 emit_insn (gen_avx2_pshufbv32qi3 (t3, op1,
22340 gen_lowpart (V32QImode, t6)));
22341 emit_insn (gen_avx2_pshufbv32qi3 (t2, op0, t1));
22342 emit_insn (gen_avx2_pshufbv32qi3 (t1, op1, t1));
22343 t7 = gen_reg_rtx (V4DImode);
22344 emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t4),
22345 const2_rtx, GEN_INT (3),
22346 const0_rtx, const1_rtx));
22347 t8 = gen_reg_rtx (V4DImode);
22348 emit_insn (gen_avx2_permv4di_1 (t8, gen_lowpart (V4DImode, t3),
22349 const2_rtx, GEN_INT (3),
22350 const0_rtx, const1_rtx));
22351 emit_insn (gen_iorv32qi3 (t4, t2, gen_lowpart (V32QImode, t7)));
22352 emit_insn (gen_iorv32qi3 (t3, t1, gen_lowpart (V32QImode, t8)));
22358 gcc_assert (GET_MODE_SIZE (mode) <= 16);
22365 /* The XOP VPPERM insn supports three inputs. By ignoring the
22366 one_operand_shuffle special case, we avoid creating another
22367 set of constant vectors in memory. */
22368 one_operand_shuffle = false;
22370 /* mask = mask & {2*w-1, ...} */
22371 vt = GEN_INT (2*w - 1);
22375 /* mask = mask & {w-1, ...} */
22376 vt = GEN_INT (w - 1);
22379 for (i = 0; i < w; i++)
22381 vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
22382 mask = expand_simple_binop (maskmode, AND, mask, vt,
22383 NULL_RTX, 0, OPTAB_DIRECT);
22385 /* For non-QImode operations, convert the word permutation control
22386 into a byte permutation control. */
22387 if (mode != V16QImode)
22389 mask = expand_simple_binop (maskmode, ASHIFT, mask,
22390 GEN_INT (exact_log2 (e)),
22391 NULL_RTX, 0, OPTAB_DIRECT);
22393 /* Convert mask to vector of chars. */
22394 mask = force_reg (V16QImode, gen_lowpart (V16QImode, mask));
22396 /* Replicate each of the input bytes into byte positions:
22397 (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
22398 (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
22399 (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
22400 for (i = 0; i < 16; ++i)
22401 vec[i] = GEN_INT (i/e * e);
22402 vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
22403 vt = validize_mem (force_const_mem (V16QImode, vt));
22405 emit_insn (gen_xop_pperm (mask, mask, mask, vt));
22407 emit_insn (gen_ssse3_pshufbv16qi3 (mask, mask, vt));
22409 /* Convert it into the byte positions by doing
22410 mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
22411 for (i = 0; i < 16; ++i)
22412 vec[i] = GEN_INT (i % e);
22413 vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
22414 vt = validize_mem (force_const_mem (V16QImode, vt));
22415 emit_insn (gen_addv16qi3 (mask, mask, vt));
22418 /* The actual shuffle operations all operate on V16QImode. */
22419 op0 = gen_lowpart (V16QImode, op0);
22420 op1 = gen_lowpart (V16QImode, op1);
22424 if (GET_MODE (target) != V16QImode)
22425 target = gen_reg_rtx (V16QImode);
22426 emit_insn (gen_xop_pperm (target, op0, op1, mask));
22427 if (target != operands[0])
22428 emit_move_insn (operands[0],
22429 gen_lowpart (GET_MODE (operands[0]), target));
22431 else if (one_operand_shuffle)
22433 if (GET_MODE (target) != V16QImode)
22434 target = gen_reg_rtx (V16QImode);
22435 emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, mask));
22436 if (target != operands[0])
22437 emit_move_insn (operands[0],
22438 gen_lowpart (GET_MODE (operands[0]), target));
22445 /* Shuffle the two input vectors independently. */
22446 t1 = gen_reg_rtx (V16QImode);
22447 t2 = gen_reg_rtx (V16QImode);
22448 emit_insn (gen_ssse3_pshufbv16qi3 (t1, op0, mask));
22449 emit_insn (gen_ssse3_pshufbv16qi3 (t2, op1, mask));
22452 /* Then merge them together. The key is whether any given control
22453 element contained a bit set that indicates the second word. */
22454 mask = operands[3];
22456 if (maskmode == V2DImode && !TARGET_SSE4_1)
22458 /* Without SSE4.1, we don't have V2DImode EQ. Perform one
22459 more shuffle to convert the V2DI input mask into a V4SI
22460 input mask. At which point the masking that expand_int_vcond
22461 will work as desired. */
22462 rtx t3 = gen_reg_rtx (V4SImode);
22463 emit_insn (gen_sse2_pshufd_1 (t3, gen_lowpart (V4SImode, mask),
22464 const0_rtx, const0_rtx,
22465 const2_rtx, const2_rtx));
22467 maskmode = V4SImode;
22471 for (i = 0; i < w; i++)
22473 vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
22474 vt = force_reg (maskmode, vt);
22475 mask = expand_simple_binop (maskmode, AND, mask, vt,
22476 NULL_RTX, 0, OPTAB_DIRECT);
22478 if (GET_MODE (target) != mode)
22479 target = gen_reg_rtx (mode);
22481 xops[1] = gen_lowpart (mode, t2);
22482 xops[2] = gen_lowpart (mode, t1);
22483 xops[3] = gen_rtx_EQ (maskmode, mask, vt);
22486 ok = ix86_expand_int_vcond (xops);
22488 if (target != operands[0])
22489 emit_move_insn (operands[0],
22490 gen_lowpart (GET_MODE (operands[0]), target));
22494 /* Unpack OP[1] into the next wider integer vector type. UNSIGNED_P is
22495 true if we should do zero extension, else sign extension. HIGH_P is
22496 true if we want the N/2 high elements, else the low elements. */
22499 ix86_expand_sse_unpack (rtx dest, rtx src, bool unsigned_p, bool high_p)
22501 machine_mode imode = GET_MODE (src);
22506 rtx (*unpack)(rtx, rtx);
22507 rtx (*extract)(rtx, rtx) = NULL;
22508 machine_mode halfmode = BLKmode;
22514 unpack = gen_avx512bw_zero_extendv32qiv32hi2;
22516 unpack = gen_avx512bw_sign_extendv32qiv32hi2;
22517 halfmode = V32QImode;
22519 = high_p ? gen_vec_extract_hi_v64qi : gen_vec_extract_lo_v64qi;
22523 unpack = gen_avx2_zero_extendv16qiv16hi2;
22525 unpack = gen_avx2_sign_extendv16qiv16hi2;
22526 halfmode = V16QImode;
22528 = high_p ? gen_vec_extract_hi_v32qi : gen_vec_extract_lo_v32qi;
22532 unpack = gen_avx512f_zero_extendv16hiv16si2;
22534 unpack = gen_avx512f_sign_extendv16hiv16si2;
22535 halfmode = V16HImode;
22537 = high_p ? gen_vec_extract_hi_v32hi : gen_vec_extract_lo_v32hi;
22541 unpack = gen_avx2_zero_extendv8hiv8si2;
22543 unpack = gen_avx2_sign_extendv8hiv8si2;
22544 halfmode = V8HImode;
22546 = high_p ? gen_vec_extract_hi_v16hi : gen_vec_extract_lo_v16hi;
22550 unpack = gen_avx512f_zero_extendv8siv8di2;
22552 unpack = gen_avx512f_sign_extendv8siv8di2;
22553 halfmode = V8SImode;
22555 = high_p ? gen_vec_extract_hi_v16si : gen_vec_extract_lo_v16si;
22559 unpack = gen_avx2_zero_extendv4siv4di2;
22561 unpack = gen_avx2_sign_extendv4siv4di2;
22562 halfmode = V4SImode;
22564 = high_p ? gen_vec_extract_hi_v8si : gen_vec_extract_lo_v8si;
22568 unpack = gen_sse4_1_zero_extendv8qiv8hi2;
22570 unpack = gen_sse4_1_sign_extendv8qiv8hi2;
22574 unpack = gen_sse4_1_zero_extendv4hiv4si2;
22576 unpack = gen_sse4_1_sign_extendv4hiv4si2;
22580 unpack = gen_sse4_1_zero_extendv2siv2di2;
22582 unpack = gen_sse4_1_sign_extendv2siv2di2;
22585 gcc_unreachable ();
22588 if (GET_MODE_SIZE (imode) >= 32)
22590 tmp = gen_reg_rtx (halfmode);
22591 emit_insn (extract (tmp, src));
22595 /* Shift higher 8 bytes to lower 8 bytes. */
22596 tmp = gen_reg_rtx (V1TImode);
22597 emit_insn (gen_sse2_lshrv1ti3 (tmp, gen_lowpart (V1TImode, src),
22599 tmp = gen_lowpart (imode, tmp);
22604 emit_insn (unpack (dest, tmp));
22608 rtx (*unpack)(rtx, rtx, rtx);
22614 unpack = gen_vec_interleave_highv16qi;
22616 unpack = gen_vec_interleave_lowv16qi;
22620 unpack = gen_vec_interleave_highv8hi;
22622 unpack = gen_vec_interleave_lowv8hi;
22626 unpack = gen_vec_interleave_highv4si;
22628 unpack = gen_vec_interleave_lowv4si;
22631 gcc_unreachable ();
22635 tmp = force_reg (imode, CONST0_RTX (imode));
22637 tmp = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
22638 src, pc_rtx, pc_rtx);
22640 rtx tmp2 = gen_reg_rtx (imode);
22641 emit_insn (unpack (tmp2, src, tmp));
22642 emit_move_insn (dest, gen_lowpart (GET_MODE (dest), tmp2));
22646 /* Expand conditional increment or decrement using adb/sbb instructions.
22647 The default case using setcc followed by the conditional move can be
22648 done by generic code. */
22650 ix86_expand_int_addcc (rtx operands[])
22652 enum rtx_code code = GET_CODE (operands[1]);
22654 rtx (*insn)(rtx, rtx, rtx, rtx, rtx);
22656 rtx val = const0_rtx;
22657 bool fpcmp = false;
22659 rtx op0 = XEXP (operands[1], 0);
22660 rtx op1 = XEXP (operands[1], 1);
22662 if (operands[3] != const1_rtx
22663 && operands[3] != constm1_rtx)
22665 if (!ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
22667 code = GET_CODE (compare_op);
22669 flags = XEXP (compare_op, 0);
22671 if (GET_MODE (flags) == CCFPmode
22672 || GET_MODE (flags) == CCFPUmode)
22675 code = ix86_fp_compare_code_to_integer (code);
22682 PUT_CODE (compare_op,
22683 reverse_condition_maybe_unordered
22684 (GET_CODE (compare_op)));
22686 PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
22689 mode = GET_MODE (operands[0]);
22691 /* Construct either adc or sbb insn. */
22692 if ((code == LTU) == (operands[3] == constm1_rtx))
22697 insn = gen_subqi3_carry;
22700 insn = gen_subhi3_carry;
22703 insn = gen_subsi3_carry;
22706 insn = gen_subdi3_carry;
22709 gcc_unreachable ();
22717 insn = gen_addqi3_carry;
22720 insn = gen_addhi3_carry;
22723 insn = gen_addsi3_carry;
22726 insn = gen_adddi3_carry;
22729 gcc_unreachable ();
22732 emit_insn (insn (operands[0], operands[2], val, flags, compare_op));
22738 /* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
22739 but works for floating pointer parameters and nonoffsetable memories.
22740 For pushes, it returns just stack offsets; the values will be saved
22741 in the right order. Maximally three parts are generated. */
22744 ix86_split_to_parts (rtx operand, rtx *parts, machine_mode mode)
22749 size = mode==XFmode ? 3 : GET_MODE_SIZE (mode) / 4;
22751 size = (GET_MODE_SIZE (mode) + 4) / 8;
22753 gcc_assert (!REG_P (operand) || !MMX_REGNO_P (REGNO (operand)));
22754 gcc_assert (size >= 2 && size <= 4);
22756 /* Optimize constant pool reference to immediates. This is used by fp
22757 moves, that force all constants to memory to allow combining. */
22758 if (MEM_P (operand) && MEM_READONLY_P (operand))
22760 rtx tmp = maybe_get_pool_constant (operand);
22765 if (MEM_P (operand) && !offsettable_memref_p (operand))
22767 /* The only non-offsetable memories we handle are pushes. */
22768 int ok = push_operand (operand, VOIDmode);
22772 operand = copy_rtx (operand);
22773 PUT_MODE (operand, word_mode);
22774 parts[0] = parts[1] = parts[2] = parts[3] = operand;
22778 if (GET_CODE (operand) == CONST_VECTOR)
22780 machine_mode imode = int_mode_for_mode (mode);
22781 /* Caution: if we looked through a constant pool memory above,
22782 the operand may actually have a different mode now. That's
22783 ok, since we want to pun this all the way back to an integer. */
22784 operand = simplify_subreg (imode, operand, GET_MODE (operand), 0);
22785 gcc_assert (operand != NULL);
22791 if (mode == DImode)
22792 split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
22797 if (REG_P (operand))
22799 gcc_assert (reload_completed);
22800 for (i = 0; i < size; i++)
22801 parts[i] = gen_rtx_REG (SImode, REGNO (operand) + i);
22803 else if (offsettable_memref_p (operand))
22805 operand = adjust_address (operand, SImode, 0);
22806 parts[0] = operand;
22807 for (i = 1; i < size; i++)
22808 parts[i] = adjust_address (operand, SImode, 4 * i);
22810 else if (GET_CODE (operand) == CONST_DOUBLE)
22815 REAL_VALUE_FROM_CONST_DOUBLE (r, operand);
22819 real_to_target (l, &r, mode);
22820 parts[3] = gen_int_mode (l[3], SImode);
22821 parts[2] = gen_int_mode (l[2], SImode);
22824 /* We can't use REAL_VALUE_TO_TARGET_LONG_DOUBLE since
22825 long double may not be 80-bit. */
22826 real_to_target (l, &r, mode);
22827 parts[2] = gen_int_mode (l[2], SImode);
22830 REAL_VALUE_TO_TARGET_DOUBLE (r, l);
22833 gcc_unreachable ();
22835 parts[1] = gen_int_mode (l[1], SImode);
22836 parts[0] = gen_int_mode (l[0], SImode);
22839 gcc_unreachable ();
22844 if (mode == TImode)
22845 split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
22846 if (mode == XFmode || mode == TFmode)
22848 machine_mode upper_mode = mode==XFmode ? SImode : DImode;
22849 if (REG_P (operand))
22851 gcc_assert (reload_completed);
22852 parts[0] = gen_rtx_REG (DImode, REGNO (operand) + 0);
22853 parts[1] = gen_rtx_REG (upper_mode, REGNO (operand) + 1);
22855 else if (offsettable_memref_p (operand))
22857 operand = adjust_address (operand, DImode, 0);
22858 parts[0] = operand;
22859 parts[1] = adjust_address (operand, upper_mode, 8);
22861 else if (GET_CODE (operand) == CONST_DOUBLE)
22866 REAL_VALUE_FROM_CONST_DOUBLE (r, operand);
22867 real_to_target (l, &r, mode);
22869 /* Do not use shift by 32 to avoid warning on 32bit systems. */
22870 if (HOST_BITS_PER_WIDE_INT >= 64)
22873 ((l[0] & (((HOST_WIDE_INT) 2 << 31) - 1))
22874 + ((((HOST_WIDE_INT) l[1]) << 31) << 1),
22877 parts[0] = immed_double_const (l[0], l[1], DImode);
22879 if (upper_mode == SImode)
22880 parts[1] = gen_int_mode (l[2], SImode);
22881 else if (HOST_BITS_PER_WIDE_INT >= 64)
22884 ((l[2] & (((HOST_WIDE_INT) 2 << 31) - 1))
22885 + ((((HOST_WIDE_INT) l[3]) << 31) << 1),
22888 parts[1] = immed_double_const (l[2], l[3], DImode);
22891 gcc_unreachable ();
22898 /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
22899 Return false when normal moves are needed; true when all required
22900 insns have been emitted. Operands 2-4 contain the input values
22901 int the correct order; operands 5-7 contain the output values. */
22904 ix86_split_long_move (rtx operands[])
22909 int collisions = 0;
22910 machine_mode mode = GET_MODE (operands[0]);
22911 bool collisionparts[4];
22913 /* The DFmode expanders may ask us to move double.
22914 For 64bit target this is single move. By hiding the fact
22915 here we simplify i386.md splitters. */
22916 if (TARGET_64BIT && GET_MODE_SIZE (GET_MODE (operands[0])) == 8)
22918 /* Optimize constant pool reference to immediates. This is used by
22919 fp moves, that force all constants to memory to allow combining. */
22921 if (MEM_P (operands[1])
22922 && GET_CODE (XEXP (operands[1], 0)) == SYMBOL_REF
22923 && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0)))
22924 operands[1] = get_pool_constant (XEXP (operands[1], 0));
22925 if (push_operand (operands[0], VOIDmode))
22927 operands[0] = copy_rtx (operands[0]);
22928 PUT_MODE (operands[0], word_mode);
22931 operands[0] = gen_lowpart (DImode, operands[0]);
22932 operands[1] = gen_lowpart (DImode, operands[1]);
22933 emit_move_insn (operands[0], operands[1]);
22937 /* The only non-offsettable memory we handle is push. */
22938 if (push_operand (operands[0], VOIDmode))
22941 gcc_assert (!MEM_P (operands[0])
22942 || offsettable_memref_p (operands[0]));
22944 nparts = ix86_split_to_parts (operands[1], part[1], GET_MODE (operands[0]));
22945 ix86_split_to_parts (operands[0], part[0], GET_MODE (operands[0]));
22947 /* When emitting push, take care for source operands on the stack. */
22948 if (push && MEM_P (operands[1])
22949 && reg_overlap_mentioned_p (stack_pointer_rtx, operands[1]))
22951 rtx src_base = XEXP (part[1][nparts - 1], 0);
22953 /* Compensate for the stack decrement by 4. */
22954 if (!TARGET_64BIT && nparts == 3
22955 && mode == XFmode && TARGET_128BIT_LONG_DOUBLE)
22956 src_base = plus_constant (Pmode, src_base, 4);
22958 /* src_base refers to the stack pointer and is
22959 automatically decreased by emitted push. */
22960 for (i = 0; i < nparts; i++)
22961 part[1][i] = change_address (part[1][i],
22962 GET_MODE (part[1][i]), src_base);
22965 /* We need to do copy in the right order in case an address register
22966 of the source overlaps the destination. */
22967 if (REG_P (part[0][0]) && MEM_P (part[1][0]))
22971 for (i = 0; i < nparts; i++)
22974 = reg_overlap_mentioned_p (part[0][i], XEXP (part[1][0], 0));
22975 if (collisionparts[i])
22979 /* Collision in the middle part can be handled by reordering. */
22980 if (collisions == 1 && nparts == 3 && collisionparts [1])
22982 std::swap (part[0][1], part[0][2]);
22983 std::swap (part[1][1], part[1][2]);
22985 else if (collisions == 1
22987 && (collisionparts [1] || collisionparts [2]))
22989 if (collisionparts [1])
22991 std::swap (part[0][1], part[0][2]);
22992 std::swap (part[1][1], part[1][2]);
22996 std::swap (part[0][2], part[0][3]);
22997 std::swap (part[1][2], part[1][3]);
23001 /* If there are more collisions, we can't handle it by reordering.
23002 Do an lea to the last part and use only one colliding move. */
23003 else if (collisions > 1)
23005 rtx base, addr, tls_base = NULL_RTX;
23009 base = part[0][nparts - 1];
23011 /* Handle the case when the last part isn't valid for lea.
23012 Happens in 64-bit mode storing the 12-byte XFmode. */
23013 if (GET_MODE (base) != Pmode)
23014 base = gen_rtx_REG (Pmode, REGNO (base));
23016 addr = XEXP (part[1][0], 0);
23017 if (TARGET_TLS_DIRECT_SEG_REFS)
23019 struct ix86_address parts;
23020 int ok = ix86_decompose_address (addr, &parts);
23022 if (parts.seg == DEFAULT_TLS_SEG_REG)
23024 /* It is not valid to use %gs: or %fs: in
23025 lea though, so we need to remove it from the
23026 address used for lea and add it to each individual
23027 memory loads instead. */
23028 addr = copy_rtx (addr);
23030 while (GET_CODE (*x) == PLUS)
23032 for (i = 0; i < 2; i++)
23034 rtx u = XEXP (*x, i);
23035 if (GET_CODE (u) == ZERO_EXTEND)
23037 if (GET_CODE (u) == UNSPEC
23038 && XINT (u, 1) == UNSPEC_TP)
23040 tls_base = XEXP (*x, i);
23041 *x = XEXP (*x, 1 - i);
23049 gcc_assert (tls_base);
23052 emit_insn (gen_rtx_SET (VOIDmode, base, addr));
23054 base = gen_rtx_PLUS (GET_MODE (base), base, tls_base);
23055 part[1][0] = replace_equiv_address (part[1][0], base);
23056 for (i = 1; i < nparts; i++)
23059 base = copy_rtx (base);
23060 tmp = plus_constant (Pmode, base, UNITS_PER_WORD * i);
23061 part[1][i] = replace_equiv_address (part[1][i], tmp);
23072 if (TARGET_128BIT_LONG_DOUBLE && mode == XFmode)
23073 emit_insn (ix86_gen_add3 (stack_pointer_rtx,
23074 stack_pointer_rtx, GEN_INT (-4)));
23075 emit_move_insn (part[0][2], part[1][2]);
23077 else if (nparts == 4)
23079 emit_move_insn (part[0][3], part[1][3]);
23080 emit_move_insn (part[0][2], part[1][2]);
23085 /* In 64bit mode we don't have 32bit push available. In case this is
23086 register, it is OK - we will just use larger counterpart. We also
23087 retype memory - these comes from attempt to avoid REX prefix on
23088 moving of second half of TFmode value. */
23089 if (GET_MODE (part[1][1]) == SImode)
23091 switch (GET_CODE (part[1][1]))
23094 part[1][1] = adjust_address (part[1][1], DImode, 0);
23098 part[1][1] = gen_rtx_REG (DImode, REGNO (part[1][1]));
23102 gcc_unreachable ();
23105 if (GET_MODE (part[1][0]) == SImode)
23106 part[1][0] = part[1][1];
23109 emit_move_insn (part[0][1], part[1][1]);
23110 emit_move_insn (part[0][0], part[1][0]);
23114 /* Choose correct order to not overwrite the source before it is copied. */
23115 if ((REG_P (part[0][0])
23116 && REG_P (part[1][1])
23117 && (REGNO (part[0][0]) == REGNO (part[1][1])
23119 && REGNO (part[0][0]) == REGNO (part[1][2]))
23121 && REGNO (part[0][0]) == REGNO (part[1][3]))))
23123 && reg_overlap_mentioned_p (part[0][0], XEXP (part[1][0], 0))))
23125 for (i = 0, j = nparts - 1; i < nparts; i++, j--)
23127 operands[2 + i] = part[0][j];
23128 operands[6 + i] = part[1][j];
23133 for (i = 0; i < nparts; i++)
23135 operands[2 + i] = part[0][i];
23136 operands[6 + i] = part[1][i];
23140 /* If optimizing for size, attempt to locally unCSE nonzero constants. */
23141 if (optimize_insn_for_size_p ())
23143 for (j = 0; j < nparts - 1; j++)
23144 if (CONST_INT_P (operands[6 + j])
23145 && operands[6 + j] != const0_rtx
23146 && REG_P (operands[2 + j]))
23147 for (i = j; i < nparts - 1; i++)
23148 if (CONST_INT_P (operands[7 + i])
23149 && INTVAL (operands[7 + i]) == INTVAL (operands[6 + j]))
23150 operands[7 + i] = operands[2 + j];
23153 for (i = 0; i < nparts; i++)
23154 emit_move_insn (operands[2 + i], operands[6 + i]);
23159 /* Helper function of ix86_split_ashl used to generate an SImode/DImode
23160 left shift by a constant, either using a single shift or
23161 a sequence of add instructions. */
23164 ix86_expand_ashl_const (rtx operand, int count, machine_mode mode)
23166 rtx (*insn)(rtx, rtx, rtx);
23169 || (count * ix86_cost->add <= ix86_cost->shift_const
23170 && !optimize_insn_for_size_p ()))
23172 insn = mode == DImode ? gen_addsi3 : gen_adddi3;
23173 while (count-- > 0)
23174 emit_insn (insn (operand, operand, operand));
23178 insn = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
23179 emit_insn (insn (operand, operand, GEN_INT (count)));
23184 ix86_split_ashl (rtx *operands, rtx scratch, machine_mode mode)
23186 rtx (*gen_ashl3)(rtx, rtx, rtx);
23187 rtx (*gen_shld)(rtx, rtx, rtx);
23188 int half_width = GET_MODE_BITSIZE (mode) >> 1;
23190 rtx low[2], high[2];
23193 if (CONST_INT_P (operands[2]))
23195 split_double_mode (mode, operands, 2, low, high);
23196 count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
23198 if (count >= half_width)
23200 emit_move_insn (high[0], low[1]);
23201 emit_move_insn (low[0], const0_rtx);
23203 if (count > half_width)
23204 ix86_expand_ashl_const (high[0], count - half_width, mode);
23208 gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
23210 if (!rtx_equal_p (operands[0], operands[1]))
23211 emit_move_insn (operands[0], operands[1]);
23213 emit_insn (gen_shld (high[0], low[0], GEN_INT (count)));
23214 ix86_expand_ashl_const (low[0], count, mode);
23219 split_double_mode (mode, operands, 1, low, high);
23221 gen_ashl3 = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
23223 if (operands[1] == const1_rtx)
23225 /* Assuming we've chosen a QImode capable registers, then 1 << N
23226 can be done with two 32/64-bit shifts, no branches, no cmoves. */
23227 if (ANY_QI_REG_P (low[0]) && ANY_QI_REG_P (high[0]))
23229 rtx s, d, flags = gen_rtx_REG (CCZmode, FLAGS_REG);
23231 ix86_expand_clear (low[0]);
23232 ix86_expand_clear (high[0]);
23233 emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (half_width)));
23235 d = gen_lowpart (QImode, low[0]);
23236 d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
23237 s = gen_rtx_EQ (QImode, flags, const0_rtx);
23238 emit_insn (gen_rtx_SET (VOIDmode, d, s));
23240 d = gen_lowpart (QImode, high[0]);
23241 d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
23242 s = gen_rtx_NE (QImode, flags, const0_rtx);
23243 emit_insn (gen_rtx_SET (VOIDmode, d, s));
23246 /* Otherwise, we can get the same results by manually performing
23247 a bit extract operation on bit 5/6, and then performing the two
23248 shifts. The two methods of getting 0/1 into low/high are exactly
23249 the same size. Avoiding the shift in the bit extract case helps
23250 pentium4 a bit; no one else seems to care much either way. */
23253 machine_mode half_mode;
23254 rtx (*gen_lshr3)(rtx, rtx, rtx);
23255 rtx (*gen_and3)(rtx, rtx, rtx);
23256 rtx (*gen_xor3)(rtx, rtx, rtx);
23257 HOST_WIDE_INT bits;
23260 if (mode == DImode)
23262 half_mode = SImode;
23263 gen_lshr3 = gen_lshrsi3;
23264 gen_and3 = gen_andsi3;
23265 gen_xor3 = gen_xorsi3;
23270 half_mode = DImode;
23271 gen_lshr3 = gen_lshrdi3;
23272 gen_and3 = gen_anddi3;
23273 gen_xor3 = gen_xordi3;
23277 if (TARGET_PARTIAL_REG_STALL && !optimize_insn_for_size_p ())
23278 x = gen_rtx_ZERO_EXTEND (half_mode, operands[2]);
23280 x = gen_lowpart (half_mode, operands[2]);
23281 emit_insn (gen_rtx_SET (VOIDmode, high[0], x));
23283 emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (bits)));
23284 emit_insn (gen_and3 (high[0], high[0], const1_rtx));
23285 emit_move_insn (low[0], high[0]);
23286 emit_insn (gen_xor3 (low[0], low[0], const1_rtx));
23289 emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
23290 emit_insn (gen_ashl3 (high[0], high[0], operands[2]));
23294 if (operands[1] == constm1_rtx)
23296 /* For -1 << N, we can avoid the shld instruction, because we
23297 know that we're shifting 0...31/63 ones into a -1. */
23298 emit_move_insn (low[0], constm1_rtx);
23299 if (optimize_insn_for_size_p ())
23300 emit_move_insn (high[0], low[0]);
23302 emit_move_insn (high[0], constm1_rtx);
23306 gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
23308 if (!rtx_equal_p (operands[0], operands[1]))
23309 emit_move_insn (operands[0], operands[1]);
23311 split_double_mode (mode, operands, 1, low, high);
23312 emit_insn (gen_shld (high[0], low[0], operands[2]));
23315 emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
23317 if (TARGET_CMOVE && scratch)
23319 rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
23320 = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;
23322 ix86_expand_clear (scratch);
23323 emit_insn (gen_x86_shift_adj_1 (high[0], low[0], operands[2], scratch));
23327 rtx (*gen_x86_shift_adj_2)(rtx, rtx, rtx)
23328 = mode == DImode ? gen_x86_shiftsi_adj_2 : gen_x86_shiftdi_adj_2;
23330 emit_insn (gen_x86_shift_adj_2 (high[0], low[0], operands[2]));
23335 ix86_split_ashr (rtx *operands, rtx scratch, machine_mode mode)
23337 rtx (*gen_ashr3)(rtx, rtx, rtx)
23338 = mode == DImode ? gen_ashrsi3 : gen_ashrdi3;
23339 rtx (*gen_shrd)(rtx, rtx, rtx);
23340 int half_width = GET_MODE_BITSIZE (mode) >> 1;
23342 rtx low[2], high[2];
23345 if (CONST_INT_P (operands[2]))
23347 split_double_mode (mode, operands, 2, low, high);
23348 count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
23350 if (count == GET_MODE_BITSIZE (mode) - 1)
23352 emit_move_insn (high[0], high[1]);
23353 emit_insn (gen_ashr3 (high[0], high[0],
23354 GEN_INT (half_width - 1)));
23355 emit_move_insn (low[0], high[0]);
23358 else if (count >= half_width)
23360 emit_move_insn (low[0], high[1]);
23361 emit_move_insn (high[0], low[0]);
23362 emit_insn (gen_ashr3 (high[0], high[0],
23363 GEN_INT (half_width - 1)));
23365 if (count > half_width)
23366 emit_insn (gen_ashr3 (low[0], low[0],
23367 GEN_INT (count - half_width)));
23371 gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
23373 if (!rtx_equal_p (operands[0], operands[1]))
23374 emit_move_insn (operands[0], operands[1]);
23376 emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
23377 emit_insn (gen_ashr3 (high[0], high[0], GEN_INT (count)));
23382 gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
23384 if (!rtx_equal_p (operands[0], operands[1]))
23385 emit_move_insn (operands[0], operands[1]);
23387 split_double_mode (mode, operands, 1, low, high);
23389 emit_insn (gen_shrd (low[0], high[0], operands[2]));
23390 emit_insn (gen_ashr3 (high[0], high[0], operands[2]));
23392 if (TARGET_CMOVE && scratch)
23394 rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
23395 = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;
23397 emit_move_insn (scratch, high[0]);
23398 emit_insn (gen_ashr3 (scratch, scratch,
23399 GEN_INT (half_width - 1)));
23400 emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2],
23405 rtx (*gen_x86_shift_adj_3)(rtx, rtx, rtx)
23406 = mode == DImode ? gen_x86_shiftsi_adj_3 : gen_x86_shiftdi_adj_3;
23408 emit_insn (gen_x86_shift_adj_3 (low[0], high[0], operands[2]));
23414 ix86_split_lshr (rtx *operands, rtx scratch, machine_mode mode)
23416 rtx (*gen_lshr3)(rtx, rtx, rtx)
23417 = mode == DImode ? gen_lshrsi3 : gen_lshrdi3;
23418 rtx (*gen_shrd)(rtx, rtx, rtx);
23419 int half_width = GET_MODE_BITSIZE (mode) >> 1;
23421 rtx low[2], high[2];
23424 if (CONST_INT_P (operands[2]))
23426 split_double_mode (mode, operands, 2, low, high);
23427 count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
23429 if (count >= half_width)
23431 emit_move_insn (low[0], high[1]);
23432 ix86_expand_clear (high[0]);
23434 if (count > half_width)
23435 emit_insn (gen_lshr3 (low[0], low[0],
23436 GEN_INT (count - half_width)));
23440 gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
23442 if (!rtx_equal_p (operands[0], operands[1]))
23443 emit_move_insn (operands[0], operands[1]);
23445 emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
23446 emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (count)));
23451 gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
23453 if (!rtx_equal_p (operands[0], operands[1]))
23454 emit_move_insn (operands[0], operands[1]);
23456 split_double_mode (mode, operands, 1, low, high);
23458 emit_insn (gen_shrd (low[0], high[0], operands[2]));
23459 emit_insn (gen_lshr3 (high[0], high[0], operands[2]));
23461 if (TARGET_CMOVE && scratch)
23463 rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
23464 = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;
23466 ix86_expand_clear (scratch);
23467 emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2],
23472 rtx (*gen_x86_shift_adj_2)(rtx, rtx, rtx)
23473 = mode == DImode ? gen_x86_shiftsi_adj_2 : gen_x86_shiftdi_adj_2;
23475 emit_insn (gen_x86_shift_adj_2 (low[0], high[0], operands[2]));
23480 /* Predict just emitted jump instruction to be taken with probability PROB. */
23482 predict_jump (int prob)
23484 rtx insn = get_last_insn ();
23485 gcc_assert (JUMP_P (insn));
23486 add_int_reg_note (insn, REG_BR_PROB, prob);
23489 /* Helper function for the string operations below. Dest VARIABLE whether
23490 it is aligned to VALUE bytes. If true, jump to the label. */
23491 static rtx_code_label *
23492 ix86_expand_aligntest (rtx variable, int value, bool epilogue)
23494 rtx_code_label *label = gen_label_rtx ();
23495 rtx tmpcount = gen_reg_rtx (GET_MODE (variable));
23496 if (GET_MODE (variable) == DImode)
23497 emit_insn (gen_anddi3 (tmpcount, variable, GEN_INT (value)));
23499 emit_insn (gen_andsi3 (tmpcount, variable, GEN_INT (value)));
23500 emit_cmp_and_jump_insns (tmpcount, const0_rtx, EQ, 0, GET_MODE (variable),
23503 predict_jump (REG_BR_PROB_BASE * 50 / 100);
23505 predict_jump (REG_BR_PROB_BASE * 90 / 100);
23509 /* Adjust COUNTER by the VALUE. */
23511 ix86_adjust_counter (rtx countreg, HOST_WIDE_INT value)
23513 rtx (*gen_add)(rtx, rtx, rtx)
23514 = GET_MODE (countreg) == DImode ? gen_adddi3 : gen_addsi3;
23516 emit_insn (gen_add (countreg, countreg, GEN_INT (-value)));
23519 /* Zero extend possibly SImode EXP to Pmode register. */
23521 ix86_zero_extend_to_Pmode (rtx exp)
23523 return force_reg (Pmode, convert_to_mode (Pmode, exp, 1));
23526 /* Divide COUNTREG by SCALE. */
23528 scale_counter (rtx countreg, int scale)
23534 if (CONST_INT_P (countreg))
23535 return GEN_INT (INTVAL (countreg) / scale);
23536 gcc_assert (REG_P (countreg));
23538 sc = expand_simple_binop (GET_MODE (countreg), LSHIFTRT, countreg,
23539 GEN_INT (exact_log2 (scale)),
23540 NULL, 1, OPTAB_DIRECT);
23544 /* Return mode for the memcpy/memset loop counter. Prefer SImode over
23545 DImode for constant loop counts. */
23547 static machine_mode
23548 counter_mode (rtx count_exp)
23550 if (GET_MODE (count_exp) != VOIDmode)
23551 return GET_MODE (count_exp);
23552 if (!CONST_INT_P (count_exp))
23554 if (TARGET_64BIT && (INTVAL (count_exp) & ~0xffffffff))
23559 /* Copy the address to a Pmode register. This is used for x32 to
23560 truncate DImode TLS address to a SImode register. */
23563 ix86_copy_addr_to_reg (rtx addr)
23566 if (GET_MODE (addr) == Pmode || GET_MODE (addr) == VOIDmode)
23568 reg = copy_addr_to_reg (addr);
23569 REG_POINTER (reg) = 1;
23574 gcc_assert (GET_MODE (addr) == DImode && Pmode == SImode);
23575 reg = copy_to_mode_reg (DImode, addr);
23576 REG_POINTER (reg) = 1;
23577 return gen_rtx_SUBREG (SImode, reg, 0);
23581 /* When ISSETMEM is FALSE, output simple loop to move memory pointer to SRCPTR
23582 to DESTPTR via chunks of MODE unrolled UNROLL times, overall size is COUNT
23583 specified in bytes. When ISSETMEM is TRUE, output the equivalent loop to set
23584 memory by VALUE (supposed to be in MODE).
23586 The size is rounded down to whole number of chunk size moved at once.
23587 SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
23591 expand_set_or_movmem_via_loop (rtx destmem, rtx srcmem,
23592 rtx destptr, rtx srcptr, rtx value,
23593 rtx count, machine_mode mode, int unroll,
23594 int expected_size, bool issetmem)
23596 rtx_code_label *out_label, *top_label;
23598 machine_mode iter_mode = counter_mode (count);
23599 int piece_size_n = GET_MODE_SIZE (mode) * unroll;
23600 rtx piece_size = GEN_INT (piece_size_n);
23601 rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
23605 top_label = gen_label_rtx ();
23606 out_label = gen_label_rtx ();
23607 iter = gen_reg_rtx (iter_mode);
23609 size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
23610 NULL, 1, OPTAB_DIRECT);
23611 /* Those two should combine. */
23612 if (piece_size == const1_rtx)
23614 emit_cmp_and_jump_insns (size, const0_rtx, EQ, NULL_RTX, iter_mode,
23616 predict_jump (REG_BR_PROB_BASE * 10 / 100);
23618 emit_move_insn (iter, const0_rtx);
23620 emit_label (top_label);
23622 tmp = convert_modes (Pmode, iter_mode, iter, true);
23624 /* This assert could be relaxed - in this case we'll need to compute
23625 smallest power of two, containing in PIECE_SIZE_N and pass it to
23627 gcc_assert ((piece_size_n & (piece_size_n - 1)) == 0);
23628 destmem = offset_address (destmem, tmp, piece_size_n);
23629 destmem = adjust_address (destmem, mode, 0);
23633 srcmem = offset_address (srcmem, copy_rtx (tmp), piece_size_n);
23634 srcmem = adjust_address (srcmem, mode, 0);
23636 /* When unrolling for chips that reorder memory reads and writes,
23637 we can save registers by using single temporary.
23638 Also using 4 temporaries is overkill in 32bit mode. */
23639 if (!TARGET_64BIT && 0)
23641 for (i = 0; i < unroll; i++)
23646 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
23648 adjust_address (copy_rtx (srcmem), mode, GET_MODE_SIZE (mode));
23650 emit_move_insn (destmem, srcmem);
23656 gcc_assert (unroll <= 4);
23657 for (i = 0; i < unroll; i++)
23659 tmpreg[i] = gen_reg_rtx (mode);
23663 adjust_address (copy_rtx (srcmem), mode, GET_MODE_SIZE (mode));
23665 emit_move_insn (tmpreg[i], srcmem);
23667 for (i = 0; i < unroll; i++)
23672 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
23674 emit_move_insn (destmem, tmpreg[i]);
23679 for (i = 0; i < unroll; i++)
23683 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
23684 emit_move_insn (destmem, value);
23687 tmp = expand_simple_binop (iter_mode, PLUS, iter, piece_size, iter,
23688 true, OPTAB_LIB_WIDEN);
23690 emit_move_insn (iter, tmp);
23692 emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
23694 if (expected_size != -1)
23696 expected_size /= GET_MODE_SIZE (mode) * unroll;
23697 if (expected_size == 0)
23699 else if (expected_size > REG_BR_PROB_BASE)
23700 predict_jump (REG_BR_PROB_BASE - 1);
23702 predict_jump (REG_BR_PROB_BASE - (REG_BR_PROB_BASE + expected_size / 2) / expected_size);
23705 predict_jump (REG_BR_PROB_BASE * 80 / 100);
23706 iter = ix86_zero_extend_to_Pmode (iter);
23707 tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
23708 true, OPTAB_LIB_WIDEN);
23709 if (tmp != destptr)
23710 emit_move_insn (destptr, tmp);
23713 tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
23714 true, OPTAB_LIB_WIDEN);
23716 emit_move_insn (srcptr, tmp);
23718 emit_label (out_label);
23721 /* Output "rep; mov" or "rep; stos" instruction depending on ISSETMEM argument.
23722 When ISSETMEM is true, arguments SRCMEM and SRCPTR are ignored.
23723 When ISSETMEM is false, arguments VALUE and ORIG_VALUE are ignored.
23724 For setmem case, VALUE is a promoted to a wider size ORIG_VALUE.
23725 ORIG_VALUE is the original value passed to memset to fill the memory with.
23726 Other arguments have same meaning as for previous function. */
23729 expand_set_or_movmem_via_rep (rtx destmem, rtx srcmem,
23730 rtx destptr, rtx srcptr, rtx value, rtx orig_value,
23732 machine_mode mode, bool issetmem)
23737 HOST_WIDE_INT rounded_count;
23739 /* If possible, it is shorter to use rep movs.
23740 TODO: Maybe it is better to move this logic to decide_alg. */
23741 if (mode == QImode && CONST_INT_P (count) && !(INTVAL (count) & 3)
23742 && (!issetmem || orig_value == const0_rtx))
23745 if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
23746 destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
23748 countreg = ix86_zero_extend_to_Pmode (scale_counter (count,
23749 GET_MODE_SIZE (mode)));
23750 if (mode != QImode)
23752 destexp = gen_rtx_ASHIFT (Pmode, countreg,
23753 GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
23754 destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
23757 destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
23758 if ((!issetmem || orig_value == const0_rtx) && CONST_INT_P (count))
23760 rounded_count = (INTVAL (count)
23761 & ~((HOST_WIDE_INT) GET_MODE_SIZE (mode) - 1));
23762 destmem = shallow_copy_rtx (destmem);
23763 set_mem_size (destmem, rounded_count);
23765 else if (MEM_SIZE_KNOWN_P (destmem))
23766 clear_mem_size (destmem);
23770 value = force_reg (mode, gen_lowpart (mode, value));
23771 emit_insn (gen_rep_stos (destptr, countreg, destmem, value, destexp));
23775 if (srcptr != XEXP (srcmem, 0) || GET_MODE (srcmem) != BLKmode)
23776 srcmem = adjust_automodify_address_nv (srcmem, BLKmode, srcptr, 0);
23777 if (mode != QImode)
23779 srcexp = gen_rtx_ASHIFT (Pmode, countreg,
23780 GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
23781 srcexp = gen_rtx_PLUS (Pmode, srcexp, srcptr);
23784 srcexp = gen_rtx_PLUS (Pmode, srcptr, countreg);
23785 if (CONST_INT_P (count))
23787 rounded_count = (INTVAL (count)
23788 & ~((HOST_WIDE_INT) GET_MODE_SIZE (mode) - 1));
23789 srcmem = shallow_copy_rtx (srcmem);
23790 set_mem_size (srcmem, rounded_count);
23794 if (MEM_SIZE_KNOWN_P (srcmem))
23795 clear_mem_size (srcmem);
23797 emit_insn (gen_rep_mov (destptr, destmem, srcptr, srcmem, countreg,
23802 /* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
23804 SRC is passed by pointer to be updated on return.
23805 Return value is updated DST. */
23807 emit_memmov (rtx destmem, rtx *srcmem, rtx destptr, rtx srcptr,
23808 HOST_WIDE_INT size_to_move)
23810 rtx dst = destmem, src = *srcmem, adjust, tempreg;
23811 enum insn_code code;
23812 machine_mode move_mode;
23815 /* Find the widest mode in which we could perform moves.
23816 Start with the biggest power of 2 less than SIZE_TO_MOVE and half
23817 it until move of such size is supported. */
23818 piece_size = 1 << floor_log2 (size_to_move);
23819 move_mode = mode_for_size (piece_size * BITS_PER_UNIT, MODE_INT, 0);
23820 code = optab_handler (mov_optab, move_mode);
23821 while (code == CODE_FOR_nothing && piece_size > 1)
23824 move_mode = mode_for_size (piece_size * BITS_PER_UNIT, MODE_INT, 0);
23825 code = optab_handler (mov_optab, move_mode);
23828 /* Find the corresponding vector mode with the same size as MOVE_MODE.
23829 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
23830 if (GET_MODE_SIZE (move_mode) > GET_MODE_SIZE (word_mode))
23832 int nunits = GET_MODE_SIZE (move_mode) / GET_MODE_SIZE (word_mode);
23833 move_mode = mode_for_vector (word_mode, nunits);
23834 code = optab_handler (mov_optab, move_mode);
23835 if (code == CODE_FOR_nothing)
23837 move_mode = word_mode;
23838 piece_size = GET_MODE_SIZE (move_mode);
23839 code = optab_handler (mov_optab, move_mode);
23842 gcc_assert (code != CODE_FOR_nothing);
23844 dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
23845 src = adjust_automodify_address_nv (src, move_mode, srcptr, 0);
23847 /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
23848 gcc_assert (size_to_move % piece_size == 0);
23849 adjust = GEN_INT (piece_size);
23850 for (i = 0; i < size_to_move; i += piece_size)
23852 /* We move from memory to memory, so we'll need to do it via
23853 a temporary register. */
23854 tempreg = gen_reg_rtx (move_mode);
23855 emit_insn (GEN_FCN (code) (tempreg, src));
23856 emit_insn (GEN_FCN (code) (dst, tempreg));
23858 emit_move_insn (destptr,
23859 gen_rtx_PLUS (Pmode, copy_rtx (destptr), adjust));
23860 emit_move_insn (srcptr,
23861 gen_rtx_PLUS (Pmode, copy_rtx (srcptr), adjust));
23863 dst = adjust_automodify_address_nv (dst, move_mode, destptr,
23865 src = adjust_automodify_address_nv (src, move_mode, srcptr,
23869 /* Update DST and SRC rtx. */
23874 /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
23876 expand_movmem_epilogue (rtx destmem, rtx srcmem,
23877 rtx destptr, rtx srcptr, rtx count, int max_size)
23880 if (CONST_INT_P (count))
23882 HOST_WIDE_INT countval = INTVAL (count);
23883 HOST_WIDE_INT epilogue_size = countval % max_size;
23886 /* For now MAX_SIZE should be a power of 2. This assert could be
23887 relaxed, but it'll require a bit more complicated epilogue
23889 gcc_assert ((max_size & (max_size - 1)) == 0);
23890 for (i = max_size; i >= 1; i >>= 1)
23892 if (epilogue_size & i)
23893 destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
23899 count = expand_simple_binop (GET_MODE (count), AND, count, GEN_INT (max_size - 1),
23900 count, 1, OPTAB_DIRECT);
23901 expand_set_or_movmem_via_loop (destmem, srcmem, destptr, srcptr, NULL,
23902 count, QImode, 1, 4, false);
23906 /* When there are stringops, we can cheaply increase dest and src pointers.
23907 Otherwise we save code size by maintaining offset (zero is readily
23908 available from preceding rep operation) and using x86 addressing modes.
23910 if (TARGET_SINGLE_STRINGOP)
23914 rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
23915 src = change_address (srcmem, SImode, srcptr);
23916 dest = change_address (destmem, SImode, destptr);
23917 emit_insn (gen_strmov (destptr, dest, srcptr, src));
23918 emit_label (label);
23919 LABEL_NUSES (label) = 1;
23923 rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
23924 src = change_address (srcmem, HImode, srcptr);
23925 dest = change_address (destmem, HImode, destptr);
23926 emit_insn (gen_strmov (destptr, dest, srcptr, src));
23927 emit_label (label);
23928 LABEL_NUSES (label) = 1;
23932 rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
23933 src = change_address (srcmem, QImode, srcptr);
23934 dest = change_address (destmem, QImode, destptr);
23935 emit_insn (gen_strmov (destptr, dest, srcptr, src));
23936 emit_label (label);
23937 LABEL_NUSES (label) = 1;
23942 rtx offset = force_reg (Pmode, const0_rtx);
23947 rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
23948 src = change_address (srcmem, SImode, srcptr);
23949 dest = change_address (destmem, SImode, destptr);
23950 emit_move_insn (dest, src);
23951 tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (4), NULL,
23952 true, OPTAB_LIB_WIDEN);
23954 emit_move_insn (offset, tmp);
23955 emit_label (label);
23956 LABEL_NUSES (label) = 1;
23960 rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
23961 tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
23962 src = change_address (srcmem, HImode, tmp);
23963 tmp = gen_rtx_PLUS (Pmode, destptr, offset);
23964 dest = change_address (destmem, HImode, tmp);
23965 emit_move_insn (dest, src);
23966 tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (2), tmp,
23967 true, OPTAB_LIB_WIDEN);
23969 emit_move_insn (offset, tmp);
23970 emit_label (label);
23971 LABEL_NUSES (label) = 1;
23975 rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
23976 tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
23977 src = change_address (srcmem, QImode, tmp);
23978 tmp = gen_rtx_PLUS (Pmode, destptr, offset);
23979 dest = change_address (destmem, QImode, tmp);
23980 emit_move_insn (dest, src);
23981 emit_label (label);
23982 LABEL_NUSES (label) = 1;
23987 /* This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM
23988 with value PROMOTED_VAL.
23989 SRC is passed by pointer to be updated on return.
23990 Return value is updated DST. */
23992 emit_memset (rtx destmem, rtx destptr, rtx promoted_val,
23993 HOST_WIDE_INT size_to_move)
23995 rtx dst = destmem, adjust;
23996 enum insn_code code;
23997 machine_mode move_mode;
24000 /* Find the widest mode in which we could perform moves.
24001 Start with the biggest power of 2 less than SIZE_TO_MOVE and half
24002 it until move of such size is supported. */
24003 move_mode = GET_MODE (promoted_val);
24004 if (move_mode == VOIDmode)
24005 move_mode = QImode;
24006 if (size_to_move < GET_MODE_SIZE (move_mode))
24008 move_mode = mode_for_size (size_to_move * BITS_PER_UNIT, MODE_INT, 0);
24009 promoted_val = gen_lowpart (move_mode, promoted_val);
24011 piece_size = GET_MODE_SIZE (move_mode);
24012 code = optab_handler (mov_optab, move_mode);
24013 gcc_assert (code != CODE_FOR_nothing && promoted_val != NULL_RTX);
24015 dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
24017 /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
24018 gcc_assert (size_to_move % piece_size == 0);
24019 adjust = GEN_INT (piece_size);
24020 for (i = 0; i < size_to_move; i += piece_size)
24022 if (piece_size <= GET_MODE_SIZE (word_mode))
24024 emit_insn (gen_strset (destptr, dst, promoted_val));
24025 dst = adjust_automodify_address_nv (dst, move_mode, destptr,
24030 emit_insn (GEN_FCN (code) (dst, promoted_val));
24032 emit_move_insn (destptr,
24033 gen_rtx_PLUS (Pmode, copy_rtx (destptr), adjust));
24035 dst = adjust_automodify_address_nv (dst, move_mode, destptr,
24039 /* Update DST rtx. */
24042 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
24044 expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
24045 rtx count, int max_size)
24048 expand_simple_binop (counter_mode (count), AND, count,
24049 GEN_INT (max_size - 1), count, 1, OPTAB_DIRECT);
24050 expand_set_or_movmem_via_loop (destmem, NULL, destptr, NULL,
24051 gen_lowpart (QImode, value), count, QImode,
24052 1, max_size / 2, true);
24055 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
24057 expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
24058 rtx count, int max_size)
24062 if (CONST_INT_P (count))
24064 HOST_WIDE_INT countval = INTVAL (count);
24065 HOST_WIDE_INT epilogue_size = countval % max_size;
24068 /* For now MAX_SIZE should be a power of 2. This assert could be
24069 relaxed, but it'll require a bit more complicated epilogue
24071 gcc_assert ((max_size & (max_size - 1)) == 0);
24072 for (i = max_size; i >= 1; i >>= 1)
24074 if (epilogue_size & i)
24076 if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
24077 destmem = emit_memset (destmem, destptr, vec_value, i);
24079 destmem = emit_memset (destmem, destptr, value, i);
24086 expand_setmem_epilogue_via_loop (destmem, destptr, value, count, max_size);
24091 rtx_code_label *label = ix86_expand_aligntest (count, 16, true);
24094 dest = change_address (destmem, DImode, destptr);
24095 emit_insn (gen_strset (destptr, dest, value));
24096 dest = adjust_automodify_address_nv (dest, DImode, destptr, 8);
24097 emit_insn (gen_strset (destptr, dest, value));
24101 dest = change_address (destmem, SImode, destptr);
24102 emit_insn (gen_strset (destptr, dest, value));
24103 dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
24104 emit_insn (gen_strset (destptr, dest, value));
24105 dest = adjust_automodify_address_nv (dest, SImode, destptr, 8);
24106 emit_insn (gen_strset (destptr, dest, value));
24107 dest = adjust_automodify_address_nv (dest, SImode, destptr, 12);
24108 emit_insn (gen_strset (destptr, dest, value));
24110 emit_label (label);
24111 LABEL_NUSES (label) = 1;
24115 rtx_code_label *label = ix86_expand_aligntest (count, 8, true);
24118 dest = change_address (destmem, DImode, destptr);
24119 emit_insn (gen_strset (destptr, dest, value));
24123 dest = change_address (destmem, SImode, destptr);
24124 emit_insn (gen_strset (destptr, dest, value));
24125 dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
24126 emit_insn (gen_strset (destptr, dest, value));
24128 emit_label (label);
24129 LABEL_NUSES (label) = 1;
24133 rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
24134 dest = change_address (destmem, SImode, destptr);
24135 emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
24136 emit_label (label);
24137 LABEL_NUSES (label) = 1;
24141 rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
24142 dest = change_address (destmem, HImode, destptr);
24143 emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
24144 emit_label (label);
24145 LABEL_NUSES (label) = 1;
24149 rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
24150 dest = change_address (destmem, QImode, destptr);
24151 emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
24152 emit_label (label);
24153 LABEL_NUSES (label) = 1;
24157 /* Depending on ISSETMEM, copy enough from SRCMEM to DESTMEM or set enough to
24158 DESTMEM to align it to DESIRED_ALIGNMENT. Original alignment is ALIGN.
24159 Depending on ISSETMEM, either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are
24161 Return value is updated DESTMEM. */
24163 expand_set_or_movmem_prologue (rtx destmem, rtx srcmem,
24164 rtx destptr, rtx srcptr, rtx value,
24165 rtx vec_value, rtx count, int align,
24166 int desired_alignment, bool issetmem)
24169 for (i = 1; i < desired_alignment; i <<= 1)
24173 rtx_code_label *label = ix86_expand_aligntest (destptr, i, false);
24176 if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
24177 destmem = emit_memset (destmem, destptr, vec_value, i);
24179 destmem = emit_memset (destmem, destptr, value, i);
24182 destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
24183 ix86_adjust_counter (count, i);
24184 emit_label (label);
24185 LABEL_NUSES (label) = 1;
24186 set_mem_align (destmem, i * 2 * BITS_PER_UNIT);
24192 /* Test if COUNT&SIZE is nonzero and if so, expand movme
24193 or setmem sequence that is valid for SIZE..2*SIZE-1 bytes
24194 and jump to DONE_LABEL. */
24196 expand_small_movmem_or_setmem (rtx destmem, rtx srcmem,
24197 rtx destptr, rtx srcptr,
24198 rtx value, rtx vec_value,
24199 rtx count, int size,
24200 rtx done_label, bool issetmem)
24202 rtx_code_label *label = ix86_expand_aligntest (count, size, false);
24203 machine_mode mode = mode_for_size (size * BITS_PER_UNIT, MODE_INT, 1);
24207 /* If we do not have vector value to copy, we must reduce size. */
24212 if (GET_MODE (value) == VOIDmode && size > 8)
24214 else if (GET_MODE_SIZE (mode) > GET_MODE_SIZE (GET_MODE (value)))
24215 mode = GET_MODE (value);
24218 mode = GET_MODE (vec_value), value = vec_value;
24222 /* Choose appropriate vector mode. */
24224 mode = TARGET_AVX ? V32QImode : TARGET_SSE ? V16QImode : DImode;
24225 else if (size >= 16)
24226 mode = TARGET_SSE ? V16QImode : DImode;
24227 srcmem = change_address (srcmem, mode, srcptr);
24229 destmem = change_address (destmem, mode, destptr);
24230 modesize = GEN_INT (GET_MODE_SIZE (mode));
24231 gcc_assert (GET_MODE_SIZE (mode) <= size);
24232 for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
24235 emit_move_insn (destmem, gen_lowpart (mode, value));
24238 emit_move_insn (destmem, srcmem);
24239 srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
24241 destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
24244 destmem = offset_address (destmem, count, 1);
24245 destmem = offset_address (destmem, GEN_INT (-2 * size),
24246 GET_MODE_SIZE (mode));
24249 srcmem = offset_address (srcmem, count, 1);
24250 srcmem = offset_address (srcmem, GEN_INT (-2 * size),
24251 GET_MODE_SIZE (mode));
24253 for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
24256 emit_move_insn (destmem, gen_lowpart (mode, value));
24259 emit_move_insn (destmem, srcmem);
24260 srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
24262 destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
24264 emit_jump_insn (gen_jump (done_label));
24267 emit_label (label);
24268 LABEL_NUSES (label) = 1;
24271 /* Handle small memcpy (up to SIZE that is supposed to be small power of 2.
24272 and get ready for the main memcpy loop by copying iniital DESIRED_ALIGN-ALIGN
24273 bytes and last SIZE bytes adjusitng DESTPTR/SRCPTR/COUNT in a way we can
24274 proceed with an loop copying SIZE bytes at once. Do moves in MODE.
24275 DONE_LABEL is a label after the whole copying sequence. The label is created
24276 on demand if *DONE_LABEL is NULL.
24277 MIN_SIZE is minimal size of block copied. This value gets adjusted for new
24278 bounds after the initial copies.
24280 DESTMEM/SRCMEM are memory expressions pointing to the copies block,
24281 DESTPTR/SRCPTR are pointers to the block. DYNAMIC_CHECK indicate whether
24282 we will dispatch to a library call for large blocks.
24284 In pseudocode we do:
24288 Assume that SIZE is 4. Bigger sizes are handled analogously
24291 copy 4 bytes from SRCPTR to DESTPTR
24292 copy 4 bytes from SRCPTR + COUNT - 4 to DESTPTR + COUNT - 4
24297 copy 1 byte from SRCPTR to DESTPTR
24300 copy 2 bytes from SRCPTR to DESTPTR
24301 copy 2 bytes from SRCPTR + COUNT - 2 to DESTPTR + COUNT - 2
24306 copy at least DESIRED_ALIGN-ALIGN bytes from SRCPTR to DESTPTR
24307 copy SIZE bytes from SRCPTR + COUNT - SIZE to DESTPTR + COUNT -SIZE
24309 OLD_DESPTR = DESTPTR;
24310 Align DESTPTR up to DESIRED_ALIGN
24311 SRCPTR += DESTPTR - OLD_DESTPTR
24312 COUNT -= DEST_PTR - OLD_DESTPTR
24314 Round COUNT down to multiple of SIZE
24315 << optional caller supplied zero size guard is here >>
24316 << optional caller suppplied dynamic check is here >>
24317 << caller supplied main copy loop is here >>
24322 expand_set_or_movmem_prologue_epilogue_by_misaligned_moves (rtx destmem, rtx srcmem,
24323 rtx *destptr, rtx *srcptr,
24325 rtx value, rtx vec_value,
24327 rtx_code_label **done_label,
24331 unsigned HOST_WIDE_INT *min_size,
24332 bool dynamic_check,
24335 rtx_code_label *loop_label = NULL, *label;
24338 int prolog_size = 0;
24341 /* Chose proper value to copy. */
24342 if (issetmem && VECTOR_MODE_P (mode))
24343 mode_value = vec_value;
24345 mode_value = value;
24346 gcc_assert (GET_MODE_SIZE (mode) <= size);
24348 /* See if block is big or small, handle small blocks. */
24349 if (!CONST_INT_P (*count) && *min_size < (unsigned HOST_WIDE_INT)size)
24352 loop_label = gen_label_rtx ();
24355 *done_label = gen_label_rtx ();
24357 emit_cmp_and_jump_insns (*count, GEN_INT (size2), GE, 0, GET_MODE (*count),
24361 /* Handle sizes > 3. */
24362 for (;size2 > 2; size2 >>= 1)
24363 expand_small_movmem_or_setmem (destmem, srcmem,
24367 size2, *done_label, issetmem);
24368 /* Nothing to copy? Jump to DONE_LABEL if so */
24369 emit_cmp_and_jump_insns (*count, const0_rtx, EQ, 0, GET_MODE (*count),
24372 /* Do a byte copy. */
24373 destmem = change_address (destmem, QImode, *destptr);
24375 emit_move_insn (destmem, gen_lowpart (QImode, value));
24378 srcmem = change_address (srcmem, QImode, *srcptr);
24379 emit_move_insn (destmem, srcmem);
24382 /* Handle sizes 2 and 3. */
24383 label = ix86_expand_aligntest (*count, 2, false);
24384 destmem = change_address (destmem, HImode, *destptr);
24385 destmem = offset_address (destmem, *count, 1);
24386 destmem = offset_address (destmem, GEN_INT (-2), 2);
24388 emit_move_insn (destmem, gen_lowpart (HImode, value));
24391 srcmem = change_address (srcmem, HImode, *srcptr);
24392 srcmem = offset_address (srcmem, *count, 1);
24393 srcmem = offset_address (srcmem, GEN_INT (-2), 2);
24394 emit_move_insn (destmem, srcmem);
24397 emit_label (label);
24398 LABEL_NUSES (label) = 1;
24399 emit_jump_insn (gen_jump (*done_label));
24403 gcc_assert (*min_size >= (unsigned HOST_WIDE_INT)size
24404 || UINTVAL (*count) >= (unsigned HOST_WIDE_INT)size);
24406 /* Start memcpy for COUNT >= SIZE. */
24409 emit_label (loop_label);
24410 LABEL_NUSES (loop_label) = 1;
24413 /* Copy first desired_align bytes. */
24415 srcmem = change_address (srcmem, mode, *srcptr);
24416 destmem = change_address (destmem, mode, *destptr);
24417 modesize = GEN_INT (GET_MODE_SIZE (mode));
24418 for (n = 0; prolog_size < desired_align - align; n++)
24421 emit_move_insn (destmem, mode_value);
24424 emit_move_insn (destmem, srcmem);
24425 srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
24427 destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
24428 prolog_size += GET_MODE_SIZE (mode);
24432 /* Copy last SIZE bytes. */
24433 destmem = offset_address (destmem, *count, 1);
24434 destmem = offset_address (destmem,
24435 GEN_INT (-size - prolog_size),
24438 emit_move_insn (destmem, mode_value);
24441 srcmem = offset_address (srcmem, *count, 1);
24442 srcmem = offset_address (srcmem,
24443 GEN_INT (-size - prolog_size),
24445 emit_move_insn (destmem, srcmem);
24447 for (n = 1; n * GET_MODE_SIZE (mode) < size; n++)
24449 destmem = offset_address (destmem, modesize, 1);
24451 emit_move_insn (destmem, mode_value);
24454 srcmem = offset_address (srcmem, modesize, 1);
24455 emit_move_insn (destmem, srcmem);
24459 /* Align destination. */
24460 if (desired_align > 1 && desired_align > align)
24462 rtx saveddest = *destptr;
24464 gcc_assert (desired_align <= size);
24465 /* Align destptr up, place it to new register. */
24466 *destptr = expand_simple_binop (GET_MODE (*destptr), PLUS, *destptr,
24467 GEN_INT (prolog_size),
24468 NULL_RTX, 1, OPTAB_DIRECT);
24469 if (REG_P (*destptr) && REG_P (saveddest) && REG_POINTER (saveddest))
24470 REG_POINTER (*destptr) = 1;
24471 *destptr = expand_simple_binop (GET_MODE (*destptr), AND, *destptr,
24472 GEN_INT (-desired_align),
24473 *destptr, 1, OPTAB_DIRECT);
24474 /* See how many bytes we skipped. */
24475 saveddest = expand_simple_binop (GET_MODE (*destptr), MINUS, saveddest,
24477 saveddest, 1, OPTAB_DIRECT);
24478 /* Adjust srcptr and count. */
24480 *srcptr = expand_simple_binop (GET_MODE (*srcptr), MINUS, *srcptr,
24481 saveddest, *srcptr, 1, OPTAB_DIRECT);
24482 *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
24483 saveddest, *count, 1, OPTAB_DIRECT);
24484 /* We copied at most size + prolog_size. */
24485 if (*min_size > (unsigned HOST_WIDE_INT)(size + prolog_size))
24486 *min_size = (*min_size - size) & ~(unsigned HOST_WIDE_INT)(size - 1);
24490 /* Our loops always round down the bock size, but for dispatch to library
24491 we need precise value. */
24493 *count = expand_simple_binop (GET_MODE (*count), AND, *count,
24494 GEN_INT (-size), *count, 1, OPTAB_DIRECT);
24498 gcc_assert (prolog_size == 0);
24499 /* Decrease count, so we won't end up copying last word twice. */
24500 if (!CONST_INT_P (*count))
24501 *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
24502 constm1_rtx, *count, 1, OPTAB_DIRECT);
24504 *count = GEN_INT ((UINTVAL (*count) - 1) & ~(unsigned HOST_WIDE_INT)(size - 1));
24506 *min_size = (*min_size - 1) & ~(unsigned HOST_WIDE_INT)(size - 1);
24511 /* This function is like the previous one, except here we know how many bytes
24512 need to be copied. That allows us to update alignment not only of DST, which
24513 is returned, but also of SRC, which is passed as a pointer for that
24516 expand_set_or_movmem_constant_prologue (rtx dst, rtx *srcp, rtx destreg,
24517 rtx srcreg, rtx value, rtx vec_value,
24518 int desired_align, int align_bytes,
24522 rtx orig_dst = dst;
24523 rtx orig_src = NULL;
24524 int piece_size = 1;
24525 int copied_bytes = 0;
24529 gcc_assert (srcp != NULL);
24534 for (piece_size = 1;
24535 piece_size <= desired_align && copied_bytes < align_bytes;
24538 if (align_bytes & piece_size)
24542 if (vec_value && piece_size > GET_MODE_SIZE (GET_MODE (value)))
24543 dst = emit_memset (dst, destreg, vec_value, piece_size);
24545 dst = emit_memset (dst, destreg, value, piece_size);
24548 dst = emit_memmov (dst, &src, destreg, srcreg, piece_size);
24549 copied_bytes += piece_size;
24552 if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
24553 set_mem_align (dst, desired_align * BITS_PER_UNIT);
24554 if (MEM_SIZE_KNOWN_P (orig_dst))
24555 set_mem_size (dst, MEM_SIZE (orig_dst) - align_bytes);
24559 int src_align_bytes = get_mem_align_offset (src, desired_align
24561 if (src_align_bytes >= 0)
24562 src_align_bytes = desired_align - src_align_bytes;
24563 if (src_align_bytes >= 0)
24565 unsigned int src_align;
24566 for (src_align = desired_align; src_align >= 2; src_align >>= 1)
24568 if ((src_align_bytes & (src_align - 1))
24569 == (align_bytes & (src_align - 1)))
24572 if (src_align > (unsigned int) desired_align)
24573 src_align = desired_align;
24574 if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
24575 set_mem_align (src, src_align * BITS_PER_UNIT);
24577 if (MEM_SIZE_KNOWN_P (orig_src))
24578 set_mem_size (src, MEM_SIZE (orig_src) - align_bytes);
24585 /* Return true if ALG can be used in current context.
24586 Assume we expand memset if MEMSET is true. */
24588 alg_usable_p (enum stringop_alg alg, bool memset)
24590 if (alg == no_stringop)
24592 if (alg == vector_loop)
24593 return TARGET_SSE || TARGET_AVX;
24594 /* Algorithms using the rep prefix want at least edi and ecx;
24595 additionally, memset wants eax and memcpy wants esi. Don't
24596 consider such algorithms if the user has appropriated those
24597 registers for their own purposes. */
24598 if (alg == rep_prefix_1_byte
24599 || alg == rep_prefix_4_byte
24600 || alg == rep_prefix_8_byte)
24601 return !(fixed_regs[CX_REG] || fixed_regs[DI_REG]
24602 || (memset ? fixed_regs[AX_REG] : fixed_regs[SI_REG]));
24606 /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
24607 static enum stringop_alg
24608 decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size,
24609 unsigned HOST_WIDE_INT min_size, unsigned HOST_WIDE_INT max_size,
24610 bool memset, bool zero_memset, int *dynamic_check, bool *noalign)
24612 const struct stringop_algs * algs;
24613 bool optimize_for_speed;
24615 const struct processor_costs *cost;
24617 bool any_alg_usable_p = false;
24620 *dynamic_check = -1;
24622 /* Even if the string operation call is cold, we still might spend a lot
24623 of time processing large blocks. */
24624 if (optimize_function_for_size_p (cfun)
24625 || (optimize_insn_for_size_p ()
24627 || (expected_size != -1 && expected_size < 256))))
24628 optimize_for_speed = false;
24630 optimize_for_speed = true;
24632 cost = optimize_for_speed ? ix86_cost : &ix86_size_cost;
24634 algs = &cost->memset[TARGET_64BIT != 0];
24636 algs = &cost->memcpy[TARGET_64BIT != 0];
24638 /* See maximal size for user defined algorithm. */
24639 for (i = 0; i < MAX_STRINGOP_ALGS; i++)
24641 enum stringop_alg candidate = algs->size[i].alg;
24642 bool usable = alg_usable_p (candidate, memset);
24643 any_alg_usable_p |= usable;
24645 if (candidate != libcall && candidate && usable)
24646 max = algs->size[i].max;
24649 /* If expected size is not known but max size is small enough
24650 so inline version is a win, set expected size into
24652 if (((max > 1 && (unsigned HOST_WIDE_INT) max >= max_size) || max == -1)
24653 && expected_size == -1)
24654 expected_size = min_size / 2 + max_size / 2;
24656 /* If user specified the algorithm, honnor it if possible. */
24657 if (ix86_stringop_alg != no_stringop
24658 && alg_usable_p (ix86_stringop_alg, memset))
24659 return ix86_stringop_alg;
24660 /* rep; movq or rep; movl is the smallest variant. */
24661 else if (!optimize_for_speed)
24664 if (!count || (count & 3) || (memset && !zero_memset))
24665 return alg_usable_p (rep_prefix_1_byte, memset)
24666 ? rep_prefix_1_byte : loop_1_byte;
24668 return alg_usable_p (rep_prefix_4_byte, memset)
24669 ? rep_prefix_4_byte : loop;
24671 /* Very tiny blocks are best handled via the loop, REP is expensive to
24673 else if (expected_size != -1 && expected_size < 4)
24674 return loop_1_byte;
24675 else if (expected_size != -1)
24677 enum stringop_alg alg = libcall;
24678 bool alg_noalign = false;
24679 for (i = 0; i < MAX_STRINGOP_ALGS; i++)
24681 /* We get here if the algorithms that were not libcall-based
24682 were rep-prefix based and we are unable to use rep prefixes
24683 based on global register usage. Break out of the loop and
24684 use the heuristic below. */
24685 if (algs->size[i].max == 0)
24687 if (algs->size[i].max >= expected_size || algs->size[i].max == -1)
24689 enum stringop_alg candidate = algs->size[i].alg;
24691 if (candidate != libcall && alg_usable_p (candidate, memset))
24694 alg_noalign = algs->size[i].noalign;
24696 /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
24697 last non-libcall inline algorithm. */
24698 if (TARGET_INLINE_ALL_STRINGOPS)
24700 /* When the current size is best to be copied by a libcall,
24701 but we are still forced to inline, run the heuristic below
24702 that will pick code for medium sized blocks. */
24703 if (alg != libcall)
24705 *noalign = alg_noalign;
24708 else if (!any_alg_usable_p)
24711 else if (alg_usable_p (candidate, memset))
24713 *noalign = algs->size[i].noalign;
24719 /* When asked to inline the call anyway, try to pick meaningful choice.
24720 We look for maximal size of block that is faster to copy by hand and
24721 take blocks of at most of that size guessing that average size will
24722 be roughly half of the block.
24724 If this turns out to be bad, we might simply specify the preferred
24725 choice in ix86_costs. */
24726 if ((TARGET_INLINE_ALL_STRINGOPS || TARGET_INLINE_STRINGOPS_DYNAMICALLY)
24727 && (algs->unknown_size == libcall
24728 || !alg_usable_p (algs->unknown_size, memset)))
24730 enum stringop_alg alg;
24732 /* If there aren't any usable algorithms, then recursing on
24733 smaller sizes isn't going to find anything. Just return the
24734 simple byte-at-a-time copy loop. */
24735 if (!any_alg_usable_p)
24737 /* Pick something reasonable. */
24738 if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
24739 *dynamic_check = 128;
24740 return loop_1_byte;
24744 alg = decide_alg (count, max / 2, min_size, max_size, memset,
24745 zero_memset, dynamic_check, noalign);
24746 gcc_assert (*dynamic_check == -1);
24747 if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
24748 *dynamic_check = max;
24750 gcc_assert (alg != libcall);
24753 return (alg_usable_p (algs->unknown_size, memset)
24754 ? algs->unknown_size : libcall);
24757 /* Decide on alignment. We know that the operand is already aligned to ALIGN
24758 (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
24760 decide_alignment (int align,
24761 enum stringop_alg alg,
24763 machine_mode move_mode)
24765 int desired_align = 0;
24767 gcc_assert (alg != no_stringop);
24769 if (alg == libcall)
24771 if (move_mode == VOIDmode)
24774 desired_align = GET_MODE_SIZE (move_mode);
24775 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
24776 copying whole cacheline at once. */
24777 if (TARGET_PENTIUMPRO
24778 && (alg == rep_prefix_4_byte || alg == rep_prefix_1_byte))
24783 if (desired_align < align)
24784 desired_align = align;
24785 if (expected_size != -1 && expected_size < 4)
24786 desired_align = align;
24788 return desired_align;
24792 /* Helper function for memcpy. For QImode value 0xXY produce
24793 0xXYXYXYXY of wide specified by MODE. This is essentially
24794 a * 0x10101010, but we can do slightly better than
24795 synth_mult by unwinding the sequence by hand on CPUs with
24798 promote_duplicated_reg (machine_mode mode, rtx val)
24800 machine_mode valmode = GET_MODE (val);
24802 int nops = mode == DImode ? 3 : 2;
24804 gcc_assert (mode == SImode || mode == DImode || val == const0_rtx);
24805 if (val == const0_rtx)
24806 return copy_to_mode_reg (mode, CONST0_RTX (mode));
24807 if (CONST_INT_P (val))
24809 HOST_WIDE_INT v = INTVAL (val) & 255;
24813 if (mode == DImode)
24814 v |= (v << 16) << 16;
24815 return copy_to_mode_reg (mode, gen_int_mode (v, mode));
24818 if (valmode == VOIDmode)
24820 if (valmode != QImode)
24821 val = gen_lowpart (QImode, val);
24822 if (mode == QImode)
24824 if (!TARGET_PARTIAL_REG_STALL)
24826 if (ix86_cost->mult_init[mode == DImode ? 3 : 2]
24827 + ix86_cost->mult_bit * (mode == DImode ? 8 : 4)
24828 <= (ix86_cost->shift_const + ix86_cost->add) * nops
24829 + (COSTS_N_INSNS (TARGET_PARTIAL_REG_STALL == 0)))
24831 rtx reg = convert_modes (mode, QImode, val, true);
24832 tmp = promote_duplicated_reg (mode, const1_rtx);
24833 return expand_simple_binop (mode, MULT, reg, tmp, NULL, 1,
24838 rtx reg = convert_modes (mode, QImode, val, true);
24840 if (!TARGET_PARTIAL_REG_STALL)
24841 if (mode == SImode)
24842 emit_insn (gen_movsi_insv_1 (reg, reg));
24844 emit_insn (gen_movdi_insv_1 (reg, reg));
24847 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (8),
24848 NULL, 1, OPTAB_DIRECT);
24850 expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
24852 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (16),
24853 NULL, 1, OPTAB_DIRECT);
24854 reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
24855 if (mode == SImode)
24857 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (32),
24858 NULL, 1, OPTAB_DIRECT);
24859 reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
24864 /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
24865 be needed by main loop copying SIZE_NEEDED chunks and prologue getting
24866 alignment from ALIGN to DESIRED_ALIGN. */
24868 promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align,
24874 && (size_needed > 4 || (desired_align > align && desired_align > 4)))
24875 promoted_val = promote_duplicated_reg (DImode, val);
24876 else if (size_needed > 2 || (desired_align > align && desired_align > 2))
24877 promoted_val = promote_duplicated_reg (SImode, val);
24878 else if (size_needed > 1 || (desired_align > align && desired_align > 1))
24879 promoted_val = promote_duplicated_reg (HImode, val);
24881 promoted_val = val;
24883 return promoted_val;
24886 /* Expand string move (memcpy) ot store (memset) operation. Use i386 string
24887 operations when profitable. The code depends upon architecture, block size
24888 and alignment, but always has one of the following overall structures:
24890 Aligned move sequence:
24892 1) Prologue guard: Conditional that jumps up to epilogues for small
24893 blocks that can be handled by epilogue alone. This is faster
24894 but also needed for correctness, since prologue assume the block
24895 is larger than the desired alignment.
24897 Optional dynamic check for size and libcall for large
24898 blocks is emitted here too, with -minline-stringops-dynamically.
24900 2) Prologue: copy first few bytes in order to get destination
24901 aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
24902 than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
24903 copied. We emit either a jump tree on power of two sized
24904 blocks, or a byte loop.
24906 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
24907 with specified algorithm.
24909 4) Epilogue: code copying tail of the block that is too small to be
24910 handled by main body (or up to size guarded by prologue guard).
24912 Misaligned move sequence
24914 1) missaligned move prologue/epilogue containing:
24915 a) Prologue handling small memory blocks and jumping to done_label
24916 (skipped if blocks are known to be large enough)
24917 b) Signle move copying first DESIRED_ALIGN-ALIGN bytes if alignment is
24918 needed by single possibly misaligned move
24919 (skipped if alignment is not needed)
24920 c) Copy of last SIZE_NEEDED bytes by possibly misaligned moves
24922 2) Zero size guard dispatching to done_label, if needed
24924 3) dispatch to library call, if needed,
24926 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
24927 with specified algorithm. */
24929 ix86_expand_set_or_movmem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
24930 rtx align_exp, rtx expected_align_exp,
24931 rtx expected_size_exp, rtx min_size_exp,
24932 rtx max_size_exp, rtx probable_max_size_exp,
24937 rtx_code_label *label = NULL;
24939 rtx_code_label *jump_around_label = NULL;
24940 HOST_WIDE_INT align = 1;
24941 unsigned HOST_WIDE_INT count = 0;
24942 HOST_WIDE_INT expected_size = -1;
24943 int size_needed = 0, epilogue_size_needed;
24944 int desired_align = 0, align_bytes = 0;
24945 enum stringop_alg alg;
24946 rtx promoted_val = NULL;
24947 rtx vec_promoted_val = NULL;
24948 bool force_loopy_epilogue = false;
24950 bool need_zero_guard = false;
24952 machine_mode move_mode = VOIDmode;
24953 int unroll_factor = 1;
24954 /* TODO: Once value ranges are available, fill in proper data. */
24955 unsigned HOST_WIDE_INT min_size = 0;
24956 unsigned HOST_WIDE_INT max_size = -1;
24957 unsigned HOST_WIDE_INT probable_max_size = -1;
24958 bool misaligned_prologue_used = false;
24960 if (CONST_INT_P (align_exp))
24961 align = INTVAL (align_exp);
24962 /* i386 can do misaligned access on reasonably increased cost. */
24963 if (CONST_INT_P (expected_align_exp)
24964 && INTVAL (expected_align_exp) > align)
24965 align = INTVAL (expected_align_exp);
24966 /* ALIGN is the minimum of destination and source alignment, but we care here
24967 just about destination alignment. */
24969 && MEM_ALIGN (dst) > (unsigned HOST_WIDE_INT) align * BITS_PER_UNIT)
24970 align = MEM_ALIGN (dst) / BITS_PER_UNIT;
24972 if (CONST_INT_P (count_exp))
24974 min_size = max_size = probable_max_size = count = expected_size
24975 = INTVAL (count_exp);
24976 /* When COUNT is 0, there is nothing to do. */
24983 min_size = INTVAL (min_size_exp);
24985 max_size = INTVAL (max_size_exp);
24986 if (probable_max_size_exp)
24987 probable_max_size = INTVAL (probable_max_size_exp);
24988 if (CONST_INT_P (expected_size_exp))
24989 expected_size = INTVAL (expected_size_exp);
24992 /* Make sure we don't need to care about overflow later on. */
24993 if (count > ((unsigned HOST_WIDE_INT) 1 << 30))
24996 /* Step 0: Decide on preferred algorithm, desired alignment and
24997 size of chunks to be copied by main loop. */
24998 alg = decide_alg (count, expected_size, min_size, probable_max_size,
25000 issetmem && val_exp == const0_rtx,
25001 &dynamic_check, &noalign);
25002 if (alg == libcall)
25004 gcc_assert (alg != no_stringop);
25006 /* For now vector-version of memset is generated only for memory zeroing, as
25007 creating of promoted vector value is very cheap in this case. */
25008 if (issetmem && alg == vector_loop && val_exp != const0_rtx)
25009 alg = unrolled_loop;
25012 count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
25013 destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
25015 srcreg = ix86_copy_addr_to_reg (XEXP (src, 0));
25018 move_mode = word_mode;
25024 gcc_unreachable ();
25026 need_zero_guard = true;
25027 move_mode = QImode;
25030 need_zero_guard = true;
25032 case unrolled_loop:
25033 need_zero_guard = true;
25034 unroll_factor = (TARGET_64BIT ? 4 : 2);
25037 need_zero_guard = true;
25039 /* Find the widest supported mode. */
25040 move_mode = word_mode;
25041 while (optab_handler (mov_optab, GET_MODE_WIDER_MODE (move_mode))
25042 != CODE_FOR_nothing)
25043 move_mode = GET_MODE_WIDER_MODE (move_mode);
25045 /* Find the corresponding vector mode with the same size as MOVE_MODE.
25046 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
25047 if (GET_MODE_SIZE (move_mode) > GET_MODE_SIZE (word_mode))
25049 int nunits = GET_MODE_SIZE (move_mode) / GET_MODE_SIZE (word_mode);
25050 move_mode = mode_for_vector (word_mode, nunits);
25051 if (optab_handler (mov_optab, move_mode) == CODE_FOR_nothing)
25052 move_mode = word_mode;
25054 gcc_assert (optab_handler (mov_optab, move_mode) != CODE_FOR_nothing);
25056 case rep_prefix_8_byte:
25057 move_mode = DImode;
25059 case rep_prefix_4_byte:
25060 move_mode = SImode;
25062 case rep_prefix_1_byte:
25063 move_mode = QImode;
25066 size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
25067 epilogue_size_needed = size_needed;
25069 desired_align = decide_alignment (align, alg, expected_size, move_mode);
25070 if (!TARGET_ALIGN_STRINGOPS || noalign)
25071 align = desired_align;
25073 /* Step 1: Prologue guard. */
25075 /* Alignment code needs count to be in register. */
25076 if (CONST_INT_P (count_exp) && desired_align > align)
25078 if (INTVAL (count_exp) > desired_align
25079 && INTVAL (count_exp) > size_needed)
25082 = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
25083 if (align_bytes <= 0)
25086 align_bytes = desired_align - align_bytes;
25088 if (align_bytes == 0)
25089 count_exp = force_reg (counter_mode (count_exp), count_exp);
25091 gcc_assert (desired_align >= 1 && align >= 1);
25093 /* Misaligned move sequences handle both prologue and epilogue at once.
25094 Default code generation results in a smaller code for large alignments
25095 and also avoids redundant job when sizes are known precisely. */
25096 misaligned_prologue_used
25097 = (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES
25098 && MAX (desired_align, epilogue_size_needed) <= 32
25099 && desired_align <= epilogue_size_needed
25100 && ((desired_align > align && !align_bytes)
25101 || (!count && epilogue_size_needed > 1)));
25103 /* Do the cheap promotion to allow better CSE across the
25104 main loop and epilogue (ie one load of the big constant in the
25106 For now the misaligned move sequences do not have fast path
25107 without broadcasting. */
25108 if (issetmem && ((CONST_INT_P (val_exp) || misaligned_prologue_used)))
25110 if (alg == vector_loop)
25112 gcc_assert (val_exp == const0_rtx);
25113 vec_promoted_val = promote_duplicated_reg (move_mode, val_exp);
25114 promoted_val = promote_duplicated_reg_to_size (val_exp,
25115 GET_MODE_SIZE (word_mode),
25116 desired_align, align);
25120 promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
25121 desired_align, align);
25124 /* Misaligned move sequences handles both prologues and epilogues at once.
25125 Default code generation results in smaller code for large alignments and
25126 also avoids redundant job when sizes are known precisely. */
25127 if (misaligned_prologue_used)
25129 /* Misaligned move prologue handled small blocks by itself. */
25130 expand_set_or_movmem_prologue_epilogue_by_misaligned_moves
25131 (dst, src, &destreg, &srcreg,
25132 move_mode, promoted_val, vec_promoted_val,
25134 &jump_around_label,
25135 desired_align < align
25136 ? MAX (desired_align, epilogue_size_needed) : epilogue_size_needed,
25137 desired_align, align, &min_size, dynamic_check, issetmem);
25139 src = change_address (src, BLKmode, srcreg);
25140 dst = change_address (dst, BLKmode, destreg);
25141 set_mem_align (dst, desired_align * BITS_PER_UNIT);
25142 epilogue_size_needed = 0;
25143 if (need_zero_guard && !min_size)
25145 /* It is possible that we copied enough so the main loop will not
25147 gcc_assert (size_needed > 1);
25148 if (jump_around_label == NULL_RTX)
25149 jump_around_label = gen_label_rtx ();
25150 emit_cmp_and_jump_insns (count_exp,
25151 GEN_INT (size_needed),
25152 LTU, 0, counter_mode (count_exp), 1, jump_around_label);
25153 if (expected_size == -1
25154 || expected_size < (desired_align - align) / 2 + size_needed)
25155 predict_jump (REG_BR_PROB_BASE * 20 / 100);
25157 predict_jump (REG_BR_PROB_BASE * 60 / 100);
25160 /* Ensure that alignment prologue won't copy past end of block. */
25161 else if (size_needed > 1 || (desired_align > 1 && desired_align > align))
25163 epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
25164 /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
25165 Make sure it is power of 2. */
25166 epilogue_size_needed = 1 << (floor_log2 (epilogue_size_needed) + 1);
25168 /* To improve performance of small blocks, we jump around the VAL
25169 promoting mode. This mean that if the promoted VAL is not constant,
25170 we might not use it in the epilogue and have to use byte
25172 if (issetmem && epilogue_size_needed > 2 && !promoted_val)
25173 force_loopy_epilogue = true;
25174 if ((count && count < (unsigned HOST_WIDE_INT) epilogue_size_needed)
25175 || max_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
25177 /* If main algorithm works on QImode, no epilogue is needed.
25178 For small sizes just don't align anything. */
25179 if (size_needed == 1)
25180 desired_align = align;
25185 && min_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
25187 label = gen_label_rtx ();
25188 emit_cmp_and_jump_insns (count_exp,
25189 GEN_INT (epilogue_size_needed),
25190 LTU, 0, counter_mode (count_exp), 1, label);
25191 if (expected_size == -1 || expected_size < epilogue_size_needed)
25192 predict_jump (REG_BR_PROB_BASE * 60 / 100);
25194 predict_jump (REG_BR_PROB_BASE * 20 / 100);
25198 /* Emit code to decide on runtime whether library call or inline should be
25200 if (dynamic_check != -1)
25202 if (!issetmem && CONST_INT_P (count_exp))
25204 if (UINTVAL (count_exp) >= (unsigned HOST_WIDE_INT)dynamic_check)
25206 emit_block_move_via_libcall (dst, src, count_exp, false);
25207 count_exp = const0_rtx;
25213 rtx_code_label *hot_label = gen_label_rtx ();
25214 if (jump_around_label == NULL_RTX)
25215 jump_around_label = gen_label_rtx ();
25216 emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
25217 LEU, 0, counter_mode (count_exp),
25219 predict_jump (REG_BR_PROB_BASE * 90 / 100);
25221 set_storage_via_libcall (dst, count_exp, val_exp, false);
25223 emit_block_move_via_libcall (dst, src, count_exp, false);
25224 emit_jump (jump_around_label);
25225 emit_label (hot_label);
25229 /* Step 2: Alignment prologue. */
25230 /* Do the expensive promotion once we branched off the small blocks. */
25231 if (issetmem && !promoted_val)
25232 promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
25233 desired_align, align);
25235 if (desired_align > align && !misaligned_prologue_used)
25237 if (align_bytes == 0)
25239 /* Except for the first move in prologue, we no longer know
25240 constant offset in aliasing info. It don't seems to worth
25241 the pain to maintain it for the first move, so throw away
25243 dst = change_address (dst, BLKmode, destreg);
25245 src = change_address (src, BLKmode, srcreg);
25246 dst = expand_set_or_movmem_prologue (dst, src, destreg, srcreg,
25247 promoted_val, vec_promoted_val,
25248 count_exp, align, desired_align,
25250 /* At most desired_align - align bytes are copied. */
25251 if (min_size < (unsigned)(desired_align - align))
25254 min_size -= desired_align - align;
25258 /* If we know how many bytes need to be stored before dst is
25259 sufficiently aligned, maintain aliasing info accurately. */
25260 dst = expand_set_or_movmem_constant_prologue (dst, &src, destreg,
25268 count_exp = plus_constant (counter_mode (count_exp),
25269 count_exp, -align_bytes);
25270 count -= align_bytes;
25271 min_size -= align_bytes;
25272 max_size -= align_bytes;
25274 if (need_zero_guard
25276 && (count < (unsigned HOST_WIDE_INT) size_needed
25277 || (align_bytes == 0
25278 && count < ((unsigned HOST_WIDE_INT) size_needed
25279 + desired_align - align))))
25281 /* It is possible that we copied enough so the main loop will not
25283 gcc_assert (size_needed > 1);
25284 if (label == NULL_RTX)
25285 label = gen_label_rtx ();
25286 emit_cmp_and_jump_insns (count_exp,
25287 GEN_INT (size_needed),
25288 LTU, 0, counter_mode (count_exp), 1, label);
25289 if (expected_size == -1
25290 || expected_size < (desired_align - align) / 2 + size_needed)
25291 predict_jump (REG_BR_PROB_BASE * 20 / 100);
25293 predict_jump (REG_BR_PROB_BASE * 60 / 100);
25296 if (label && size_needed == 1)
25298 emit_label (label);
25299 LABEL_NUSES (label) = 1;
25301 epilogue_size_needed = 1;
25303 promoted_val = val_exp;
25305 else if (label == NULL_RTX && !misaligned_prologue_used)
25306 epilogue_size_needed = size_needed;
25308 /* Step 3: Main loop. */
25315 gcc_unreachable ();
25318 case unrolled_loop:
25319 expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, promoted_val,
25320 count_exp, move_mode, unroll_factor,
25321 expected_size, issetmem);
25324 expand_set_or_movmem_via_loop (dst, src, destreg, srcreg,
25325 vec_promoted_val, count_exp, move_mode,
25326 unroll_factor, expected_size, issetmem);
25328 case rep_prefix_8_byte:
25329 case rep_prefix_4_byte:
25330 case rep_prefix_1_byte:
25331 expand_set_or_movmem_via_rep (dst, src, destreg, srcreg, promoted_val,
25332 val_exp, count_exp, move_mode, issetmem);
25335 /* Adjust properly the offset of src and dest memory for aliasing. */
25336 if (CONST_INT_P (count_exp))
25339 src = adjust_automodify_address_nv (src, BLKmode, srcreg,
25340 (count / size_needed) * size_needed);
25341 dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
25342 (count / size_needed) * size_needed);
25347 src = change_address (src, BLKmode, srcreg);
25348 dst = change_address (dst, BLKmode, destreg);
25351 /* Step 4: Epilogue to copy the remaining bytes. */
25355 /* When the main loop is done, COUNT_EXP might hold original count,
25356 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
25357 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
25358 bytes. Compensate if needed. */
25360 if (size_needed < epilogue_size_needed)
25363 expand_simple_binop (counter_mode (count_exp), AND, count_exp,
25364 GEN_INT (size_needed - 1), count_exp, 1,
25366 if (tmp != count_exp)
25367 emit_move_insn (count_exp, tmp);
25369 emit_label (label);
25370 LABEL_NUSES (label) = 1;
25373 if (count_exp != const0_rtx && epilogue_size_needed > 1)
25375 if (force_loopy_epilogue)
25376 expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
25377 epilogue_size_needed);
25381 expand_setmem_epilogue (dst, destreg, promoted_val,
25382 vec_promoted_val, count_exp,
25383 epilogue_size_needed);
25385 expand_movmem_epilogue (dst, src, destreg, srcreg, count_exp,
25386 epilogue_size_needed);
25389 if (jump_around_label)
25390 emit_label (jump_around_label);
25395 /* Expand the appropriate insns for doing strlen if not just doing
25398 out = result, initialized with the start address
25399 align_rtx = alignment of the address.
25400 scratch = scratch register, initialized with the startaddress when
25401 not aligned, otherwise undefined
25403 This is just the body. It needs the initializations mentioned above and
25404 some address computing at the end. These things are done in i386.md. */
25407 ix86_expand_strlensi_unroll_1 (rtx out, rtx src, rtx align_rtx)
25411 rtx_code_label *align_2_label = NULL;
25412 rtx_code_label *align_3_label = NULL;
25413 rtx_code_label *align_4_label = gen_label_rtx ();
25414 rtx_code_label *end_0_label = gen_label_rtx ();
25416 rtx tmpreg = gen_reg_rtx (SImode);
25417 rtx scratch = gen_reg_rtx (SImode);
25421 if (CONST_INT_P (align_rtx))
25422 align = INTVAL (align_rtx);
25424 /* Loop to check 1..3 bytes for null to get an aligned pointer. */
25426 /* Is there a known alignment and is it less than 4? */
25429 rtx scratch1 = gen_reg_rtx (Pmode);
25430 emit_move_insn (scratch1, out);
25431 /* Is there a known alignment and is it not 2? */
25434 align_3_label = gen_label_rtx (); /* Label when aligned to 3-byte */
25435 align_2_label = gen_label_rtx (); /* Label when aligned to 2-byte */
25437 /* Leave just the 3 lower bits. */
25438 align_rtx = expand_binop (Pmode, and_optab, scratch1, GEN_INT (3),
25439 NULL_RTX, 0, OPTAB_WIDEN);
25441 emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
25442 Pmode, 1, align_4_label);
25443 emit_cmp_and_jump_insns (align_rtx, const2_rtx, EQ, NULL,
25444 Pmode, 1, align_2_label);
25445 emit_cmp_and_jump_insns (align_rtx, const2_rtx, GTU, NULL,
25446 Pmode, 1, align_3_label);
25450 /* Since the alignment is 2, we have to check 2 or 0 bytes;
25451 check if is aligned to 4 - byte. */
25453 align_rtx = expand_binop (Pmode, and_optab, scratch1, const2_rtx,
25454 NULL_RTX, 0, OPTAB_WIDEN);
25456 emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
25457 Pmode, 1, align_4_label);
25460 mem = change_address (src, QImode, out);
25462 /* Now compare the bytes. */
25464 /* Compare the first n unaligned byte on a byte per byte basis. */
25465 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL,
25466 QImode, 1, end_0_label);
25468 /* Increment the address. */
25469 emit_insn (ix86_gen_add3 (out, out, const1_rtx));
25471 /* Not needed with an alignment of 2 */
25474 emit_label (align_2_label);
25476 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
25479 emit_insn (ix86_gen_add3 (out, out, const1_rtx));
25481 emit_label (align_3_label);
25484 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
25487 emit_insn (ix86_gen_add3 (out, out, const1_rtx));
25490 /* Generate loop to check 4 bytes at a time. It is not a good idea to
25491 align this loop. It gives only huge programs, but does not help to
25493 emit_label (align_4_label);
25495 mem = change_address (src, SImode, out);
25496 emit_move_insn (scratch, mem);
25497 emit_insn (ix86_gen_add3 (out, out, GEN_INT (4)));
25499 /* This formula yields a nonzero result iff one of the bytes is zero.
25500 This saves three branches inside loop and many cycles. */
25502 emit_insn (gen_addsi3 (tmpreg, scratch, GEN_INT (-0x01010101)));
25503 emit_insn (gen_one_cmplsi2 (scratch, scratch));
25504 emit_insn (gen_andsi3 (tmpreg, tmpreg, scratch));
25505 emit_insn (gen_andsi3 (tmpreg, tmpreg,
25506 gen_int_mode (0x80808080, SImode)));
25507 emit_cmp_and_jump_insns (tmpreg, const0_rtx, EQ, 0, SImode, 1,
25512 rtx reg = gen_reg_rtx (SImode);
25513 rtx reg2 = gen_reg_rtx (Pmode);
25514 emit_move_insn (reg, tmpreg);
25515 emit_insn (gen_lshrsi3 (reg, reg, GEN_INT (16)));
25517 /* If zero is not in the first two bytes, move two bytes forward. */
25518 emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
25519 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
25520 tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
25521 emit_insn (gen_rtx_SET (VOIDmode, tmpreg,
25522 gen_rtx_IF_THEN_ELSE (SImode, tmp,
25525 /* Emit lea manually to avoid clobbering of flags. */
25526 emit_insn (gen_rtx_SET (SImode, reg2,
25527 gen_rtx_PLUS (Pmode, out, const2_rtx)));
25529 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
25530 tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
25531 emit_insn (gen_rtx_SET (VOIDmode, out,
25532 gen_rtx_IF_THEN_ELSE (Pmode, tmp,
25538 rtx_code_label *end_2_label = gen_label_rtx ();
25539 /* Is zero in the first two bytes? */
25541 emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
25542 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
25543 tmp = gen_rtx_NE (VOIDmode, tmp, const0_rtx);
25544 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
25545 gen_rtx_LABEL_REF (VOIDmode, end_2_label),
25547 tmp = emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp));
25548 JUMP_LABEL (tmp) = end_2_label;
25550 /* Not in the first two. Move two bytes forward. */
25551 emit_insn (gen_lshrsi3 (tmpreg, tmpreg, GEN_INT (16)));
25552 emit_insn (ix86_gen_add3 (out, out, const2_rtx));
25554 emit_label (end_2_label);
25558 /* Avoid branch in fixing the byte. */
25559 tmpreg = gen_lowpart (QImode, tmpreg);
25560 emit_insn (gen_addqi3_cc (tmpreg, tmpreg, tmpreg));
25561 tmp = gen_rtx_REG (CCmode, FLAGS_REG);
25562 cmp = gen_rtx_LTU (VOIDmode, tmp, const0_rtx);
25563 emit_insn (ix86_gen_sub3_carry (out, out, GEN_INT (3), tmp, cmp));
25565 emit_label (end_0_label);
25568 /* Expand strlen. */
25571 ix86_expand_strlen (rtx out, rtx src, rtx eoschar, rtx align)
25573 rtx addr, scratch1, scratch2, scratch3, scratch4;
25575 /* The generic case of strlen expander is long. Avoid it's
25576 expanding unless TARGET_INLINE_ALL_STRINGOPS. */
25578 if (TARGET_UNROLL_STRLEN && eoschar == const0_rtx && optimize > 1
25579 && !TARGET_INLINE_ALL_STRINGOPS
25580 && !optimize_insn_for_size_p ()
25581 && (!CONST_INT_P (align) || INTVAL (align) < 4))
25584 addr = force_reg (Pmode, XEXP (src, 0));
25585 scratch1 = gen_reg_rtx (Pmode);
25587 if (TARGET_UNROLL_STRLEN && eoschar == const0_rtx && optimize > 1
25588 && !optimize_insn_for_size_p ())
25590 /* Well it seems that some optimizer does not combine a call like
25591 foo(strlen(bar), strlen(bar));
25592 when the move and the subtraction is done here. It does calculate
25593 the length just once when these instructions are done inside of
25594 output_strlen_unroll(). But I think since &bar[strlen(bar)] is
25595 often used and I use one fewer register for the lifetime of
25596 output_strlen_unroll() this is better. */
25598 emit_move_insn (out, addr);
25600 ix86_expand_strlensi_unroll_1 (out, src, align);
25602 /* strlensi_unroll_1 returns the address of the zero at the end of
25603 the string, like memchr(), so compute the length by subtracting
25604 the start address. */
25605 emit_insn (ix86_gen_sub3 (out, out, addr));
25611 /* Can't use this if the user has appropriated eax, ecx, or edi. */
25612 if (fixed_regs[AX_REG] || fixed_regs[CX_REG] || fixed_regs[DI_REG])
25615 scratch2 = gen_reg_rtx (Pmode);
25616 scratch3 = gen_reg_rtx (Pmode);
25617 scratch4 = force_reg (Pmode, constm1_rtx);
25619 emit_move_insn (scratch3, addr);
25620 eoschar = force_reg (QImode, eoschar);
25622 src = replace_equiv_address_nv (src, scratch3);
25624 /* If .md starts supporting :P, this can be done in .md. */
25625 unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, src, eoschar, align,
25626 scratch4), UNSPEC_SCAS);
25627 emit_insn (gen_strlenqi_1 (scratch1, scratch3, unspec));
25628 emit_insn (ix86_gen_one_cmpl2 (scratch2, scratch1));
25629 emit_insn (ix86_gen_add3 (out, scratch2, constm1_rtx));
25634 /* For given symbol (function) construct code to compute address of it's PLT
25635 entry in large x86-64 PIC model. */
25637 construct_plt_address (rtx symbol)
25641 gcc_assert (GET_CODE (symbol) == SYMBOL_REF);
25642 gcc_assert (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF);
25643 gcc_assert (Pmode == DImode);
25645 tmp = gen_reg_rtx (Pmode);
25646 unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, symbol), UNSPEC_PLTOFF);
25648 emit_move_insn (tmp, gen_rtx_CONST (Pmode, unspec));
25649 emit_insn (ix86_gen_add3 (tmp, tmp, pic_offset_table_rtx));
25654 ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
25656 rtx pop, bool sibcall)
25659 rtx use = NULL, call;
25660 unsigned int vec_len = 0;
25662 if (pop == const0_rtx)
25664 gcc_assert (!TARGET_64BIT || !pop);
25666 if (TARGET_MACHO && !TARGET_64BIT)
25669 if (flag_pic && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
25670 fnaddr = machopic_indirect_call_target (fnaddr);
25675 /* Static functions and indirect calls don't need the pic register. */
25678 || (ix86_cmodel == CM_LARGE_PIC
25679 && DEFAULT_ABI != MS_ABI))
25680 && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF
25681 && ! SYMBOL_REF_LOCAL_P (XEXP (fnaddr, 0)))
25683 use_reg (&use, gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM));
25684 if (ix86_use_pseudo_pic_reg ())
25685 emit_move_insn (gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM),
25686 pic_offset_table_rtx);
25690 /* Skip setting up RAX register for -mskip-rax-setup when there are no
25691 parameters passed in vector registers. */
25693 && (INTVAL (callarg2) > 0
25694 || (INTVAL (callarg2) == 0
25695 && (TARGET_SSE || !flag_skip_rax_setup))))
25697 rtx al = gen_rtx_REG (QImode, AX_REG);
25698 emit_move_insn (al, callarg2);
25699 use_reg (&use, al);
25702 if (ix86_cmodel == CM_LARGE_PIC
25705 && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF
25706 && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode))
25707 fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0)));
25709 ? !sibcall_insn_operand (XEXP (fnaddr, 0), word_mode)
25710 : !call_insn_operand (XEXP (fnaddr, 0), word_mode))
25712 fnaddr = convert_to_mode (word_mode, XEXP (fnaddr, 0), 1);
25713 fnaddr = gen_rtx_MEM (QImode, copy_to_mode_reg (word_mode, fnaddr));
25716 call = gen_rtx_CALL (VOIDmode, fnaddr, callarg1);
25720 /* We should add bounds as destination register in case
25721 pointer with bounds may be returned. */
25722 if (TARGET_MPX && SCALAR_INT_MODE_P (GET_MODE (retval)))
25724 rtx b0 = gen_rtx_REG (BND64mode, FIRST_BND_REG);
25725 rtx b1 = gen_rtx_REG (BND64mode, FIRST_BND_REG + 1);
25726 if (GET_CODE (retval) == PARALLEL)
25728 b0 = gen_rtx_EXPR_LIST (VOIDmode, b0, const0_rtx);
25729 b1 = gen_rtx_EXPR_LIST (VOIDmode, b1, const0_rtx);
25730 rtx par = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, b0, b1));
25731 retval = chkp_join_splitted_slot (retval, par);
25735 retval = gen_rtx_PARALLEL (VOIDmode,
25736 gen_rtvec (3, retval, b0, b1));
25737 chkp_put_regs_to_expr_list (retval);
25741 call = gen_rtx_SET (VOIDmode, retval, call);
25743 vec[vec_len++] = call;
25747 pop = gen_rtx_PLUS (Pmode, stack_pointer_rtx, pop);
25748 pop = gen_rtx_SET (VOIDmode, stack_pointer_rtx, pop);
25749 vec[vec_len++] = pop;
25752 if (TARGET_64BIT_MS_ABI
25753 && (!callarg2 || INTVAL (callarg2) != -2))
25755 int const cregs_size
25756 = ARRAY_SIZE (x86_64_ms_sysv_extra_clobbered_registers);
25759 for (i = 0; i < cregs_size; i++)
25761 int regno = x86_64_ms_sysv_extra_clobbered_registers[i];
25762 machine_mode mode = SSE_REGNO_P (regno) ? TImode : DImode;
25764 clobber_reg (&use, gen_rtx_REG (mode, regno));
25769 call = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (vec_len, vec));
25770 call = emit_call_insn (call);
25772 CALL_INSN_FUNCTION_USAGE (call) = use;
25777 /* Output the assembly for a call instruction. */
25780 ix86_output_call_insn (rtx_insn *insn, rtx call_op)
25782 bool direct_p = constant_call_address_operand (call_op, VOIDmode);
25783 bool seh_nop_p = false;
25786 if (SIBLING_CALL_P (insn))
25789 xasm = "%!jmp\t%P0";
25790 /* SEH epilogue detection requires the indirect branch case
25791 to include REX.W. */
25792 else if (TARGET_SEH)
25793 xasm = "%!rex.W jmp %A0";
25795 xasm = "%!jmp\t%A0";
25797 output_asm_insn (xasm, &call_op);
25801 /* SEH unwinding can require an extra nop to be emitted in several
25802 circumstances. Determine if we have one of those. */
25807 for (i = NEXT_INSN (insn); i ; i = NEXT_INSN (i))
25809 /* If we get to another real insn, we don't need the nop. */
25813 /* If we get to the epilogue note, prevent a catch region from
25814 being adjacent to the standard epilogue sequence. If non-
25815 call-exceptions, we'll have done this during epilogue emission. */
25816 if (NOTE_P (i) && NOTE_KIND (i) == NOTE_INSN_EPILOGUE_BEG
25817 && !flag_non_call_exceptions
25818 && !can_throw_internal (insn))
25825 /* If we didn't find a real insn following the call, prevent the
25826 unwinder from looking into the next function. */
25832 xasm = "%!call\t%P0";
25834 xasm = "%!call\t%A0";
25836 output_asm_insn (xasm, &call_op);
25844 /* Clear stack slot assignments remembered from previous functions.
25845 This is called from INIT_EXPANDERS once before RTL is emitted for each
25848 static struct machine_function *
25849 ix86_init_machine_status (void)
25851 struct machine_function *f;
25853 f = ggc_cleared_alloc<machine_function> ();
25854 f->use_fast_prologue_epilogue_nregs = -1;
25855 f->call_abi = ix86_abi;
25860 /* Return a MEM corresponding to a stack slot with mode MODE.
25861 Allocate a new slot if necessary.
25863 The RTL for a function can have several slots available: N is
25864 which slot to use. */
25867 assign_386_stack_local (machine_mode mode, enum ix86_stack_slot n)
25869 struct stack_local_entry *s;
25871 gcc_assert (n < MAX_386_STACK_LOCALS);
25873 for (s = ix86_stack_locals; s; s = s->next)
25874 if (s->mode == mode && s->n == n)
25875 return validize_mem (copy_rtx (s->rtl));
25877 s = ggc_alloc<stack_local_entry> ();
25880 s->rtl = assign_stack_local (mode, GET_MODE_SIZE (mode), 0);
25882 s->next = ix86_stack_locals;
25883 ix86_stack_locals = s;
25884 return validize_mem (copy_rtx (s->rtl));
25888 ix86_instantiate_decls (void)
25890 struct stack_local_entry *s;
25892 for (s = ix86_stack_locals; s; s = s->next)
25893 if (s->rtl != NULL_RTX)
25894 instantiate_decl_rtl (s->rtl);
25897 /* Check whether x86 address PARTS is a pc-relative address. */
25900 rip_relative_addr_p (struct ix86_address *parts)
25902 rtx base, index, disp;
25904 base = parts->base;
25905 index = parts->index;
25906 disp = parts->disp;
25908 if (disp && !base && !index)
25914 if (GET_CODE (disp) == CONST)
25915 symbol = XEXP (disp, 0);
25916 if (GET_CODE (symbol) == PLUS
25917 && CONST_INT_P (XEXP (symbol, 1)))
25918 symbol = XEXP (symbol, 0);
25920 if (GET_CODE (symbol) == LABEL_REF
25921 || (GET_CODE (symbol) == SYMBOL_REF
25922 && SYMBOL_REF_TLS_MODEL (symbol) == 0)
25923 || (GET_CODE (symbol) == UNSPEC
25924 && (XINT (symbol, 1) == UNSPEC_GOTPCREL
25925 || XINT (symbol, 1) == UNSPEC_PCREL
25926 || XINT (symbol, 1) == UNSPEC_GOTNTPOFF)))
25933 /* Calculate the length of the memory address in the instruction encoding.
25934 Includes addr32 prefix, does not include the one-byte modrm, opcode,
25935 or other prefixes. We never generate addr32 prefix for LEA insn. */
25938 memory_address_length (rtx addr, bool lea)
25940 struct ix86_address parts;
25941 rtx base, index, disp;
25945 if (GET_CODE (addr) == PRE_DEC
25946 || GET_CODE (addr) == POST_INC
25947 || GET_CODE (addr) == PRE_MODIFY
25948 || GET_CODE (addr) == POST_MODIFY)
25951 ok = ix86_decompose_address (addr, &parts);
25954 len = (parts.seg == SEG_DEFAULT) ? 0 : 1;
25956 /* If this is not LEA instruction, add the length of addr32 prefix. */
25957 if (TARGET_64BIT && !lea
25958 && (SImode_address_operand (addr, VOIDmode)
25959 || (parts.base && GET_MODE (parts.base) == SImode)
25960 || (parts.index && GET_MODE (parts.index) == SImode)))
25964 index = parts.index;
25967 if (base && GET_CODE (base) == SUBREG)
25968 base = SUBREG_REG (base);
25969 if (index && GET_CODE (index) == SUBREG)
25970 index = SUBREG_REG (index);
25972 gcc_assert (base == NULL_RTX || REG_P (base));
25973 gcc_assert (index == NULL_RTX || REG_P (index));
25976 - esp as the base always wants an index,
25977 - ebp as the base always wants a displacement,
25978 - r12 as the base always wants an index,
25979 - r13 as the base always wants a displacement. */
25981 /* Register Indirect. */
25982 if (base && !index && !disp)
25984 /* esp (for its index) and ebp (for its displacement) need
25985 the two-byte modrm form. Similarly for r12 and r13 in 64-bit
25987 if (base == arg_pointer_rtx
25988 || base == frame_pointer_rtx
25989 || REGNO (base) == SP_REG
25990 || REGNO (base) == BP_REG
25991 || REGNO (base) == R12_REG
25992 || REGNO (base) == R13_REG)
25996 /* Direct Addressing. In 64-bit mode mod 00 r/m 5
25997 is not disp32, but disp32(%rip), so for disp32
25998 SIB byte is needed, unless print_operand_address
25999 optimizes it into disp32(%rip) or (%rip) is implied
26001 else if (disp && !base && !index)
26004 if (rip_relative_addr_p (&parts))
26009 /* Find the length of the displacement constant. */
26012 if (base && satisfies_constraint_K (disp))
26017 /* ebp always wants a displacement. Similarly r13. */
26018 else if (base && (REGNO (base) == BP_REG || REGNO (base) == R13_REG))
26021 /* An index requires the two-byte modrm form.... */
26023 /* ...like esp (or r12), which always wants an index. */
26024 || base == arg_pointer_rtx
26025 || base == frame_pointer_rtx
26026 || (base && (REGNO (base) == SP_REG || REGNO (base) == R12_REG)))
26033 /* Compute default value for "length_immediate" attribute. When SHORTFORM
26034 is set, expect that insn have 8bit immediate alternative. */
26036 ix86_attr_length_immediate_default (rtx_insn *insn, bool shortform)
26040 extract_insn_cached (insn);
26041 for (i = recog_data.n_operands - 1; i >= 0; --i)
26042 if (CONSTANT_P (recog_data.operand[i]))
26044 enum attr_mode mode = get_attr_mode (insn);
26047 if (shortform && CONST_INT_P (recog_data.operand[i]))
26049 HOST_WIDE_INT ival = INTVAL (recog_data.operand[i]);
26056 ival = trunc_int_for_mode (ival, HImode);
26059 ival = trunc_int_for_mode (ival, SImode);
26064 if (IN_RANGE (ival, -128, 127))
26081 /* Immediates for DImode instructions are encoded
26082 as 32bit sign extended values. */
26087 fatal_insn ("unknown insn mode", insn);
26093 /* Compute default value for "length_address" attribute. */
26095 ix86_attr_length_address_default (rtx_insn *insn)
26099 if (get_attr_type (insn) == TYPE_LEA)
26101 rtx set = PATTERN (insn), addr;
26103 if (GET_CODE (set) == PARALLEL)
26104 set = XVECEXP (set, 0, 0);
26106 gcc_assert (GET_CODE (set) == SET);
26108 addr = SET_SRC (set);
26110 return memory_address_length (addr, true);
26113 extract_insn_cached (insn);
26114 for (i = recog_data.n_operands - 1; i >= 0; --i)
26115 if (MEM_P (recog_data.operand[i]))
26117 constrain_operands_cached (insn, reload_completed);
26118 if (which_alternative != -1)
26120 const char *constraints = recog_data.constraints[i];
26121 int alt = which_alternative;
26123 while (*constraints == '=' || *constraints == '+')
26126 while (*constraints++ != ',')
26128 /* Skip ignored operands. */
26129 if (*constraints == 'X')
26132 return memory_address_length (XEXP (recog_data.operand[i], 0), false);
26137 /* Compute default value for "length_vex" attribute. It includes
26138 2 or 3 byte VEX prefix and 1 opcode byte. */
26141 ix86_attr_length_vex_default (rtx_insn *insn, bool has_0f_opcode,
26146 /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
26147 byte VEX prefix. */
26148 if (!has_0f_opcode || has_vex_w)
26151 /* We can always use 2 byte VEX prefix in 32bit. */
26155 extract_insn_cached (insn);
26157 for (i = recog_data.n_operands - 1; i >= 0; --i)
26158 if (REG_P (recog_data.operand[i]))
26160 /* REX.W bit uses 3 byte VEX prefix. */
26161 if (GET_MODE (recog_data.operand[i]) == DImode
26162 && GENERAL_REG_P (recog_data.operand[i]))
26167 /* REX.X or REX.B bits use 3 byte VEX prefix. */
26168 if (MEM_P (recog_data.operand[i])
26169 && x86_extended_reg_mentioned_p (recog_data.operand[i]))
26176 /* Return the maximum number of instructions a cpu can issue. */
26179 ix86_issue_rate (void)
26183 case PROCESSOR_PENTIUM:
26184 case PROCESSOR_BONNELL:
26185 case PROCESSOR_SILVERMONT:
26186 case PROCESSOR_KNL:
26187 case PROCESSOR_INTEL:
26189 case PROCESSOR_BTVER2:
26190 case PROCESSOR_PENTIUM4:
26191 case PROCESSOR_NOCONA:
26194 case PROCESSOR_PENTIUMPRO:
26195 case PROCESSOR_ATHLON:
26197 case PROCESSOR_AMDFAM10:
26198 case PROCESSOR_GENERIC:
26199 case PROCESSOR_BTVER1:
26202 case PROCESSOR_BDVER1:
26203 case PROCESSOR_BDVER2:
26204 case PROCESSOR_BDVER3:
26205 case PROCESSOR_BDVER4:
26206 case PROCESSOR_CORE2:
26207 case PROCESSOR_NEHALEM:
26208 case PROCESSOR_SANDYBRIDGE:
26209 case PROCESSOR_HASWELL:
26217 /* A subroutine of ix86_adjust_cost -- return TRUE iff INSN reads flags set
26218 by DEP_INSN and nothing set by DEP_INSN. */
26221 ix86_flags_dependent (rtx_insn *insn, rtx_insn *dep_insn, enum attr_type insn_type)
26225 /* Simplify the test for uninteresting insns. */
26226 if (insn_type != TYPE_SETCC
26227 && insn_type != TYPE_ICMOV
26228 && insn_type != TYPE_FCMOV
26229 && insn_type != TYPE_IBR)
26232 if ((set = single_set (dep_insn)) != 0)
26234 set = SET_DEST (set);
26237 else if (GET_CODE (PATTERN (dep_insn)) == PARALLEL
26238 && XVECLEN (PATTERN (dep_insn), 0) == 2
26239 && GET_CODE (XVECEXP (PATTERN (dep_insn), 0, 0)) == SET
26240 && GET_CODE (XVECEXP (PATTERN (dep_insn), 0, 1)) == SET)
26242 set = SET_DEST (XVECEXP (PATTERN (dep_insn), 0, 0));
26243 set2 = SET_DEST (XVECEXP (PATTERN (dep_insn), 0, 0));
26248 if (!REG_P (set) || REGNO (set) != FLAGS_REG)
26251 /* This test is true if the dependent insn reads the flags but
26252 not any other potentially set register. */
26253 if (!reg_overlap_mentioned_p (set, PATTERN (insn)))
26256 if (set2 && reg_overlap_mentioned_p (set2, PATTERN (insn)))
26262 /* Return true iff USE_INSN has a memory address with operands set by
26266 ix86_agi_dependent (rtx_insn *set_insn, rtx_insn *use_insn)
26269 extract_insn_cached (use_insn);
26270 for (i = recog_data.n_operands - 1; i >= 0; --i)
26271 if (MEM_P (recog_data.operand[i]))
26273 rtx addr = XEXP (recog_data.operand[i], 0);
26274 return modified_in_p (addr, set_insn) != 0;
26279 /* Helper function for exact_store_load_dependency.
26280 Return true if addr is found in insn. */
26282 exact_dependency_1 (rtx addr, rtx insn)
26284 enum rtx_code code;
26285 const char *format_ptr;
26288 code = GET_CODE (insn);
26292 if (rtx_equal_p (addr, insn))
26307 format_ptr = GET_RTX_FORMAT (code);
26308 for (i = 0; i < GET_RTX_LENGTH (code); i++)
26310 switch (*format_ptr++)
26313 if (exact_dependency_1 (addr, XEXP (insn, i)))
26317 for (j = 0; j < XVECLEN (insn, i); j++)
26318 if (exact_dependency_1 (addr, XVECEXP (insn, i, j)))
26326 /* Return true if there exists exact dependency for store & load, i.e.
26327 the same memory address is used in them. */
26329 exact_store_load_dependency (rtx_insn *store, rtx_insn *load)
26333 set1 = single_set (store);
26336 if (!MEM_P (SET_DEST (set1)))
26338 set2 = single_set (load);
26341 if (exact_dependency_1 (SET_DEST (set1), SET_SRC (set2)))
26347 ix86_adjust_cost (rtx_insn *insn, rtx link, rtx_insn *dep_insn, int cost)
26349 enum attr_type insn_type, dep_insn_type;
26350 enum attr_memory memory;
26352 int dep_insn_code_number;
26354 /* Anti and output dependencies have zero cost on all CPUs. */
26355 if (REG_NOTE_KIND (link) != 0)
26358 dep_insn_code_number = recog_memoized (dep_insn);
26360 /* If we can't recognize the insns, we can't really do anything. */
26361 if (dep_insn_code_number < 0 || recog_memoized (insn) < 0)
26364 insn_type = get_attr_type (insn);
26365 dep_insn_type = get_attr_type (dep_insn);
26369 case PROCESSOR_PENTIUM:
26370 /* Address Generation Interlock adds a cycle of latency. */
26371 if (insn_type == TYPE_LEA)
26373 rtx addr = PATTERN (insn);
26375 if (GET_CODE (addr) == PARALLEL)
26376 addr = XVECEXP (addr, 0, 0);
26378 gcc_assert (GET_CODE (addr) == SET);
26380 addr = SET_SRC (addr);
26381 if (modified_in_p (addr, dep_insn))
26384 else if (ix86_agi_dependent (dep_insn, insn))
26387 /* ??? Compares pair with jump/setcc. */
26388 if (ix86_flags_dependent (insn, dep_insn, insn_type))
26391 /* Floating point stores require value to be ready one cycle earlier. */
26392 if (insn_type == TYPE_FMOV
26393 && get_attr_memory (insn) == MEMORY_STORE
26394 && !ix86_agi_dependent (dep_insn, insn))
26398 case PROCESSOR_PENTIUMPRO:
26399 /* INT->FP conversion is expensive. */
26400 if (get_attr_fp_int_src (dep_insn))
26403 /* There is one cycle extra latency between an FP op and a store. */
26404 if (insn_type == TYPE_FMOV
26405 && (set = single_set (dep_insn)) != NULL_RTX
26406 && (set2 = single_set (insn)) != NULL_RTX
26407 && rtx_equal_p (SET_DEST (set), SET_SRC (set2))
26408 && MEM_P (SET_DEST (set2)))
26411 memory = get_attr_memory (insn);
26413 /* Show ability of reorder buffer to hide latency of load by executing
26414 in parallel with previous instruction in case
26415 previous instruction is not needed to compute the address. */
26416 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
26417 && !ix86_agi_dependent (dep_insn, insn))
26419 /* Claim moves to take one cycle, as core can issue one load
26420 at time and the next load can start cycle later. */
26421 if (dep_insn_type == TYPE_IMOV
26422 || dep_insn_type == TYPE_FMOV)
26430 /* The esp dependency is resolved before
26431 the instruction is really finished. */
26432 if ((insn_type == TYPE_PUSH || insn_type == TYPE_POP)
26433 && (dep_insn_type == TYPE_PUSH || dep_insn_type == TYPE_POP))
26436 /* INT->FP conversion is expensive. */
26437 if (get_attr_fp_int_src (dep_insn))
26440 memory = get_attr_memory (insn);
26442 /* Show ability of reorder buffer to hide latency of load by executing
26443 in parallel with previous instruction in case
26444 previous instruction is not needed to compute the address. */
26445 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
26446 && !ix86_agi_dependent (dep_insn, insn))
26448 /* Claim moves to take one cycle, as core can issue one load
26449 at time and the next load can start cycle later. */
26450 if (dep_insn_type == TYPE_IMOV
26451 || dep_insn_type == TYPE_FMOV)
26460 case PROCESSOR_AMDFAM10:
26461 case PROCESSOR_BDVER1:
26462 case PROCESSOR_BDVER2:
26463 case PROCESSOR_BDVER3:
26464 case PROCESSOR_BDVER4:
26465 case PROCESSOR_BTVER1:
26466 case PROCESSOR_BTVER2:
26467 case PROCESSOR_GENERIC:
26468 /* Stack engine allows to execute push&pop instructions in parall. */
26469 if ((insn_type == TYPE_PUSH || insn_type == TYPE_POP)
26470 && (dep_insn_type == TYPE_PUSH || dep_insn_type == TYPE_POP))
26474 case PROCESSOR_ATHLON:
26476 memory = get_attr_memory (insn);
26478 /* Show ability of reorder buffer to hide latency of load by executing
26479 in parallel with previous instruction in case
26480 previous instruction is not needed to compute the address. */
26481 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
26482 && !ix86_agi_dependent (dep_insn, insn))
26484 enum attr_unit unit = get_attr_unit (insn);
26487 /* Because of the difference between the length of integer and
26488 floating unit pipeline preparation stages, the memory operands
26489 for floating point are cheaper.
26491 ??? For Athlon it the difference is most probably 2. */
26492 if (unit == UNIT_INTEGER || unit == UNIT_UNKNOWN)
26495 loadcost = TARGET_ATHLON ? 2 : 0;
26497 if (cost >= loadcost)
26504 case PROCESSOR_CORE2:
26505 case PROCESSOR_NEHALEM:
26506 case PROCESSOR_SANDYBRIDGE:
26507 case PROCESSOR_HASWELL:
26508 /* Stack engine allows to execute push&pop instructions in parall. */
26509 if ((insn_type == TYPE_PUSH || insn_type == TYPE_POP)
26510 && (dep_insn_type == TYPE_PUSH || dep_insn_type == TYPE_POP))
26513 memory = get_attr_memory (insn);
26515 /* Show ability of reorder buffer to hide latency of load by executing
26516 in parallel with previous instruction in case
26517 previous instruction is not needed to compute the address. */
26518 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
26519 && !ix86_agi_dependent (dep_insn, insn))
26528 case PROCESSOR_SILVERMONT:
26529 case PROCESSOR_KNL:
26530 case PROCESSOR_INTEL:
26531 if (!reload_completed)
26534 /* Increase cost of integer loads. */
26535 memory = get_attr_memory (dep_insn);
26536 if (memory == MEMORY_LOAD || memory == MEMORY_BOTH)
26538 enum attr_unit unit = get_attr_unit (dep_insn);
26539 if (unit == UNIT_INTEGER && cost == 1)
26541 if (memory == MEMORY_LOAD)
26545 /* Increase cost of ld/st for short int types only
26546 because of store forwarding issue. */
26547 rtx set = single_set (dep_insn);
26548 if (set && (GET_MODE (SET_DEST (set)) == QImode
26549 || GET_MODE (SET_DEST (set)) == HImode))
26551 /* Increase cost of store/load insn if exact
26552 dependence exists and it is load insn. */
26553 enum attr_memory insn_memory = get_attr_memory (insn);
26554 if (insn_memory == MEMORY_LOAD
26555 && exact_store_load_dependency (dep_insn, insn))
26569 /* How many alternative schedules to try. This should be as wide as the
26570 scheduling freedom in the DFA, but no wider. Making this value too
26571 large results extra work for the scheduler. */
26574 ia32_multipass_dfa_lookahead (void)
26578 case PROCESSOR_PENTIUM:
26581 case PROCESSOR_PENTIUMPRO:
26585 case PROCESSOR_BDVER1:
26586 case PROCESSOR_BDVER2:
26587 case PROCESSOR_BDVER3:
26588 case PROCESSOR_BDVER4:
26589 /* We use lookahead value 4 for BD both before and after reload
26590 schedules. Plan is to have value 8 included for O3. */
26593 case PROCESSOR_CORE2:
26594 case PROCESSOR_NEHALEM:
26595 case PROCESSOR_SANDYBRIDGE:
26596 case PROCESSOR_HASWELL:
26597 case PROCESSOR_BONNELL:
26598 case PROCESSOR_SILVERMONT:
26599 case PROCESSOR_KNL:
26600 case PROCESSOR_INTEL:
26601 /* Generally, we want haifa-sched:max_issue() to look ahead as far
26602 as many instructions can be executed on a cycle, i.e.,
26603 issue_rate. I wonder why tuning for many CPUs does not do this. */
26604 if (reload_completed)
26605 return ix86_issue_rate ();
26606 /* Don't use lookahead for pre-reload schedule to save compile time. */
26614 /* Return true if target platform supports macro-fusion. */
26617 ix86_macro_fusion_p ()
26619 return TARGET_FUSE_CMP_AND_BRANCH;
26622 /* Check whether current microarchitecture support macro fusion
26623 for insn pair "CONDGEN + CONDJMP". Refer to
26624 "Intel Architectures Optimization Reference Manual". */
26627 ix86_macro_fusion_pair_p (rtx_insn *condgen, rtx_insn *condjmp)
26630 enum rtx_code ccode;
26631 rtx compare_set = NULL_RTX, test_if, cond;
26632 rtx alu_set = NULL_RTX, addr = NULL_RTX;
26634 if (!any_condjump_p (condjmp))
26637 if (get_attr_type (condgen) != TYPE_TEST
26638 && get_attr_type (condgen) != TYPE_ICMP
26639 && get_attr_type (condgen) != TYPE_INCDEC
26640 && get_attr_type (condgen) != TYPE_ALU)
26643 compare_set = single_set (condgen);
26644 if (compare_set == NULL_RTX
26645 && !TARGET_FUSE_ALU_AND_BRANCH)
26648 if (compare_set == NULL_RTX)
26651 rtx pat = PATTERN (condgen);
26652 for (i = 0; i < XVECLEN (pat, 0); i++)
26653 if (GET_CODE (XVECEXP (pat, 0, i)) == SET)
26655 rtx set_src = SET_SRC (XVECEXP (pat, 0, i));
26656 if (GET_CODE (set_src) == COMPARE)
26657 compare_set = XVECEXP (pat, 0, i);
26659 alu_set = XVECEXP (pat, 0, i);
26662 if (compare_set == NULL_RTX)
26664 src = SET_SRC (compare_set);
26665 if (GET_CODE (src) != COMPARE)
26668 /* Macro-fusion for cmp/test MEM-IMM + conditional jmp is not
26670 if ((MEM_P (XEXP (src, 0))
26671 && CONST_INT_P (XEXP (src, 1)))
26672 || (MEM_P (XEXP (src, 1))
26673 && CONST_INT_P (XEXP (src, 0))))
26676 /* No fusion for RIP-relative address. */
26677 if (MEM_P (XEXP (src, 0)))
26678 addr = XEXP (XEXP (src, 0), 0);
26679 else if (MEM_P (XEXP (src, 1)))
26680 addr = XEXP (XEXP (src, 1), 0);
26683 ix86_address parts;
26684 int ok = ix86_decompose_address (addr, &parts);
26687 if (rip_relative_addr_p (&parts))
26691 test_if = SET_SRC (pc_set (condjmp));
26692 cond = XEXP (test_if, 0);
26693 ccode = GET_CODE (cond);
26694 /* Check whether conditional jump use Sign or Overflow Flags. */
26695 if (!TARGET_FUSE_CMP_AND_BRANCH_SOFLAGS
26702 /* Return true for TYPE_TEST and TYPE_ICMP. */
26703 if (get_attr_type (condgen) == TYPE_TEST
26704 || get_attr_type (condgen) == TYPE_ICMP)
26707 /* The following is the case that macro-fusion for alu + jmp. */
26708 if (!TARGET_FUSE_ALU_AND_BRANCH || !alu_set)
26711 /* No fusion for alu op with memory destination operand. */
26712 dest = SET_DEST (alu_set);
26716 /* Macro-fusion for inc/dec + unsigned conditional jump is not
26718 if (get_attr_type (condgen) == TYPE_INCDEC
26728 /* Try to reorder ready list to take advantage of Atom pipelined IMUL
26729 execution. It is applied if
26730 (1) IMUL instruction is on the top of list;
26731 (2) There exists the only producer of independent IMUL instruction in
26733 Return index of IMUL producer if it was found and -1 otherwise. */
26735 do_reorder_for_imul (rtx_insn **ready, int n_ready)
26738 rtx set, insn1, insn2;
26739 sd_iterator_def sd_it;
26744 if (!TARGET_BONNELL)
26747 /* Check that IMUL instruction is on the top of ready list. */
26748 insn = ready[n_ready - 1];
26749 set = single_set (insn);
26752 if (!(GET_CODE (SET_SRC (set)) == MULT
26753 && GET_MODE (SET_SRC (set)) == SImode))
26756 /* Search for producer of independent IMUL instruction. */
26757 for (i = n_ready - 2; i >= 0; i--)
26760 if (!NONDEBUG_INSN_P (insn))
26762 /* Skip IMUL instruction. */
26763 insn2 = PATTERN (insn);
26764 if (GET_CODE (insn2) == PARALLEL)
26765 insn2 = XVECEXP (insn2, 0, 0);
26766 if (GET_CODE (insn2) == SET
26767 && GET_CODE (SET_SRC (insn2)) == MULT
26768 && GET_MODE (SET_SRC (insn2)) == SImode)
26771 FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep)
26774 con = DEP_CON (dep);
26775 if (!NONDEBUG_INSN_P (con))
26777 insn1 = PATTERN (con);
26778 if (GET_CODE (insn1) == PARALLEL)
26779 insn1 = XVECEXP (insn1, 0, 0);
26781 if (GET_CODE (insn1) == SET
26782 && GET_CODE (SET_SRC (insn1)) == MULT
26783 && GET_MODE (SET_SRC (insn1)) == SImode)
26785 sd_iterator_def sd_it1;
26787 /* Check if there is no other dependee for IMUL. */
26789 FOR_EACH_DEP (con, SD_LIST_BACK, sd_it1, dep1)
26792 pro = DEP_PRO (dep1);
26793 if (!NONDEBUG_INSN_P (pro))
26808 /* Try to find the best candidate on the top of ready list if two insns
26809 have the same priority - candidate is best if its dependees were
26810 scheduled earlier. Applied for Silvermont only.
26811 Return true if top 2 insns must be interchanged. */
26813 swap_top_of_ready_list (rtx_insn **ready, int n_ready)
26815 rtx_insn *top = ready[n_ready - 1];
26816 rtx_insn *next = ready[n_ready - 2];
26818 sd_iterator_def sd_it;
26822 #define INSN_TICK(INSN) (HID (INSN)->tick)
26824 if (!TARGET_SILVERMONT && !TARGET_INTEL)
26827 if (!NONDEBUG_INSN_P (top))
26829 if (!NONJUMP_INSN_P (top))
26831 if (!NONDEBUG_INSN_P (next))
26833 if (!NONJUMP_INSN_P (next))
26835 set = single_set (top);
26838 set = single_set (next);
26842 if (INSN_PRIORITY_KNOWN (top) && INSN_PRIORITY_KNOWN (next))
26844 if (INSN_PRIORITY (top) != INSN_PRIORITY (next))
26846 /* Determine winner more precise. */
26847 FOR_EACH_DEP (top, SD_LIST_RES_BACK, sd_it, dep)
26850 pro = DEP_PRO (dep);
26851 if (!NONDEBUG_INSN_P (pro))
26853 if (INSN_TICK (pro) > clock1)
26854 clock1 = INSN_TICK (pro);
26856 FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep)
26859 pro = DEP_PRO (dep);
26860 if (!NONDEBUG_INSN_P (pro))
26862 if (INSN_TICK (pro) > clock2)
26863 clock2 = INSN_TICK (pro);
26866 if (clock1 == clock2)
26868 /* Determine winner - load must win. */
26869 enum attr_memory memory1, memory2;
26870 memory1 = get_attr_memory (top);
26871 memory2 = get_attr_memory (next);
26872 if (memory2 == MEMORY_LOAD && memory1 != MEMORY_LOAD)
26875 return (bool) (clock2 < clock1);
26881 /* Perform possible reodering of ready list for Atom/Silvermont only.
26882 Return issue rate. */
26884 ix86_sched_reorder (FILE *dump, int sched_verbose, rtx_insn **ready,
26885 int *pn_ready, int clock_var)
26887 int issue_rate = -1;
26888 int n_ready = *pn_ready;
26893 /* Set up issue rate. */
26894 issue_rate = ix86_issue_rate ();
26896 /* Do reodering for BONNELL/SILVERMONT only. */
26897 if (!TARGET_BONNELL && !TARGET_SILVERMONT && !TARGET_INTEL)
26900 /* Nothing to do if ready list contains only 1 instruction. */
26904 /* Do reodering for post-reload scheduler only. */
26905 if (!reload_completed)
26908 if ((index = do_reorder_for_imul (ready, n_ready)) >= 0)
26910 if (sched_verbose > 1)
26911 fprintf (dump, ";;\tatom sched_reorder: put %d insn on top\n",
26912 INSN_UID (ready[index]));
26914 /* Put IMUL producer (ready[index]) at the top of ready list. */
26915 insn = ready[index];
26916 for (i = index; i < n_ready - 1; i++)
26917 ready[i] = ready[i + 1];
26918 ready[n_ready - 1] = insn;
26922 /* Skip selective scheduling since HID is not populated in it. */
26925 && swap_top_of_ready_list (ready, n_ready))
26927 if (sched_verbose > 1)
26928 fprintf (dump, ";;\tslm sched_reorder: swap %d and %d insns\n",
26929 INSN_UID (ready[n_ready - 1]), INSN_UID (ready[n_ready - 2]));
26930 /* Swap 2 top elements of ready list. */
26931 insn = ready[n_ready - 1];
26932 ready[n_ready - 1] = ready[n_ready - 2];
26933 ready[n_ready - 2] = insn;
26939 ix86_class_likely_spilled_p (reg_class_t);
26941 /* Returns true if lhs of insn is HW function argument register and set up
26942 is_spilled to true if it is likely spilled HW register. */
26944 insn_is_function_arg (rtx insn, bool* is_spilled)
26948 if (!NONDEBUG_INSN_P (insn))
26950 /* Call instructions are not movable, ignore it. */
26953 insn = PATTERN (insn);
26954 if (GET_CODE (insn) == PARALLEL)
26955 insn = XVECEXP (insn, 0, 0);
26956 if (GET_CODE (insn) != SET)
26958 dst = SET_DEST (insn);
26959 if (REG_P (dst) && HARD_REGISTER_P (dst)
26960 && ix86_function_arg_regno_p (REGNO (dst)))
26962 /* Is it likely spilled HW register? */
26963 if (!TEST_HARD_REG_BIT (fixed_reg_set, REGNO (dst))
26964 && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (dst))))
26965 *is_spilled = true;
26971 /* Add output dependencies for chain of function adjacent arguments if only
26972 there is a move to likely spilled HW register. Return first argument
26973 if at least one dependence was added or NULL otherwise. */
26975 add_parameter_dependencies (rtx_insn *call, rtx_insn *head)
26978 rtx_insn *last = call;
26979 rtx_insn *first_arg = NULL;
26980 bool is_spilled = false;
26982 head = PREV_INSN (head);
26984 /* Find nearest to call argument passing instruction. */
26987 last = PREV_INSN (last);
26990 if (!NONDEBUG_INSN_P (last))
26992 if (insn_is_function_arg (last, &is_spilled))
27000 insn = PREV_INSN (last);
27001 if (!INSN_P (insn))
27005 if (!NONDEBUG_INSN_P (insn))
27010 if (insn_is_function_arg (insn, &is_spilled))
27012 /* Add output depdendence between two function arguments if chain
27013 of output arguments contains likely spilled HW registers. */
27015 add_dependence (first_arg, insn, REG_DEP_OUTPUT);
27016 first_arg = last = insn;
27026 /* Add output or anti dependency from insn to first_arg to restrict its code
27029 avoid_func_arg_motion (rtx_insn *first_arg, rtx_insn *insn)
27034 /* Add anti dependencies for bounds stores. */
27036 && GET_CODE (PATTERN (insn)) == PARALLEL
27037 && GET_CODE (XVECEXP (PATTERN (insn), 0, 0)) == UNSPEC
27038 && XINT (XVECEXP (PATTERN (insn), 0, 0), 1) == UNSPEC_BNDSTX)
27040 add_dependence (first_arg, insn, REG_DEP_ANTI);
27044 set = single_set (insn);
27047 tmp = SET_DEST (set);
27050 /* Add output dependency to the first function argument. */
27051 add_dependence (first_arg, insn, REG_DEP_OUTPUT);
27054 /* Add anti dependency. */
27055 add_dependence (first_arg, insn, REG_DEP_ANTI);
27058 /* Avoid cross block motion of function argument through adding dependency
27059 from the first non-jump instruction in bb. */
27061 add_dependee_for_func_arg (rtx_insn *arg, basic_block bb)
27063 rtx_insn *insn = BB_END (bb);
27067 if (NONDEBUG_INSN_P (insn) && NONJUMP_INSN_P (insn))
27069 rtx set = single_set (insn);
27072 avoid_func_arg_motion (arg, insn);
27076 if (insn == BB_HEAD (bb))
27078 insn = PREV_INSN (insn);
27082 /* Hook for pre-reload schedule - avoid motion of function arguments
27083 passed in likely spilled HW registers. */
27085 ix86_dependencies_evaluation_hook (rtx_insn *head, rtx_insn *tail)
27088 rtx_insn *first_arg = NULL;
27089 if (reload_completed)
27091 while (head != tail && DEBUG_INSN_P (head))
27092 head = NEXT_INSN (head);
27093 for (insn = tail; insn != head; insn = PREV_INSN (insn))
27094 if (INSN_P (insn) && CALL_P (insn))
27096 first_arg = add_parameter_dependencies (insn, head);
27099 /* Add dependee for first argument to predecessors if only
27100 region contains more than one block. */
27101 basic_block bb = BLOCK_FOR_INSN (insn);
27102 int rgn = CONTAINING_RGN (bb->index);
27103 int nr_blks = RGN_NR_BLOCKS (rgn);
27104 /* Skip trivial regions and region head blocks that can have
27105 predecessors outside of region. */
27106 if (nr_blks > 1 && BLOCK_TO_BB (bb->index) != 0)
27111 /* Regions are SCCs with the exception of selective
27112 scheduling with pipelining of outer blocks enabled.
27113 So also check that immediate predecessors of a non-head
27114 block are in the same region. */
27115 FOR_EACH_EDGE (e, ei, bb->preds)
27117 /* Avoid creating of loop-carried dependencies through
27118 using topological ordering in the region. */
27119 if (rgn == CONTAINING_RGN (e->src->index)
27120 && BLOCK_TO_BB (bb->index) > BLOCK_TO_BB (e->src->index))
27121 add_dependee_for_func_arg (first_arg, e->src);
27129 else if (first_arg)
27130 avoid_func_arg_motion (first_arg, insn);
27133 /* Hook for pre-reload schedule - set priority of moves from likely spilled
27134 HW registers to maximum, to schedule them at soon as possible. These are
27135 moves from function argument registers at the top of the function entry
27136 and moves from function return value registers after call. */
27138 ix86_adjust_priority (rtx_insn *insn, int priority)
27142 if (reload_completed)
27145 if (!NONDEBUG_INSN_P (insn))
27148 set = single_set (insn);
27151 rtx tmp = SET_SRC (set);
27153 && HARD_REGISTER_P (tmp)
27154 && !TEST_HARD_REG_BIT (fixed_reg_set, REGNO (tmp))
27155 && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (tmp))))
27156 return current_sched_info->sched_max_insns_priority;
27162 /* Model decoder of Core 2/i7.
27163 Below hooks for multipass scheduling (see haifa-sched.c:max_issue)
27164 track the instruction fetch block boundaries and make sure that long
27165 (9+ bytes) instructions are assigned to D0. */
27167 /* Maximum length of an insn that can be handled by
27168 a secondary decoder unit. '8' for Core 2/i7. */
27169 static int core2i7_secondary_decoder_max_insn_size;
27171 /* Ifetch block size, i.e., number of bytes decoder reads per cycle.
27172 '16' for Core 2/i7. */
27173 static int core2i7_ifetch_block_size;
27175 /* Maximum number of instructions decoder can handle per cycle.
27176 '6' for Core 2/i7. */
27177 static int core2i7_ifetch_block_max_insns;
27179 typedef struct ix86_first_cycle_multipass_data_ *
27180 ix86_first_cycle_multipass_data_t;
27181 typedef const struct ix86_first_cycle_multipass_data_ *
27182 const_ix86_first_cycle_multipass_data_t;
27184 /* A variable to store target state across calls to max_issue within
27186 static struct ix86_first_cycle_multipass_data_ _ix86_first_cycle_multipass_data,
27187 *ix86_first_cycle_multipass_data = &_ix86_first_cycle_multipass_data;
27189 /* Initialize DATA. */
27191 core2i7_first_cycle_multipass_init (void *_data)
27193 ix86_first_cycle_multipass_data_t data
27194 = (ix86_first_cycle_multipass_data_t) _data;
27196 data->ifetch_block_len = 0;
27197 data->ifetch_block_n_insns = 0;
27198 data->ready_try_change = NULL;
27199 data->ready_try_change_size = 0;
27202 /* Advancing the cycle; reset ifetch block counts. */
27204 core2i7_dfa_post_advance_cycle (void)
27206 ix86_first_cycle_multipass_data_t data = ix86_first_cycle_multipass_data;
27208 gcc_assert (data->ifetch_block_n_insns <= core2i7_ifetch_block_max_insns);
27210 data->ifetch_block_len = 0;
27211 data->ifetch_block_n_insns = 0;
27214 static int min_insn_size (rtx_insn *);
27216 /* Filter out insns from ready_try that the core will not be able to issue
27217 on current cycle due to decoder. */
27219 core2i7_first_cycle_multipass_filter_ready_try
27220 (const_ix86_first_cycle_multipass_data_t data,
27221 signed char *ready_try, int n_ready, bool first_cycle_insn_p)
27228 if (ready_try[n_ready])
27231 insn = get_ready_element (n_ready);
27232 insn_size = min_insn_size (insn);
27234 if (/* If this is a too long an insn for a secondary decoder ... */
27235 (!first_cycle_insn_p
27236 && insn_size > core2i7_secondary_decoder_max_insn_size)
27237 /* ... or it would not fit into the ifetch block ... */
27238 || data->ifetch_block_len + insn_size > core2i7_ifetch_block_size
27239 /* ... or the decoder is full already ... */
27240 || data->ifetch_block_n_insns + 1 > core2i7_ifetch_block_max_insns)
27241 /* ... mask the insn out. */
27243 ready_try[n_ready] = 1;
27245 if (data->ready_try_change)
27246 bitmap_set_bit (data->ready_try_change, n_ready);
27251 /* Prepare for a new round of multipass lookahead scheduling. */
27253 core2i7_first_cycle_multipass_begin (void *_data,
27254 signed char *ready_try, int n_ready,
27255 bool first_cycle_insn_p)
27257 ix86_first_cycle_multipass_data_t data
27258 = (ix86_first_cycle_multipass_data_t) _data;
27259 const_ix86_first_cycle_multipass_data_t prev_data
27260 = ix86_first_cycle_multipass_data;
27262 /* Restore the state from the end of the previous round. */
27263 data->ifetch_block_len = prev_data->ifetch_block_len;
27264 data->ifetch_block_n_insns = prev_data->ifetch_block_n_insns;
27266 /* Filter instructions that cannot be issued on current cycle due to
27267 decoder restrictions. */
27268 core2i7_first_cycle_multipass_filter_ready_try (data, ready_try, n_ready,
27269 first_cycle_insn_p);
27272 /* INSN is being issued in current solution. Account for its impact on
27273 the decoder model. */
27275 core2i7_first_cycle_multipass_issue (void *_data,
27276 signed char *ready_try, int n_ready,
27277 rtx_insn *insn, const void *_prev_data)
27279 ix86_first_cycle_multipass_data_t data
27280 = (ix86_first_cycle_multipass_data_t) _data;
27281 const_ix86_first_cycle_multipass_data_t prev_data
27282 = (const_ix86_first_cycle_multipass_data_t) _prev_data;
27284 int insn_size = min_insn_size (insn);
27286 data->ifetch_block_len = prev_data->ifetch_block_len + insn_size;
27287 data->ifetch_block_n_insns = prev_data->ifetch_block_n_insns + 1;
27288 gcc_assert (data->ifetch_block_len <= core2i7_ifetch_block_size
27289 && data->ifetch_block_n_insns <= core2i7_ifetch_block_max_insns);
27291 /* Allocate or resize the bitmap for storing INSN's effect on ready_try. */
27292 if (!data->ready_try_change)
27294 data->ready_try_change = sbitmap_alloc (n_ready);
27295 data->ready_try_change_size = n_ready;
27297 else if (data->ready_try_change_size < n_ready)
27299 data->ready_try_change = sbitmap_resize (data->ready_try_change,
27301 data->ready_try_change_size = n_ready;
27303 bitmap_clear (data->ready_try_change);
27305 /* Filter out insns from ready_try that the core will not be able to issue
27306 on current cycle due to decoder. */
27307 core2i7_first_cycle_multipass_filter_ready_try (data, ready_try, n_ready,
27311 /* Revert the effect on ready_try. */
27313 core2i7_first_cycle_multipass_backtrack (const void *_data,
27314 signed char *ready_try,
27315 int n_ready ATTRIBUTE_UNUSED)
27317 const_ix86_first_cycle_multipass_data_t data
27318 = (const_ix86_first_cycle_multipass_data_t) _data;
27319 unsigned int i = 0;
27320 sbitmap_iterator sbi;
27322 gcc_assert (bitmap_last_set_bit (data->ready_try_change) < n_ready);
27323 EXECUTE_IF_SET_IN_BITMAP (data->ready_try_change, 0, i, sbi)
27329 /* Save the result of multipass lookahead scheduling for the next round. */
27331 core2i7_first_cycle_multipass_end (const void *_data)
27333 const_ix86_first_cycle_multipass_data_t data
27334 = (const_ix86_first_cycle_multipass_data_t) _data;
27335 ix86_first_cycle_multipass_data_t next_data
27336 = ix86_first_cycle_multipass_data;
27340 next_data->ifetch_block_len = data->ifetch_block_len;
27341 next_data->ifetch_block_n_insns = data->ifetch_block_n_insns;
27345 /* Deallocate target data. */
27347 core2i7_first_cycle_multipass_fini (void *_data)
27349 ix86_first_cycle_multipass_data_t data
27350 = (ix86_first_cycle_multipass_data_t) _data;
27352 if (data->ready_try_change)
27354 sbitmap_free (data->ready_try_change);
27355 data->ready_try_change = NULL;
27356 data->ready_try_change_size = 0;
27360 /* Prepare for scheduling pass. */
27362 ix86_sched_init_global (FILE *, int, int)
27364 /* Install scheduling hooks for current CPU. Some of these hooks are used
27365 in time-critical parts of the scheduler, so we only set them up when
27366 they are actually used. */
27369 case PROCESSOR_CORE2:
27370 case PROCESSOR_NEHALEM:
27371 case PROCESSOR_SANDYBRIDGE:
27372 case PROCESSOR_HASWELL:
27373 /* Do not perform multipass scheduling for pre-reload schedule
27374 to save compile time. */
27375 if (reload_completed)
27377 targetm.sched.dfa_post_advance_cycle
27378 = core2i7_dfa_post_advance_cycle;
27379 targetm.sched.first_cycle_multipass_init
27380 = core2i7_first_cycle_multipass_init;
27381 targetm.sched.first_cycle_multipass_begin
27382 = core2i7_first_cycle_multipass_begin;
27383 targetm.sched.first_cycle_multipass_issue
27384 = core2i7_first_cycle_multipass_issue;
27385 targetm.sched.first_cycle_multipass_backtrack
27386 = core2i7_first_cycle_multipass_backtrack;
27387 targetm.sched.first_cycle_multipass_end
27388 = core2i7_first_cycle_multipass_end;
27389 targetm.sched.first_cycle_multipass_fini
27390 = core2i7_first_cycle_multipass_fini;
27392 /* Set decoder parameters. */
27393 core2i7_secondary_decoder_max_insn_size = 8;
27394 core2i7_ifetch_block_size = 16;
27395 core2i7_ifetch_block_max_insns = 6;
27398 /* ... Fall through ... */
27400 targetm.sched.dfa_post_advance_cycle = NULL;
27401 targetm.sched.first_cycle_multipass_init = NULL;
27402 targetm.sched.first_cycle_multipass_begin = NULL;
27403 targetm.sched.first_cycle_multipass_issue = NULL;
27404 targetm.sched.first_cycle_multipass_backtrack = NULL;
27405 targetm.sched.first_cycle_multipass_end = NULL;
27406 targetm.sched.first_cycle_multipass_fini = NULL;
27412 /* Compute the alignment given to a constant that is being placed in memory.
27413 EXP is the constant and ALIGN is the alignment that the object would
27415 The value of this function is used instead of that alignment to align
27419 ix86_constant_alignment (tree exp, int align)
27421 if (TREE_CODE (exp) == REAL_CST || TREE_CODE (exp) == VECTOR_CST
27422 || TREE_CODE (exp) == INTEGER_CST)
27424 if (TYPE_MODE (TREE_TYPE (exp)) == DFmode && align < 64)
27426 else if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (exp))) && align < 128)
27429 else if (!optimize_size && TREE_CODE (exp) == STRING_CST
27430 && TREE_STRING_LENGTH (exp) >= 31 && align < BITS_PER_WORD)
27431 return BITS_PER_WORD;
27436 /* Compute the alignment for a static variable.
27437 TYPE is the data type, and ALIGN is the alignment that
27438 the object would ordinarily have. The value of this function is used
27439 instead of that alignment to align the object. */
27442 ix86_data_alignment (tree type, int align, bool opt)
27444 /* GCC 4.8 and earlier used to incorrectly assume this alignment even
27445 for symbols from other compilation units or symbols that don't need
27446 to bind locally. In order to preserve some ABI compatibility with
27447 those compilers, ensure we don't decrease alignment from what we
27450 int max_align_compat = MIN (256, MAX_OFILE_ALIGNMENT);
27452 /* A data structure, equal or greater than the size of a cache line
27453 (64 bytes in the Pentium 4 and other recent Intel processors, including
27454 processors based on Intel Core microarchitecture) should be aligned
27455 so that its base address is a multiple of a cache line size. */
27458 = MIN ((unsigned) ix86_tune_cost->prefetch_block * 8, MAX_OFILE_ALIGNMENT);
27460 if (max_align < BITS_PER_WORD)
27461 max_align = BITS_PER_WORD;
27463 switch (ix86_align_data_type)
27465 case ix86_align_data_type_abi: opt = false; break;
27466 case ix86_align_data_type_compat: max_align = BITS_PER_WORD; break;
27467 case ix86_align_data_type_cacheline: break;
27471 && AGGREGATE_TYPE_P (type)
27472 && TYPE_SIZE (type)
27473 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST)
27475 if (wi::geu_p (TYPE_SIZE (type), max_align_compat)
27476 && align < max_align_compat)
27477 align = max_align_compat;
27478 if (wi::geu_p (TYPE_SIZE (type), max_align)
27479 && align < max_align)
27483 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
27484 to 16byte boundary. */
27487 if ((opt ? AGGREGATE_TYPE_P (type) : TREE_CODE (type) == ARRAY_TYPE)
27488 && TYPE_SIZE (type)
27489 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
27490 && wi::geu_p (TYPE_SIZE (type), 128)
27498 if (TREE_CODE (type) == ARRAY_TYPE)
27500 if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
27502 if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
27505 else if (TREE_CODE (type) == COMPLEX_TYPE)
27508 if (TYPE_MODE (type) == DCmode && align < 64)
27510 if ((TYPE_MODE (type) == XCmode
27511 || TYPE_MODE (type) == TCmode) && align < 128)
27514 else if ((TREE_CODE (type) == RECORD_TYPE
27515 || TREE_CODE (type) == UNION_TYPE
27516 || TREE_CODE (type) == QUAL_UNION_TYPE)
27517 && TYPE_FIELDS (type))
27519 if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
27521 if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
27524 else if (TREE_CODE (type) == REAL_TYPE || TREE_CODE (type) == VECTOR_TYPE
27525 || TREE_CODE (type) == INTEGER_TYPE)
27527 if (TYPE_MODE (type) == DFmode && align < 64)
27529 if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
27536 /* Compute the alignment for a local variable or a stack slot. EXP is
27537 the data type or decl itself, MODE is the widest mode available and
27538 ALIGN is the alignment that the object would ordinarily have. The
27539 value of this macro is used instead of that alignment to align the
27543 ix86_local_alignment (tree exp, machine_mode mode,
27544 unsigned int align)
27548 if (exp && DECL_P (exp))
27550 type = TREE_TYPE (exp);
27559 /* Don't do dynamic stack realignment for long long objects with
27560 -mpreferred-stack-boundary=2. */
27563 && ix86_preferred_stack_boundary < 64
27564 && (mode == DImode || (type && TYPE_MODE (type) == DImode))
27565 && (!type || !TYPE_USER_ALIGN (type))
27566 && (!decl || !DECL_USER_ALIGN (decl)))
27569 /* If TYPE is NULL, we are allocating a stack slot for caller-save
27570 register in MODE. We will return the largest alignment of XF
27574 if (mode == XFmode && align < GET_MODE_ALIGNMENT (DFmode))
27575 align = GET_MODE_ALIGNMENT (DFmode);
27579 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
27580 to 16byte boundary. Exact wording is:
27582 An array uses the same alignment as its elements, except that a local or
27583 global array variable of length at least 16 bytes or
27584 a C99 variable-length array variable always has alignment of at least 16 bytes.
27586 This was added to allow use of aligned SSE instructions at arrays. This
27587 rule is meant for static storage (where compiler can not do the analysis
27588 by itself). We follow it for automatic variables only when convenient.
27589 We fully control everything in the function compiled and functions from
27590 other unit can not rely on the alignment.
27592 Exclude va_list type. It is the common case of local array where
27593 we can not benefit from the alignment.
27595 TODO: Probably one should optimize for size only when var is not escaping. */
27596 if (TARGET_64BIT && optimize_function_for_speed_p (cfun)
27599 if (AGGREGATE_TYPE_P (type)
27600 && (va_list_type_node == NULL_TREE
27601 || (TYPE_MAIN_VARIANT (type)
27602 != TYPE_MAIN_VARIANT (va_list_type_node)))
27603 && TYPE_SIZE (type)
27604 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
27605 && wi::geu_p (TYPE_SIZE (type), 16)
27609 if (TREE_CODE (type) == ARRAY_TYPE)
27611 if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
27613 if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
27616 else if (TREE_CODE (type) == COMPLEX_TYPE)
27618 if (TYPE_MODE (type) == DCmode && align < 64)
27620 if ((TYPE_MODE (type) == XCmode
27621 || TYPE_MODE (type) == TCmode) && align < 128)
27624 else if ((TREE_CODE (type) == RECORD_TYPE
27625 || TREE_CODE (type) == UNION_TYPE
27626 || TREE_CODE (type) == QUAL_UNION_TYPE)
27627 && TYPE_FIELDS (type))
27629 if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
27631 if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
27634 else if (TREE_CODE (type) == REAL_TYPE || TREE_CODE (type) == VECTOR_TYPE
27635 || TREE_CODE (type) == INTEGER_TYPE)
27638 if (TYPE_MODE (type) == DFmode && align < 64)
27640 if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
27646 /* Compute the minimum required alignment for dynamic stack realignment
27647 purposes for a local variable, parameter or a stack slot. EXP is
27648 the data type or decl itself, MODE is its mode and ALIGN is the
27649 alignment that the object would ordinarily have. */
27652 ix86_minimum_alignment (tree exp, machine_mode mode,
27653 unsigned int align)
27657 if (exp && DECL_P (exp))
27659 type = TREE_TYPE (exp);
27668 if (TARGET_64BIT || align != 64 || ix86_preferred_stack_boundary >= 64)
27671 /* Don't do dynamic stack realignment for long long objects with
27672 -mpreferred-stack-boundary=2. */
27673 if ((mode == DImode || (type && TYPE_MODE (type) == DImode))
27674 && (!type || !TYPE_USER_ALIGN (type))
27675 && (!decl || !DECL_USER_ALIGN (decl)))
27681 /* Find a location for the static chain incoming to a nested function.
27682 This is a register, unless all free registers are used by arguments. */
27685 ix86_static_chain (const_tree fndecl_or_type, bool incoming_p)
27689 /* While this function won't be called by the middle-end when a static
27690 chain isn't needed, it's also used throughout the backend so it's
27691 easiest to keep this check centralized. */
27692 if (DECL_P (fndecl_or_type) && !DECL_STATIC_CHAIN (fndecl_or_type))
27697 /* We always use R10 in 64-bit mode. */
27702 const_tree fntype, fndecl;
27705 /* By default in 32-bit mode we use ECX to pass the static chain. */
27708 if (TREE_CODE (fndecl_or_type) == FUNCTION_DECL)
27710 fntype = TREE_TYPE (fndecl_or_type);
27711 fndecl = fndecl_or_type;
27715 fntype = fndecl_or_type;
27719 ccvt = ix86_get_callcvt (fntype);
27720 if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
27722 /* Fastcall functions use ecx/edx for arguments, which leaves
27723 us with EAX for the static chain.
27724 Thiscall functions use ecx for arguments, which also
27725 leaves us with EAX for the static chain. */
27728 else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
27730 /* Thiscall functions use ecx for arguments, which leaves
27731 us with EAX and EDX for the static chain.
27732 We are using for abi-compatibility EAX. */
27735 else if (ix86_function_regparm (fntype, fndecl) == 3)
27737 /* For regparm 3, we have no free call-clobbered registers in
27738 which to store the static chain. In order to implement this,
27739 we have the trampoline push the static chain to the stack.
27740 However, we can't push a value below the return address when
27741 we call the nested function directly, so we have to use an
27742 alternate entry point. For this we use ESI, and have the
27743 alternate entry point push ESI, so that things appear the
27744 same once we're executing the nested function. */
27747 if (fndecl == current_function_decl)
27748 ix86_static_chain_on_stack = true;
27749 return gen_frame_mem (SImode,
27750 plus_constant (Pmode,
27751 arg_pointer_rtx, -8));
27757 return gen_rtx_REG (Pmode, regno);
27760 /* Emit RTL insns to initialize the variable parts of a trampoline.
27761 FNDECL is the decl of the target address; M_TRAMP is a MEM for
27762 the trampoline, and CHAIN_VALUE is an RTX for the static chain
27763 to be passed to the target function. */
27766 ix86_trampoline_init (rtx m_tramp, tree fndecl, rtx chain_value)
27772 fnaddr = XEXP (DECL_RTL (fndecl), 0);
27778 /* Load the function address to r11. Try to load address using
27779 the shorter movl instead of movabs. We may want to support
27780 movq for kernel mode, but kernel does not use trampolines at
27781 the moment. FNADDR is a 32bit address and may not be in
27782 DImode when ptr_mode == SImode. Always use movl in this
27784 if (ptr_mode == SImode
27785 || x86_64_zext_immediate_operand (fnaddr, VOIDmode))
27787 fnaddr = copy_addr_to_reg (fnaddr);
27789 mem = adjust_address (m_tramp, HImode, offset);
27790 emit_move_insn (mem, gen_int_mode (0xbb41, HImode));
27792 mem = adjust_address (m_tramp, SImode, offset + 2);
27793 emit_move_insn (mem, gen_lowpart (SImode, fnaddr));
27798 mem = adjust_address (m_tramp, HImode, offset);
27799 emit_move_insn (mem, gen_int_mode (0xbb49, HImode));
27801 mem = adjust_address (m_tramp, DImode, offset + 2);
27802 emit_move_insn (mem, fnaddr);
27806 /* Load static chain using movabs to r10. Use the shorter movl
27807 instead of movabs when ptr_mode == SImode. */
27808 if (ptr_mode == SImode)
27819 mem = adjust_address (m_tramp, HImode, offset);
27820 emit_move_insn (mem, gen_int_mode (opcode, HImode));
27822 mem = adjust_address (m_tramp, ptr_mode, offset + 2);
27823 emit_move_insn (mem, chain_value);
27826 /* Jump to r11; the last (unused) byte is a nop, only there to
27827 pad the write out to a single 32-bit store. */
27828 mem = adjust_address (m_tramp, SImode, offset);
27829 emit_move_insn (mem, gen_int_mode (0x90e3ff49, SImode));
27836 /* Depending on the static chain location, either load a register
27837 with a constant, or push the constant to the stack. All of the
27838 instructions are the same size. */
27839 chain = ix86_static_chain (fndecl, true);
27842 switch (REGNO (chain))
27845 opcode = 0xb8; break;
27847 opcode = 0xb9; break;
27849 gcc_unreachable ();
27855 mem = adjust_address (m_tramp, QImode, offset);
27856 emit_move_insn (mem, gen_int_mode (opcode, QImode));
27858 mem = adjust_address (m_tramp, SImode, offset + 1);
27859 emit_move_insn (mem, chain_value);
27862 mem = adjust_address (m_tramp, QImode, offset);
27863 emit_move_insn (mem, gen_int_mode (0xe9, QImode));
27865 mem = adjust_address (m_tramp, SImode, offset + 1);
27867 /* Compute offset from the end of the jmp to the target function.
27868 In the case in which the trampoline stores the static chain on
27869 the stack, we need to skip the first insn which pushes the
27870 (call-saved) register static chain; this push is 1 byte. */
27872 disp = expand_binop (SImode, sub_optab, fnaddr,
27873 plus_constant (Pmode, XEXP (m_tramp, 0),
27874 offset - (MEM_P (chain) ? 1 : 0)),
27875 NULL_RTX, 1, OPTAB_DIRECT);
27876 emit_move_insn (mem, disp);
27879 gcc_assert (offset <= TRAMPOLINE_SIZE);
27881 #ifdef HAVE_ENABLE_EXECUTE_STACK
27882 #ifdef CHECK_EXECUTE_STACK_ENABLED
27883 if (CHECK_EXECUTE_STACK_ENABLED)
27885 emit_library_call (gen_rtx_SYMBOL_REF (Pmode, "__enable_execute_stack"),
27886 LCT_NORMAL, VOIDmode, 1, XEXP (m_tramp, 0), Pmode);
27890 /* The following file contains several enumerations and data structures
27891 built from the definitions in i386-builtin-types.def. */
27893 #include "i386-builtin-types.inc"
27895 /* Table for the ix86 builtin non-function types. */
27896 static GTY(()) tree ix86_builtin_type_tab[(int) IX86_BT_LAST_CPTR + 1];
27898 /* Retrieve an element from the above table, building some of
27899 the types lazily. */
27902 ix86_get_builtin_type (enum ix86_builtin_type tcode)
27904 unsigned int index;
27907 gcc_assert ((unsigned)tcode < ARRAY_SIZE(ix86_builtin_type_tab));
27909 type = ix86_builtin_type_tab[(int) tcode];
27913 gcc_assert (tcode > IX86_BT_LAST_PRIM);
27914 if (tcode <= IX86_BT_LAST_VECT)
27918 index = tcode - IX86_BT_LAST_PRIM - 1;
27919 itype = ix86_get_builtin_type (ix86_builtin_type_vect_base[index]);
27920 mode = ix86_builtin_type_vect_mode[index];
27922 type = build_vector_type_for_mode (itype, mode);
27928 index = tcode - IX86_BT_LAST_VECT - 1;
27929 if (tcode <= IX86_BT_LAST_PTR)
27930 quals = TYPE_UNQUALIFIED;
27932 quals = TYPE_QUAL_CONST;
27934 itype = ix86_get_builtin_type (ix86_builtin_type_ptr_base[index]);
27935 if (quals != TYPE_UNQUALIFIED)
27936 itype = build_qualified_type (itype, quals);
27938 type = build_pointer_type (itype);
27941 ix86_builtin_type_tab[(int) tcode] = type;
27945 /* Table for the ix86 builtin function types. */
27946 static GTY(()) tree ix86_builtin_func_type_tab[(int) IX86_BT_LAST_ALIAS + 1];
27948 /* Retrieve an element from the above table, building some of
27949 the types lazily. */
27952 ix86_get_builtin_func_type (enum ix86_builtin_func_type tcode)
27956 gcc_assert ((unsigned)tcode < ARRAY_SIZE (ix86_builtin_func_type_tab));
27958 type = ix86_builtin_func_type_tab[(int) tcode];
27962 if (tcode <= IX86_BT_LAST_FUNC)
27964 unsigned start = ix86_builtin_func_start[(int) tcode];
27965 unsigned after = ix86_builtin_func_start[(int) tcode + 1];
27966 tree rtype, atype, args = void_list_node;
27969 rtype = ix86_get_builtin_type (ix86_builtin_func_args[start]);
27970 for (i = after - 1; i > start; --i)
27972 atype = ix86_get_builtin_type (ix86_builtin_func_args[i]);
27973 args = tree_cons (NULL, atype, args);
27976 type = build_function_type (rtype, args);
27980 unsigned index = tcode - IX86_BT_LAST_FUNC - 1;
27981 enum ix86_builtin_func_type icode;
27983 icode = ix86_builtin_func_alias_base[index];
27984 type = ix86_get_builtin_func_type (icode);
27987 ix86_builtin_func_type_tab[(int) tcode] = type;
27992 /* Codes for all the SSE/MMX builtins. */
27995 IX86_BUILTIN_ADDPS,
27996 IX86_BUILTIN_ADDSS,
27997 IX86_BUILTIN_DIVPS,
27998 IX86_BUILTIN_DIVSS,
27999 IX86_BUILTIN_MULPS,
28000 IX86_BUILTIN_MULSS,
28001 IX86_BUILTIN_SUBPS,
28002 IX86_BUILTIN_SUBSS,
28004 IX86_BUILTIN_CMPEQPS,
28005 IX86_BUILTIN_CMPLTPS,
28006 IX86_BUILTIN_CMPLEPS,
28007 IX86_BUILTIN_CMPGTPS,
28008 IX86_BUILTIN_CMPGEPS,
28009 IX86_BUILTIN_CMPNEQPS,
28010 IX86_BUILTIN_CMPNLTPS,
28011 IX86_BUILTIN_CMPNLEPS,
28012 IX86_BUILTIN_CMPNGTPS,
28013 IX86_BUILTIN_CMPNGEPS,
28014 IX86_BUILTIN_CMPORDPS,
28015 IX86_BUILTIN_CMPUNORDPS,
28016 IX86_BUILTIN_CMPEQSS,
28017 IX86_BUILTIN_CMPLTSS,
28018 IX86_BUILTIN_CMPLESS,
28019 IX86_BUILTIN_CMPNEQSS,
28020 IX86_BUILTIN_CMPNLTSS,
28021 IX86_BUILTIN_CMPNLESS,
28022 IX86_BUILTIN_CMPORDSS,
28023 IX86_BUILTIN_CMPUNORDSS,
28025 IX86_BUILTIN_COMIEQSS,
28026 IX86_BUILTIN_COMILTSS,
28027 IX86_BUILTIN_COMILESS,
28028 IX86_BUILTIN_COMIGTSS,
28029 IX86_BUILTIN_COMIGESS,
28030 IX86_BUILTIN_COMINEQSS,
28031 IX86_BUILTIN_UCOMIEQSS,
28032 IX86_BUILTIN_UCOMILTSS,
28033 IX86_BUILTIN_UCOMILESS,
28034 IX86_BUILTIN_UCOMIGTSS,
28035 IX86_BUILTIN_UCOMIGESS,
28036 IX86_BUILTIN_UCOMINEQSS,
28038 IX86_BUILTIN_CVTPI2PS,
28039 IX86_BUILTIN_CVTPS2PI,
28040 IX86_BUILTIN_CVTSI2SS,
28041 IX86_BUILTIN_CVTSI642SS,
28042 IX86_BUILTIN_CVTSS2SI,
28043 IX86_BUILTIN_CVTSS2SI64,
28044 IX86_BUILTIN_CVTTPS2PI,
28045 IX86_BUILTIN_CVTTSS2SI,
28046 IX86_BUILTIN_CVTTSS2SI64,
28048 IX86_BUILTIN_MAXPS,
28049 IX86_BUILTIN_MAXSS,
28050 IX86_BUILTIN_MINPS,
28051 IX86_BUILTIN_MINSS,
28053 IX86_BUILTIN_LOADUPS,
28054 IX86_BUILTIN_STOREUPS,
28055 IX86_BUILTIN_MOVSS,
28057 IX86_BUILTIN_MOVHLPS,
28058 IX86_BUILTIN_MOVLHPS,
28059 IX86_BUILTIN_LOADHPS,
28060 IX86_BUILTIN_LOADLPS,
28061 IX86_BUILTIN_STOREHPS,
28062 IX86_BUILTIN_STORELPS,
28064 IX86_BUILTIN_MASKMOVQ,
28065 IX86_BUILTIN_MOVMSKPS,
28066 IX86_BUILTIN_PMOVMSKB,
28068 IX86_BUILTIN_MOVNTPS,
28069 IX86_BUILTIN_MOVNTQ,
28071 IX86_BUILTIN_LOADDQU,
28072 IX86_BUILTIN_STOREDQU,
28074 IX86_BUILTIN_PACKSSWB,
28075 IX86_BUILTIN_PACKSSDW,
28076 IX86_BUILTIN_PACKUSWB,
28078 IX86_BUILTIN_PADDB,
28079 IX86_BUILTIN_PADDW,
28080 IX86_BUILTIN_PADDD,
28081 IX86_BUILTIN_PADDQ,
28082 IX86_BUILTIN_PADDSB,
28083 IX86_BUILTIN_PADDSW,
28084 IX86_BUILTIN_PADDUSB,
28085 IX86_BUILTIN_PADDUSW,
28086 IX86_BUILTIN_PSUBB,
28087 IX86_BUILTIN_PSUBW,
28088 IX86_BUILTIN_PSUBD,
28089 IX86_BUILTIN_PSUBQ,
28090 IX86_BUILTIN_PSUBSB,
28091 IX86_BUILTIN_PSUBSW,
28092 IX86_BUILTIN_PSUBUSB,
28093 IX86_BUILTIN_PSUBUSW,
28096 IX86_BUILTIN_PANDN,
28100 IX86_BUILTIN_PAVGB,
28101 IX86_BUILTIN_PAVGW,
28103 IX86_BUILTIN_PCMPEQB,
28104 IX86_BUILTIN_PCMPEQW,
28105 IX86_BUILTIN_PCMPEQD,
28106 IX86_BUILTIN_PCMPGTB,
28107 IX86_BUILTIN_PCMPGTW,
28108 IX86_BUILTIN_PCMPGTD,
28110 IX86_BUILTIN_PMADDWD,
28112 IX86_BUILTIN_PMAXSW,
28113 IX86_BUILTIN_PMAXUB,
28114 IX86_BUILTIN_PMINSW,
28115 IX86_BUILTIN_PMINUB,
28117 IX86_BUILTIN_PMULHUW,
28118 IX86_BUILTIN_PMULHW,
28119 IX86_BUILTIN_PMULLW,
28121 IX86_BUILTIN_PSADBW,
28122 IX86_BUILTIN_PSHUFW,
28124 IX86_BUILTIN_PSLLW,
28125 IX86_BUILTIN_PSLLD,
28126 IX86_BUILTIN_PSLLQ,
28127 IX86_BUILTIN_PSRAW,
28128 IX86_BUILTIN_PSRAD,
28129 IX86_BUILTIN_PSRLW,
28130 IX86_BUILTIN_PSRLD,
28131 IX86_BUILTIN_PSRLQ,
28132 IX86_BUILTIN_PSLLWI,
28133 IX86_BUILTIN_PSLLDI,
28134 IX86_BUILTIN_PSLLQI,
28135 IX86_BUILTIN_PSRAWI,
28136 IX86_BUILTIN_PSRADI,
28137 IX86_BUILTIN_PSRLWI,
28138 IX86_BUILTIN_PSRLDI,
28139 IX86_BUILTIN_PSRLQI,
28141 IX86_BUILTIN_PUNPCKHBW,
28142 IX86_BUILTIN_PUNPCKHWD,
28143 IX86_BUILTIN_PUNPCKHDQ,
28144 IX86_BUILTIN_PUNPCKLBW,
28145 IX86_BUILTIN_PUNPCKLWD,
28146 IX86_BUILTIN_PUNPCKLDQ,
28148 IX86_BUILTIN_SHUFPS,
28150 IX86_BUILTIN_RCPPS,
28151 IX86_BUILTIN_RCPSS,
28152 IX86_BUILTIN_RSQRTPS,
28153 IX86_BUILTIN_RSQRTPS_NR,
28154 IX86_BUILTIN_RSQRTSS,
28155 IX86_BUILTIN_RSQRTF,
28156 IX86_BUILTIN_SQRTPS,
28157 IX86_BUILTIN_SQRTPS_NR,
28158 IX86_BUILTIN_SQRTSS,
28160 IX86_BUILTIN_UNPCKHPS,
28161 IX86_BUILTIN_UNPCKLPS,
28163 IX86_BUILTIN_ANDPS,
28164 IX86_BUILTIN_ANDNPS,
28166 IX86_BUILTIN_XORPS,
28169 IX86_BUILTIN_LDMXCSR,
28170 IX86_BUILTIN_STMXCSR,
28171 IX86_BUILTIN_SFENCE,
28173 IX86_BUILTIN_FXSAVE,
28174 IX86_BUILTIN_FXRSTOR,
28175 IX86_BUILTIN_FXSAVE64,
28176 IX86_BUILTIN_FXRSTOR64,
28178 IX86_BUILTIN_XSAVE,
28179 IX86_BUILTIN_XRSTOR,
28180 IX86_BUILTIN_XSAVE64,
28181 IX86_BUILTIN_XRSTOR64,
28183 IX86_BUILTIN_XSAVEOPT,
28184 IX86_BUILTIN_XSAVEOPT64,
28186 IX86_BUILTIN_XSAVEC,
28187 IX86_BUILTIN_XSAVEC64,
28189 IX86_BUILTIN_XSAVES,
28190 IX86_BUILTIN_XRSTORS,
28191 IX86_BUILTIN_XSAVES64,
28192 IX86_BUILTIN_XRSTORS64,
28194 /* 3DNow! Original */
28195 IX86_BUILTIN_FEMMS,
28196 IX86_BUILTIN_PAVGUSB,
28197 IX86_BUILTIN_PF2ID,
28198 IX86_BUILTIN_PFACC,
28199 IX86_BUILTIN_PFADD,
28200 IX86_BUILTIN_PFCMPEQ,
28201 IX86_BUILTIN_PFCMPGE,
28202 IX86_BUILTIN_PFCMPGT,
28203 IX86_BUILTIN_PFMAX,
28204 IX86_BUILTIN_PFMIN,
28205 IX86_BUILTIN_PFMUL,
28206 IX86_BUILTIN_PFRCP,
28207 IX86_BUILTIN_PFRCPIT1,
28208 IX86_BUILTIN_PFRCPIT2,
28209 IX86_BUILTIN_PFRSQIT1,
28210 IX86_BUILTIN_PFRSQRT,
28211 IX86_BUILTIN_PFSUB,
28212 IX86_BUILTIN_PFSUBR,
28213 IX86_BUILTIN_PI2FD,
28214 IX86_BUILTIN_PMULHRW,
28216 /* 3DNow! Athlon Extensions */
28217 IX86_BUILTIN_PF2IW,
28218 IX86_BUILTIN_PFNACC,
28219 IX86_BUILTIN_PFPNACC,
28220 IX86_BUILTIN_PI2FW,
28221 IX86_BUILTIN_PSWAPDSI,
28222 IX86_BUILTIN_PSWAPDSF,
28225 IX86_BUILTIN_ADDPD,
28226 IX86_BUILTIN_ADDSD,
28227 IX86_BUILTIN_DIVPD,
28228 IX86_BUILTIN_DIVSD,
28229 IX86_BUILTIN_MULPD,
28230 IX86_BUILTIN_MULSD,
28231 IX86_BUILTIN_SUBPD,
28232 IX86_BUILTIN_SUBSD,
28234 IX86_BUILTIN_CMPEQPD,
28235 IX86_BUILTIN_CMPLTPD,
28236 IX86_BUILTIN_CMPLEPD,
28237 IX86_BUILTIN_CMPGTPD,
28238 IX86_BUILTIN_CMPGEPD,
28239 IX86_BUILTIN_CMPNEQPD,
28240 IX86_BUILTIN_CMPNLTPD,
28241 IX86_BUILTIN_CMPNLEPD,
28242 IX86_BUILTIN_CMPNGTPD,
28243 IX86_BUILTIN_CMPNGEPD,
28244 IX86_BUILTIN_CMPORDPD,
28245 IX86_BUILTIN_CMPUNORDPD,
28246 IX86_BUILTIN_CMPEQSD,
28247 IX86_BUILTIN_CMPLTSD,
28248 IX86_BUILTIN_CMPLESD,
28249 IX86_BUILTIN_CMPNEQSD,
28250 IX86_BUILTIN_CMPNLTSD,
28251 IX86_BUILTIN_CMPNLESD,
28252 IX86_BUILTIN_CMPORDSD,
28253 IX86_BUILTIN_CMPUNORDSD,
28255 IX86_BUILTIN_COMIEQSD,
28256 IX86_BUILTIN_COMILTSD,
28257 IX86_BUILTIN_COMILESD,
28258 IX86_BUILTIN_COMIGTSD,
28259 IX86_BUILTIN_COMIGESD,
28260 IX86_BUILTIN_COMINEQSD,
28261 IX86_BUILTIN_UCOMIEQSD,
28262 IX86_BUILTIN_UCOMILTSD,
28263 IX86_BUILTIN_UCOMILESD,
28264 IX86_BUILTIN_UCOMIGTSD,
28265 IX86_BUILTIN_UCOMIGESD,
28266 IX86_BUILTIN_UCOMINEQSD,
28268 IX86_BUILTIN_MAXPD,
28269 IX86_BUILTIN_MAXSD,
28270 IX86_BUILTIN_MINPD,
28271 IX86_BUILTIN_MINSD,
28273 IX86_BUILTIN_ANDPD,
28274 IX86_BUILTIN_ANDNPD,
28276 IX86_BUILTIN_XORPD,
28278 IX86_BUILTIN_SQRTPD,
28279 IX86_BUILTIN_SQRTSD,
28281 IX86_BUILTIN_UNPCKHPD,
28282 IX86_BUILTIN_UNPCKLPD,
28284 IX86_BUILTIN_SHUFPD,
28286 IX86_BUILTIN_LOADUPD,
28287 IX86_BUILTIN_STOREUPD,
28288 IX86_BUILTIN_MOVSD,
28290 IX86_BUILTIN_LOADHPD,
28291 IX86_BUILTIN_LOADLPD,
28293 IX86_BUILTIN_CVTDQ2PD,
28294 IX86_BUILTIN_CVTDQ2PS,
28296 IX86_BUILTIN_CVTPD2DQ,
28297 IX86_BUILTIN_CVTPD2PI,
28298 IX86_BUILTIN_CVTPD2PS,
28299 IX86_BUILTIN_CVTTPD2DQ,
28300 IX86_BUILTIN_CVTTPD2PI,
28302 IX86_BUILTIN_CVTPI2PD,
28303 IX86_BUILTIN_CVTSI2SD,
28304 IX86_BUILTIN_CVTSI642SD,
28306 IX86_BUILTIN_CVTSD2SI,
28307 IX86_BUILTIN_CVTSD2SI64,
28308 IX86_BUILTIN_CVTSD2SS,
28309 IX86_BUILTIN_CVTSS2SD,
28310 IX86_BUILTIN_CVTTSD2SI,
28311 IX86_BUILTIN_CVTTSD2SI64,
28313 IX86_BUILTIN_CVTPS2DQ,
28314 IX86_BUILTIN_CVTPS2PD,
28315 IX86_BUILTIN_CVTTPS2DQ,
28317 IX86_BUILTIN_MOVNTI,
28318 IX86_BUILTIN_MOVNTI64,
28319 IX86_BUILTIN_MOVNTPD,
28320 IX86_BUILTIN_MOVNTDQ,
28322 IX86_BUILTIN_MOVQ128,
28325 IX86_BUILTIN_MASKMOVDQU,
28326 IX86_BUILTIN_MOVMSKPD,
28327 IX86_BUILTIN_PMOVMSKB128,
28329 IX86_BUILTIN_PACKSSWB128,
28330 IX86_BUILTIN_PACKSSDW128,
28331 IX86_BUILTIN_PACKUSWB128,
28333 IX86_BUILTIN_PADDB128,
28334 IX86_BUILTIN_PADDW128,
28335 IX86_BUILTIN_PADDD128,
28336 IX86_BUILTIN_PADDQ128,
28337 IX86_BUILTIN_PADDSB128,
28338 IX86_BUILTIN_PADDSW128,
28339 IX86_BUILTIN_PADDUSB128,
28340 IX86_BUILTIN_PADDUSW128,
28341 IX86_BUILTIN_PSUBB128,
28342 IX86_BUILTIN_PSUBW128,
28343 IX86_BUILTIN_PSUBD128,
28344 IX86_BUILTIN_PSUBQ128,
28345 IX86_BUILTIN_PSUBSB128,
28346 IX86_BUILTIN_PSUBSW128,
28347 IX86_BUILTIN_PSUBUSB128,
28348 IX86_BUILTIN_PSUBUSW128,
28350 IX86_BUILTIN_PAND128,
28351 IX86_BUILTIN_PANDN128,
28352 IX86_BUILTIN_POR128,
28353 IX86_BUILTIN_PXOR128,
28355 IX86_BUILTIN_PAVGB128,
28356 IX86_BUILTIN_PAVGW128,
28358 IX86_BUILTIN_PCMPEQB128,
28359 IX86_BUILTIN_PCMPEQW128,
28360 IX86_BUILTIN_PCMPEQD128,
28361 IX86_BUILTIN_PCMPGTB128,
28362 IX86_BUILTIN_PCMPGTW128,
28363 IX86_BUILTIN_PCMPGTD128,
28365 IX86_BUILTIN_PMADDWD128,
28367 IX86_BUILTIN_PMAXSW128,
28368 IX86_BUILTIN_PMAXUB128,
28369 IX86_BUILTIN_PMINSW128,
28370 IX86_BUILTIN_PMINUB128,
28372 IX86_BUILTIN_PMULUDQ,
28373 IX86_BUILTIN_PMULUDQ128,
28374 IX86_BUILTIN_PMULHUW128,
28375 IX86_BUILTIN_PMULHW128,
28376 IX86_BUILTIN_PMULLW128,
28378 IX86_BUILTIN_PSADBW128,
28379 IX86_BUILTIN_PSHUFHW,
28380 IX86_BUILTIN_PSHUFLW,
28381 IX86_BUILTIN_PSHUFD,
28383 IX86_BUILTIN_PSLLDQI128,
28384 IX86_BUILTIN_PSLLWI128,
28385 IX86_BUILTIN_PSLLDI128,
28386 IX86_BUILTIN_PSLLQI128,
28387 IX86_BUILTIN_PSRAWI128,
28388 IX86_BUILTIN_PSRADI128,
28389 IX86_BUILTIN_PSRLDQI128,
28390 IX86_BUILTIN_PSRLWI128,
28391 IX86_BUILTIN_PSRLDI128,
28392 IX86_BUILTIN_PSRLQI128,
28394 IX86_BUILTIN_PSLLDQ128,
28395 IX86_BUILTIN_PSLLW128,
28396 IX86_BUILTIN_PSLLD128,
28397 IX86_BUILTIN_PSLLQ128,
28398 IX86_BUILTIN_PSRAW128,
28399 IX86_BUILTIN_PSRAD128,
28400 IX86_BUILTIN_PSRLW128,
28401 IX86_BUILTIN_PSRLD128,
28402 IX86_BUILTIN_PSRLQ128,
28404 IX86_BUILTIN_PUNPCKHBW128,
28405 IX86_BUILTIN_PUNPCKHWD128,
28406 IX86_BUILTIN_PUNPCKHDQ128,
28407 IX86_BUILTIN_PUNPCKHQDQ128,
28408 IX86_BUILTIN_PUNPCKLBW128,
28409 IX86_BUILTIN_PUNPCKLWD128,
28410 IX86_BUILTIN_PUNPCKLDQ128,
28411 IX86_BUILTIN_PUNPCKLQDQ128,
28413 IX86_BUILTIN_CLFLUSH,
28414 IX86_BUILTIN_MFENCE,
28415 IX86_BUILTIN_LFENCE,
28416 IX86_BUILTIN_PAUSE,
28418 IX86_BUILTIN_FNSTENV,
28419 IX86_BUILTIN_FLDENV,
28420 IX86_BUILTIN_FNSTSW,
28421 IX86_BUILTIN_FNCLEX,
28423 IX86_BUILTIN_BSRSI,
28424 IX86_BUILTIN_BSRDI,
28425 IX86_BUILTIN_RDPMC,
28426 IX86_BUILTIN_RDTSC,
28427 IX86_BUILTIN_RDTSCP,
28428 IX86_BUILTIN_ROLQI,
28429 IX86_BUILTIN_ROLHI,
28430 IX86_BUILTIN_RORQI,
28431 IX86_BUILTIN_RORHI,
28434 IX86_BUILTIN_ADDSUBPS,
28435 IX86_BUILTIN_HADDPS,
28436 IX86_BUILTIN_HSUBPS,
28437 IX86_BUILTIN_MOVSHDUP,
28438 IX86_BUILTIN_MOVSLDUP,
28439 IX86_BUILTIN_ADDSUBPD,
28440 IX86_BUILTIN_HADDPD,
28441 IX86_BUILTIN_HSUBPD,
28442 IX86_BUILTIN_LDDQU,
28444 IX86_BUILTIN_MONITOR,
28445 IX86_BUILTIN_MWAIT,
28448 IX86_BUILTIN_PHADDW,
28449 IX86_BUILTIN_PHADDD,
28450 IX86_BUILTIN_PHADDSW,
28451 IX86_BUILTIN_PHSUBW,
28452 IX86_BUILTIN_PHSUBD,
28453 IX86_BUILTIN_PHSUBSW,
28454 IX86_BUILTIN_PMADDUBSW,
28455 IX86_BUILTIN_PMULHRSW,
28456 IX86_BUILTIN_PSHUFB,
28457 IX86_BUILTIN_PSIGNB,
28458 IX86_BUILTIN_PSIGNW,
28459 IX86_BUILTIN_PSIGND,
28460 IX86_BUILTIN_PALIGNR,
28461 IX86_BUILTIN_PABSB,
28462 IX86_BUILTIN_PABSW,
28463 IX86_BUILTIN_PABSD,
28465 IX86_BUILTIN_PHADDW128,
28466 IX86_BUILTIN_PHADDD128,
28467 IX86_BUILTIN_PHADDSW128,
28468 IX86_BUILTIN_PHSUBW128,
28469 IX86_BUILTIN_PHSUBD128,
28470 IX86_BUILTIN_PHSUBSW128,
28471 IX86_BUILTIN_PMADDUBSW128,
28472 IX86_BUILTIN_PMULHRSW128,
28473 IX86_BUILTIN_PSHUFB128,
28474 IX86_BUILTIN_PSIGNB128,
28475 IX86_BUILTIN_PSIGNW128,
28476 IX86_BUILTIN_PSIGND128,
28477 IX86_BUILTIN_PALIGNR128,
28478 IX86_BUILTIN_PABSB128,
28479 IX86_BUILTIN_PABSW128,
28480 IX86_BUILTIN_PABSD128,
28482 /* AMDFAM10 - SSE4A New Instructions. */
28483 IX86_BUILTIN_MOVNTSD,
28484 IX86_BUILTIN_MOVNTSS,
28485 IX86_BUILTIN_EXTRQI,
28486 IX86_BUILTIN_EXTRQ,
28487 IX86_BUILTIN_INSERTQI,
28488 IX86_BUILTIN_INSERTQ,
28491 IX86_BUILTIN_BLENDPD,
28492 IX86_BUILTIN_BLENDPS,
28493 IX86_BUILTIN_BLENDVPD,
28494 IX86_BUILTIN_BLENDVPS,
28495 IX86_BUILTIN_PBLENDVB128,
28496 IX86_BUILTIN_PBLENDW128,
28501 IX86_BUILTIN_INSERTPS128,
28503 IX86_BUILTIN_MOVNTDQA,
28504 IX86_BUILTIN_MPSADBW128,
28505 IX86_BUILTIN_PACKUSDW128,
28506 IX86_BUILTIN_PCMPEQQ,
28507 IX86_BUILTIN_PHMINPOSUW128,
28509 IX86_BUILTIN_PMAXSB128,
28510 IX86_BUILTIN_PMAXSD128,
28511 IX86_BUILTIN_PMAXUD128,
28512 IX86_BUILTIN_PMAXUW128,
28514 IX86_BUILTIN_PMINSB128,
28515 IX86_BUILTIN_PMINSD128,
28516 IX86_BUILTIN_PMINUD128,
28517 IX86_BUILTIN_PMINUW128,
28519 IX86_BUILTIN_PMOVSXBW128,
28520 IX86_BUILTIN_PMOVSXBD128,
28521 IX86_BUILTIN_PMOVSXBQ128,
28522 IX86_BUILTIN_PMOVSXWD128,
28523 IX86_BUILTIN_PMOVSXWQ128,
28524 IX86_BUILTIN_PMOVSXDQ128,
28526 IX86_BUILTIN_PMOVZXBW128,
28527 IX86_BUILTIN_PMOVZXBD128,
28528 IX86_BUILTIN_PMOVZXBQ128,
28529 IX86_BUILTIN_PMOVZXWD128,
28530 IX86_BUILTIN_PMOVZXWQ128,
28531 IX86_BUILTIN_PMOVZXDQ128,
28533 IX86_BUILTIN_PMULDQ128,
28534 IX86_BUILTIN_PMULLD128,
28536 IX86_BUILTIN_ROUNDSD,
28537 IX86_BUILTIN_ROUNDSS,
28539 IX86_BUILTIN_ROUNDPD,
28540 IX86_BUILTIN_ROUNDPS,
28542 IX86_BUILTIN_FLOORPD,
28543 IX86_BUILTIN_CEILPD,
28544 IX86_BUILTIN_TRUNCPD,
28545 IX86_BUILTIN_RINTPD,
28546 IX86_BUILTIN_ROUNDPD_AZ,
28548 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX,
28549 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX,
28550 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX,
28552 IX86_BUILTIN_FLOORPS,
28553 IX86_BUILTIN_CEILPS,
28554 IX86_BUILTIN_TRUNCPS,
28555 IX86_BUILTIN_RINTPS,
28556 IX86_BUILTIN_ROUNDPS_AZ,
28558 IX86_BUILTIN_FLOORPS_SFIX,
28559 IX86_BUILTIN_CEILPS_SFIX,
28560 IX86_BUILTIN_ROUNDPS_AZ_SFIX,
28562 IX86_BUILTIN_PTESTZ,
28563 IX86_BUILTIN_PTESTC,
28564 IX86_BUILTIN_PTESTNZC,
28566 IX86_BUILTIN_VEC_INIT_V2SI,
28567 IX86_BUILTIN_VEC_INIT_V4HI,
28568 IX86_BUILTIN_VEC_INIT_V8QI,
28569 IX86_BUILTIN_VEC_EXT_V2DF,
28570 IX86_BUILTIN_VEC_EXT_V2DI,
28571 IX86_BUILTIN_VEC_EXT_V4SF,
28572 IX86_BUILTIN_VEC_EXT_V4SI,
28573 IX86_BUILTIN_VEC_EXT_V8HI,
28574 IX86_BUILTIN_VEC_EXT_V2SI,
28575 IX86_BUILTIN_VEC_EXT_V4HI,
28576 IX86_BUILTIN_VEC_EXT_V16QI,
28577 IX86_BUILTIN_VEC_SET_V2DI,
28578 IX86_BUILTIN_VEC_SET_V4SF,
28579 IX86_BUILTIN_VEC_SET_V4SI,
28580 IX86_BUILTIN_VEC_SET_V8HI,
28581 IX86_BUILTIN_VEC_SET_V4HI,
28582 IX86_BUILTIN_VEC_SET_V16QI,
28584 IX86_BUILTIN_VEC_PACK_SFIX,
28585 IX86_BUILTIN_VEC_PACK_SFIX256,
28588 IX86_BUILTIN_CRC32QI,
28589 IX86_BUILTIN_CRC32HI,
28590 IX86_BUILTIN_CRC32SI,
28591 IX86_BUILTIN_CRC32DI,
28593 IX86_BUILTIN_PCMPESTRI128,
28594 IX86_BUILTIN_PCMPESTRM128,
28595 IX86_BUILTIN_PCMPESTRA128,
28596 IX86_BUILTIN_PCMPESTRC128,
28597 IX86_BUILTIN_PCMPESTRO128,
28598 IX86_BUILTIN_PCMPESTRS128,
28599 IX86_BUILTIN_PCMPESTRZ128,
28600 IX86_BUILTIN_PCMPISTRI128,
28601 IX86_BUILTIN_PCMPISTRM128,
28602 IX86_BUILTIN_PCMPISTRA128,
28603 IX86_BUILTIN_PCMPISTRC128,
28604 IX86_BUILTIN_PCMPISTRO128,
28605 IX86_BUILTIN_PCMPISTRS128,
28606 IX86_BUILTIN_PCMPISTRZ128,
28608 IX86_BUILTIN_PCMPGTQ,
28610 /* AES instructions */
28611 IX86_BUILTIN_AESENC128,
28612 IX86_BUILTIN_AESENCLAST128,
28613 IX86_BUILTIN_AESDEC128,
28614 IX86_BUILTIN_AESDECLAST128,
28615 IX86_BUILTIN_AESIMC128,
28616 IX86_BUILTIN_AESKEYGENASSIST128,
28618 /* PCLMUL instruction */
28619 IX86_BUILTIN_PCLMULQDQ128,
28622 IX86_BUILTIN_ADDPD256,
28623 IX86_BUILTIN_ADDPS256,
28624 IX86_BUILTIN_ADDSUBPD256,
28625 IX86_BUILTIN_ADDSUBPS256,
28626 IX86_BUILTIN_ANDPD256,
28627 IX86_BUILTIN_ANDPS256,
28628 IX86_BUILTIN_ANDNPD256,
28629 IX86_BUILTIN_ANDNPS256,
28630 IX86_BUILTIN_BLENDPD256,
28631 IX86_BUILTIN_BLENDPS256,
28632 IX86_BUILTIN_BLENDVPD256,
28633 IX86_BUILTIN_BLENDVPS256,
28634 IX86_BUILTIN_DIVPD256,
28635 IX86_BUILTIN_DIVPS256,
28636 IX86_BUILTIN_DPPS256,
28637 IX86_BUILTIN_HADDPD256,
28638 IX86_BUILTIN_HADDPS256,
28639 IX86_BUILTIN_HSUBPD256,
28640 IX86_BUILTIN_HSUBPS256,
28641 IX86_BUILTIN_MAXPD256,
28642 IX86_BUILTIN_MAXPS256,
28643 IX86_BUILTIN_MINPD256,
28644 IX86_BUILTIN_MINPS256,
28645 IX86_BUILTIN_MULPD256,
28646 IX86_BUILTIN_MULPS256,
28647 IX86_BUILTIN_ORPD256,
28648 IX86_BUILTIN_ORPS256,
28649 IX86_BUILTIN_SHUFPD256,
28650 IX86_BUILTIN_SHUFPS256,
28651 IX86_BUILTIN_SUBPD256,
28652 IX86_BUILTIN_SUBPS256,
28653 IX86_BUILTIN_XORPD256,
28654 IX86_BUILTIN_XORPS256,
28655 IX86_BUILTIN_CMPSD,
28656 IX86_BUILTIN_CMPSS,
28657 IX86_BUILTIN_CMPPD,
28658 IX86_BUILTIN_CMPPS,
28659 IX86_BUILTIN_CMPPD256,
28660 IX86_BUILTIN_CMPPS256,
28661 IX86_BUILTIN_CVTDQ2PD256,
28662 IX86_BUILTIN_CVTDQ2PS256,
28663 IX86_BUILTIN_CVTPD2PS256,
28664 IX86_BUILTIN_CVTPS2DQ256,
28665 IX86_BUILTIN_CVTPS2PD256,
28666 IX86_BUILTIN_CVTTPD2DQ256,
28667 IX86_BUILTIN_CVTPD2DQ256,
28668 IX86_BUILTIN_CVTTPS2DQ256,
28669 IX86_BUILTIN_EXTRACTF128PD256,
28670 IX86_BUILTIN_EXTRACTF128PS256,
28671 IX86_BUILTIN_EXTRACTF128SI256,
28672 IX86_BUILTIN_VZEROALL,
28673 IX86_BUILTIN_VZEROUPPER,
28674 IX86_BUILTIN_VPERMILVARPD,
28675 IX86_BUILTIN_VPERMILVARPS,
28676 IX86_BUILTIN_VPERMILVARPD256,
28677 IX86_BUILTIN_VPERMILVARPS256,
28678 IX86_BUILTIN_VPERMILPD,
28679 IX86_BUILTIN_VPERMILPS,
28680 IX86_BUILTIN_VPERMILPD256,
28681 IX86_BUILTIN_VPERMILPS256,
28682 IX86_BUILTIN_VPERMIL2PD,
28683 IX86_BUILTIN_VPERMIL2PS,
28684 IX86_BUILTIN_VPERMIL2PD256,
28685 IX86_BUILTIN_VPERMIL2PS256,
28686 IX86_BUILTIN_VPERM2F128PD256,
28687 IX86_BUILTIN_VPERM2F128PS256,
28688 IX86_BUILTIN_VPERM2F128SI256,
28689 IX86_BUILTIN_VBROADCASTSS,
28690 IX86_BUILTIN_VBROADCASTSD256,
28691 IX86_BUILTIN_VBROADCASTSS256,
28692 IX86_BUILTIN_VBROADCASTPD256,
28693 IX86_BUILTIN_VBROADCASTPS256,
28694 IX86_BUILTIN_VINSERTF128PD256,
28695 IX86_BUILTIN_VINSERTF128PS256,
28696 IX86_BUILTIN_VINSERTF128SI256,
28697 IX86_BUILTIN_LOADUPD256,
28698 IX86_BUILTIN_LOADUPS256,
28699 IX86_BUILTIN_STOREUPD256,
28700 IX86_BUILTIN_STOREUPS256,
28701 IX86_BUILTIN_LDDQU256,
28702 IX86_BUILTIN_MOVNTDQ256,
28703 IX86_BUILTIN_MOVNTPD256,
28704 IX86_BUILTIN_MOVNTPS256,
28705 IX86_BUILTIN_LOADDQU256,
28706 IX86_BUILTIN_STOREDQU256,
28707 IX86_BUILTIN_MASKLOADPD,
28708 IX86_BUILTIN_MASKLOADPS,
28709 IX86_BUILTIN_MASKSTOREPD,
28710 IX86_BUILTIN_MASKSTOREPS,
28711 IX86_BUILTIN_MASKLOADPD256,
28712 IX86_BUILTIN_MASKLOADPS256,
28713 IX86_BUILTIN_MASKSTOREPD256,
28714 IX86_BUILTIN_MASKSTOREPS256,
28715 IX86_BUILTIN_MOVSHDUP256,
28716 IX86_BUILTIN_MOVSLDUP256,
28717 IX86_BUILTIN_MOVDDUP256,
28719 IX86_BUILTIN_SQRTPD256,
28720 IX86_BUILTIN_SQRTPS256,
28721 IX86_BUILTIN_SQRTPS_NR256,
28722 IX86_BUILTIN_RSQRTPS256,
28723 IX86_BUILTIN_RSQRTPS_NR256,
28725 IX86_BUILTIN_RCPPS256,
28727 IX86_BUILTIN_ROUNDPD256,
28728 IX86_BUILTIN_ROUNDPS256,
28730 IX86_BUILTIN_FLOORPD256,
28731 IX86_BUILTIN_CEILPD256,
28732 IX86_BUILTIN_TRUNCPD256,
28733 IX86_BUILTIN_RINTPD256,
28734 IX86_BUILTIN_ROUNDPD_AZ256,
28736 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256,
28737 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256,
28738 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256,
28740 IX86_BUILTIN_FLOORPS256,
28741 IX86_BUILTIN_CEILPS256,
28742 IX86_BUILTIN_TRUNCPS256,
28743 IX86_BUILTIN_RINTPS256,
28744 IX86_BUILTIN_ROUNDPS_AZ256,
28746 IX86_BUILTIN_FLOORPS_SFIX256,
28747 IX86_BUILTIN_CEILPS_SFIX256,
28748 IX86_BUILTIN_ROUNDPS_AZ_SFIX256,
28750 IX86_BUILTIN_UNPCKHPD256,
28751 IX86_BUILTIN_UNPCKLPD256,
28752 IX86_BUILTIN_UNPCKHPS256,
28753 IX86_BUILTIN_UNPCKLPS256,
28755 IX86_BUILTIN_SI256_SI,
28756 IX86_BUILTIN_PS256_PS,
28757 IX86_BUILTIN_PD256_PD,
28758 IX86_BUILTIN_SI_SI256,
28759 IX86_BUILTIN_PS_PS256,
28760 IX86_BUILTIN_PD_PD256,
28762 IX86_BUILTIN_VTESTZPD,
28763 IX86_BUILTIN_VTESTCPD,
28764 IX86_BUILTIN_VTESTNZCPD,
28765 IX86_BUILTIN_VTESTZPS,
28766 IX86_BUILTIN_VTESTCPS,
28767 IX86_BUILTIN_VTESTNZCPS,
28768 IX86_BUILTIN_VTESTZPD256,
28769 IX86_BUILTIN_VTESTCPD256,
28770 IX86_BUILTIN_VTESTNZCPD256,
28771 IX86_BUILTIN_VTESTZPS256,
28772 IX86_BUILTIN_VTESTCPS256,
28773 IX86_BUILTIN_VTESTNZCPS256,
28774 IX86_BUILTIN_PTESTZ256,
28775 IX86_BUILTIN_PTESTC256,
28776 IX86_BUILTIN_PTESTNZC256,
28778 IX86_BUILTIN_MOVMSKPD256,
28779 IX86_BUILTIN_MOVMSKPS256,
28782 IX86_BUILTIN_MPSADBW256,
28783 IX86_BUILTIN_PABSB256,
28784 IX86_BUILTIN_PABSW256,
28785 IX86_BUILTIN_PABSD256,
28786 IX86_BUILTIN_PACKSSDW256,
28787 IX86_BUILTIN_PACKSSWB256,
28788 IX86_BUILTIN_PACKUSDW256,
28789 IX86_BUILTIN_PACKUSWB256,
28790 IX86_BUILTIN_PADDB256,
28791 IX86_BUILTIN_PADDW256,
28792 IX86_BUILTIN_PADDD256,
28793 IX86_BUILTIN_PADDQ256,
28794 IX86_BUILTIN_PADDSB256,
28795 IX86_BUILTIN_PADDSW256,
28796 IX86_BUILTIN_PADDUSB256,
28797 IX86_BUILTIN_PADDUSW256,
28798 IX86_BUILTIN_PALIGNR256,
28799 IX86_BUILTIN_AND256I,
28800 IX86_BUILTIN_ANDNOT256I,
28801 IX86_BUILTIN_PAVGB256,
28802 IX86_BUILTIN_PAVGW256,
28803 IX86_BUILTIN_PBLENDVB256,
28804 IX86_BUILTIN_PBLENDVW256,
28805 IX86_BUILTIN_PCMPEQB256,
28806 IX86_BUILTIN_PCMPEQW256,
28807 IX86_BUILTIN_PCMPEQD256,
28808 IX86_BUILTIN_PCMPEQQ256,
28809 IX86_BUILTIN_PCMPGTB256,
28810 IX86_BUILTIN_PCMPGTW256,
28811 IX86_BUILTIN_PCMPGTD256,
28812 IX86_BUILTIN_PCMPGTQ256,
28813 IX86_BUILTIN_PHADDW256,
28814 IX86_BUILTIN_PHADDD256,
28815 IX86_BUILTIN_PHADDSW256,
28816 IX86_BUILTIN_PHSUBW256,
28817 IX86_BUILTIN_PHSUBD256,
28818 IX86_BUILTIN_PHSUBSW256,
28819 IX86_BUILTIN_PMADDUBSW256,
28820 IX86_BUILTIN_PMADDWD256,
28821 IX86_BUILTIN_PMAXSB256,
28822 IX86_BUILTIN_PMAXSW256,
28823 IX86_BUILTIN_PMAXSD256,
28824 IX86_BUILTIN_PMAXUB256,
28825 IX86_BUILTIN_PMAXUW256,
28826 IX86_BUILTIN_PMAXUD256,
28827 IX86_BUILTIN_PMINSB256,
28828 IX86_BUILTIN_PMINSW256,
28829 IX86_BUILTIN_PMINSD256,
28830 IX86_BUILTIN_PMINUB256,
28831 IX86_BUILTIN_PMINUW256,
28832 IX86_BUILTIN_PMINUD256,
28833 IX86_BUILTIN_PMOVMSKB256,
28834 IX86_BUILTIN_PMOVSXBW256,
28835 IX86_BUILTIN_PMOVSXBD256,
28836 IX86_BUILTIN_PMOVSXBQ256,
28837 IX86_BUILTIN_PMOVSXWD256,
28838 IX86_BUILTIN_PMOVSXWQ256,
28839 IX86_BUILTIN_PMOVSXDQ256,
28840 IX86_BUILTIN_PMOVZXBW256,
28841 IX86_BUILTIN_PMOVZXBD256,
28842 IX86_BUILTIN_PMOVZXBQ256,
28843 IX86_BUILTIN_PMOVZXWD256,
28844 IX86_BUILTIN_PMOVZXWQ256,
28845 IX86_BUILTIN_PMOVZXDQ256,
28846 IX86_BUILTIN_PMULDQ256,
28847 IX86_BUILTIN_PMULHRSW256,
28848 IX86_BUILTIN_PMULHUW256,
28849 IX86_BUILTIN_PMULHW256,
28850 IX86_BUILTIN_PMULLW256,
28851 IX86_BUILTIN_PMULLD256,
28852 IX86_BUILTIN_PMULUDQ256,
28853 IX86_BUILTIN_POR256,
28854 IX86_BUILTIN_PSADBW256,
28855 IX86_BUILTIN_PSHUFB256,
28856 IX86_BUILTIN_PSHUFD256,
28857 IX86_BUILTIN_PSHUFHW256,
28858 IX86_BUILTIN_PSHUFLW256,
28859 IX86_BUILTIN_PSIGNB256,
28860 IX86_BUILTIN_PSIGNW256,
28861 IX86_BUILTIN_PSIGND256,
28862 IX86_BUILTIN_PSLLDQI256,
28863 IX86_BUILTIN_PSLLWI256,
28864 IX86_BUILTIN_PSLLW256,
28865 IX86_BUILTIN_PSLLDI256,
28866 IX86_BUILTIN_PSLLD256,
28867 IX86_BUILTIN_PSLLQI256,
28868 IX86_BUILTIN_PSLLQ256,
28869 IX86_BUILTIN_PSRAWI256,
28870 IX86_BUILTIN_PSRAW256,
28871 IX86_BUILTIN_PSRADI256,
28872 IX86_BUILTIN_PSRAD256,
28873 IX86_BUILTIN_PSRLDQI256,
28874 IX86_BUILTIN_PSRLWI256,
28875 IX86_BUILTIN_PSRLW256,
28876 IX86_BUILTIN_PSRLDI256,
28877 IX86_BUILTIN_PSRLD256,
28878 IX86_BUILTIN_PSRLQI256,
28879 IX86_BUILTIN_PSRLQ256,
28880 IX86_BUILTIN_PSUBB256,
28881 IX86_BUILTIN_PSUBW256,
28882 IX86_BUILTIN_PSUBD256,
28883 IX86_BUILTIN_PSUBQ256,
28884 IX86_BUILTIN_PSUBSB256,
28885 IX86_BUILTIN_PSUBSW256,
28886 IX86_BUILTIN_PSUBUSB256,
28887 IX86_BUILTIN_PSUBUSW256,
28888 IX86_BUILTIN_PUNPCKHBW256,
28889 IX86_BUILTIN_PUNPCKHWD256,
28890 IX86_BUILTIN_PUNPCKHDQ256,
28891 IX86_BUILTIN_PUNPCKHQDQ256,
28892 IX86_BUILTIN_PUNPCKLBW256,
28893 IX86_BUILTIN_PUNPCKLWD256,
28894 IX86_BUILTIN_PUNPCKLDQ256,
28895 IX86_BUILTIN_PUNPCKLQDQ256,
28896 IX86_BUILTIN_PXOR256,
28897 IX86_BUILTIN_MOVNTDQA256,
28898 IX86_BUILTIN_VBROADCASTSS_PS,
28899 IX86_BUILTIN_VBROADCASTSS_PS256,
28900 IX86_BUILTIN_VBROADCASTSD_PD256,
28901 IX86_BUILTIN_VBROADCASTSI256,
28902 IX86_BUILTIN_PBLENDD256,
28903 IX86_BUILTIN_PBLENDD128,
28904 IX86_BUILTIN_PBROADCASTB256,
28905 IX86_BUILTIN_PBROADCASTW256,
28906 IX86_BUILTIN_PBROADCASTD256,
28907 IX86_BUILTIN_PBROADCASTQ256,
28908 IX86_BUILTIN_PBROADCASTB128,
28909 IX86_BUILTIN_PBROADCASTW128,
28910 IX86_BUILTIN_PBROADCASTD128,
28911 IX86_BUILTIN_PBROADCASTQ128,
28912 IX86_BUILTIN_VPERMVARSI256,
28913 IX86_BUILTIN_VPERMDF256,
28914 IX86_BUILTIN_VPERMVARSF256,
28915 IX86_BUILTIN_VPERMDI256,
28916 IX86_BUILTIN_VPERMTI256,
28917 IX86_BUILTIN_VEXTRACT128I256,
28918 IX86_BUILTIN_VINSERT128I256,
28919 IX86_BUILTIN_MASKLOADD,
28920 IX86_BUILTIN_MASKLOADQ,
28921 IX86_BUILTIN_MASKLOADD256,
28922 IX86_BUILTIN_MASKLOADQ256,
28923 IX86_BUILTIN_MASKSTORED,
28924 IX86_BUILTIN_MASKSTOREQ,
28925 IX86_BUILTIN_MASKSTORED256,
28926 IX86_BUILTIN_MASKSTOREQ256,
28927 IX86_BUILTIN_PSLLVV4DI,
28928 IX86_BUILTIN_PSLLVV2DI,
28929 IX86_BUILTIN_PSLLVV8SI,
28930 IX86_BUILTIN_PSLLVV4SI,
28931 IX86_BUILTIN_PSRAVV8SI,
28932 IX86_BUILTIN_PSRAVV4SI,
28933 IX86_BUILTIN_PSRLVV4DI,
28934 IX86_BUILTIN_PSRLVV2DI,
28935 IX86_BUILTIN_PSRLVV8SI,
28936 IX86_BUILTIN_PSRLVV4SI,
28938 IX86_BUILTIN_GATHERSIV2DF,
28939 IX86_BUILTIN_GATHERSIV4DF,
28940 IX86_BUILTIN_GATHERDIV2DF,
28941 IX86_BUILTIN_GATHERDIV4DF,
28942 IX86_BUILTIN_GATHERSIV4SF,
28943 IX86_BUILTIN_GATHERSIV8SF,
28944 IX86_BUILTIN_GATHERDIV4SF,
28945 IX86_BUILTIN_GATHERDIV8SF,
28946 IX86_BUILTIN_GATHERSIV2DI,
28947 IX86_BUILTIN_GATHERSIV4DI,
28948 IX86_BUILTIN_GATHERDIV2DI,
28949 IX86_BUILTIN_GATHERDIV4DI,
28950 IX86_BUILTIN_GATHERSIV4SI,
28951 IX86_BUILTIN_GATHERSIV8SI,
28952 IX86_BUILTIN_GATHERDIV4SI,
28953 IX86_BUILTIN_GATHERDIV8SI,
28956 IX86_BUILTIN_SI512_SI256,
28957 IX86_BUILTIN_PD512_PD256,
28958 IX86_BUILTIN_PS512_PS256,
28959 IX86_BUILTIN_SI512_SI,
28960 IX86_BUILTIN_PD512_PD,
28961 IX86_BUILTIN_PS512_PS,
28962 IX86_BUILTIN_ADDPD512,
28963 IX86_BUILTIN_ADDPS512,
28964 IX86_BUILTIN_ADDSD_ROUND,
28965 IX86_BUILTIN_ADDSS_ROUND,
28966 IX86_BUILTIN_ALIGND512,
28967 IX86_BUILTIN_ALIGNQ512,
28968 IX86_BUILTIN_BLENDMD512,
28969 IX86_BUILTIN_BLENDMPD512,
28970 IX86_BUILTIN_BLENDMPS512,
28971 IX86_BUILTIN_BLENDMQ512,
28972 IX86_BUILTIN_BROADCASTF32X4_512,
28973 IX86_BUILTIN_BROADCASTF64X4_512,
28974 IX86_BUILTIN_BROADCASTI32X4_512,
28975 IX86_BUILTIN_BROADCASTI64X4_512,
28976 IX86_BUILTIN_BROADCASTSD512,
28977 IX86_BUILTIN_BROADCASTSS512,
28978 IX86_BUILTIN_CMPD512,
28979 IX86_BUILTIN_CMPPD512,
28980 IX86_BUILTIN_CMPPS512,
28981 IX86_BUILTIN_CMPQ512,
28982 IX86_BUILTIN_CMPSD_MASK,
28983 IX86_BUILTIN_CMPSS_MASK,
28984 IX86_BUILTIN_COMIDF,
28985 IX86_BUILTIN_COMISF,
28986 IX86_BUILTIN_COMPRESSPD512,
28987 IX86_BUILTIN_COMPRESSPDSTORE512,
28988 IX86_BUILTIN_COMPRESSPS512,
28989 IX86_BUILTIN_COMPRESSPSSTORE512,
28990 IX86_BUILTIN_CVTDQ2PD512,
28991 IX86_BUILTIN_CVTDQ2PS512,
28992 IX86_BUILTIN_CVTPD2DQ512,
28993 IX86_BUILTIN_CVTPD2PS512,
28994 IX86_BUILTIN_CVTPD2UDQ512,
28995 IX86_BUILTIN_CVTPH2PS512,
28996 IX86_BUILTIN_CVTPS2DQ512,
28997 IX86_BUILTIN_CVTPS2PD512,
28998 IX86_BUILTIN_CVTPS2PH512,
28999 IX86_BUILTIN_CVTPS2UDQ512,
29000 IX86_BUILTIN_CVTSD2SS_ROUND,
29001 IX86_BUILTIN_CVTSI2SD64,
29002 IX86_BUILTIN_CVTSI2SS32,
29003 IX86_BUILTIN_CVTSI2SS64,
29004 IX86_BUILTIN_CVTSS2SD_ROUND,
29005 IX86_BUILTIN_CVTTPD2DQ512,
29006 IX86_BUILTIN_CVTTPD2UDQ512,
29007 IX86_BUILTIN_CVTTPS2DQ512,
29008 IX86_BUILTIN_CVTTPS2UDQ512,
29009 IX86_BUILTIN_CVTUDQ2PD512,
29010 IX86_BUILTIN_CVTUDQ2PS512,
29011 IX86_BUILTIN_CVTUSI2SD32,
29012 IX86_BUILTIN_CVTUSI2SD64,
29013 IX86_BUILTIN_CVTUSI2SS32,
29014 IX86_BUILTIN_CVTUSI2SS64,
29015 IX86_BUILTIN_DIVPD512,
29016 IX86_BUILTIN_DIVPS512,
29017 IX86_BUILTIN_DIVSD_ROUND,
29018 IX86_BUILTIN_DIVSS_ROUND,
29019 IX86_BUILTIN_EXPANDPD512,
29020 IX86_BUILTIN_EXPANDPD512Z,
29021 IX86_BUILTIN_EXPANDPDLOAD512,
29022 IX86_BUILTIN_EXPANDPDLOAD512Z,
29023 IX86_BUILTIN_EXPANDPS512,
29024 IX86_BUILTIN_EXPANDPS512Z,
29025 IX86_BUILTIN_EXPANDPSLOAD512,
29026 IX86_BUILTIN_EXPANDPSLOAD512Z,
29027 IX86_BUILTIN_EXTRACTF32X4,
29028 IX86_BUILTIN_EXTRACTF64X4,
29029 IX86_BUILTIN_EXTRACTI32X4,
29030 IX86_BUILTIN_EXTRACTI64X4,
29031 IX86_BUILTIN_FIXUPIMMPD512_MASK,
29032 IX86_BUILTIN_FIXUPIMMPD512_MASKZ,
29033 IX86_BUILTIN_FIXUPIMMPS512_MASK,
29034 IX86_BUILTIN_FIXUPIMMPS512_MASKZ,
29035 IX86_BUILTIN_FIXUPIMMSD128_MASK,
29036 IX86_BUILTIN_FIXUPIMMSD128_MASKZ,
29037 IX86_BUILTIN_FIXUPIMMSS128_MASK,
29038 IX86_BUILTIN_FIXUPIMMSS128_MASKZ,
29039 IX86_BUILTIN_GETEXPPD512,
29040 IX86_BUILTIN_GETEXPPS512,
29041 IX86_BUILTIN_GETEXPSD128,
29042 IX86_BUILTIN_GETEXPSS128,
29043 IX86_BUILTIN_GETMANTPD512,
29044 IX86_BUILTIN_GETMANTPS512,
29045 IX86_BUILTIN_GETMANTSD128,
29046 IX86_BUILTIN_GETMANTSS128,
29047 IX86_BUILTIN_INSERTF32X4,
29048 IX86_BUILTIN_INSERTF64X4,
29049 IX86_BUILTIN_INSERTI32X4,
29050 IX86_BUILTIN_INSERTI64X4,
29051 IX86_BUILTIN_LOADAPD512,
29052 IX86_BUILTIN_LOADAPS512,
29053 IX86_BUILTIN_LOADDQUDI512,
29054 IX86_BUILTIN_LOADDQUSI512,
29055 IX86_BUILTIN_LOADUPD512,
29056 IX86_BUILTIN_LOADUPS512,
29057 IX86_BUILTIN_MAXPD512,
29058 IX86_BUILTIN_MAXPS512,
29059 IX86_BUILTIN_MAXSD_ROUND,
29060 IX86_BUILTIN_MAXSS_ROUND,
29061 IX86_BUILTIN_MINPD512,
29062 IX86_BUILTIN_MINPS512,
29063 IX86_BUILTIN_MINSD_ROUND,
29064 IX86_BUILTIN_MINSS_ROUND,
29065 IX86_BUILTIN_MOVAPD512,
29066 IX86_BUILTIN_MOVAPS512,
29067 IX86_BUILTIN_MOVDDUP512,
29068 IX86_BUILTIN_MOVDQA32LOAD512,
29069 IX86_BUILTIN_MOVDQA32STORE512,
29070 IX86_BUILTIN_MOVDQA32_512,
29071 IX86_BUILTIN_MOVDQA64LOAD512,
29072 IX86_BUILTIN_MOVDQA64STORE512,
29073 IX86_BUILTIN_MOVDQA64_512,
29074 IX86_BUILTIN_MOVNTDQ512,
29075 IX86_BUILTIN_MOVNTDQA512,
29076 IX86_BUILTIN_MOVNTPD512,
29077 IX86_BUILTIN_MOVNTPS512,
29078 IX86_BUILTIN_MOVSHDUP512,
29079 IX86_BUILTIN_MOVSLDUP512,
29080 IX86_BUILTIN_MULPD512,
29081 IX86_BUILTIN_MULPS512,
29082 IX86_BUILTIN_MULSD_ROUND,
29083 IX86_BUILTIN_MULSS_ROUND,
29084 IX86_BUILTIN_PABSD512,
29085 IX86_BUILTIN_PABSQ512,
29086 IX86_BUILTIN_PADDD512,
29087 IX86_BUILTIN_PADDQ512,
29088 IX86_BUILTIN_PANDD512,
29089 IX86_BUILTIN_PANDND512,
29090 IX86_BUILTIN_PANDNQ512,
29091 IX86_BUILTIN_PANDQ512,
29092 IX86_BUILTIN_PBROADCASTD512,
29093 IX86_BUILTIN_PBROADCASTD512_GPR,
29094 IX86_BUILTIN_PBROADCASTMB512,
29095 IX86_BUILTIN_PBROADCASTMW512,
29096 IX86_BUILTIN_PBROADCASTQ512,
29097 IX86_BUILTIN_PBROADCASTQ512_GPR,
29098 IX86_BUILTIN_PCMPEQD512_MASK,
29099 IX86_BUILTIN_PCMPEQQ512_MASK,
29100 IX86_BUILTIN_PCMPGTD512_MASK,
29101 IX86_BUILTIN_PCMPGTQ512_MASK,
29102 IX86_BUILTIN_PCOMPRESSD512,
29103 IX86_BUILTIN_PCOMPRESSDSTORE512,
29104 IX86_BUILTIN_PCOMPRESSQ512,
29105 IX86_BUILTIN_PCOMPRESSQSTORE512,
29106 IX86_BUILTIN_PEXPANDD512,
29107 IX86_BUILTIN_PEXPANDD512Z,
29108 IX86_BUILTIN_PEXPANDDLOAD512,
29109 IX86_BUILTIN_PEXPANDDLOAD512Z,
29110 IX86_BUILTIN_PEXPANDQ512,
29111 IX86_BUILTIN_PEXPANDQ512Z,
29112 IX86_BUILTIN_PEXPANDQLOAD512,
29113 IX86_BUILTIN_PEXPANDQLOAD512Z,
29114 IX86_BUILTIN_PMAXSD512,
29115 IX86_BUILTIN_PMAXSQ512,
29116 IX86_BUILTIN_PMAXUD512,
29117 IX86_BUILTIN_PMAXUQ512,
29118 IX86_BUILTIN_PMINSD512,
29119 IX86_BUILTIN_PMINSQ512,
29120 IX86_BUILTIN_PMINUD512,
29121 IX86_BUILTIN_PMINUQ512,
29122 IX86_BUILTIN_PMOVDB512,
29123 IX86_BUILTIN_PMOVDB512_MEM,
29124 IX86_BUILTIN_PMOVDW512,
29125 IX86_BUILTIN_PMOVDW512_MEM,
29126 IX86_BUILTIN_PMOVQB512,
29127 IX86_BUILTIN_PMOVQB512_MEM,
29128 IX86_BUILTIN_PMOVQD512,
29129 IX86_BUILTIN_PMOVQD512_MEM,
29130 IX86_BUILTIN_PMOVQW512,
29131 IX86_BUILTIN_PMOVQW512_MEM,
29132 IX86_BUILTIN_PMOVSDB512,
29133 IX86_BUILTIN_PMOVSDB512_MEM,
29134 IX86_BUILTIN_PMOVSDW512,
29135 IX86_BUILTIN_PMOVSDW512_MEM,
29136 IX86_BUILTIN_PMOVSQB512,
29137 IX86_BUILTIN_PMOVSQB512_MEM,
29138 IX86_BUILTIN_PMOVSQD512,
29139 IX86_BUILTIN_PMOVSQD512_MEM,
29140 IX86_BUILTIN_PMOVSQW512,
29141 IX86_BUILTIN_PMOVSQW512_MEM,
29142 IX86_BUILTIN_PMOVSXBD512,
29143 IX86_BUILTIN_PMOVSXBQ512,
29144 IX86_BUILTIN_PMOVSXDQ512,
29145 IX86_BUILTIN_PMOVSXWD512,
29146 IX86_BUILTIN_PMOVSXWQ512,
29147 IX86_BUILTIN_PMOVUSDB512,
29148 IX86_BUILTIN_PMOVUSDB512_MEM,
29149 IX86_BUILTIN_PMOVUSDW512,
29150 IX86_BUILTIN_PMOVUSDW512_MEM,
29151 IX86_BUILTIN_PMOVUSQB512,
29152 IX86_BUILTIN_PMOVUSQB512_MEM,
29153 IX86_BUILTIN_PMOVUSQD512,
29154 IX86_BUILTIN_PMOVUSQD512_MEM,
29155 IX86_BUILTIN_PMOVUSQW512,
29156 IX86_BUILTIN_PMOVUSQW512_MEM,
29157 IX86_BUILTIN_PMOVZXBD512,
29158 IX86_BUILTIN_PMOVZXBQ512,
29159 IX86_BUILTIN_PMOVZXDQ512,
29160 IX86_BUILTIN_PMOVZXWD512,
29161 IX86_BUILTIN_PMOVZXWQ512,
29162 IX86_BUILTIN_PMULDQ512,
29163 IX86_BUILTIN_PMULLD512,
29164 IX86_BUILTIN_PMULUDQ512,
29165 IX86_BUILTIN_PORD512,
29166 IX86_BUILTIN_PORQ512,
29167 IX86_BUILTIN_PROLD512,
29168 IX86_BUILTIN_PROLQ512,
29169 IX86_BUILTIN_PROLVD512,
29170 IX86_BUILTIN_PROLVQ512,
29171 IX86_BUILTIN_PRORD512,
29172 IX86_BUILTIN_PRORQ512,
29173 IX86_BUILTIN_PRORVD512,
29174 IX86_BUILTIN_PRORVQ512,
29175 IX86_BUILTIN_PSHUFD512,
29176 IX86_BUILTIN_PSLLD512,
29177 IX86_BUILTIN_PSLLDI512,
29178 IX86_BUILTIN_PSLLQ512,
29179 IX86_BUILTIN_PSLLQI512,
29180 IX86_BUILTIN_PSLLVV16SI,
29181 IX86_BUILTIN_PSLLVV8DI,
29182 IX86_BUILTIN_PSRAD512,
29183 IX86_BUILTIN_PSRADI512,
29184 IX86_BUILTIN_PSRAQ512,
29185 IX86_BUILTIN_PSRAQI512,
29186 IX86_BUILTIN_PSRAVV16SI,
29187 IX86_BUILTIN_PSRAVV8DI,
29188 IX86_BUILTIN_PSRLD512,
29189 IX86_BUILTIN_PSRLDI512,
29190 IX86_BUILTIN_PSRLQ512,
29191 IX86_BUILTIN_PSRLQI512,
29192 IX86_BUILTIN_PSRLVV16SI,
29193 IX86_BUILTIN_PSRLVV8DI,
29194 IX86_BUILTIN_PSUBD512,
29195 IX86_BUILTIN_PSUBQ512,
29196 IX86_BUILTIN_PTESTMD512,
29197 IX86_BUILTIN_PTESTMQ512,
29198 IX86_BUILTIN_PTESTNMD512,
29199 IX86_BUILTIN_PTESTNMQ512,
29200 IX86_BUILTIN_PUNPCKHDQ512,
29201 IX86_BUILTIN_PUNPCKHQDQ512,
29202 IX86_BUILTIN_PUNPCKLDQ512,
29203 IX86_BUILTIN_PUNPCKLQDQ512,
29204 IX86_BUILTIN_PXORD512,
29205 IX86_BUILTIN_PXORQ512,
29206 IX86_BUILTIN_RCP14PD512,
29207 IX86_BUILTIN_RCP14PS512,
29208 IX86_BUILTIN_RCP14SD,
29209 IX86_BUILTIN_RCP14SS,
29210 IX86_BUILTIN_RNDSCALEPD,
29211 IX86_BUILTIN_RNDSCALEPS,
29212 IX86_BUILTIN_RNDSCALESD,
29213 IX86_BUILTIN_RNDSCALESS,
29214 IX86_BUILTIN_RSQRT14PD512,
29215 IX86_BUILTIN_RSQRT14PS512,
29216 IX86_BUILTIN_RSQRT14SD,
29217 IX86_BUILTIN_RSQRT14SS,
29218 IX86_BUILTIN_SCALEFPD512,
29219 IX86_BUILTIN_SCALEFPS512,
29220 IX86_BUILTIN_SCALEFSD,
29221 IX86_BUILTIN_SCALEFSS,
29222 IX86_BUILTIN_SHUFPD512,
29223 IX86_BUILTIN_SHUFPS512,
29224 IX86_BUILTIN_SHUF_F32x4,
29225 IX86_BUILTIN_SHUF_F64x2,
29226 IX86_BUILTIN_SHUF_I32x4,
29227 IX86_BUILTIN_SHUF_I64x2,
29228 IX86_BUILTIN_SQRTPD512,
29229 IX86_BUILTIN_SQRTPD512_MASK,
29230 IX86_BUILTIN_SQRTPS512_MASK,
29231 IX86_BUILTIN_SQRTPS_NR512,
29232 IX86_BUILTIN_SQRTSD_ROUND,
29233 IX86_BUILTIN_SQRTSS_ROUND,
29234 IX86_BUILTIN_STOREAPD512,
29235 IX86_BUILTIN_STOREAPS512,
29236 IX86_BUILTIN_STOREDQUDI512,
29237 IX86_BUILTIN_STOREDQUSI512,
29238 IX86_BUILTIN_STOREUPD512,
29239 IX86_BUILTIN_STOREUPS512,
29240 IX86_BUILTIN_SUBPD512,
29241 IX86_BUILTIN_SUBPS512,
29242 IX86_BUILTIN_SUBSD_ROUND,
29243 IX86_BUILTIN_SUBSS_ROUND,
29244 IX86_BUILTIN_UCMPD512,
29245 IX86_BUILTIN_UCMPQ512,
29246 IX86_BUILTIN_UNPCKHPD512,
29247 IX86_BUILTIN_UNPCKHPS512,
29248 IX86_BUILTIN_UNPCKLPD512,
29249 IX86_BUILTIN_UNPCKLPS512,
29250 IX86_BUILTIN_VCVTSD2SI32,
29251 IX86_BUILTIN_VCVTSD2SI64,
29252 IX86_BUILTIN_VCVTSD2USI32,
29253 IX86_BUILTIN_VCVTSD2USI64,
29254 IX86_BUILTIN_VCVTSS2SI32,
29255 IX86_BUILTIN_VCVTSS2SI64,
29256 IX86_BUILTIN_VCVTSS2USI32,
29257 IX86_BUILTIN_VCVTSS2USI64,
29258 IX86_BUILTIN_VCVTTSD2SI32,
29259 IX86_BUILTIN_VCVTTSD2SI64,
29260 IX86_BUILTIN_VCVTTSD2USI32,
29261 IX86_BUILTIN_VCVTTSD2USI64,
29262 IX86_BUILTIN_VCVTTSS2SI32,
29263 IX86_BUILTIN_VCVTTSS2SI64,
29264 IX86_BUILTIN_VCVTTSS2USI32,
29265 IX86_BUILTIN_VCVTTSS2USI64,
29266 IX86_BUILTIN_VFMADDPD512_MASK,
29267 IX86_BUILTIN_VFMADDPD512_MASK3,
29268 IX86_BUILTIN_VFMADDPD512_MASKZ,
29269 IX86_BUILTIN_VFMADDPS512_MASK,
29270 IX86_BUILTIN_VFMADDPS512_MASK3,
29271 IX86_BUILTIN_VFMADDPS512_MASKZ,
29272 IX86_BUILTIN_VFMADDSD3_ROUND,
29273 IX86_BUILTIN_VFMADDSS3_ROUND,
29274 IX86_BUILTIN_VFMADDSUBPD512_MASK,
29275 IX86_BUILTIN_VFMADDSUBPD512_MASK3,
29276 IX86_BUILTIN_VFMADDSUBPD512_MASKZ,
29277 IX86_BUILTIN_VFMADDSUBPS512_MASK,
29278 IX86_BUILTIN_VFMADDSUBPS512_MASK3,
29279 IX86_BUILTIN_VFMADDSUBPS512_MASKZ,
29280 IX86_BUILTIN_VFMSUBADDPD512_MASK3,
29281 IX86_BUILTIN_VFMSUBADDPS512_MASK3,
29282 IX86_BUILTIN_VFMSUBPD512_MASK3,
29283 IX86_BUILTIN_VFMSUBPS512_MASK3,
29284 IX86_BUILTIN_VFMSUBSD3_MASK3,
29285 IX86_BUILTIN_VFMSUBSS3_MASK3,
29286 IX86_BUILTIN_VFNMADDPD512_MASK,
29287 IX86_BUILTIN_VFNMADDPS512_MASK,
29288 IX86_BUILTIN_VFNMSUBPD512_MASK,
29289 IX86_BUILTIN_VFNMSUBPD512_MASK3,
29290 IX86_BUILTIN_VFNMSUBPS512_MASK,
29291 IX86_BUILTIN_VFNMSUBPS512_MASK3,
29292 IX86_BUILTIN_VPCLZCNTD512,
29293 IX86_BUILTIN_VPCLZCNTQ512,
29294 IX86_BUILTIN_VPCONFLICTD512,
29295 IX86_BUILTIN_VPCONFLICTQ512,
29296 IX86_BUILTIN_VPERMDF512,
29297 IX86_BUILTIN_VPERMDI512,
29298 IX86_BUILTIN_VPERMI2VARD512,
29299 IX86_BUILTIN_VPERMI2VARPD512,
29300 IX86_BUILTIN_VPERMI2VARPS512,
29301 IX86_BUILTIN_VPERMI2VARQ512,
29302 IX86_BUILTIN_VPERMILPD512,
29303 IX86_BUILTIN_VPERMILPS512,
29304 IX86_BUILTIN_VPERMILVARPD512,
29305 IX86_BUILTIN_VPERMILVARPS512,
29306 IX86_BUILTIN_VPERMT2VARD512,
29307 IX86_BUILTIN_VPERMT2VARD512_MASKZ,
29308 IX86_BUILTIN_VPERMT2VARPD512,
29309 IX86_BUILTIN_VPERMT2VARPD512_MASKZ,
29310 IX86_BUILTIN_VPERMT2VARPS512,
29311 IX86_BUILTIN_VPERMT2VARPS512_MASKZ,
29312 IX86_BUILTIN_VPERMT2VARQ512,
29313 IX86_BUILTIN_VPERMT2VARQ512_MASKZ,
29314 IX86_BUILTIN_VPERMVARDF512,
29315 IX86_BUILTIN_VPERMVARDI512,
29316 IX86_BUILTIN_VPERMVARSF512,
29317 IX86_BUILTIN_VPERMVARSI512,
29318 IX86_BUILTIN_VTERNLOGD512_MASK,
29319 IX86_BUILTIN_VTERNLOGD512_MASKZ,
29320 IX86_BUILTIN_VTERNLOGQ512_MASK,
29321 IX86_BUILTIN_VTERNLOGQ512_MASKZ,
29323 /* Mask arithmetic operations */
29324 IX86_BUILTIN_KAND16,
29325 IX86_BUILTIN_KANDN16,
29326 IX86_BUILTIN_KNOT16,
29327 IX86_BUILTIN_KOR16,
29328 IX86_BUILTIN_KORTESTC16,
29329 IX86_BUILTIN_KORTESTZ16,
29330 IX86_BUILTIN_KUNPCKBW,
29331 IX86_BUILTIN_KXNOR16,
29332 IX86_BUILTIN_KXOR16,
29333 IX86_BUILTIN_KMOV16,
29336 IX86_BUILTIN_PMOVUSQD256_MEM,
29337 IX86_BUILTIN_PMOVUSQD128_MEM,
29338 IX86_BUILTIN_PMOVSQD256_MEM,
29339 IX86_BUILTIN_PMOVSQD128_MEM,
29340 IX86_BUILTIN_PMOVQD256_MEM,
29341 IX86_BUILTIN_PMOVQD128_MEM,
29342 IX86_BUILTIN_PMOVUSQW256_MEM,
29343 IX86_BUILTIN_PMOVUSQW128_MEM,
29344 IX86_BUILTIN_PMOVSQW256_MEM,
29345 IX86_BUILTIN_PMOVSQW128_MEM,
29346 IX86_BUILTIN_PMOVQW256_MEM,
29347 IX86_BUILTIN_PMOVQW128_MEM,
29348 IX86_BUILTIN_PMOVUSQB256_MEM,
29349 IX86_BUILTIN_PMOVUSQB128_MEM,
29350 IX86_BUILTIN_PMOVSQB256_MEM,
29351 IX86_BUILTIN_PMOVSQB128_MEM,
29352 IX86_BUILTIN_PMOVQB256_MEM,
29353 IX86_BUILTIN_PMOVQB128_MEM,
29354 IX86_BUILTIN_PMOVUSDW256_MEM,
29355 IX86_BUILTIN_PMOVUSDW128_MEM,
29356 IX86_BUILTIN_PMOVSDW256_MEM,
29357 IX86_BUILTIN_PMOVSDW128_MEM,
29358 IX86_BUILTIN_PMOVDW256_MEM,
29359 IX86_BUILTIN_PMOVDW128_MEM,
29360 IX86_BUILTIN_PMOVUSDB256_MEM,
29361 IX86_BUILTIN_PMOVUSDB128_MEM,
29362 IX86_BUILTIN_PMOVSDB256_MEM,
29363 IX86_BUILTIN_PMOVSDB128_MEM,
29364 IX86_BUILTIN_PMOVDB256_MEM,
29365 IX86_BUILTIN_PMOVDB128_MEM,
29366 IX86_BUILTIN_MOVDQA64LOAD256_MASK,
29367 IX86_BUILTIN_MOVDQA64LOAD128_MASK,
29368 IX86_BUILTIN_MOVDQA32LOAD256_MASK,
29369 IX86_BUILTIN_MOVDQA32LOAD128_MASK,
29370 IX86_BUILTIN_MOVDQA64STORE256_MASK,
29371 IX86_BUILTIN_MOVDQA64STORE128_MASK,
29372 IX86_BUILTIN_MOVDQA32STORE256_MASK,
29373 IX86_BUILTIN_MOVDQA32STORE128_MASK,
29374 IX86_BUILTIN_LOADAPD256_MASK,
29375 IX86_BUILTIN_LOADAPD128_MASK,
29376 IX86_BUILTIN_LOADAPS256_MASK,
29377 IX86_BUILTIN_LOADAPS128_MASK,
29378 IX86_BUILTIN_STOREAPD256_MASK,
29379 IX86_BUILTIN_STOREAPD128_MASK,
29380 IX86_BUILTIN_STOREAPS256_MASK,
29381 IX86_BUILTIN_STOREAPS128_MASK,
29382 IX86_BUILTIN_LOADUPD256_MASK,
29383 IX86_BUILTIN_LOADUPD128_MASK,
29384 IX86_BUILTIN_LOADUPS256_MASK,
29385 IX86_BUILTIN_LOADUPS128_MASK,
29386 IX86_BUILTIN_STOREUPD256_MASK,
29387 IX86_BUILTIN_STOREUPD128_MASK,
29388 IX86_BUILTIN_STOREUPS256_MASK,
29389 IX86_BUILTIN_STOREUPS128_MASK,
29390 IX86_BUILTIN_LOADDQUDI256_MASK,
29391 IX86_BUILTIN_LOADDQUDI128_MASK,
29392 IX86_BUILTIN_LOADDQUSI256_MASK,
29393 IX86_BUILTIN_LOADDQUSI128_MASK,
29394 IX86_BUILTIN_LOADDQUHI256_MASK,
29395 IX86_BUILTIN_LOADDQUHI128_MASK,
29396 IX86_BUILTIN_LOADDQUQI256_MASK,
29397 IX86_BUILTIN_LOADDQUQI128_MASK,
29398 IX86_BUILTIN_STOREDQUDI256_MASK,
29399 IX86_BUILTIN_STOREDQUDI128_MASK,
29400 IX86_BUILTIN_STOREDQUSI256_MASK,
29401 IX86_BUILTIN_STOREDQUSI128_MASK,
29402 IX86_BUILTIN_STOREDQUHI256_MASK,
29403 IX86_BUILTIN_STOREDQUHI128_MASK,
29404 IX86_BUILTIN_STOREDQUQI256_MASK,
29405 IX86_BUILTIN_STOREDQUQI128_MASK,
29406 IX86_BUILTIN_COMPRESSPDSTORE256,
29407 IX86_BUILTIN_COMPRESSPDSTORE128,
29408 IX86_BUILTIN_COMPRESSPSSTORE256,
29409 IX86_BUILTIN_COMPRESSPSSTORE128,
29410 IX86_BUILTIN_PCOMPRESSQSTORE256,
29411 IX86_BUILTIN_PCOMPRESSQSTORE128,
29412 IX86_BUILTIN_PCOMPRESSDSTORE256,
29413 IX86_BUILTIN_PCOMPRESSDSTORE128,
29414 IX86_BUILTIN_EXPANDPDLOAD256,
29415 IX86_BUILTIN_EXPANDPDLOAD128,
29416 IX86_BUILTIN_EXPANDPSLOAD256,
29417 IX86_BUILTIN_EXPANDPSLOAD128,
29418 IX86_BUILTIN_PEXPANDQLOAD256,
29419 IX86_BUILTIN_PEXPANDQLOAD128,
29420 IX86_BUILTIN_PEXPANDDLOAD256,
29421 IX86_BUILTIN_PEXPANDDLOAD128,
29422 IX86_BUILTIN_EXPANDPDLOAD256Z,
29423 IX86_BUILTIN_EXPANDPDLOAD128Z,
29424 IX86_BUILTIN_EXPANDPSLOAD256Z,
29425 IX86_BUILTIN_EXPANDPSLOAD128Z,
29426 IX86_BUILTIN_PEXPANDQLOAD256Z,
29427 IX86_BUILTIN_PEXPANDQLOAD128Z,
29428 IX86_BUILTIN_PEXPANDDLOAD256Z,
29429 IX86_BUILTIN_PEXPANDDLOAD128Z,
29430 IX86_BUILTIN_PALIGNR256_MASK,
29431 IX86_BUILTIN_PALIGNR128_MASK,
29432 IX86_BUILTIN_MOVDQA64_256_MASK,
29433 IX86_BUILTIN_MOVDQA64_128_MASK,
29434 IX86_BUILTIN_MOVDQA32_256_MASK,
29435 IX86_BUILTIN_MOVDQA32_128_MASK,
29436 IX86_BUILTIN_MOVAPD256_MASK,
29437 IX86_BUILTIN_MOVAPD128_MASK,
29438 IX86_BUILTIN_MOVAPS256_MASK,
29439 IX86_BUILTIN_MOVAPS128_MASK,
29440 IX86_BUILTIN_MOVDQUHI256_MASK,
29441 IX86_BUILTIN_MOVDQUHI128_MASK,
29442 IX86_BUILTIN_MOVDQUQI256_MASK,
29443 IX86_BUILTIN_MOVDQUQI128_MASK,
29444 IX86_BUILTIN_MINPS128_MASK,
29445 IX86_BUILTIN_MAXPS128_MASK,
29446 IX86_BUILTIN_MINPD128_MASK,
29447 IX86_BUILTIN_MAXPD128_MASK,
29448 IX86_BUILTIN_MAXPD256_MASK,
29449 IX86_BUILTIN_MAXPS256_MASK,
29450 IX86_BUILTIN_MINPD256_MASK,
29451 IX86_BUILTIN_MINPS256_MASK,
29452 IX86_BUILTIN_MULPS128_MASK,
29453 IX86_BUILTIN_DIVPS128_MASK,
29454 IX86_BUILTIN_MULPD128_MASK,
29455 IX86_BUILTIN_DIVPD128_MASK,
29456 IX86_BUILTIN_DIVPD256_MASK,
29457 IX86_BUILTIN_DIVPS256_MASK,
29458 IX86_BUILTIN_MULPD256_MASK,
29459 IX86_BUILTIN_MULPS256_MASK,
29460 IX86_BUILTIN_ADDPD128_MASK,
29461 IX86_BUILTIN_ADDPD256_MASK,
29462 IX86_BUILTIN_ADDPS128_MASK,
29463 IX86_BUILTIN_ADDPS256_MASK,
29464 IX86_BUILTIN_SUBPD128_MASK,
29465 IX86_BUILTIN_SUBPD256_MASK,
29466 IX86_BUILTIN_SUBPS128_MASK,
29467 IX86_BUILTIN_SUBPS256_MASK,
29468 IX86_BUILTIN_XORPD256_MASK,
29469 IX86_BUILTIN_XORPD128_MASK,
29470 IX86_BUILTIN_XORPS256_MASK,
29471 IX86_BUILTIN_XORPS128_MASK,
29472 IX86_BUILTIN_ORPD256_MASK,
29473 IX86_BUILTIN_ORPD128_MASK,
29474 IX86_BUILTIN_ORPS256_MASK,
29475 IX86_BUILTIN_ORPS128_MASK,
29476 IX86_BUILTIN_BROADCASTF32x2_256,
29477 IX86_BUILTIN_BROADCASTI32x2_256,
29478 IX86_BUILTIN_BROADCASTI32x2_128,
29479 IX86_BUILTIN_BROADCASTF64X2_256,
29480 IX86_BUILTIN_BROADCASTI64X2_256,
29481 IX86_BUILTIN_BROADCASTF32X4_256,
29482 IX86_BUILTIN_BROADCASTI32X4_256,
29483 IX86_BUILTIN_EXTRACTF32X4_256,
29484 IX86_BUILTIN_EXTRACTI32X4_256,
29485 IX86_BUILTIN_DBPSADBW256,
29486 IX86_BUILTIN_DBPSADBW128,
29487 IX86_BUILTIN_CVTTPD2QQ256,
29488 IX86_BUILTIN_CVTTPD2QQ128,
29489 IX86_BUILTIN_CVTTPD2UQQ256,
29490 IX86_BUILTIN_CVTTPD2UQQ128,
29491 IX86_BUILTIN_CVTPD2QQ256,
29492 IX86_BUILTIN_CVTPD2QQ128,
29493 IX86_BUILTIN_CVTPD2UQQ256,
29494 IX86_BUILTIN_CVTPD2UQQ128,
29495 IX86_BUILTIN_CVTPD2UDQ256_MASK,
29496 IX86_BUILTIN_CVTPD2UDQ128_MASK,
29497 IX86_BUILTIN_CVTTPS2QQ256,
29498 IX86_BUILTIN_CVTTPS2QQ128,
29499 IX86_BUILTIN_CVTTPS2UQQ256,
29500 IX86_BUILTIN_CVTTPS2UQQ128,
29501 IX86_BUILTIN_CVTTPS2DQ256_MASK,
29502 IX86_BUILTIN_CVTTPS2DQ128_MASK,
29503 IX86_BUILTIN_CVTTPS2UDQ256,
29504 IX86_BUILTIN_CVTTPS2UDQ128,
29505 IX86_BUILTIN_CVTTPD2DQ256_MASK,
29506 IX86_BUILTIN_CVTTPD2DQ128_MASK,
29507 IX86_BUILTIN_CVTTPD2UDQ256_MASK,
29508 IX86_BUILTIN_CVTTPD2UDQ128_MASK,
29509 IX86_BUILTIN_CVTPD2DQ256_MASK,
29510 IX86_BUILTIN_CVTPD2DQ128_MASK,
29511 IX86_BUILTIN_CVTDQ2PD256_MASK,
29512 IX86_BUILTIN_CVTDQ2PD128_MASK,
29513 IX86_BUILTIN_CVTUDQ2PD256_MASK,
29514 IX86_BUILTIN_CVTUDQ2PD128_MASK,
29515 IX86_BUILTIN_CVTDQ2PS256_MASK,
29516 IX86_BUILTIN_CVTDQ2PS128_MASK,
29517 IX86_BUILTIN_CVTUDQ2PS256_MASK,
29518 IX86_BUILTIN_CVTUDQ2PS128_MASK,
29519 IX86_BUILTIN_CVTPS2PD256_MASK,
29520 IX86_BUILTIN_CVTPS2PD128_MASK,
29521 IX86_BUILTIN_PBROADCASTB256_MASK,
29522 IX86_BUILTIN_PBROADCASTB256_GPR_MASK,
29523 IX86_BUILTIN_PBROADCASTB128_MASK,
29524 IX86_BUILTIN_PBROADCASTB128_GPR_MASK,
29525 IX86_BUILTIN_PBROADCASTW256_MASK,
29526 IX86_BUILTIN_PBROADCASTW256_GPR_MASK,
29527 IX86_BUILTIN_PBROADCASTW128_MASK,
29528 IX86_BUILTIN_PBROADCASTW128_GPR_MASK,
29529 IX86_BUILTIN_PBROADCASTD256_MASK,
29530 IX86_BUILTIN_PBROADCASTD256_GPR_MASK,
29531 IX86_BUILTIN_PBROADCASTD128_MASK,
29532 IX86_BUILTIN_PBROADCASTD128_GPR_MASK,
29533 IX86_BUILTIN_PBROADCASTQ256_MASK,
29534 IX86_BUILTIN_PBROADCASTQ256_GPR_MASK,
29535 IX86_BUILTIN_PBROADCASTQ128_MASK,
29536 IX86_BUILTIN_PBROADCASTQ128_GPR_MASK,
29537 IX86_BUILTIN_BROADCASTSS256,
29538 IX86_BUILTIN_BROADCASTSS128,
29539 IX86_BUILTIN_BROADCASTSD256,
29540 IX86_BUILTIN_EXTRACTF64X2_256,
29541 IX86_BUILTIN_EXTRACTI64X2_256,
29542 IX86_BUILTIN_INSERTF32X4_256,
29543 IX86_BUILTIN_INSERTI32X4_256,
29544 IX86_BUILTIN_PMOVSXBW256_MASK,
29545 IX86_BUILTIN_PMOVSXBW128_MASK,
29546 IX86_BUILTIN_PMOVSXBD256_MASK,
29547 IX86_BUILTIN_PMOVSXBD128_MASK,
29548 IX86_BUILTIN_PMOVSXBQ256_MASK,
29549 IX86_BUILTIN_PMOVSXBQ128_MASK,
29550 IX86_BUILTIN_PMOVSXWD256_MASK,
29551 IX86_BUILTIN_PMOVSXWD128_MASK,
29552 IX86_BUILTIN_PMOVSXWQ256_MASK,
29553 IX86_BUILTIN_PMOVSXWQ128_MASK,
29554 IX86_BUILTIN_PMOVSXDQ256_MASK,
29555 IX86_BUILTIN_PMOVSXDQ128_MASK,
29556 IX86_BUILTIN_PMOVZXBW256_MASK,
29557 IX86_BUILTIN_PMOVZXBW128_MASK,
29558 IX86_BUILTIN_PMOVZXBD256_MASK,
29559 IX86_BUILTIN_PMOVZXBD128_MASK,
29560 IX86_BUILTIN_PMOVZXBQ256_MASK,
29561 IX86_BUILTIN_PMOVZXBQ128_MASK,
29562 IX86_BUILTIN_PMOVZXWD256_MASK,
29563 IX86_BUILTIN_PMOVZXWD128_MASK,
29564 IX86_BUILTIN_PMOVZXWQ256_MASK,
29565 IX86_BUILTIN_PMOVZXWQ128_MASK,
29566 IX86_BUILTIN_PMOVZXDQ256_MASK,
29567 IX86_BUILTIN_PMOVZXDQ128_MASK,
29568 IX86_BUILTIN_REDUCEPD256_MASK,
29569 IX86_BUILTIN_REDUCEPD128_MASK,
29570 IX86_BUILTIN_REDUCEPS256_MASK,
29571 IX86_BUILTIN_REDUCEPS128_MASK,
29572 IX86_BUILTIN_REDUCESD_MASK,
29573 IX86_BUILTIN_REDUCESS_MASK,
29574 IX86_BUILTIN_VPERMVARHI256_MASK,
29575 IX86_BUILTIN_VPERMVARHI128_MASK,
29576 IX86_BUILTIN_VPERMT2VARHI256,
29577 IX86_BUILTIN_VPERMT2VARHI256_MASKZ,
29578 IX86_BUILTIN_VPERMT2VARHI128,
29579 IX86_BUILTIN_VPERMT2VARHI128_MASKZ,
29580 IX86_BUILTIN_VPERMI2VARHI256,
29581 IX86_BUILTIN_VPERMI2VARHI128,
29582 IX86_BUILTIN_RCP14PD256,
29583 IX86_BUILTIN_RCP14PD128,
29584 IX86_BUILTIN_RCP14PS256,
29585 IX86_BUILTIN_RCP14PS128,
29586 IX86_BUILTIN_RSQRT14PD256_MASK,
29587 IX86_BUILTIN_RSQRT14PD128_MASK,
29588 IX86_BUILTIN_RSQRT14PS256_MASK,
29589 IX86_BUILTIN_RSQRT14PS128_MASK,
29590 IX86_BUILTIN_SQRTPD256_MASK,
29591 IX86_BUILTIN_SQRTPD128_MASK,
29592 IX86_BUILTIN_SQRTPS256_MASK,
29593 IX86_BUILTIN_SQRTPS128_MASK,
29594 IX86_BUILTIN_PADDB128_MASK,
29595 IX86_BUILTIN_PADDW128_MASK,
29596 IX86_BUILTIN_PADDD128_MASK,
29597 IX86_BUILTIN_PADDQ128_MASK,
29598 IX86_BUILTIN_PSUBB128_MASK,
29599 IX86_BUILTIN_PSUBW128_MASK,
29600 IX86_BUILTIN_PSUBD128_MASK,
29601 IX86_BUILTIN_PSUBQ128_MASK,
29602 IX86_BUILTIN_PADDSB128_MASK,
29603 IX86_BUILTIN_PADDSW128_MASK,
29604 IX86_BUILTIN_PSUBSB128_MASK,
29605 IX86_BUILTIN_PSUBSW128_MASK,
29606 IX86_BUILTIN_PADDUSB128_MASK,
29607 IX86_BUILTIN_PADDUSW128_MASK,
29608 IX86_BUILTIN_PSUBUSB128_MASK,
29609 IX86_BUILTIN_PSUBUSW128_MASK,
29610 IX86_BUILTIN_PADDB256_MASK,
29611 IX86_BUILTIN_PADDW256_MASK,
29612 IX86_BUILTIN_PADDD256_MASK,
29613 IX86_BUILTIN_PADDQ256_MASK,
29614 IX86_BUILTIN_PADDSB256_MASK,
29615 IX86_BUILTIN_PADDSW256_MASK,
29616 IX86_BUILTIN_PADDUSB256_MASK,
29617 IX86_BUILTIN_PADDUSW256_MASK,
29618 IX86_BUILTIN_PSUBB256_MASK,
29619 IX86_BUILTIN_PSUBW256_MASK,
29620 IX86_BUILTIN_PSUBD256_MASK,
29621 IX86_BUILTIN_PSUBQ256_MASK,
29622 IX86_BUILTIN_PSUBSB256_MASK,
29623 IX86_BUILTIN_PSUBSW256_MASK,
29624 IX86_BUILTIN_PSUBUSB256_MASK,
29625 IX86_BUILTIN_PSUBUSW256_MASK,
29626 IX86_BUILTIN_SHUF_F64x2_256,
29627 IX86_BUILTIN_SHUF_I64x2_256,
29628 IX86_BUILTIN_SHUF_I32x4_256,
29629 IX86_BUILTIN_SHUF_F32x4_256,
29630 IX86_BUILTIN_PMOVWB128,
29631 IX86_BUILTIN_PMOVWB256,
29632 IX86_BUILTIN_PMOVSWB128,
29633 IX86_BUILTIN_PMOVSWB256,
29634 IX86_BUILTIN_PMOVUSWB128,
29635 IX86_BUILTIN_PMOVUSWB256,
29636 IX86_BUILTIN_PMOVDB128,
29637 IX86_BUILTIN_PMOVDB256,
29638 IX86_BUILTIN_PMOVSDB128,
29639 IX86_BUILTIN_PMOVSDB256,
29640 IX86_BUILTIN_PMOVUSDB128,
29641 IX86_BUILTIN_PMOVUSDB256,
29642 IX86_BUILTIN_PMOVDW128,
29643 IX86_BUILTIN_PMOVDW256,
29644 IX86_BUILTIN_PMOVSDW128,
29645 IX86_BUILTIN_PMOVSDW256,
29646 IX86_BUILTIN_PMOVUSDW128,
29647 IX86_BUILTIN_PMOVUSDW256,
29648 IX86_BUILTIN_PMOVQB128,
29649 IX86_BUILTIN_PMOVQB256,
29650 IX86_BUILTIN_PMOVSQB128,
29651 IX86_BUILTIN_PMOVSQB256,
29652 IX86_BUILTIN_PMOVUSQB128,
29653 IX86_BUILTIN_PMOVUSQB256,
29654 IX86_BUILTIN_PMOVQW128,
29655 IX86_BUILTIN_PMOVQW256,
29656 IX86_BUILTIN_PMOVSQW128,
29657 IX86_BUILTIN_PMOVSQW256,
29658 IX86_BUILTIN_PMOVUSQW128,
29659 IX86_BUILTIN_PMOVUSQW256,
29660 IX86_BUILTIN_PMOVQD128,
29661 IX86_BUILTIN_PMOVQD256,
29662 IX86_BUILTIN_PMOVSQD128,
29663 IX86_BUILTIN_PMOVSQD256,
29664 IX86_BUILTIN_PMOVUSQD128,
29665 IX86_BUILTIN_PMOVUSQD256,
29666 IX86_BUILTIN_RANGEPD256,
29667 IX86_BUILTIN_RANGEPD128,
29668 IX86_BUILTIN_RANGEPS256,
29669 IX86_BUILTIN_RANGEPS128,
29670 IX86_BUILTIN_GETEXPPS256,
29671 IX86_BUILTIN_GETEXPPD256,
29672 IX86_BUILTIN_GETEXPPS128,
29673 IX86_BUILTIN_GETEXPPD128,
29674 IX86_BUILTIN_FIXUPIMMPD256_MASK,
29675 IX86_BUILTIN_FIXUPIMMPD256_MASKZ,
29676 IX86_BUILTIN_FIXUPIMMPS256_MASK,
29677 IX86_BUILTIN_FIXUPIMMPS256_MASKZ,
29678 IX86_BUILTIN_FIXUPIMMPD128_MASK,
29679 IX86_BUILTIN_FIXUPIMMPD128_MASKZ,
29680 IX86_BUILTIN_FIXUPIMMPS128_MASK,
29681 IX86_BUILTIN_FIXUPIMMPS128_MASKZ,
29682 IX86_BUILTIN_PABSQ256,
29683 IX86_BUILTIN_PABSQ128,
29684 IX86_BUILTIN_PABSD256_MASK,
29685 IX86_BUILTIN_PABSD128_MASK,
29686 IX86_BUILTIN_PMULHRSW256_MASK,
29687 IX86_BUILTIN_PMULHRSW128_MASK,
29688 IX86_BUILTIN_PMULHUW128_MASK,
29689 IX86_BUILTIN_PMULHUW256_MASK,
29690 IX86_BUILTIN_PMULHW256_MASK,
29691 IX86_BUILTIN_PMULHW128_MASK,
29692 IX86_BUILTIN_PMULLW256_MASK,
29693 IX86_BUILTIN_PMULLW128_MASK,
29694 IX86_BUILTIN_PMULLQ256,
29695 IX86_BUILTIN_PMULLQ128,
29696 IX86_BUILTIN_ANDPD256_MASK,
29697 IX86_BUILTIN_ANDPD128_MASK,
29698 IX86_BUILTIN_ANDPS256_MASK,
29699 IX86_BUILTIN_ANDPS128_MASK,
29700 IX86_BUILTIN_ANDNPD256_MASK,
29701 IX86_BUILTIN_ANDNPD128_MASK,
29702 IX86_BUILTIN_ANDNPS256_MASK,
29703 IX86_BUILTIN_ANDNPS128_MASK,
29704 IX86_BUILTIN_PSLLWI128_MASK,
29705 IX86_BUILTIN_PSLLDI128_MASK,
29706 IX86_BUILTIN_PSLLQI128_MASK,
29707 IX86_BUILTIN_PSLLW128_MASK,
29708 IX86_BUILTIN_PSLLD128_MASK,
29709 IX86_BUILTIN_PSLLQ128_MASK,
29710 IX86_BUILTIN_PSLLWI256_MASK ,
29711 IX86_BUILTIN_PSLLW256_MASK,
29712 IX86_BUILTIN_PSLLDI256_MASK,
29713 IX86_BUILTIN_PSLLD256_MASK,
29714 IX86_BUILTIN_PSLLQI256_MASK,
29715 IX86_BUILTIN_PSLLQ256_MASK,
29716 IX86_BUILTIN_PSRADI128_MASK,
29717 IX86_BUILTIN_PSRAD128_MASK,
29718 IX86_BUILTIN_PSRADI256_MASK,
29719 IX86_BUILTIN_PSRAD256_MASK,
29720 IX86_BUILTIN_PSRAQI128_MASK,
29721 IX86_BUILTIN_PSRAQ128_MASK,
29722 IX86_BUILTIN_PSRAQI256_MASK,
29723 IX86_BUILTIN_PSRAQ256_MASK,
29724 IX86_BUILTIN_PANDD256,
29725 IX86_BUILTIN_PANDD128,
29726 IX86_BUILTIN_PSRLDI128_MASK,
29727 IX86_BUILTIN_PSRLD128_MASK,
29728 IX86_BUILTIN_PSRLDI256_MASK,
29729 IX86_BUILTIN_PSRLD256_MASK,
29730 IX86_BUILTIN_PSRLQI128_MASK,
29731 IX86_BUILTIN_PSRLQ128_MASK,
29732 IX86_BUILTIN_PSRLQI256_MASK,
29733 IX86_BUILTIN_PSRLQ256_MASK,
29734 IX86_BUILTIN_PANDQ256,
29735 IX86_BUILTIN_PANDQ128,
29736 IX86_BUILTIN_PANDND256,
29737 IX86_BUILTIN_PANDND128,
29738 IX86_BUILTIN_PANDNQ256,
29739 IX86_BUILTIN_PANDNQ128,
29740 IX86_BUILTIN_PORD256,
29741 IX86_BUILTIN_PORD128,
29742 IX86_BUILTIN_PORQ256,
29743 IX86_BUILTIN_PORQ128,
29744 IX86_BUILTIN_PXORD256,
29745 IX86_BUILTIN_PXORD128,
29746 IX86_BUILTIN_PXORQ256,
29747 IX86_BUILTIN_PXORQ128,
29748 IX86_BUILTIN_PACKSSWB256_MASK,
29749 IX86_BUILTIN_PACKSSWB128_MASK,
29750 IX86_BUILTIN_PACKUSWB256_MASK,
29751 IX86_BUILTIN_PACKUSWB128_MASK,
29752 IX86_BUILTIN_RNDSCALEPS256,
29753 IX86_BUILTIN_RNDSCALEPD256,
29754 IX86_BUILTIN_RNDSCALEPS128,
29755 IX86_BUILTIN_RNDSCALEPD128,
29756 IX86_BUILTIN_VTERNLOGQ256_MASK,
29757 IX86_BUILTIN_VTERNLOGQ256_MASKZ,
29758 IX86_BUILTIN_VTERNLOGD256_MASK,
29759 IX86_BUILTIN_VTERNLOGD256_MASKZ,
29760 IX86_BUILTIN_VTERNLOGQ128_MASK,
29761 IX86_BUILTIN_VTERNLOGQ128_MASKZ,
29762 IX86_BUILTIN_VTERNLOGD128_MASK,
29763 IX86_BUILTIN_VTERNLOGD128_MASKZ,
29764 IX86_BUILTIN_SCALEFPD256,
29765 IX86_BUILTIN_SCALEFPS256,
29766 IX86_BUILTIN_SCALEFPD128,
29767 IX86_BUILTIN_SCALEFPS128,
29768 IX86_BUILTIN_VFMADDPD256_MASK,
29769 IX86_BUILTIN_VFMADDPD256_MASK3,
29770 IX86_BUILTIN_VFMADDPD256_MASKZ,
29771 IX86_BUILTIN_VFMADDPD128_MASK,
29772 IX86_BUILTIN_VFMADDPD128_MASK3,
29773 IX86_BUILTIN_VFMADDPD128_MASKZ,
29774 IX86_BUILTIN_VFMADDPS256_MASK,
29775 IX86_BUILTIN_VFMADDPS256_MASK3,
29776 IX86_BUILTIN_VFMADDPS256_MASKZ,
29777 IX86_BUILTIN_VFMADDPS128_MASK,
29778 IX86_BUILTIN_VFMADDPS128_MASK3,
29779 IX86_BUILTIN_VFMADDPS128_MASKZ,
29780 IX86_BUILTIN_VFMSUBPD256_MASK3,
29781 IX86_BUILTIN_VFMSUBPD128_MASK3,
29782 IX86_BUILTIN_VFMSUBPS256_MASK3,
29783 IX86_BUILTIN_VFMSUBPS128_MASK3,
29784 IX86_BUILTIN_VFNMADDPD256_MASK,
29785 IX86_BUILTIN_VFNMADDPD128_MASK,
29786 IX86_BUILTIN_VFNMADDPS256_MASK,
29787 IX86_BUILTIN_VFNMADDPS128_MASK,
29788 IX86_BUILTIN_VFNMSUBPD256_MASK,
29789 IX86_BUILTIN_VFNMSUBPD256_MASK3,
29790 IX86_BUILTIN_VFNMSUBPD128_MASK,
29791 IX86_BUILTIN_VFNMSUBPD128_MASK3,
29792 IX86_BUILTIN_VFNMSUBPS256_MASK,
29793 IX86_BUILTIN_VFNMSUBPS256_MASK3,
29794 IX86_BUILTIN_VFNMSUBPS128_MASK,
29795 IX86_BUILTIN_VFNMSUBPS128_MASK3,
29796 IX86_BUILTIN_VFMADDSUBPD256_MASK,
29797 IX86_BUILTIN_VFMADDSUBPD256_MASK3,
29798 IX86_BUILTIN_VFMADDSUBPD256_MASKZ,
29799 IX86_BUILTIN_VFMADDSUBPD128_MASK,
29800 IX86_BUILTIN_VFMADDSUBPD128_MASK3,
29801 IX86_BUILTIN_VFMADDSUBPD128_MASKZ,
29802 IX86_BUILTIN_VFMADDSUBPS256_MASK,
29803 IX86_BUILTIN_VFMADDSUBPS256_MASK3,
29804 IX86_BUILTIN_VFMADDSUBPS256_MASKZ,
29805 IX86_BUILTIN_VFMADDSUBPS128_MASK,
29806 IX86_BUILTIN_VFMADDSUBPS128_MASK3,
29807 IX86_BUILTIN_VFMADDSUBPS128_MASKZ,
29808 IX86_BUILTIN_VFMSUBADDPD256_MASK3,
29809 IX86_BUILTIN_VFMSUBADDPD128_MASK3,
29810 IX86_BUILTIN_VFMSUBADDPS256_MASK3,
29811 IX86_BUILTIN_VFMSUBADDPS128_MASK3,
29812 IX86_BUILTIN_INSERTF64X2_256,
29813 IX86_BUILTIN_INSERTI64X2_256,
29814 IX86_BUILTIN_PSRAVV16HI,
29815 IX86_BUILTIN_PSRAVV8HI,
29816 IX86_BUILTIN_PMADDUBSW256_MASK,
29817 IX86_BUILTIN_PMADDUBSW128_MASK,
29818 IX86_BUILTIN_PMADDWD256_MASK,
29819 IX86_BUILTIN_PMADDWD128_MASK,
29820 IX86_BUILTIN_PSRLVV16HI,
29821 IX86_BUILTIN_PSRLVV8HI,
29822 IX86_BUILTIN_CVTPS2DQ256_MASK,
29823 IX86_BUILTIN_CVTPS2DQ128_MASK,
29824 IX86_BUILTIN_CVTPS2UDQ256,
29825 IX86_BUILTIN_CVTPS2UDQ128,
29826 IX86_BUILTIN_CVTPS2QQ256,
29827 IX86_BUILTIN_CVTPS2QQ128,
29828 IX86_BUILTIN_CVTPS2UQQ256,
29829 IX86_BUILTIN_CVTPS2UQQ128,
29830 IX86_BUILTIN_GETMANTPS256,
29831 IX86_BUILTIN_GETMANTPS128,
29832 IX86_BUILTIN_GETMANTPD256,
29833 IX86_BUILTIN_GETMANTPD128,
29834 IX86_BUILTIN_MOVDDUP256_MASK,
29835 IX86_BUILTIN_MOVDDUP128_MASK,
29836 IX86_BUILTIN_MOVSHDUP256_MASK,
29837 IX86_BUILTIN_MOVSHDUP128_MASK,
29838 IX86_BUILTIN_MOVSLDUP256_MASK,
29839 IX86_BUILTIN_MOVSLDUP128_MASK,
29840 IX86_BUILTIN_CVTQQ2PS256,
29841 IX86_BUILTIN_CVTQQ2PS128,
29842 IX86_BUILTIN_CVTUQQ2PS256,
29843 IX86_BUILTIN_CVTUQQ2PS128,
29844 IX86_BUILTIN_CVTQQ2PD256,
29845 IX86_BUILTIN_CVTQQ2PD128,
29846 IX86_BUILTIN_CVTUQQ2PD256,
29847 IX86_BUILTIN_CVTUQQ2PD128,
29848 IX86_BUILTIN_VPERMT2VARQ256,
29849 IX86_BUILTIN_VPERMT2VARQ256_MASKZ,
29850 IX86_BUILTIN_VPERMT2VARD256,
29851 IX86_BUILTIN_VPERMT2VARD256_MASKZ,
29852 IX86_BUILTIN_VPERMI2VARQ256,
29853 IX86_BUILTIN_VPERMI2VARD256,
29854 IX86_BUILTIN_VPERMT2VARPD256,
29855 IX86_BUILTIN_VPERMT2VARPD256_MASKZ,
29856 IX86_BUILTIN_VPERMT2VARPS256,
29857 IX86_BUILTIN_VPERMT2VARPS256_MASKZ,
29858 IX86_BUILTIN_VPERMI2VARPD256,
29859 IX86_BUILTIN_VPERMI2VARPS256,
29860 IX86_BUILTIN_VPERMT2VARQ128,
29861 IX86_BUILTIN_VPERMT2VARQ128_MASKZ,
29862 IX86_BUILTIN_VPERMT2VARD128,
29863 IX86_BUILTIN_VPERMT2VARD128_MASKZ,
29864 IX86_BUILTIN_VPERMI2VARQ128,
29865 IX86_BUILTIN_VPERMI2VARD128,
29866 IX86_BUILTIN_VPERMT2VARPD128,
29867 IX86_BUILTIN_VPERMT2VARPD128_MASKZ,
29868 IX86_BUILTIN_VPERMT2VARPS128,
29869 IX86_BUILTIN_VPERMT2VARPS128_MASKZ,
29870 IX86_BUILTIN_VPERMI2VARPD128,
29871 IX86_BUILTIN_VPERMI2VARPS128,
29872 IX86_BUILTIN_PSHUFB256_MASK,
29873 IX86_BUILTIN_PSHUFB128_MASK,
29874 IX86_BUILTIN_PSHUFHW256_MASK,
29875 IX86_BUILTIN_PSHUFHW128_MASK,
29876 IX86_BUILTIN_PSHUFLW256_MASK,
29877 IX86_BUILTIN_PSHUFLW128_MASK,
29878 IX86_BUILTIN_PSHUFD256_MASK,
29879 IX86_BUILTIN_PSHUFD128_MASK,
29880 IX86_BUILTIN_SHUFPD256_MASK,
29881 IX86_BUILTIN_SHUFPD128_MASK,
29882 IX86_BUILTIN_SHUFPS256_MASK,
29883 IX86_BUILTIN_SHUFPS128_MASK,
29884 IX86_BUILTIN_PROLVQ256,
29885 IX86_BUILTIN_PROLVQ128,
29886 IX86_BUILTIN_PROLQ256,
29887 IX86_BUILTIN_PROLQ128,
29888 IX86_BUILTIN_PRORVQ256,
29889 IX86_BUILTIN_PRORVQ128,
29890 IX86_BUILTIN_PRORQ256,
29891 IX86_BUILTIN_PRORQ128,
29892 IX86_BUILTIN_PSRAVQ128,
29893 IX86_BUILTIN_PSRAVQ256,
29894 IX86_BUILTIN_PSLLVV4DI_MASK,
29895 IX86_BUILTIN_PSLLVV2DI_MASK,
29896 IX86_BUILTIN_PSLLVV8SI_MASK,
29897 IX86_BUILTIN_PSLLVV4SI_MASK,
29898 IX86_BUILTIN_PSRAVV8SI_MASK,
29899 IX86_BUILTIN_PSRAVV4SI_MASK,
29900 IX86_BUILTIN_PSRLVV4DI_MASK,
29901 IX86_BUILTIN_PSRLVV2DI_MASK,
29902 IX86_BUILTIN_PSRLVV8SI_MASK,
29903 IX86_BUILTIN_PSRLVV4SI_MASK,
29904 IX86_BUILTIN_PSRAWI256_MASK,
29905 IX86_BUILTIN_PSRAW256_MASK,
29906 IX86_BUILTIN_PSRAWI128_MASK,
29907 IX86_BUILTIN_PSRAW128_MASK,
29908 IX86_BUILTIN_PSRLWI256_MASK,
29909 IX86_BUILTIN_PSRLW256_MASK,
29910 IX86_BUILTIN_PSRLWI128_MASK,
29911 IX86_BUILTIN_PSRLW128_MASK,
29912 IX86_BUILTIN_PRORVD256,
29913 IX86_BUILTIN_PROLVD256,
29914 IX86_BUILTIN_PRORD256,
29915 IX86_BUILTIN_PROLD256,
29916 IX86_BUILTIN_PRORVD128,
29917 IX86_BUILTIN_PROLVD128,
29918 IX86_BUILTIN_PRORD128,
29919 IX86_BUILTIN_PROLD128,
29920 IX86_BUILTIN_FPCLASSPD256,
29921 IX86_BUILTIN_FPCLASSPD128,
29922 IX86_BUILTIN_FPCLASSSD,
29923 IX86_BUILTIN_FPCLASSPS256,
29924 IX86_BUILTIN_FPCLASSPS128,
29925 IX86_BUILTIN_FPCLASSSS,
29926 IX86_BUILTIN_CVTB2MASK128,
29927 IX86_BUILTIN_CVTB2MASK256,
29928 IX86_BUILTIN_CVTW2MASK128,
29929 IX86_BUILTIN_CVTW2MASK256,
29930 IX86_BUILTIN_CVTD2MASK128,
29931 IX86_BUILTIN_CVTD2MASK256,
29932 IX86_BUILTIN_CVTQ2MASK128,
29933 IX86_BUILTIN_CVTQ2MASK256,
29934 IX86_BUILTIN_CVTMASK2B128,
29935 IX86_BUILTIN_CVTMASK2B256,
29936 IX86_BUILTIN_CVTMASK2W128,
29937 IX86_BUILTIN_CVTMASK2W256,
29938 IX86_BUILTIN_CVTMASK2D128,
29939 IX86_BUILTIN_CVTMASK2D256,
29940 IX86_BUILTIN_CVTMASK2Q128,
29941 IX86_BUILTIN_CVTMASK2Q256,
29942 IX86_BUILTIN_PCMPEQB128_MASK,
29943 IX86_BUILTIN_PCMPEQB256_MASK,
29944 IX86_BUILTIN_PCMPEQW128_MASK,
29945 IX86_BUILTIN_PCMPEQW256_MASK,
29946 IX86_BUILTIN_PCMPEQD128_MASK,
29947 IX86_BUILTIN_PCMPEQD256_MASK,
29948 IX86_BUILTIN_PCMPEQQ128_MASK,
29949 IX86_BUILTIN_PCMPEQQ256_MASK,
29950 IX86_BUILTIN_PCMPGTB128_MASK,
29951 IX86_BUILTIN_PCMPGTB256_MASK,
29952 IX86_BUILTIN_PCMPGTW128_MASK,
29953 IX86_BUILTIN_PCMPGTW256_MASK,
29954 IX86_BUILTIN_PCMPGTD128_MASK,
29955 IX86_BUILTIN_PCMPGTD256_MASK,
29956 IX86_BUILTIN_PCMPGTQ128_MASK,
29957 IX86_BUILTIN_PCMPGTQ256_MASK,
29958 IX86_BUILTIN_PTESTMB128,
29959 IX86_BUILTIN_PTESTMB256,
29960 IX86_BUILTIN_PTESTMW128,
29961 IX86_BUILTIN_PTESTMW256,
29962 IX86_BUILTIN_PTESTMD128,
29963 IX86_BUILTIN_PTESTMD256,
29964 IX86_BUILTIN_PTESTMQ128,
29965 IX86_BUILTIN_PTESTMQ256,
29966 IX86_BUILTIN_PTESTNMB128,
29967 IX86_BUILTIN_PTESTNMB256,
29968 IX86_BUILTIN_PTESTNMW128,
29969 IX86_BUILTIN_PTESTNMW256,
29970 IX86_BUILTIN_PTESTNMD128,
29971 IX86_BUILTIN_PTESTNMD256,
29972 IX86_BUILTIN_PTESTNMQ128,
29973 IX86_BUILTIN_PTESTNMQ256,
29974 IX86_BUILTIN_PBROADCASTMB128,
29975 IX86_BUILTIN_PBROADCASTMB256,
29976 IX86_BUILTIN_PBROADCASTMW128,
29977 IX86_BUILTIN_PBROADCASTMW256,
29978 IX86_BUILTIN_COMPRESSPD256,
29979 IX86_BUILTIN_COMPRESSPD128,
29980 IX86_BUILTIN_COMPRESSPS256,
29981 IX86_BUILTIN_COMPRESSPS128,
29982 IX86_BUILTIN_PCOMPRESSQ256,
29983 IX86_BUILTIN_PCOMPRESSQ128,
29984 IX86_BUILTIN_PCOMPRESSD256,
29985 IX86_BUILTIN_PCOMPRESSD128,
29986 IX86_BUILTIN_EXPANDPD256,
29987 IX86_BUILTIN_EXPANDPD128,
29988 IX86_BUILTIN_EXPANDPS256,
29989 IX86_BUILTIN_EXPANDPS128,
29990 IX86_BUILTIN_PEXPANDQ256,
29991 IX86_BUILTIN_PEXPANDQ128,
29992 IX86_BUILTIN_PEXPANDD256,
29993 IX86_BUILTIN_PEXPANDD128,
29994 IX86_BUILTIN_EXPANDPD256Z,
29995 IX86_BUILTIN_EXPANDPD128Z,
29996 IX86_BUILTIN_EXPANDPS256Z,
29997 IX86_BUILTIN_EXPANDPS128Z,
29998 IX86_BUILTIN_PEXPANDQ256Z,
29999 IX86_BUILTIN_PEXPANDQ128Z,
30000 IX86_BUILTIN_PEXPANDD256Z,
30001 IX86_BUILTIN_PEXPANDD128Z,
30002 IX86_BUILTIN_PMAXSD256_MASK,
30003 IX86_BUILTIN_PMINSD256_MASK,
30004 IX86_BUILTIN_PMAXUD256_MASK,
30005 IX86_BUILTIN_PMINUD256_MASK,
30006 IX86_BUILTIN_PMAXSD128_MASK,
30007 IX86_BUILTIN_PMINSD128_MASK,
30008 IX86_BUILTIN_PMAXUD128_MASK,
30009 IX86_BUILTIN_PMINUD128_MASK,
30010 IX86_BUILTIN_PMAXSQ256_MASK,
30011 IX86_BUILTIN_PMINSQ256_MASK,
30012 IX86_BUILTIN_PMAXUQ256_MASK,
30013 IX86_BUILTIN_PMINUQ256_MASK,
30014 IX86_BUILTIN_PMAXSQ128_MASK,
30015 IX86_BUILTIN_PMINSQ128_MASK,
30016 IX86_BUILTIN_PMAXUQ128_MASK,
30017 IX86_BUILTIN_PMINUQ128_MASK,
30018 IX86_BUILTIN_PMINSB256_MASK,
30019 IX86_BUILTIN_PMINUB256_MASK,
30020 IX86_BUILTIN_PMAXSB256_MASK,
30021 IX86_BUILTIN_PMAXUB256_MASK,
30022 IX86_BUILTIN_PMINSB128_MASK,
30023 IX86_BUILTIN_PMINUB128_MASK,
30024 IX86_BUILTIN_PMAXSB128_MASK,
30025 IX86_BUILTIN_PMAXUB128_MASK,
30026 IX86_BUILTIN_PMINSW256_MASK,
30027 IX86_BUILTIN_PMINUW256_MASK,
30028 IX86_BUILTIN_PMAXSW256_MASK,
30029 IX86_BUILTIN_PMAXUW256_MASK,
30030 IX86_BUILTIN_PMINSW128_MASK,
30031 IX86_BUILTIN_PMINUW128_MASK,
30032 IX86_BUILTIN_PMAXSW128_MASK,
30033 IX86_BUILTIN_PMAXUW128_MASK,
30034 IX86_BUILTIN_VPCONFLICTQ256,
30035 IX86_BUILTIN_VPCONFLICTD256,
30036 IX86_BUILTIN_VPCLZCNTQ256,
30037 IX86_BUILTIN_VPCLZCNTD256,
30038 IX86_BUILTIN_UNPCKHPD256_MASK,
30039 IX86_BUILTIN_UNPCKHPD128_MASK,
30040 IX86_BUILTIN_UNPCKHPS256_MASK,
30041 IX86_BUILTIN_UNPCKHPS128_MASK,
30042 IX86_BUILTIN_UNPCKLPD256_MASK,
30043 IX86_BUILTIN_UNPCKLPD128_MASK,
30044 IX86_BUILTIN_UNPCKLPS256_MASK,
30045 IX86_BUILTIN_VPCONFLICTQ128,
30046 IX86_BUILTIN_VPCONFLICTD128,
30047 IX86_BUILTIN_VPCLZCNTQ128,
30048 IX86_BUILTIN_VPCLZCNTD128,
30049 IX86_BUILTIN_UNPCKLPS128_MASK,
30050 IX86_BUILTIN_ALIGND256,
30051 IX86_BUILTIN_ALIGNQ256,
30052 IX86_BUILTIN_ALIGND128,
30053 IX86_BUILTIN_ALIGNQ128,
30054 IX86_BUILTIN_CVTPS2PH256_MASK,
30055 IX86_BUILTIN_CVTPS2PH_MASK,
30056 IX86_BUILTIN_CVTPH2PS_MASK,
30057 IX86_BUILTIN_CVTPH2PS256_MASK,
30058 IX86_BUILTIN_PUNPCKHDQ128_MASK,
30059 IX86_BUILTIN_PUNPCKHDQ256_MASK,
30060 IX86_BUILTIN_PUNPCKHQDQ128_MASK,
30061 IX86_BUILTIN_PUNPCKHQDQ256_MASK,
30062 IX86_BUILTIN_PUNPCKLDQ128_MASK,
30063 IX86_BUILTIN_PUNPCKLDQ256_MASK,
30064 IX86_BUILTIN_PUNPCKLQDQ128_MASK,
30065 IX86_BUILTIN_PUNPCKLQDQ256_MASK,
30066 IX86_BUILTIN_PUNPCKHBW128_MASK,
30067 IX86_BUILTIN_PUNPCKHBW256_MASK,
30068 IX86_BUILTIN_PUNPCKHWD128_MASK,
30069 IX86_BUILTIN_PUNPCKHWD256_MASK,
30070 IX86_BUILTIN_PUNPCKLBW128_MASK,
30071 IX86_BUILTIN_PUNPCKLBW256_MASK,
30072 IX86_BUILTIN_PUNPCKLWD128_MASK,
30073 IX86_BUILTIN_PUNPCKLWD256_MASK,
30074 IX86_BUILTIN_PSLLVV16HI,
30075 IX86_BUILTIN_PSLLVV8HI,
30076 IX86_BUILTIN_PACKSSDW256_MASK,
30077 IX86_BUILTIN_PACKSSDW128_MASK,
30078 IX86_BUILTIN_PACKUSDW256_MASK,
30079 IX86_BUILTIN_PACKUSDW128_MASK,
30080 IX86_BUILTIN_PAVGB256_MASK,
30081 IX86_BUILTIN_PAVGW256_MASK,
30082 IX86_BUILTIN_PAVGB128_MASK,
30083 IX86_BUILTIN_PAVGW128_MASK,
30084 IX86_BUILTIN_VPERMVARSF256_MASK,
30085 IX86_BUILTIN_VPERMVARDF256_MASK,
30086 IX86_BUILTIN_VPERMDF256_MASK,
30087 IX86_BUILTIN_PABSB256_MASK,
30088 IX86_BUILTIN_PABSB128_MASK,
30089 IX86_BUILTIN_PABSW256_MASK,
30090 IX86_BUILTIN_PABSW128_MASK,
30091 IX86_BUILTIN_VPERMILVARPD_MASK,
30092 IX86_BUILTIN_VPERMILVARPS_MASK,
30093 IX86_BUILTIN_VPERMILVARPD256_MASK,
30094 IX86_BUILTIN_VPERMILVARPS256_MASK,
30095 IX86_BUILTIN_VPERMILPD_MASK,
30096 IX86_BUILTIN_VPERMILPS_MASK,
30097 IX86_BUILTIN_VPERMILPD256_MASK,
30098 IX86_BUILTIN_VPERMILPS256_MASK,
30099 IX86_BUILTIN_BLENDMQ256,
30100 IX86_BUILTIN_BLENDMD256,
30101 IX86_BUILTIN_BLENDMPD256,
30102 IX86_BUILTIN_BLENDMPS256,
30103 IX86_BUILTIN_BLENDMQ128,
30104 IX86_BUILTIN_BLENDMD128,
30105 IX86_BUILTIN_BLENDMPD128,
30106 IX86_BUILTIN_BLENDMPS128,
30107 IX86_BUILTIN_BLENDMW256,
30108 IX86_BUILTIN_BLENDMB256,
30109 IX86_BUILTIN_BLENDMW128,
30110 IX86_BUILTIN_BLENDMB128,
30111 IX86_BUILTIN_PMULLD256_MASK,
30112 IX86_BUILTIN_PMULLD128_MASK,
30113 IX86_BUILTIN_PMULUDQ256_MASK,
30114 IX86_BUILTIN_PMULDQ256_MASK,
30115 IX86_BUILTIN_PMULDQ128_MASK,
30116 IX86_BUILTIN_PMULUDQ128_MASK,
30117 IX86_BUILTIN_CVTPD2PS256_MASK,
30118 IX86_BUILTIN_CVTPD2PS_MASK,
30119 IX86_BUILTIN_VPERMVARSI256_MASK,
30120 IX86_BUILTIN_VPERMVARDI256_MASK,
30121 IX86_BUILTIN_VPERMDI256_MASK,
30122 IX86_BUILTIN_CMPQ256,
30123 IX86_BUILTIN_CMPD256,
30124 IX86_BUILTIN_UCMPQ256,
30125 IX86_BUILTIN_UCMPD256,
30126 IX86_BUILTIN_CMPB256,
30127 IX86_BUILTIN_CMPW256,
30128 IX86_BUILTIN_UCMPB256,
30129 IX86_BUILTIN_UCMPW256,
30130 IX86_BUILTIN_CMPPD256_MASK,
30131 IX86_BUILTIN_CMPPS256_MASK,
30132 IX86_BUILTIN_CMPQ128,
30133 IX86_BUILTIN_CMPD128,
30134 IX86_BUILTIN_UCMPQ128,
30135 IX86_BUILTIN_UCMPD128,
30136 IX86_BUILTIN_CMPB128,
30137 IX86_BUILTIN_CMPW128,
30138 IX86_BUILTIN_UCMPB128,
30139 IX86_BUILTIN_UCMPW128,
30140 IX86_BUILTIN_CMPPD128_MASK,
30141 IX86_BUILTIN_CMPPS128_MASK,
30143 IX86_BUILTIN_GATHER3SIV8SF,
30144 IX86_BUILTIN_GATHER3SIV4SF,
30145 IX86_BUILTIN_GATHER3SIV4DF,
30146 IX86_BUILTIN_GATHER3SIV2DF,
30147 IX86_BUILTIN_GATHER3DIV8SF,
30148 IX86_BUILTIN_GATHER3DIV4SF,
30149 IX86_BUILTIN_GATHER3DIV4DF,
30150 IX86_BUILTIN_GATHER3DIV2DF,
30151 IX86_BUILTIN_GATHER3SIV8SI,
30152 IX86_BUILTIN_GATHER3SIV4SI,
30153 IX86_BUILTIN_GATHER3SIV4DI,
30154 IX86_BUILTIN_GATHER3SIV2DI,
30155 IX86_BUILTIN_GATHER3DIV8SI,
30156 IX86_BUILTIN_GATHER3DIV4SI,
30157 IX86_BUILTIN_GATHER3DIV4DI,
30158 IX86_BUILTIN_GATHER3DIV2DI,
30159 IX86_BUILTIN_SCATTERSIV8SF,
30160 IX86_BUILTIN_SCATTERSIV4SF,
30161 IX86_BUILTIN_SCATTERSIV4DF,
30162 IX86_BUILTIN_SCATTERSIV2DF,
30163 IX86_BUILTIN_SCATTERDIV8SF,
30164 IX86_BUILTIN_SCATTERDIV4SF,
30165 IX86_BUILTIN_SCATTERDIV4DF,
30166 IX86_BUILTIN_SCATTERDIV2DF,
30167 IX86_BUILTIN_SCATTERSIV8SI,
30168 IX86_BUILTIN_SCATTERSIV4SI,
30169 IX86_BUILTIN_SCATTERSIV4DI,
30170 IX86_BUILTIN_SCATTERSIV2DI,
30171 IX86_BUILTIN_SCATTERDIV8SI,
30172 IX86_BUILTIN_SCATTERDIV4SI,
30173 IX86_BUILTIN_SCATTERDIV4DI,
30174 IX86_BUILTIN_SCATTERDIV2DI,
30177 IX86_BUILTIN_RANGESD128,
30178 IX86_BUILTIN_RANGESS128,
30179 IX86_BUILTIN_KUNPCKWD,
30180 IX86_BUILTIN_KUNPCKDQ,
30181 IX86_BUILTIN_BROADCASTF32x2_512,
30182 IX86_BUILTIN_BROADCASTI32x2_512,
30183 IX86_BUILTIN_BROADCASTF64X2_512,
30184 IX86_BUILTIN_BROADCASTI64X2_512,
30185 IX86_BUILTIN_BROADCASTF32X8_512,
30186 IX86_BUILTIN_BROADCASTI32X8_512,
30187 IX86_BUILTIN_EXTRACTF64X2_512,
30188 IX86_BUILTIN_EXTRACTF32X8,
30189 IX86_BUILTIN_EXTRACTI64X2_512,
30190 IX86_BUILTIN_EXTRACTI32X8,
30191 IX86_BUILTIN_REDUCEPD512_MASK,
30192 IX86_BUILTIN_REDUCEPS512_MASK,
30193 IX86_BUILTIN_PMULLQ512,
30194 IX86_BUILTIN_XORPD512,
30195 IX86_BUILTIN_XORPS512,
30196 IX86_BUILTIN_ORPD512,
30197 IX86_BUILTIN_ORPS512,
30198 IX86_BUILTIN_ANDPD512,
30199 IX86_BUILTIN_ANDPS512,
30200 IX86_BUILTIN_ANDNPD512,
30201 IX86_BUILTIN_ANDNPS512,
30202 IX86_BUILTIN_INSERTF32X8,
30203 IX86_BUILTIN_INSERTI32X8,
30204 IX86_BUILTIN_INSERTF64X2_512,
30205 IX86_BUILTIN_INSERTI64X2_512,
30206 IX86_BUILTIN_FPCLASSPD512,
30207 IX86_BUILTIN_FPCLASSPS512,
30208 IX86_BUILTIN_CVTD2MASK512,
30209 IX86_BUILTIN_CVTQ2MASK512,
30210 IX86_BUILTIN_CVTMASK2D512,
30211 IX86_BUILTIN_CVTMASK2Q512,
30212 IX86_BUILTIN_CVTPD2QQ512,
30213 IX86_BUILTIN_CVTPS2QQ512,
30214 IX86_BUILTIN_CVTPD2UQQ512,
30215 IX86_BUILTIN_CVTPS2UQQ512,
30216 IX86_BUILTIN_CVTQQ2PS512,
30217 IX86_BUILTIN_CVTUQQ2PS512,
30218 IX86_BUILTIN_CVTQQ2PD512,
30219 IX86_BUILTIN_CVTUQQ2PD512,
30220 IX86_BUILTIN_CVTTPS2QQ512,
30221 IX86_BUILTIN_CVTTPS2UQQ512,
30222 IX86_BUILTIN_CVTTPD2QQ512,
30223 IX86_BUILTIN_CVTTPD2UQQ512,
30224 IX86_BUILTIN_RANGEPS512,
30225 IX86_BUILTIN_RANGEPD512,
30228 IX86_BUILTIN_PACKUSDW512,
30229 IX86_BUILTIN_PACKSSDW512,
30230 IX86_BUILTIN_LOADDQUHI512_MASK,
30231 IX86_BUILTIN_LOADDQUQI512_MASK,
30232 IX86_BUILTIN_PSLLDQ512,
30233 IX86_BUILTIN_PSRLDQ512,
30234 IX86_BUILTIN_STOREDQUHI512_MASK,
30235 IX86_BUILTIN_STOREDQUQI512_MASK,
30236 IX86_BUILTIN_PALIGNR512,
30237 IX86_BUILTIN_PALIGNR512_MASK,
30238 IX86_BUILTIN_MOVDQUHI512_MASK,
30239 IX86_BUILTIN_MOVDQUQI512_MASK,
30240 IX86_BUILTIN_PSADBW512,
30241 IX86_BUILTIN_DBPSADBW512,
30242 IX86_BUILTIN_PBROADCASTB512,
30243 IX86_BUILTIN_PBROADCASTB512_GPR,
30244 IX86_BUILTIN_PBROADCASTW512,
30245 IX86_BUILTIN_PBROADCASTW512_GPR,
30246 IX86_BUILTIN_PMOVSXBW512_MASK,
30247 IX86_BUILTIN_PMOVZXBW512_MASK,
30248 IX86_BUILTIN_VPERMVARHI512_MASK,
30249 IX86_BUILTIN_VPERMT2VARHI512,
30250 IX86_BUILTIN_VPERMT2VARHI512_MASKZ,
30251 IX86_BUILTIN_VPERMI2VARHI512,
30252 IX86_BUILTIN_PAVGB512,
30253 IX86_BUILTIN_PAVGW512,
30254 IX86_BUILTIN_PADDB512,
30255 IX86_BUILTIN_PSUBB512,
30256 IX86_BUILTIN_PSUBSB512,
30257 IX86_BUILTIN_PADDSB512,
30258 IX86_BUILTIN_PSUBUSB512,
30259 IX86_BUILTIN_PADDUSB512,
30260 IX86_BUILTIN_PSUBW512,
30261 IX86_BUILTIN_PADDW512,
30262 IX86_BUILTIN_PSUBSW512,
30263 IX86_BUILTIN_PADDSW512,
30264 IX86_BUILTIN_PSUBUSW512,
30265 IX86_BUILTIN_PADDUSW512,
30266 IX86_BUILTIN_PMAXUW512,
30267 IX86_BUILTIN_PMAXSW512,
30268 IX86_BUILTIN_PMINUW512,
30269 IX86_BUILTIN_PMINSW512,
30270 IX86_BUILTIN_PMAXUB512,
30271 IX86_BUILTIN_PMAXSB512,
30272 IX86_BUILTIN_PMINUB512,
30273 IX86_BUILTIN_PMINSB512,
30274 IX86_BUILTIN_PMOVWB512,
30275 IX86_BUILTIN_PMOVSWB512,
30276 IX86_BUILTIN_PMOVUSWB512,
30277 IX86_BUILTIN_PMULHRSW512_MASK,
30278 IX86_BUILTIN_PMULHUW512_MASK,
30279 IX86_BUILTIN_PMULHW512_MASK,
30280 IX86_BUILTIN_PMULLW512_MASK,
30281 IX86_BUILTIN_PSLLWI512_MASK,
30282 IX86_BUILTIN_PSLLW512_MASK,
30283 IX86_BUILTIN_PACKSSWB512,
30284 IX86_BUILTIN_PACKUSWB512,
30285 IX86_BUILTIN_PSRAVV32HI,
30286 IX86_BUILTIN_PMADDUBSW512_MASK,
30287 IX86_BUILTIN_PMADDWD512_MASK,
30288 IX86_BUILTIN_PSRLVV32HI,
30289 IX86_BUILTIN_PUNPCKHBW512,
30290 IX86_BUILTIN_PUNPCKHWD512,
30291 IX86_BUILTIN_PUNPCKLBW512,
30292 IX86_BUILTIN_PUNPCKLWD512,
30293 IX86_BUILTIN_PSHUFB512,
30294 IX86_BUILTIN_PSHUFHW512,
30295 IX86_BUILTIN_PSHUFLW512,
30296 IX86_BUILTIN_PSRAWI512,
30297 IX86_BUILTIN_PSRAW512,
30298 IX86_BUILTIN_PSRLWI512,
30299 IX86_BUILTIN_PSRLW512,
30300 IX86_BUILTIN_CVTB2MASK512,
30301 IX86_BUILTIN_CVTW2MASK512,
30302 IX86_BUILTIN_CVTMASK2B512,
30303 IX86_BUILTIN_CVTMASK2W512,
30304 IX86_BUILTIN_PCMPEQB512_MASK,
30305 IX86_BUILTIN_PCMPEQW512_MASK,
30306 IX86_BUILTIN_PCMPGTB512_MASK,
30307 IX86_BUILTIN_PCMPGTW512_MASK,
30308 IX86_BUILTIN_PTESTMB512,
30309 IX86_BUILTIN_PTESTMW512,
30310 IX86_BUILTIN_PTESTNMB512,
30311 IX86_BUILTIN_PTESTNMW512,
30312 IX86_BUILTIN_PSLLVV32HI,
30313 IX86_BUILTIN_PABSB512,
30314 IX86_BUILTIN_PABSW512,
30315 IX86_BUILTIN_BLENDMW512,
30316 IX86_BUILTIN_BLENDMB512,
30317 IX86_BUILTIN_CMPB512,
30318 IX86_BUILTIN_CMPW512,
30319 IX86_BUILTIN_UCMPB512,
30320 IX86_BUILTIN_UCMPW512,
30322 /* Alternate 4 and 8 element gather/scatter for the vectorizer
30323 where all operands are 32-byte or 64-byte wide respectively. */
30324 IX86_BUILTIN_GATHERALTSIV4DF,
30325 IX86_BUILTIN_GATHERALTDIV8SF,
30326 IX86_BUILTIN_GATHERALTSIV4DI,
30327 IX86_BUILTIN_GATHERALTDIV8SI,
30328 IX86_BUILTIN_GATHER3ALTDIV16SF,
30329 IX86_BUILTIN_GATHER3ALTDIV16SI,
30330 IX86_BUILTIN_GATHER3ALTSIV4DF,
30331 IX86_BUILTIN_GATHER3ALTDIV8SF,
30332 IX86_BUILTIN_GATHER3ALTSIV4DI,
30333 IX86_BUILTIN_GATHER3ALTDIV8SI,
30334 IX86_BUILTIN_GATHER3ALTSIV8DF,
30335 IX86_BUILTIN_GATHER3ALTSIV8DI,
30336 IX86_BUILTIN_GATHER3DIV16SF,
30337 IX86_BUILTIN_GATHER3DIV16SI,
30338 IX86_BUILTIN_GATHER3DIV8DF,
30339 IX86_BUILTIN_GATHER3DIV8DI,
30340 IX86_BUILTIN_GATHER3SIV16SF,
30341 IX86_BUILTIN_GATHER3SIV16SI,
30342 IX86_BUILTIN_GATHER3SIV8DF,
30343 IX86_BUILTIN_GATHER3SIV8DI,
30344 IX86_BUILTIN_SCATTERDIV16SF,
30345 IX86_BUILTIN_SCATTERDIV16SI,
30346 IX86_BUILTIN_SCATTERDIV8DF,
30347 IX86_BUILTIN_SCATTERDIV8DI,
30348 IX86_BUILTIN_SCATTERSIV16SF,
30349 IX86_BUILTIN_SCATTERSIV16SI,
30350 IX86_BUILTIN_SCATTERSIV8DF,
30351 IX86_BUILTIN_SCATTERSIV8DI,
30354 IX86_BUILTIN_GATHERPFQPD,
30355 IX86_BUILTIN_GATHERPFDPS,
30356 IX86_BUILTIN_GATHERPFDPD,
30357 IX86_BUILTIN_GATHERPFQPS,
30358 IX86_BUILTIN_SCATTERPFDPD,
30359 IX86_BUILTIN_SCATTERPFDPS,
30360 IX86_BUILTIN_SCATTERPFQPD,
30361 IX86_BUILTIN_SCATTERPFQPS,
30364 IX86_BUILTIN_EXP2PD_MASK,
30365 IX86_BUILTIN_EXP2PS_MASK,
30366 IX86_BUILTIN_EXP2PS,
30367 IX86_BUILTIN_RCP28PD,
30368 IX86_BUILTIN_RCP28PS,
30369 IX86_BUILTIN_RCP28SD,
30370 IX86_BUILTIN_RCP28SS,
30371 IX86_BUILTIN_RSQRT28PD,
30372 IX86_BUILTIN_RSQRT28PS,
30373 IX86_BUILTIN_RSQRT28SD,
30374 IX86_BUILTIN_RSQRT28SS,
30377 IX86_BUILTIN_VPMADD52LUQ512,
30378 IX86_BUILTIN_VPMADD52HUQ512,
30379 IX86_BUILTIN_VPMADD52LUQ256,
30380 IX86_BUILTIN_VPMADD52HUQ256,
30381 IX86_BUILTIN_VPMADD52LUQ128,
30382 IX86_BUILTIN_VPMADD52HUQ128,
30383 IX86_BUILTIN_VPMADD52LUQ512_MASKZ,
30384 IX86_BUILTIN_VPMADD52HUQ512_MASKZ,
30385 IX86_BUILTIN_VPMADD52LUQ256_MASKZ,
30386 IX86_BUILTIN_VPMADD52HUQ256_MASKZ,
30387 IX86_BUILTIN_VPMADD52LUQ128_MASKZ,
30388 IX86_BUILTIN_VPMADD52HUQ128_MASKZ,
30391 IX86_BUILTIN_VPMULTISHIFTQB512,
30392 IX86_BUILTIN_VPMULTISHIFTQB256,
30393 IX86_BUILTIN_VPMULTISHIFTQB128,
30394 IX86_BUILTIN_VPERMVARQI512_MASK,
30395 IX86_BUILTIN_VPERMT2VARQI512,
30396 IX86_BUILTIN_VPERMT2VARQI512_MASKZ,
30397 IX86_BUILTIN_VPERMI2VARQI512,
30398 IX86_BUILTIN_VPERMVARQI256_MASK,
30399 IX86_BUILTIN_VPERMVARQI128_MASK,
30400 IX86_BUILTIN_VPERMT2VARQI256,
30401 IX86_BUILTIN_VPERMT2VARQI256_MASKZ,
30402 IX86_BUILTIN_VPERMT2VARQI128,
30403 IX86_BUILTIN_VPERMT2VARQI128_MASKZ,
30404 IX86_BUILTIN_VPERMI2VARQI256,
30405 IX86_BUILTIN_VPERMI2VARQI128,
30407 /* SHA builtins. */
30408 IX86_BUILTIN_SHA1MSG1,
30409 IX86_BUILTIN_SHA1MSG2,
30410 IX86_BUILTIN_SHA1NEXTE,
30411 IX86_BUILTIN_SHA1RNDS4,
30412 IX86_BUILTIN_SHA256MSG1,
30413 IX86_BUILTIN_SHA256MSG2,
30414 IX86_BUILTIN_SHA256RNDS2,
30416 /* CLWB instructions. */
30419 /* PCOMMIT instructions. */
30420 IX86_BUILTIN_PCOMMIT,
30422 /* CLFLUSHOPT instructions. */
30423 IX86_BUILTIN_CLFLUSHOPT,
30425 /* TFmode support builtins. */
30427 IX86_BUILTIN_HUGE_VALQ,
30428 IX86_BUILTIN_FABSQ,
30429 IX86_BUILTIN_COPYSIGNQ,
30431 /* Vectorizer support builtins. */
30432 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX512,
30433 IX86_BUILTIN_CPYSGNPS,
30434 IX86_BUILTIN_CPYSGNPD,
30435 IX86_BUILTIN_CPYSGNPS256,
30436 IX86_BUILTIN_CPYSGNPS512,
30437 IX86_BUILTIN_CPYSGNPD256,
30438 IX86_BUILTIN_CPYSGNPD512,
30439 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX512,
30440 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX512,
30443 /* FMA4 instructions. */
30444 IX86_BUILTIN_VFMADDSS,
30445 IX86_BUILTIN_VFMADDSD,
30446 IX86_BUILTIN_VFMADDPS,
30447 IX86_BUILTIN_VFMADDPD,
30448 IX86_BUILTIN_VFMADDPS256,
30449 IX86_BUILTIN_VFMADDPD256,
30450 IX86_BUILTIN_VFMADDSUBPS,
30451 IX86_BUILTIN_VFMADDSUBPD,
30452 IX86_BUILTIN_VFMADDSUBPS256,
30453 IX86_BUILTIN_VFMADDSUBPD256,
30455 /* FMA3 instructions. */
30456 IX86_BUILTIN_VFMADDSS3,
30457 IX86_BUILTIN_VFMADDSD3,
30459 /* XOP instructions. */
30460 IX86_BUILTIN_VPCMOV,
30461 IX86_BUILTIN_VPCMOV_V2DI,
30462 IX86_BUILTIN_VPCMOV_V4SI,
30463 IX86_BUILTIN_VPCMOV_V8HI,
30464 IX86_BUILTIN_VPCMOV_V16QI,
30465 IX86_BUILTIN_VPCMOV_V4SF,
30466 IX86_BUILTIN_VPCMOV_V2DF,
30467 IX86_BUILTIN_VPCMOV256,
30468 IX86_BUILTIN_VPCMOV_V4DI256,
30469 IX86_BUILTIN_VPCMOV_V8SI256,
30470 IX86_BUILTIN_VPCMOV_V16HI256,
30471 IX86_BUILTIN_VPCMOV_V32QI256,
30472 IX86_BUILTIN_VPCMOV_V8SF256,
30473 IX86_BUILTIN_VPCMOV_V4DF256,
30475 IX86_BUILTIN_VPPERM,
30477 IX86_BUILTIN_VPMACSSWW,
30478 IX86_BUILTIN_VPMACSWW,
30479 IX86_BUILTIN_VPMACSSWD,
30480 IX86_BUILTIN_VPMACSWD,
30481 IX86_BUILTIN_VPMACSSDD,
30482 IX86_BUILTIN_VPMACSDD,
30483 IX86_BUILTIN_VPMACSSDQL,
30484 IX86_BUILTIN_VPMACSSDQH,
30485 IX86_BUILTIN_VPMACSDQL,
30486 IX86_BUILTIN_VPMACSDQH,
30487 IX86_BUILTIN_VPMADCSSWD,
30488 IX86_BUILTIN_VPMADCSWD,
30490 IX86_BUILTIN_VPHADDBW,
30491 IX86_BUILTIN_VPHADDBD,
30492 IX86_BUILTIN_VPHADDBQ,
30493 IX86_BUILTIN_VPHADDWD,
30494 IX86_BUILTIN_VPHADDWQ,
30495 IX86_BUILTIN_VPHADDDQ,
30496 IX86_BUILTIN_VPHADDUBW,
30497 IX86_BUILTIN_VPHADDUBD,
30498 IX86_BUILTIN_VPHADDUBQ,
30499 IX86_BUILTIN_VPHADDUWD,
30500 IX86_BUILTIN_VPHADDUWQ,
30501 IX86_BUILTIN_VPHADDUDQ,
30502 IX86_BUILTIN_VPHSUBBW,
30503 IX86_BUILTIN_VPHSUBWD,
30504 IX86_BUILTIN_VPHSUBDQ,
30506 IX86_BUILTIN_VPROTB,
30507 IX86_BUILTIN_VPROTW,
30508 IX86_BUILTIN_VPROTD,
30509 IX86_BUILTIN_VPROTQ,
30510 IX86_BUILTIN_VPROTB_IMM,
30511 IX86_BUILTIN_VPROTW_IMM,
30512 IX86_BUILTIN_VPROTD_IMM,
30513 IX86_BUILTIN_VPROTQ_IMM,
30515 IX86_BUILTIN_VPSHLB,
30516 IX86_BUILTIN_VPSHLW,
30517 IX86_BUILTIN_VPSHLD,
30518 IX86_BUILTIN_VPSHLQ,
30519 IX86_BUILTIN_VPSHAB,
30520 IX86_BUILTIN_VPSHAW,
30521 IX86_BUILTIN_VPSHAD,
30522 IX86_BUILTIN_VPSHAQ,
30524 IX86_BUILTIN_VFRCZSS,
30525 IX86_BUILTIN_VFRCZSD,
30526 IX86_BUILTIN_VFRCZPS,
30527 IX86_BUILTIN_VFRCZPD,
30528 IX86_BUILTIN_VFRCZPS256,
30529 IX86_BUILTIN_VFRCZPD256,
30531 IX86_BUILTIN_VPCOMEQUB,
30532 IX86_BUILTIN_VPCOMNEUB,
30533 IX86_BUILTIN_VPCOMLTUB,
30534 IX86_BUILTIN_VPCOMLEUB,
30535 IX86_BUILTIN_VPCOMGTUB,
30536 IX86_BUILTIN_VPCOMGEUB,
30537 IX86_BUILTIN_VPCOMFALSEUB,
30538 IX86_BUILTIN_VPCOMTRUEUB,
30540 IX86_BUILTIN_VPCOMEQUW,
30541 IX86_BUILTIN_VPCOMNEUW,
30542 IX86_BUILTIN_VPCOMLTUW,
30543 IX86_BUILTIN_VPCOMLEUW,
30544 IX86_BUILTIN_VPCOMGTUW,
30545 IX86_BUILTIN_VPCOMGEUW,
30546 IX86_BUILTIN_VPCOMFALSEUW,
30547 IX86_BUILTIN_VPCOMTRUEUW,
30549 IX86_BUILTIN_VPCOMEQUD,
30550 IX86_BUILTIN_VPCOMNEUD,
30551 IX86_BUILTIN_VPCOMLTUD,
30552 IX86_BUILTIN_VPCOMLEUD,
30553 IX86_BUILTIN_VPCOMGTUD,
30554 IX86_BUILTIN_VPCOMGEUD,
30555 IX86_BUILTIN_VPCOMFALSEUD,
30556 IX86_BUILTIN_VPCOMTRUEUD,
30558 IX86_BUILTIN_VPCOMEQUQ,
30559 IX86_BUILTIN_VPCOMNEUQ,
30560 IX86_BUILTIN_VPCOMLTUQ,
30561 IX86_BUILTIN_VPCOMLEUQ,
30562 IX86_BUILTIN_VPCOMGTUQ,
30563 IX86_BUILTIN_VPCOMGEUQ,
30564 IX86_BUILTIN_VPCOMFALSEUQ,
30565 IX86_BUILTIN_VPCOMTRUEUQ,
30567 IX86_BUILTIN_VPCOMEQB,
30568 IX86_BUILTIN_VPCOMNEB,
30569 IX86_BUILTIN_VPCOMLTB,
30570 IX86_BUILTIN_VPCOMLEB,
30571 IX86_BUILTIN_VPCOMGTB,
30572 IX86_BUILTIN_VPCOMGEB,
30573 IX86_BUILTIN_VPCOMFALSEB,
30574 IX86_BUILTIN_VPCOMTRUEB,
30576 IX86_BUILTIN_VPCOMEQW,
30577 IX86_BUILTIN_VPCOMNEW,
30578 IX86_BUILTIN_VPCOMLTW,
30579 IX86_BUILTIN_VPCOMLEW,
30580 IX86_BUILTIN_VPCOMGTW,
30581 IX86_BUILTIN_VPCOMGEW,
30582 IX86_BUILTIN_VPCOMFALSEW,
30583 IX86_BUILTIN_VPCOMTRUEW,
30585 IX86_BUILTIN_VPCOMEQD,
30586 IX86_BUILTIN_VPCOMNED,
30587 IX86_BUILTIN_VPCOMLTD,
30588 IX86_BUILTIN_VPCOMLED,
30589 IX86_BUILTIN_VPCOMGTD,
30590 IX86_BUILTIN_VPCOMGED,
30591 IX86_BUILTIN_VPCOMFALSED,
30592 IX86_BUILTIN_VPCOMTRUED,
30594 IX86_BUILTIN_VPCOMEQQ,
30595 IX86_BUILTIN_VPCOMNEQ,
30596 IX86_BUILTIN_VPCOMLTQ,
30597 IX86_BUILTIN_VPCOMLEQ,
30598 IX86_BUILTIN_VPCOMGTQ,
30599 IX86_BUILTIN_VPCOMGEQ,
30600 IX86_BUILTIN_VPCOMFALSEQ,
30601 IX86_BUILTIN_VPCOMTRUEQ,
30603 /* LWP instructions. */
30604 IX86_BUILTIN_LLWPCB,
30605 IX86_BUILTIN_SLWPCB,
30606 IX86_BUILTIN_LWPVAL32,
30607 IX86_BUILTIN_LWPVAL64,
30608 IX86_BUILTIN_LWPINS32,
30609 IX86_BUILTIN_LWPINS64,
30614 IX86_BUILTIN_XBEGIN,
30616 IX86_BUILTIN_XABORT,
30617 IX86_BUILTIN_XTEST,
30620 IX86_BUILTIN_BNDMK,
30621 IX86_BUILTIN_BNDSTX,
30622 IX86_BUILTIN_BNDLDX,
30623 IX86_BUILTIN_BNDCL,
30624 IX86_BUILTIN_BNDCU,
30625 IX86_BUILTIN_BNDRET,
30626 IX86_BUILTIN_BNDNARROW,
30627 IX86_BUILTIN_BNDINT,
30628 IX86_BUILTIN_SIZEOF,
30629 IX86_BUILTIN_BNDLOWER,
30630 IX86_BUILTIN_BNDUPPER,
30632 /* BMI instructions. */
30633 IX86_BUILTIN_BEXTR32,
30634 IX86_BUILTIN_BEXTR64,
30637 /* TBM instructions. */
30638 IX86_BUILTIN_BEXTRI32,
30639 IX86_BUILTIN_BEXTRI64,
30641 /* BMI2 instructions. */
30642 IX86_BUILTIN_BZHI32,
30643 IX86_BUILTIN_BZHI64,
30644 IX86_BUILTIN_PDEP32,
30645 IX86_BUILTIN_PDEP64,
30646 IX86_BUILTIN_PEXT32,
30647 IX86_BUILTIN_PEXT64,
30649 /* ADX instructions. */
30650 IX86_BUILTIN_ADDCARRYX32,
30651 IX86_BUILTIN_ADDCARRYX64,
30653 /* SBB instructions. */
30654 IX86_BUILTIN_SBB32,
30655 IX86_BUILTIN_SBB64,
30657 /* FSGSBASE instructions. */
30658 IX86_BUILTIN_RDFSBASE32,
30659 IX86_BUILTIN_RDFSBASE64,
30660 IX86_BUILTIN_RDGSBASE32,
30661 IX86_BUILTIN_RDGSBASE64,
30662 IX86_BUILTIN_WRFSBASE32,
30663 IX86_BUILTIN_WRFSBASE64,
30664 IX86_BUILTIN_WRGSBASE32,
30665 IX86_BUILTIN_WRGSBASE64,
30667 /* RDRND instructions. */
30668 IX86_BUILTIN_RDRAND16_STEP,
30669 IX86_BUILTIN_RDRAND32_STEP,
30670 IX86_BUILTIN_RDRAND64_STEP,
30672 /* RDSEED instructions. */
30673 IX86_BUILTIN_RDSEED16_STEP,
30674 IX86_BUILTIN_RDSEED32_STEP,
30675 IX86_BUILTIN_RDSEED64_STEP,
30677 /* F16C instructions. */
30678 IX86_BUILTIN_CVTPH2PS,
30679 IX86_BUILTIN_CVTPH2PS256,
30680 IX86_BUILTIN_CVTPS2PH,
30681 IX86_BUILTIN_CVTPS2PH256,
30683 /* MONITORX and MWAITX instrucions. */
30684 IX86_BUILTIN_MONITORX,
30685 IX86_BUILTIN_MWAITX,
30687 /* CFString built-in for darwin */
30688 IX86_BUILTIN_CFSTRING,
30690 /* Builtins to get CPU type and supported features. */
30691 IX86_BUILTIN_CPU_INIT,
30692 IX86_BUILTIN_CPU_IS,
30693 IX86_BUILTIN_CPU_SUPPORTS,
30695 /* Read/write FLAGS register built-ins. */
30696 IX86_BUILTIN_READ_FLAGS,
30697 IX86_BUILTIN_WRITE_FLAGS,
30702 /* Table for the ix86 builtin decls. */
30703 static GTY(()) tree ix86_builtins[(int) IX86_BUILTIN_MAX];
30705 /* Table of all of the builtin functions that are possible with different ISA's
30706 but are waiting to be built until a function is declared to use that
30708 struct builtin_isa {
30709 const char *name; /* function name */
30710 enum ix86_builtin_func_type tcode; /* type to use in the declaration */
30711 HOST_WIDE_INT isa; /* isa_flags this builtin is defined for */
30712 bool const_p; /* true if the declaration is constant */
30713 bool leaf_p; /* true if the declaration has leaf attribute */
30714 bool nothrow_p; /* true if the declaration has nothrow attribute */
30715 bool set_and_not_built_p;
30718 static struct builtin_isa ix86_builtins_isa[(int) IX86_BUILTIN_MAX];
30720 /* Bits that can still enable any inclusion of a builtin. */
30721 static HOST_WIDE_INT deferred_isa_values = 0;
30723 /* Add an ix86 target builtin function with CODE, NAME and TYPE. Save the MASK
30724 of which isa_flags to use in the ix86_builtins_isa array. Stores the
30725 function decl in the ix86_builtins array. Returns the function decl or
30726 NULL_TREE, if the builtin was not added.
30728 If the front end has a special hook for builtin functions, delay adding
30729 builtin functions that aren't in the current ISA until the ISA is changed
30730 with function specific optimization. Doing so, can save about 300K for the
30731 default compiler. When the builtin is expanded, check at that time whether
30734 If the front end doesn't have a special hook, record all builtins, even if
30735 it isn't an instruction set in the current ISA in case the user uses
30736 function specific options for a different ISA, so that we don't get scope
30737 errors if a builtin is added in the middle of a function scope. */
30740 def_builtin (HOST_WIDE_INT mask, const char *name,
30741 enum ix86_builtin_func_type tcode,
30742 enum ix86_builtins code)
30744 tree decl = NULL_TREE;
30746 if (!(mask & OPTION_MASK_ISA_64BIT) || TARGET_64BIT)
30748 ix86_builtins_isa[(int) code].isa = mask;
30750 mask &= ~OPTION_MASK_ISA_64BIT;
30752 || (mask & ix86_isa_flags) != 0
30753 || (lang_hooks.builtin_function
30754 == lang_hooks.builtin_function_ext_scope))
30757 tree type = ix86_get_builtin_func_type (tcode);
30758 decl = add_builtin_function (name, type, code, BUILT_IN_MD,
30760 ix86_builtins[(int) code] = decl;
30761 ix86_builtins_isa[(int) code].set_and_not_built_p = false;
30765 /* Just a MASK where set_and_not_built_p == true can potentially
30766 include a builtin. */
30767 deferred_isa_values |= mask;
30768 ix86_builtins[(int) code] = NULL_TREE;
30769 ix86_builtins_isa[(int) code].tcode = tcode;
30770 ix86_builtins_isa[(int) code].name = name;
30771 ix86_builtins_isa[(int) code].leaf_p = false;
30772 ix86_builtins_isa[(int) code].nothrow_p = false;
30773 ix86_builtins_isa[(int) code].const_p = false;
30774 ix86_builtins_isa[(int) code].set_and_not_built_p = true;
30781 /* Like def_builtin, but also marks the function decl "const". */
30784 def_builtin_const (HOST_WIDE_INT mask, const char *name,
30785 enum ix86_builtin_func_type tcode, enum ix86_builtins code)
30787 tree decl = def_builtin (mask, name, tcode, code);
30789 TREE_READONLY (decl) = 1;
30791 ix86_builtins_isa[(int) code].const_p = true;
30796 /* Add any new builtin functions for a given ISA that may not have been
30797 declared. This saves a bit of space compared to adding all of the
30798 declarations to the tree, even if we didn't use them. */
30801 ix86_add_new_builtins (HOST_WIDE_INT isa)
30803 if ((isa & deferred_isa_values) == 0)
30806 /* Bits in ISA value can be removed from potential isa values. */
30807 deferred_isa_values &= ~isa;
30810 tree saved_current_target_pragma = current_target_pragma;
30811 current_target_pragma = NULL_TREE;
30813 for (i = 0; i < (int)IX86_BUILTIN_MAX; i++)
30815 if ((ix86_builtins_isa[i].isa & isa) != 0
30816 && ix86_builtins_isa[i].set_and_not_built_p)
30820 /* Don't define the builtin again. */
30821 ix86_builtins_isa[i].set_and_not_built_p = false;
30823 type = ix86_get_builtin_func_type (ix86_builtins_isa[i].tcode);
30824 decl = add_builtin_function_ext_scope (ix86_builtins_isa[i].name,
30825 type, i, BUILT_IN_MD, NULL,
30828 ix86_builtins[i] = decl;
30829 if (ix86_builtins_isa[i].const_p)
30830 TREE_READONLY (decl) = 1;
30831 if (ix86_builtins_isa[i].leaf_p)
30832 DECL_ATTRIBUTES (decl) = build_tree_list (get_identifier ("leaf"),
30834 if (ix86_builtins_isa[i].nothrow_p)
30835 TREE_NOTHROW (decl) = 1;
30839 current_target_pragma = saved_current_target_pragma;
30842 /* Bits for builtin_description.flag. */
30844 /* Set when we don't support the comparison natively, and should
30845 swap_comparison in order to support it. */
30846 #define BUILTIN_DESC_SWAP_OPERANDS 1
30848 struct builtin_description
30850 const HOST_WIDE_INT mask;
30851 const enum insn_code icode;
30852 const char *const name;
30853 const enum ix86_builtins code;
30854 const enum rtx_code comparison;
30858 static const struct builtin_description bdesc_comi[] =
30860 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comieq", IX86_BUILTIN_COMIEQSS, UNEQ, 0 },
30861 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comilt", IX86_BUILTIN_COMILTSS, UNLT, 0 },
30862 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comile", IX86_BUILTIN_COMILESS, UNLE, 0 },
30863 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comigt", IX86_BUILTIN_COMIGTSS, GT, 0 },
30864 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comige", IX86_BUILTIN_COMIGESS, GE, 0 },
30865 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comineq", IX86_BUILTIN_COMINEQSS, LTGT, 0 },
30866 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomieq", IX86_BUILTIN_UCOMIEQSS, UNEQ, 0 },
30867 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomilt", IX86_BUILTIN_UCOMILTSS, UNLT, 0 },
30868 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomile", IX86_BUILTIN_UCOMILESS, UNLE, 0 },
30869 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomigt", IX86_BUILTIN_UCOMIGTSS, GT, 0 },
30870 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomige", IX86_BUILTIN_UCOMIGESS, GE, 0 },
30871 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomineq", IX86_BUILTIN_UCOMINEQSS, LTGT, 0 },
30872 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdeq", IX86_BUILTIN_COMIEQSD, UNEQ, 0 },
30873 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdlt", IX86_BUILTIN_COMILTSD, UNLT, 0 },
30874 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdle", IX86_BUILTIN_COMILESD, UNLE, 0 },
30875 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdgt", IX86_BUILTIN_COMIGTSD, GT, 0 },
30876 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdge", IX86_BUILTIN_COMIGESD, GE, 0 },
30877 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdneq", IX86_BUILTIN_COMINEQSD, LTGT, 0 },
30878 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdeq", IX86_BUILTIN_UCOMIEQSD, UNEQ, 0 },
30879 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdlt", IX86_BUILTIN_UCOMILTSD, UNLT, 0 },
30880 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdle", IX86_BUILTIN_UCOMILESD, UNLE, 0 },
30881 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdgt", IX86_BUILTIN_UCOMIGTSD, GT, 0 },
30882 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdge", IX86_BUILTIN_UCOMIGESD, GE, 0 },
30883 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdneq", IX86_BUILTIN_UCOMINEQSD, LTGT, 0 },
30886 static const struct builtin_description bdesc_pcmpestr[] =
30889 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestri128", IX86_BUILTIN_PCMPESTRI128, UNKNOWN, 0 },
30890 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrm128", IX86_BUILTIN_PCMPESTRM128, UNKNOWN, 0 },
30891 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestria128", IX86_BUILTIN_PCMPESTRA128, UNKNOWN, (int) CCAmode },
30892 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestric128", IX86_BUILTIN_PCMPESTRC128, UNKNOWN, (int) CCCmode },
30893 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrio128", IX86_BUILTIN_PCMPESTRO128, UNKNOWN, (int) CCOmode },
30894 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestris128", IX86_BUILTIN_PCMPESTRS128, UNKNOWN, (int) CCSmode },
30895 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestriz128", IX86_BUILTIN_PCMPESTRZ128, UNKNOWN, (int) CCZmode },
30898 static const struct builtin_description bdesc_pcmpistr[] =
30901 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistri128", IX86_BUILTIN_PCMPISTRI128, UNKNOWN, 0 },
30902 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrm128", IX86_BUILTIN_PCMPISTRM128, UNKNOWN, 0 },
30903 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistria128", IX86_BUILTIN_PCMPISTRA128, UNKNOWN, (int) CCAmode },
30904 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistric128", IX86_BUILTIN_PCMPISTRC128, UNKNOWN, (int) CCCmode },
30905 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrio128", IX86_BUILTIN_PCMPISTRO128, UNKNOWN, (int) CCOmode },
30906 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistris128", IX86_BUILTIN_PCMPISTRS128, UNKNOWN, (int) CCSmode },
30907 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistriz128", IX86_BUILTIN_PCMPISTRZ128, UNKNOWN, (int) CCZmode },
30910 /* Special builtins with variable number of arguments. */
30911 static const struct builtin_description bdesc_special_args[] =
30913 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtsc", IX86_BUILTIN_RDTSC, UNKNOWN, (int) UINT64_FTYPE_VOID },
30914 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtscp", IX86_BUILTIN_RDTSCP, UNKNOWN, (int) UINT64_FTYPE_PUNSIGNED },
30915 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_pause, "__builtin_ia32_pause", IX86_BUILTIN_PAUSE, UNKNOWN, (int) VOID_FTYPE_VOID },
30917 /* 80387 (for use internally for atomic compound assignment). */
30918 { 0, CODE_FOR_fnstenv, "__builtin_ia32_fnstenv", IX86_BUILTIN_FNSTENV, UNKNOWN, (int) VOID_FTYPE_PVOID },
30919 { 0, CODE_FOR_fldenv, "__builtin_ia32_fldenv", IX86_BUILTIN_FLDENV, UNKNOWN, (int) VOID_FTYPE_PCVOID },
30920 { 0, CODE_FOR_fnstsw, "__builtin_ia32_fnstsw", IX86_BUILTIN_FNSTSW, UNKNOWN, (int) USHORT_FTYPE_VOID },
30921 { 0, CODE_FOR_fnclex, "__builtin_ia32_fnclex", IX86_BUILTIN_FNCLEX, UNKNOWN, (int) VOID_FTYPE_VOID },
30924 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_emms, "__builtin_ia32_emms", IX86_BUILTIN_EMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
30927 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_femms, "__builtin_ia32_femms", IX86_BUILTIN_FEMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
30929 /* FXSR, XSAVE, XSAVEOPT, XSAVEC and XSAVES. */
30930 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxsave", IX86_BUILTIN_FXSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID },
30931 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxrstor", IX86_BUILTIN_FXRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID },
30932 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xsave", IX86_BUILTIN_XSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30933 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xrstor", IX86_BUILTIN_XRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30934 { OPTION_MASK_ISA_XSAVEOPT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt", IX86_BUILTIN_XSAVEOPT, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30935 { OPTION_MASK_ISA_XSAVES, CODE_FOR_nothing, "__builtin_ia32_xsaves", IX86_BUILTIN_XSAVES, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30936 { OPTION_MASK_ISA_XSAVES, CODE_FOR_nothing, "__builtin_ia32_xrstors", IX86_BUILTIN_XRSTORS, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30937 { OPTION_MASK_ISA_XSAVEC, CODE_FOR_nothing, "__builtin_ia32_xsavec", IX86_BUILTIN_XSAVEC, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30939 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxsave64", IX86_BUILTIN_FXSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID },
30940 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxrstor64", IX86_BUILTIN_FXRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID },
30941 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsave64", IX86_BUILTIN_XSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30942 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xrstor64", IX86_BUILTIN_XRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30943 { OPTION_MASK_ISA_XSAVEOPT | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt64", IX86_BUILTIN_XSAVEOPT64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30944 { OPTION_MASK_ISA_XSAVES | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsaves64", IX86_BUILTIN_XSAVES64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30945 { OPTION_MASK_ISA_XSAVES | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xrstors64", IX86_BUILTIN_XRSTORS64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30946 { OPTION_MASK_ISA_XSAVEC | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsavec64", IX86_BUILTIN_XSAVEC64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30949 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storeups, "__builtin_ia32_storeups", IX86_BUILTIN_STOREUPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
30950 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movntv4sf, "__builtin_ia32_movntps", IX86_BUILTIN_MOVNTPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
30951 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadups, "__builtin_ia32_loadups", IX86_BUILTIN_LOADUPS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
30953 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadhps_exp, "__builtin_ia32_loadhps", IX86_BUILTIN_LOADHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
30954 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadlps_exp, "__builtin_ia32_loadlps", IX86_BUILTIN_LOADLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
30955 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storehps, "__builtin_ia32_storehps", IX86_BUILTIN_STOREHPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
30956 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storelps, "__builtin_ia32_storelps", IX86_BUILTIN_STORELPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
30958 /* SSE or 3DNow!A */
30959 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_sfence, "__builtin_ia32_sfence", IX86_BUILTIN_SFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
30960 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_movntq, "__builtin_ia32_movntq", IX86_BUILTIN_MOVNTQ, UNKNOWN, (int) VOID_FTYPE_PULONGLONG_ULONGLONG },
30963 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lfence, "__builtin_ia32_lfence", IX86_BUILTIN_LFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
30964 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_mfence, 0, IX86_BUILTIN_MFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
30965 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storeupd, "__builtin_ia32_storeupd", IX86_BUILTIN_STOREUPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
30966 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storedquv16qi, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
30967 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2df, "__builtin_ia32_movntpd", IX86_BUILTIN_MOVNTPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
30968 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2di, "__builtin_ia32_movntdq", IX86_BUILTIN_MOVNTDQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI },
30969 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntisi, "__builtin_ia32_movnti", IX86_BUILTIN_MOVNTI, UNKNOWN, (int) VOID_FTYPE_PINT_INT },
30970 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_movntidi, "__builtin_ia32_movnti64", IX86_BUILTIN_MOVNTI64, UNKNOWN, (int) VOID_FTYPE_PLONGLONG_LONGLONG },
30971 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadupd, "__builtin_ia32_loadupd", IX86_BUILTIN_LOADUPD, UNKNOWN, (int) V2DF_FTYPE_PCDOUBLE },
30972 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loaddquv16qi, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
30974 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadhpd_exp, "__builtin_ia32_loadhpd", IX86_BUILTIN_LOADHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
30975 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadlpd_exp, "__builtin_ia32_loadlpd", IX86_BUILTIN_LOADLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
30978 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_lddqu, "__builtin_ia32_lddqu", IX86_BUILTIN_LDDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
30981 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_movntdqa, "__builtin_ia32_movntdqa", IX86_BUILTIN_MOVNTDQA, UNKNOWN, (int) V2DI_FTYPE_PV2DI },
30984 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv2df, "__builtin_ia32_movntsd", IX86_BUILTIN_MOVNTSD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
30985 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv4sf, "__builtin_ia32_movntss", IX86_BUILTIN_MOVNTSS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
30988 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroall, "__builtin_ia32_vzeroall", IX86_BUILTIN_VZEROALL, UNKNOWN, (int) VOID_FTYPE_VOID },
30989 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroupper, "__builtin_ia32_vzeroupper", IX86_BUILTIN_VZEROUPPER, UNKNOWN, (int) VOID_FTYPE_VOID },
30991 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4sf, "__builtin_ia32_vbroadcastss", IX86_BUILTIN_VBROADCASTSS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
30992 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4df, "__builtin_ia32_vbroadcastsd256", IX86_BUILTIN_VBROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
30993 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv8sf, "__builtin_ia32_vbroadcastss256", IX86_BUILTIN_VBROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
30994 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v4df, "__builtin_ia32_vbroadcastf128_pd256", IX86_BUILTIN_VBROADCASTPD256, UNKNOWN, (int) V4DF_FTYPE_PCV2DF },
30995 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v8sf, "__builtin_ia32_vbroadcastf128_ps256", IX86_BUILTIN_VBROADCASTPS256, UNKNOWN, (int) V8SF_FTYPE_PCV4SF },
30997 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadupd256, "__builtin_ia32_loadupd256", IX86_BUILTIN_LOADUPD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
30998 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadups256, "__builtin_ia32_loadups256", IX86_BUILTIN_LOADUPS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
30999 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeupd256, "__builtin_ia32_storeupd256", IX86_BUILTIN_STOREUPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
31000 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeups256, "__builtin_ia32_storeups256", IX86_BUILTIN_STOREUPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
31001 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loaddquv32qi, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
31002 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storedquv32qi, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
31003 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_lddqu256, "__builtin_ia32_lddqu256", IX86_BUILTIN_LDDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
31005 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4di, "__builtin_ia32_movntdq256", IX86_BUILTIN_MOVNTDQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI },
31006 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4df, "__builtin_ia32_movntpd256", IX86_BUILTIN_MOVNTPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
31007 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv8sf, "__builtin_ia32_movntps256", IX86_BUILTIN_MOVNTPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
31009 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DI },
31010 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SI },
31011 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DI },
31012 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SI },
31013 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DI_V2DF },
31014 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SI_V4SF },
31015 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DI_V4DF },
31016 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SI_V8SF },
31019 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_movntdqa, "__builtin_ia32_movntdqa256", IX86_BUILTIN_MOVNTDQA256, UNKNOWN, (int) V4DI_FTYPE_PV4DI },
31020 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd, "__builtin_ia32_maskloadd", IX86_BUILTIN_MASKLOADD, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI },
31021 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq, "__builtin_ia32_maskloadq", IX86_BUILTIN_MASKLOADQ, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI },
31022 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd256, "__builtin_ia32_maskloadd256", IX86_BUILTIN_MASKLOADD256, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI },
31023 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq256, "__builtin_ia32_maskloadq256", IX86_BUILTIN_MASKLOADQ256, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI },
31024 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored, "__builtin_ia32_maskstored", IX86_BUILTIN_MASKSTORED, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_V4SI },
31025 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq, "__builtin_ia32_maskstoreq", IX86_BUILTIN_MASKSTOREQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_V2DI },
31026 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored256, "__builtin_ia32_maskstored256", IX86_BUILTIN_MASKSTORED256, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_V8SI },
31027 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq256, "__builtin_ia32_maskstoreq256", IX86_BUILTIN_MASKSTOREQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_V4DI },
31030 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressstorev16sf_mask, "__builtin_ia32_compressstoresf512_mask", IX86_BUILTIN_COMPRESSPSSTORE512, UNKNOWN, (int) VOID_FTYPE_PV16SF_V16SF_HI },
31031 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressstorev16si_mask, "__builtin_ia32_compressstoresi512_mask", IX86_BUILTIN_PCOMPRESSDSTORE512, UNKNOWN, (int) VOID_FTYPE_PV16SI_V16SI_HI },
31032 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressstorev8df_mask, "__builtin_ia32_compressstoredf512_mask", IX86_BUILTIN_COMPRESSPDSTORE512, UNKNOWN, (int) VOID_FTYPE_PV8DF_V8DF_QI },
31033 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressstorev8di_mask, "__builtin_ia32_compressstoredi512_mask", IX86_BUILTIN_PCOMPRESSQSTORE512, UNKNOWN, (int) VOID_FTYPE_PV8DI_V8DI_QI },
31034 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16sf_mask, "__builtin_ia32_expandloadsf512_mask", IX86_BUILTIN_EXPANDPSLOAD512, UNKNOWN, (int) V16SF_FTYPE_PCV16SF_V16SF_HI },
31035 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16sf_maskz, "__builtin_ia32_expandloadsf512_maskz", IX86_BUILTIN_EXPANDPSLOAD512Z, UNKNOWN, (int) V16SF_FTYPE_PCV16SF_V16SF_HI },
31036 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16si_mask, "__builtin_ia32_expandloadsi512_mask", IX86_BUILTIN_PEXPANDDLOAD512, UNKNOWN, (int) V16SI_FTYPE_PCV16SI_V16SI_HI },
31037 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16si_maskz, "__builtin_ia32_expandloadsi512_maskz", IX86_BUILTIN_PEXPANDDLOAD512Z, UNKNOWN, (int) V16SI_FTYPE_PCV16SI_V16SI_HI },
31038 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8df_mask, "__builtin_ia32_expandloaddf512_mask", IX86_BUILTIN_EXPANDPDLOAD512, UNKNOWN, (int) V8DF_FTYPE_PCV8DF_V8DF_QI },
31039 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8df_maskz, "__builtin_ia32_expandloaddf512_maskz", IX86_BUILTIN_EXPANDPDLOAD512Z, UNKNOWN, (int) V8DF_FTYPE_PCV8DF_V8DF_QI },
31040 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8di_mask, "__builtin_ia32_expandloaddi512_mask", IX86_BUILTIN_PEXPANDQLOAD512, UNKNOWN, (int) V8DI_FTYPE_PCV8DI_V8DI_QI },
31041 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8di_maskz, "__builtin_ia32_expandloaddi512_maskz", IX86_BUILTIN_PEXPANDQLOAD512Z, UNKNOWN, (int) V8DI_FTYPE_PCV8DI_V8DI_QI },
31042 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loaddquv16si_mask, "__builtin_ia32_loaddqusi512_mask", IX86_BUILTIN_LOADDQUSI512, UNKNOWN, (int) V16SI_FTYPE_PCV16SI_V16SI_HI },
31043 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loaddquv8di_mask, "__builtin_ia32_loaddqudi512_mask", IX86_BUILTIN_LOADDQUDI512, UNKNOWN, (int) V8DI_FTYPE_PCV8DI_V8DI_QI },
31044 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadupd512_mask, "__builtin_ia32_loadupd512_mask", IX86_BUILTIN_LOADUPD512, UNKNOWN, (int) V8DF_FTYPE_PCV8DF_V8DF_QI },
31045 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadups512_mask, "__builtin_ia32_loadups512_mask", IX86_BUILTIN_LOADUPS512, UNKNOWN, (int) V16SF_FTYPE_PCV16SF_V16SF_HI },
31046 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv16sf_mask, "__builtin_ia32_loadaps512_mask", IX86_BUILTIN_LOADAPS512, UNKNOWN, (int) V16SF_FTYPE_PCV16SF_V16SF_HI },
31047 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv16si_mask, "__builtin_ia32_movdqa32load512_mask", IX86_BUILTIN_MOVDQA32LOAD512, UNKNOWN, (int) V16SI_FTYPE_PCV16SI_V16SI_HI },
31048 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv8df_mask, "__builtin_ia32_loadapd512_mask", IX86_BUILTIN_LOADAPD512, UNKNOWN, (int) V8DF_FTYPE_PCV8DF_V8DF_QI },
31049 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv8di_mask, "__builtin_ia32_movdqa64load512_mask", IX86_BUILTIN_MOVDQA64LOAD512, UNKNOWN, (int) V8DI_FTYPE_PCV8DI_V8DI_QI },
31050 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movntv16sf, "__builtin_ia32_movntps512", IX86_BUILTIN_MOVNTPS512, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V16SF },
31051 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movntv8df, "__builtin_ia32_movntpd512", IX86_BUILTIN_MOVNTPD512, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V8DF },
31052 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movntv8di, "__builtin_ia32_movntdq512", IX86_BUILTIN_MOVNTDQ512, UNKNOWN, (int) VOID_FTYPE_PV8DI_V8DI },
31053 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movntdqa, "__builtin_ia32_movntdqa512", IX86_BUILTIN_MOVNTDQA512, UNKNOWN, (int) V8DI_FTYPE_PV8DI },
31054 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storedquv16si_mask, "__builtin_ia32_storedqusi512_mask", IX86_BUILTIN_STOREDQUSI512, UNKNOWN, (int) VOID_FTYPE_PV16SI_V16SI_HI },
31055 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storedquv8di_mask, "__builtin_ia32_storedqudi512_mask", IX86_BUILTIN_STOREDQUDI512, UNKNOWN, (int) VOID_FTYPE_PV8DI_V8DI_QI },
31056 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storeupd512_mask, "__builtin_ia32_storeupd512_mask", IX86_BUILTIN_STOREUPD512, UNKNOWN, (int) VOID_FTYPE_PV8DF_V8DF_QI },
31057 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div8si2_mask_store, "__builtin_ia32_pmovusqd512mem_mask", IX86_BUILTIN_PMOVUSQD512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8DI_QI },
31058 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div8si2_mask_store, "__builtin_ia32_pmovsqd512mem_mask", IX86_BUILTIN_PMOVSQD512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8DI_QI },
31059 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div8si2_mask_store, "__builtin_ia32_pmovqd512mem_mask", IX86_BUILTIN_PMOVQD512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8DI_QI },
31060 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div8hi2_mask_store, "__builtin_ia32_pmovusqw512mem_mask", IX86_BUILTIN_PMOVUSQW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8DI_QI },
31061 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div8hi2_mask_store, "__builtin_ia32_pmovsqw512mem_mask", IX86_BUILTIN_PMOVSQW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8DI_QI },
31062 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div8hi2_mask_store, "__builtin_ia32_pmovqw512mem_mask", IX86_BUILTIN_PMOVQW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8DI_QI },
31063 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev16siv16hi2_mask_store, "__builtin_ia32_pmovusdw512mem_mask", IX86_BUILTIN_PMOVUSDW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16HI_V16SI_HI },
31064 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev16siv16hi2_mask_store, "__builtin_ia32_pmovsdw512mem_mask", IX86_BUILTIN_PMOVSDW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16HI_V16SI_HI },
31065 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev16siv16hi2_mask_store, "__builtin_ia32_pmovdw512mem_mask", IX86_BUILTIN_PMOVDW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16HI_V16SI_HI },
31066 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div16qi2_mask_store, "__builtin_ia32_pmovqb512mem_mask", IX86_BUILTIN_PMOVQB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8DI_QI },
31067 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div16qi2_mask_store, "__builtin_ia32_pmovusqb512mem_mask", IX86_BUILTIN_PMOVUSQB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8DI_QI },
31068 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div16qi2_mask_store, "__builtin_ia32_pmovsqb512mem_mask", IX86_BUILTIN_PMOVSQB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8DI_QI },
31069 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev16siv16qi2_mask_store, "__builtin_ia32_pmovusdb512mem_mask", IX86_BUILTIN_PMOVUSDB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V16SI_HI },
31070 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev16siv16qi2_mask_store, "__builtin_ia32_pmovsdb512mem_mask", IX86_BUILTIN_PMOVSDB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V16SI_HI },
31071 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev16siv16qi2_mask_store, "__builtin_ia32_pmovdb512mem_mask", IX86_BUILTIN_PMOVDB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V16SI_HI },
31072 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storeups512_mask, "__builtin_ia32_storeups512_mask", IX86_BUILTIN_STOREUPS512, UNKNOWN, (int) VOID_FTYPE_PV16SF_V16SF_HI },
31073 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storev16sf_mask, "__builtin_ia32_storeaps512_mask", IX86_BUILTIN_STOREAPS512, UNKNOWN, (int) VOID_FTYPE_PV16SF_V16SF_HI },
31074 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storev16si_mask, "__builtin_ia32_movdqa32store512_mask", IX86_BUILTIN_MOVDQA32STORE512, UNKNOWN, (int) VOID_FTYPE_PV16SI_V16SI_HI },
31075 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storev8df_mask, "__builtin_ia32_storeapd512_mask", IX86_BUILTIN_STOREAPD512, UNKNOWN, (int) VOID_FTYPE_PV8DF_V8DF_QI },
31076 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storev8di_mask, "__builtin_ia32_movdqa64store512_mask", IX86_BUILTIN_MOVDQA64STORE512, UNKNOWN, (int) VOID_FTYPE_PV8DI_V8DI_QI },
31078 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_llwpcb, "__builtin_ia32_llwpcb", IX86_BUILTIN_LLWPCB, UNKNOWN, (int) VOID_FTYPE_PVOID },
31079 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_slwpcb, "__builtin_ia32_slwpcb", IX86_BUILTIN_SLWPCB, UNKNOWN, (int) PVOID_FTYPE_VOID },
31080 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvalsi3, "__builtin_ia32_lwpval32", IX86_BUILTIN_LWPVAL32, UNKNOWN, (int) VOID_FTYPE_UINT_UINT_UINT },
31081 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvaldi3, "__builtin_ia32_lwpval64", IX86_BUILTIN_LWPVAL64, UNKNOWN, (int) VOID_FTYPE_UINT64_UINT_UINT },
31082 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinssi3, "__builtin_ia32_lwpins32", IX86_BUILTIN_LWPINS32, UNKNOWN, (int) UCHAR_FTYPE_UINT_UINT_UINT },
31083 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinsdi3, "__builtin_ia32_lwpins64", IX86_BUILTIN_LWPINS64, UNKNOWN, (int) UCHAR_FTYPE_UINT64_UINT_UINT },
31086 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasesi, "__builtin_ia32_rdfsbase32", IX86_BUILTIN_RDFSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
31087 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasedi, "__builtin_ia32_rdfsbase64", IX86_BUILTIN_RDFSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
31088 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasesi, "__builtin_ia32_rdgsbase32", IX86_BUILTIN_RDGSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
31089 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasedi, "__builtin_ia32_rdgsbase64", IX86_BUILTIN_RDGSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
31090 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasesi, "__builtin_ia32_wrfsbase32", IX86_BUILTIN_WRFSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
31091 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasedi, "__builtin_ia32_wrfsbase64", IX86_BUILTIN_WRFSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
31092 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasesi, "__builtin_ia32_wrgsbase32", IX86_BUILTIN_WRGSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
31093 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasedi, "__builtin_ia32_wrgsbase64", IX86_BUILTIN_WRGSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
31096 { OPTION_MASK_ISA_RTM, CODE_FOR_xbegin, "__builtin_ia32_xbegin", IX86_BUILTIN_XBEGIN, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
31097 { OPTION_MASK_ISA_RTM, CODE_FOR_xend, "__builtin_ia32_xend", IX86_BUILTIN_XEND, UNKNOWN, (int) VOID_FTYPE_VOID },
31098 { OPTION_MASK_ISA_RTM, CODE_FOR_xtest, "__builtin_ia32_xtest", IX86_BUILTIN_XTEST, UNKNOWN, (int) INT_FTYPE_VOID },
31101 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_loaddquv32hi_mask, "__builtin_ia32_loaddquhi512_mask", IX86_BUILTIN_LOADDQUHI512_MASK, UNKNOWN, (int) V32HI_FTYPE_PCV32HI_V32HI_SI },
31102 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512f_loaddquv64qi_mask, "__builtin_ia32_loaddquqi512_mask", IX86_BUILTIN_LOADDQUQI512_MASK, UNKNOWN, (int) V64QI_FTYPE_PCV64QI_V64QI_DI },
31103 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_storedquv32hi_mask, "__builtin_ia32_storedquhi512_mask", IX86_BUILTIN_STOREDQUHI512_MASK, UNKNOWN, (int) VOID_FTYPE_PV32HI_V32HI_SI },
31104 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_storedquv64qi_mask, "__builtin_ia32_storedquqi512_mask", IX86_BUILTIN_STOREDQUQI512_MASK, UNKNOWN, (int) VOID_FTYPE_PV64QI_V64QI_DI },
31107 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv16hi_mask, "__builtin_ia32_loaddquhi256_mask", IX86_BUILTIN_LOADDQUHI256_MASK, UNKNOWN, (int) V16HI_FTYPE_PCV16HI_V16HI_HI },
31108 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv8hi_mask, "__builtin_ia32_loaddquhi128_mask", IX86_BUILTIN_LOADDQUHI128_MASK, UNKNOWN, (int) V8HI_FTYPE_PCV8HI_V8HI_QI },
31109 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_loaddquv32qi_mask, "__builtin_ia32_loaddquqi256_mask", IX86_BUILTIN_LOADDQUQI256_MASK, UNKNOWN, (int) V32QI_FTYPE_PCV32QI_V32QI_SI },
31110 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_loaddquv16qi_mask, "__builtin_ia32_loaddquqi128_mask", IX86_BUILTIN_LOADDQUQI128_MASK, UNKNOWN, (int) V16QI_FTYPE_PCV16QI_V16QI_HI },
31111 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4di_mask, "__builtin_ia32_movdqa64load256_mask", IX86_BUILTIN_MOVDQA64LOAD256_MASK, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI_QI },
31112 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv2di_mask, "__builtin_ia32_movdqa64load128_mask", IX86_BUILTIN_MOVDQA64LOAD128_MASK, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI_QI },
31113 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv8si_mask, "__builtin_ia32_movdqa32load256_mask", IX86_BUILTIN_MOVDQA32LOAD256_MASK, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI_QI },
31114 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4si_mask, "__builtin_ia32_movdqa32load128_mask", IX86_BUILTIN_MOVDQA32LOAD128_MASK, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI_QI },
31115 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev4di_mask, "__builtin_ia32_movdqa64store256_mask", IX86_BUILTIN_MOVDQA64STORE256_MASK, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_QI },
31116 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev2di_mask, "__builtin_ia32_movdqa64store128_mask", IX86_BUILTIN_MOVDQA64STORE128_MASK, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_QI },
31117 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev8si_mask, "__builtin_ia32_movdqa32store256_mask", IX86_BUILTIN_MOVDQA32STORE256_MASK, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_QI },
31118 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev4si_mask, "__builtin_ia32_movdqa32store128_mask", IX86_BUILTIN_MOVDQA32STORE128_MASK, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_QI },
31119 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4df_mask, "__builtin_ia32_loadapd256_mask", IX86_BUILTIN_LOADAPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DF_QI },
31120 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv2df_mask, "__builtin_ia32_loadapd128_mask", IX86_BUILTIN_LOADAPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DF_QI },
31121 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv8sf_mask, "__builtin_ia32_loadaps256_mask", IX86_BUILTIN_LOADAPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SF_QI },
31122 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4sf_mask, "__builtin_ia32_loadaps128_mask", IX86_BUILTIN_LOADAPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SF_QI },
31123 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev4df_mask, "__builtin_ia32_storeapd256_mask", IX86_BUILTIN_STOREAPD256_MASK, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DF_QI },
31124 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev2df_mask, "__builtin_ia32_storeapd128_mask", IX86_BUILTIN_STOREAPD128_MASK, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DF_QI },
31125 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev8sf_mask, "__builtin_ia32_storeaps256_mask", IX86_BUILTIN_STOREAPS256_MASK, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SF_QI },
31126 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev4sf_mask, "__builtin_ia32_storeaps128_mask", IX86_BUILTIN_STOREAPS128_MASK, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SF_QI },
31127 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_loadupd256_mask, "__builtin_ia32_loadupd256_mask", IX86_BUILTIN_LOADUPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DF_QI },
31128 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_loadupd_mask, "__builtin_ia32_loadupd128_mask", IX86_BUILTIN_LOADUPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DF_QI },
31129 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_loadups256_mask, "__builtin_ia32_loadups256_mask", IX86_BUILTIN_LOADUPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SF_QI },
31130 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse_loadups_mask, "__builtin_ia32_loadups128_mask", IX86_BUILTIN_LOADUPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SF_QI },
31131 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storeupd256_mask, "__builtin_ia32_storeupd256_mask", IX86_BUILTIN_STOREUPD256_MASK, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DF_QI },
31132 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storeupd_mask, "__builtin_ia32_storeupd128_mask", IX86_BUILTIN_STOREUPD128_MASK, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DF_QI },
31133 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storeups256_mask, "__builtin_ia32_storeups256_mask", IX86_BUILTIN_STOREUPS256_MASK, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SF_QI },
31134 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storeups_mask, "__builtin_ia32_storeups128_mask", IX86_BUILTIN_STOREUPS128_MASK, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SF_QI },
31135 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv4di_mask, "__builtin_ia32_loaddqudi256_mask", IX86_BUILTIN_LOADDQUDI256_MASK, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI_QI },
31136 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv2di_mask, "__builtin_ia32_loaddqudi128_mask", IX86_BUILTIN_LOADDQUDI128_MASK, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI_QI },
31137 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_loaddquv8si_mask, "__builtin_ia32_loaddqusi256_mask", IX86_BUILTIN_LOADDQUSI256_MASK, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI_QI },
31138 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_loaddquv4si_mask, "__builtin_ia32_loaddqusi128_mask", IX86_BUILTIN_LOADDQUSI128_MASK, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI_QI },
31139 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv4di_mask, "__builtin_ia32_storedqudi256_mask", IX86_BUILTIN_STOREDQUDI256_MASK, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_QI },
31140 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv2di_mask, "__builtin_ia32_storedqudi128_mask", IX86_BUILTIN_STOREDQUDI128_MASK, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_QI },
31141 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv8si_mask, "__builtin_ia32_storedqusi256_mask", IX86_BUILTIN_STOREDQUSI256_MASK, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_QI },
31142 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv4si_mask, "__builtin_ia32_storedqusi128_mask", IX86_BUILTIN_STOREDQUSI128_MASK, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_QI },
31143 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv16hi_mask, "__builtin_ia32_storedquhi256_mask", IX86_BUILTIN_STOREDQUHI256_MASK, UNKNOWN, (int) VOID_FTYPE_PV16HI_V16HI_HI },
31144 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv8hi_mask, "__builtin_ia32_storedquhi128_mask", IX86_BUILTIN_STOREDQUHI128_MASK, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8HI_QI },
31145 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv32qi_mask, "__builtin_ia32_storedquqi256_mask", IX86_BUILTIN_STOREDQUQI256_MASK, UNKNOWN, (int) VOID_FTYPE_PV32QI_V32QI_SI },
31146 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv16qi_mask, "__builtin_ia32_storedquqi128_mask", IX86_BUILTIN_STOREDQUQI128_MASK, UNKNOWN, (int) VOID_FTYPE_PV16QI_V16QI_HI },
31147 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev4df_mask, "__builtin_ia32_compressstoredf256_mask", IX86_BUILTIN_COMPRESSPDSTORE256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DF_QI },
31148 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev2df_mask, "__builtin_ia32_compressstoredf128_mask", IX86_BUILTIN_COMPRESSPDSTORE128, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DF_QI },
31149 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev8sf_mask, "__builtin_ia32_compressstoresf256_mask", IX86_BUILTIN_COMPRESSPSSTORE256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SF_QI },
31150 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev4sf_mask, "__builtin_ia32_compressstoresf128_mask", IX86_BUILTIN_COMPRESSPSSTORE128, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SF_QI },
31151 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev4di_mask, "__builtin_ia32_compressstoredi256_mask", IX86_BUILTIN_PCOMPRESSQSTORE256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_QI },
31152 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev2di_mask, "__builtin_ia32_compressstoredi128_mask", IX86_BUILTIN_PCOMPRESSQSTORE128, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_QI },
31153 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev8si_mask, "__builtin_ia32_compressstoresi256_mask", IX86_BUILTIN_PCOMPRESSDSTORE256, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_QI },
31154 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev4si_mask, "__builtin_ia32_compressstoresi128_mask", IX86_BUILTIN_PCOMPRESSDSTORE128, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_QI },
31155 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4df_mask, "__builtin_ia32_expandloaddf256_mask", IX86_BUILTIN_EXPANDPDLOAD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DF_QI },
31156 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2df_mask, "__builtin_ia32_expandloaddf128_mask", IX86_BUILTIN_EXPANDPDLOAD128, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DF_QI },
31157 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8sf_mask, "__builtin_ia32_expandloadsf256_mask", IX86_BUILTIN_EXPANDPSLOAD256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SF_QI },
31158 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4sf_mask, "__builtin_ia32_expandloadsf128_mask", IX86_BUILTIN_EXPANDPSLOAD128, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SF_QI },
31159 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4di_mask, "__builtin_ia32_expandloaddi256_mask", IX86_BUILTIN_PEXPANDQLOAD256, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI_QI },
31160 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2di_mask, "__builtin_ia32_expandloaddi128_mask", IX86_BUILTIN_PEXPANDQLOAD128, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI_QI },
31161 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8si_mask, "__builtin_ia32_expandloadsi256_mask", IX86_BUILTIN_PEXPANDDLOAD256, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI_QI },
31162 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4si_mask, "__builtin_ia32_expandloadsi128_mask", IX86_BUILTIN_PEXPANDDLOAD128, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI_QI },
31163 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4df_maskz, "__builtin_ia32_expandloaddf256_maskz", IX86_BUILTIN_EXPANDPDLOAD256Z, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DF_QI },
31164 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2df_maskz, "__builtin_ia32_expandloaddf128_maskz", IX86_BUILTIN_EXPANDPDLOAD128Z, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DF_QI },
31165 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8sf_maskz, "__builtin_ia32_expandloadsf256_maskz", IX86_BUILTIN_EXPANDPSLOAD256Z, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SF_QI },
31166 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4sf_maskz, "__builtin_ia32_expandloadsf128_maskz", IX86_BUILTIN_EXPANDPSLOAD128Z, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SF_QI },
31167 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4di_maskz, "__builtin_ia32_expandloaddi256_maskz", IX86_BUILTIN_PEXPANDQLOAD256Z, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI_QI },
31168 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2di_maskz, "__builtin_ia32_expandloaddi128_maskz", IX86_BUILTIN_PEXPANDQLOAD128Z, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI_QI },
31169 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8si_maskz, "__builtin_ia32_expandloadsi256_maskz", IX86_BUILTIN_PEXPANDDLOAD256Z, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI_QI },
31170 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4si_maskz, "__builtin_ia32_expandloadsi128_maskz", IX86_BUILTIN_PEXPANDDLOAD128Z, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI_QI },
31171 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4si2_mask_store, "__builtin_ia32_pmovqd256mem_mask", IX86_BUILTIN_PMOVQD256_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4DI_QI },
31172 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2si2_mask_store, "__builtin_ia32_pmovqd128mem_mask", IX86_BUILTIN_PMOVQD128_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V2DI_QI },
31173 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4si2_mask_store, "__builtin_ia32_pmovsqd256mem_mask", IX86_BUILTIN_PMOVSQD256_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4DI_QI },
31174 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2si2_mask_store, "__builtin_ia32_pmovsqd128mem_mask", IX86_BUILTIN_PMOVSQD128_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V2DI_QI },
31175 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4si2_mask_store, "__builtin_ia32_pmovusqd256mem_mask", IX86_BUILTIN_PMOVUSQD256_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4DI_QI },
31176 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2si2_mask_store, "__builtin_ia32_pmovusqd128mem_mask", IX86_BUILTIN_PMOVUSQD128_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V2DI_QI },
31177 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4hi2_mask_store, "__builtin_ia32_pmovqw256mem_mask", IX86_BUILTIN_PMOVQW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4DI_QI },
31178 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2hi2_mask_store, "__builtin_ia32_pmovqw128mem_mask", IX86_BUILTIN_PMOVQW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V2DI_QI },
31179 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4hi2_mask_store, "__builtin_ia32_pmovsqw256mem_mask", IX86_BUILTIN_PMOVSQW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4DI_QI },
31180 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2hi2_mask_store, "__builtin_ia32_pmovsqw128mem_mask", IX86_BUILTIN_PMOVSQW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V2DI_QI },
31181 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4hi2_mask_store, "__builtin_ia32_pmovusqw256mem_mask", IX86_BUILTIN_PMOVUSQW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4DI_QI },
31182 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2hi2_mask_store, "__builtin_ia32_pmovusqw128mem_mask", IX86_BUILTIN_PMOVUSQW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V2DI_QI },
31183 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4qi2_mask_store, "__builtin_ia32_pmovqb256mem_mask", IX86_BUILTIN_PMOVQB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4DI_QI },
31184 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2qi2_mask_store, "__builtin_ia32_pmovqb128mem_mask", IX86_BUILTIN_PMOVQB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V2DI_QI },
31185 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4qi2_mask_store, "__builtin_ia32_pmovsqb256mem_mask", IX86_BUILTIN_PMOVSQB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4DI_QI },
31186 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2qi2_mask_store, "__builtin_ia32_pmovsqb128mem_mask", IX86_BUILTIN_PMOVSQB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V2DI_QI },
31187 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4qi2_mask_store, "__builtin_ia32_pmovusqb256mem_mask", IX86_BUILTIN_PMOVUSQB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4DI_QI },
31188 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2qi2_mask_store, "__builtin_ia32_pmovusqb128mem_mask", IX86_BUILTIN_PMOVUSQB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V2DI_QI },
31189 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev8siv8qi2_mask_store, "__builtin_ia32_pmovdb256mem_mask", IX86_BUILTIN_PMOVDB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8SI_QI },
31190 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4siv4qi2_mask_store, "__builtin_ia32_pmovdb128mem_mask", IX86_BUILTIN_PMOVDB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4SI_QI },
31191 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev8siv8qi2_mask_store, "__builtin_ia32_pmovsdb256mem_mask", IX86_BUILTIN_PMOVSDB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8SI_QI },
31192 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4siv4qi2_mask_store, "__builtin_ia32_pmovsdb128mem_mask", IX86_BUILTIN_PMOVSDB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4SI_QI },
31193 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev8siv8qi2_mask_store, "__builtin_ia32_pmovusdb256mem_mask", IX86_BUILTIN_PMOVUSDB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8SI_QI },
31194 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4siv4qi2_mask_store, "__builtin_ia32_pmovusdb128mem_mask", IX86_BUILTIN_PMOVUSDB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4SI_QI },
31195 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev8siv8hi2_mask_store, "__builtin_ia32_pmovdw256mem_mask", IX86_BUILTIN_PMOVDW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8SI_QI },
31196 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4siv4hi2_mask_store, "__builtin_ia32_pmovdw128mem_mask", IX86_BUILTIN_PMOVDW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4SI_QI },
31197 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev8siv8hi2_mask_store, "__builtin_ia32_pmovsdw256mem_mask", IX86_BUILTIN_PMOVSDW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8SI_QI },
31198 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4siv4hi2_mask_store, "__builtin_ia32_pmovsdw128mem_mask", IX86_BUILTIN_PMOVSDW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4SI_QI },
31199 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev8siv8hi2_mask_store, "__builtin_ia32_pmovusdw256mem_mask", IX86_BUILTIN_PMOVUSDW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8SI_QI },
31200 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4siv4hi2_mask_store, "__builtin_ia32_pmovusdw128mem_mask", IX86_BUILTIN_PMOVUSDW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4SI_QI },
31203 { OPTION_MASK_ISA_PCOMMIT, CODE_FOR_pcommit, "__builtin_ia32_pcommit", IX86_BUILTIN_PCOMMIT, UNKNOWN, (int) VOID_FTYPE_VOID },
31206 /* Builtins with variable number of arguments. */
31207 static const struct builtin_description bdesc_args[] =
31209 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_bsr, "__builtin_ia32_bsrsi", IX86_BUILTIN_BSRSI, UNKNOWN, (int) INT_FTYPE_INT },
31210 { OPTION_MASK_ISA_64BIT, CODE_FOR_bsr_rex64, "__builtin_ia32_bsrdi", IX86_BUILTIN_BSRDI, UNKNOWN, (int) INT64_FTYPE_INT64 },
31211 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdpmc", IX86_BUILTIN_RDPMC, UNKNOWN, (int) UINT64_FTYPE_INT },
31212 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlqi3, "__builtin_ia32_rolqi", IX86_BUILTIN_ROLQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
31213 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlhi3, "__builtin_ia32_rolhi", IX86_BUILTIN_ROLHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
31214 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrqi3, "__builtin_ia32_rorqi", IX86_BUILTIN_RORQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
31215 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrhi3, "__builtin_ia32_rorhi", IX86_BUILTIN_RORHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
31218 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv8qi3, "__builtin_ia32_paddb", IX86_BUILTIN_PADDB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31219 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv4hi3, "__builtin_ia32_paddw", IX86_BUILTIN_PADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31220 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv2si3, "__builtin_ia32_paddd", IX86_BUILTIN_PADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31221 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv8qi3, "__builtin_ia32_psubb", IX86_BUILTIN_PSUBB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31222 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv4hi3, "__builtin_ia32_psubw", IX86_BUILTIN_PSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31223 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv2si3, "__builtin_ia32_psubd", IX86_BUILTIN_PSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31225 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv8qi3, "__builtin_ia32_paddsb", IX86_BUILTIN_PADDSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31226 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv4hi3, "__builtin_ia32_paddsw", IX86_BUILTIN_PADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31227 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv8qi3, "__builtin_ia32_psubsb", IX86_BUILTIN_PSUBSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31228 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv4hi3, "__builtin_ia32_psubsw", IX86_BUILTIN_PSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31229 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv8qi3, "__builtin_ia32_paddusb", IX86_BUILTIN_PADDUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31230 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv4hi3, "__builtin_ia32_paddusw", IX86_BUILTIN_PADDUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31231 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv8qi3, "__builtin_ia32_psubusb", IX86_BUILTIN_PSUBUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31232 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv4hi3, "__builtin_ia32_psubusw", IX86_BUILTIN_PSUBUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31234 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_mulv4hi3, "__builtin_ia32_pmullw", IX86_BUILTIN_PMULLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31235 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_smulv4hi3_highpart, "__builtin_ia32_pmulhw", IX86_BUILTIN_PMULHW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31237 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andv2si3, "__builtin_ia32_pand", IX86_BUILTIN_PAND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31238 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andnotv2si3, "__builtin_ia32_pandn", IX86_BUILTIN_PANDN, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31239 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_iorv2si3, "__builtin_ia32_por", IX86_BUILTIN_POR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31240 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_xorv2si3, "__builtin_ia32_pxor", IX86_BUILTIN_PXOR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31242 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv8qi3, "__builtin_ia32_pcmpeqb", IX86_BUILTIN_PCMPEQB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31243 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv4hi3, "__builtin_ia32_pcmpeqw", IX86_BUILTIN_PCMPEQW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31244 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv2si3, "__builtin_ia32_pcmpeqd", IX86_BUILTIN_PCMPEQD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31245 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv8qi3, "__builtin_ia32_pcmpgtb", IX86_BUILTIN_PCMPGTB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31246 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv4hi3, "__builtin_ia32_pcmpgtw", IX86_BUILTIN_PCMPGTW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31247 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv2si3, "__builtin_ia32_pcmpgtd", IX86_BUILTIN_PCMPGTD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31249 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhbw, "__builtin_ia32_punpckhbw", IX86_BUILTIN_PUNPCKHBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31250 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhwd, "__builtin_ia32_punpckhwd", IX86_BUILTIN_PUNPCKHWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31251 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhdq, "__builtin_ia32_punpckhdq", IX86_BUILTIN_PUNPCKHDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31252 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklbw, "__builtin_ia32_punpcklbw", IX86_BUILTIN_PUNPCKLBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31253 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklwd, "__builtin_ia32_punpcklwd", IX86_BUILTIN_PUNPCKLWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI},
31254 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckldq, "__builtin_ia32_punpckldq", IX86_BUILTIN_PUNPCKLDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI},
31256 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packsswb, "__builtin_ia32_packsswb", IX86_BUILTIN_PACKSSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
31257 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packssdw, "__builtin_ia32_packssdw", IX86_BUILTIN_PACKSSDW, UNKNOWN, (int) V4HI_FTYPE_V2SI_V2SI },
31258 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packuswb, "__builtin_ia32_packuswb", IX86_BUILTIN_PACKUSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
31260 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_pmaddwd, "__builtin_ia32_pmaddwd", IX86_BUILTIN_PMADDWD, UNKNOWN, (int) V2SI_FTYPE_V4HI_V4HI },
31262 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllwi", IX86_BUILTIN_PSLLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
31263 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslldi", IX86_BUILTIN_PSLLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
31264 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllqi", IX86_BUILTIN_PSLLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
31265 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllw", IX86_BUILTIN_PSLLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
31266 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslld", IX86_BUILTIN_PSLLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
31267 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllq", IX86_BUILTIN_PSLLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
31269 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlwi", IX86_BUILTIN_PSRLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
31270 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrldi", IX86_BUILTIN_PSRLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
31271 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlqi", IX86_BUILTIN_PSRLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
31272 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlw", IX86_BUILTIN_PSRLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
31273 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrld", IX86_BUILTIN_PSRLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
31274 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlq", IX86_BUILTIN_PSRLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
31276 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psrawi", IX86_BUILTIN_PSRAWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
31277 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psradi", IX86_BUILTIN_PSRADI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
31278 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psraw", IX86_BUILTIN_PSRAW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
31279 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psrad", IX86_BUILTIN_PSRAD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
31282 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pf2id, "__builtin_ia32_pf2id", IX86_BUILTIN_PF2ID, UNKNOWN, (int) V2SI_FTYPE_V2SF },
31283 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_floatv2si2, "__builtin_ia32_pi2fd", IX86_BUILTIN_PI2FD, UNKNOWN, (int) V2SF_FTYPE_V2SI },
31284 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpv2sf2, "__builtin_ia32_pfrcp", IX86_BUILTIN_PFRCP, UNKNOWN, (int) V2SF_FTYPE_V2SF },
31285 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqrtv2sf2, "__builtin_ia32_pfrsqrt", IX86_BUILTIN_PFRSQRT, UNKNOWN, (int) V2SF_FTYPE_V2SF },
31287 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgusb", IX86_BUILTIN_PAVGUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31288 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_haddv2sf3, "__builtin_ia32_pfacc", IX86_BUILTIN_PFACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31289 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_addv2sf3, "__builtin_ia32_pfadd", IX86_BUILTIN_PFADD, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31290 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_eqv2sf3, "__builtin_ia32_pfcmpeq", IX86_BUILTIN_PFCMPEQ, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
31291 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gev2sf3, "__builtin_ia32_pfcmpge", IX86_BUILTIN_PFCMPGE, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
31292 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gtv2sf3, "__builtin_ia32_pfcmpgt", IX86_BUILTIN_PFCMPGT, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
31293 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_smaxv2sf3, "__builtin_ia32_pfmax", IX86_BUILTIN_PFMAX, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31294 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_sminv2sf3, "__builtin_ia32_pfmin", IX86_BUILTIN_PFMIN, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31295 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_mulv2sf3, "__builtin_ia32_pfmul", IX86_BUILTIN_PFMUL, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31296 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit1v2sf3, "__builtin_ia32_pfrcpit1", IX86_BUILTIN_PFRCPIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31297 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit2v2sf3, "__builtin_ia32_pfrcpit2", IX86_BUILTIN_PFRCPIT2, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31298 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqit1v2sf3, "__builtin_ia32_pfrsqit1", IX86_BUILTIN_PFRSQIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31299 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subv2sf3, "__builtin_ia32_pfsub", IX86_BUILTIN_PFSUB, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31300 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subrv2sf3, "__builtin_ia32_pfsubr", IX86_BUILTIN_PFSUBR, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31301 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pmulhrwv4hi3, "__builtin_ia32_pmulhrw", IX86_BUILTIN_PMULHRW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31304 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pf2iw, "__builtin_ia32_pf2iw", IX86_BUILTIN_PF2IW, UNKNOWN, (int) V2SI_FTYPE_V2SF },
31305 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pi2fw, "__builtin_ia32_pi2fw", IX86_BUILTIN_PI2FW, UNKNOWN, (int) V2SF_FTYPE_V2SI },
31306 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2si2, "__builtin_ia32_pswapdsi", IX86_BUILTIN_PSWAPDSI, UNKNOWN, (int) V2SI_FTYPE_V2SI },
31307 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2sf2, "__builtin_ia32_pswapdsf", IX86_BUILTIN_PSWAPDSF, UNKNOWN, (int) V2SF_FTYPE_V2SF },
31308 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_hsubv2sf3, "__builtin_ia32_pfnacc", IX86_BUILTIN_PFNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31309 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_addsubv2sf3, "__builtin_ia32_pfpnacc", IX86_BUILTIN_PFPNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31312 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movmskps, "__builtin_ia32_movmskps", IX86_BUILTIN_MOVMSKPS, UNKNOWN, (int) INT_FTYPE_V4SF },
31313 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_sqrtv4sf2, "__builtin_ia32_sqrtps", IX86_BUILTIN_SQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31314 { OPTION_MASK_ISA_SSE, CODE_FOR_sqrtv4sf2, "__builtin_ia32_sqrtps_nr", IX86_BUILTIN_SQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31315 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rsqrtv4sf2, "__builtin_ia32_rsqrtps", IX86_BUILTIN_RSQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31316 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtv4sf2, "__builtin_ia32_rsqrtps_nr", IX86_BUILTIN_RSQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31317 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rcpv4sf2, "__builtin_ia32_rcpps", IX86_BUILTIN_RCPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31318 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtps2pi, "__builtin_ia32_cvtps2pi", IX86_BUILTIN_CVTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
31319 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtss2si, "__builtin_ia32_cvtss2si", IX86_BUILTIN_CVTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
31320 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq, "__builtin_ia32_cvtss2si64", IX86_BUILTIN_CVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
31321 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttps2pi, "__builtin_ia32_cvttps2pi", IX86_BUILTIN_CVTTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
31322 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttss2si, "__builtin_ia32_cvttss2si", IX86_BUILTIN_CVTTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
31323 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq, "__builtin_ia32_cvttss2si64", IX86_BUILTIN_CVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
31325 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_shufps, "__builtin_ia32_shufps", IX86_BUILTIN_SHUFPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31327 { OPTION_MASK_ISA_SSE, CODE_FOR_addv4sf3, "__builtin_ia32_addps", IX86_BUILTIN_ADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31328 { OPTION_MASK_ISA_SSE, CODE_FOR_subv4sf3, "__builtin_ia32_subps", IX86_BUILTIN_SUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31329 { OPTION_MASK_ISA_SSE, CODE_FOR_mulv4sf3, "__builtin_ia32_mulps", IX86_BUILTIN_MULPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31330 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_divv4sf3, "__builtin_ia32_divps", IX86_BUILTIN_DIVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31331 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmaddv4sf3, "__builtin_ia32_addss", IX86_BUILTIN_ADDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31332 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsubv4sf3, "__builtin_ia32_subss", IX86_BUILTIN_SUBSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31333 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmulv4sf3, "__builtin_ia32_mulss", IX86_BUILTIN_MULSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31334 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmdivv4sf3, "__builtin_ia32_divss", IX86_BUILTIN_DIVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31336 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpeqps", IX86_BUILTIN_CMPEQPS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
31337 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpltps", IX86_BUILTIN_CMPLTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
31338 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpleps", IX86_BUILTIN_CMPLEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
31339 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgtps", IX86_BUILTIN_CMPGTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
31340 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgeps", IX86_BUILTIN_CMPGEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
31341 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpunordps", IX86_BUILTIN_CMPUNORDPS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
31342 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpneqps", IX86_BUILTIN_CMPNEQPS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
31343 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnltps", IX86_BUILTIN_CMPNLTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
31344 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnleps", IX86_BUILTIN_CMPNLEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
31345 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngtps", IX86_BUILTIN_CMPNGTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
31346 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngeps", IX86_BUILTIN_CMPNGEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP},
31347 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpordps", IX86_BUILTIN_CMPORDPS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
31348 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpeqss", IX86_BUILTIN_CMPEQSS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
31349 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpltss", IX86_BUILTIN_CMPLTSS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
31350 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpless", IX86_BUILTIN_CMPLESS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
31351 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpunordss", IX86_BUILTIN_CMPUNORDSS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
31352 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpneqss", IX86_BUILTIN_CMPNEQSS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
31353 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnltss", IX86_BUILTIN_CMPNLTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
31354 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnless", IX86_BUILTIN_CMPNLESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
31355 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpordss", IX86_BUILTIN_CMPORDSS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
31357 { OPTION_MASK_ISA_SSE, CODE_FOR_sminv4sf3, "__builtin_ia32_minps", IX86_BUILTIN_MINPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31358 { OPTION_MASK_ISA_SSE, CODE_FOR_smaxv4sf3, "__builtin_ia32_maxps", IX86_BUILTIN_MAXPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31359 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsminv4sf3, "__builtin_ia32_minss", IX86_BUILTIN_MINSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31360 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsmaxv4sf3, "__builtin_ia32_maxss", IX86_BUILTIN_MAXSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31362 { OPTION_MASK_ISA_SSE, CODE_FOR_andv4sf3, "__builtin_ia32_andps", IX86_BUILTIN_ANDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31363 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_andnotv4sf3, "__builtin_ia32_andnps", IX86_BUILTIN_ANDNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31364 { OPTION_MASK_ISA_SSE, CODE_FOR_iorv4sf3, "__builtin_ia32_orps", IX86_BUILTIN_ORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31365 { OPTION_MASK_ISA_SSE, CODE_FOR_xorv4sf3, "__builtin_ia32_xorps", IX86_BUILTIN_XORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31367 { OPTION_MASK_ISA_SSE, CODE_FOR_copysignv4sf3, "__builtin_ia32_copysignps", IX86_BUILTIN_CPYSGNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31369 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movss, "__builtin_ia32_movss", IX86_BUILTIN_MOVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31370 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movhlps_exp, "__builtin_ia32_movhlps", IX86_BUILTIN_MOVHLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31371 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movlhps_exp, "__builtin_ia32_movlhps", IX86_BUILTIN_MOVLHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31372 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_highv4sf, "__builtin_ia32_unpckhps", IX86_BUILTIN_UNPCKHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31373 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_lowv4sf, "__builtin_ia32_unpcklps", IX86_BUILTIN_UNPCKLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31375 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtpi2ps, "__builtin_ia32_cvtpi2ps", IX86_BUILTIN_CVTPI2PS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2SI },
31376 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtsi2ss, "__builtin_ia32_cvtsi2ss", IX86_BUILTIN_CVTSI2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_SI },
31377 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq, "__builtin_ia32_cvtsi642ss", IX86_BUILTIN_CVTSI642SS, UNKNOWN, V4SF_FTYPE_V4SF_DI },
31379 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtsf2, "__builtin_ia32_rsqrtf", IX86_BUILTIN_RSQRTF, UNKNOWN, (int) FLOAT_FTYPE_FLOAT },
31381 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsqrtv4sf2, "__builtin_ia32_sqrtss", IX86_BUILTIN_SQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
31382 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrsqrtv4sf2, "__builtin_ia32_rsqrtss", IX86_BUILTIN_RSQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
31383 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrcpv4sf2, "__builtin_ia32_rcpss", IX86_BUILTIN_RCPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
31385 { OPTION_MASK_ISA_SSE, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
31386 { OPTION_MASK_ISA_SSE, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
31388 /* SSE MMX or 3Dnow!A */
31389 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgb", IX86_BUILTIN_PAVGB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31390 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv4hi3, "__builtin_ia32_pavgw", IX86_BUILTIN_PAVGW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31391 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umulv4hi3_highpart, "__builtin_ia32_pmulhuw", IX86_BUILTIN_PMULHUW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31393 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umaxv8qi3, "__builtin_ia32_pmaxub", IX86_BUILTIN_PMAXUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31394 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_smaxv4hi3, "__builtin_ia32_pmaxsw", IX86_BUILTIN_PMAXSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31395 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uminv8qi3, "__builtin_ia32_pminub", IX86_BUILTIN_PMINUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31396 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_sminv4hi3, "__builtin_ia32_pminsw", IX86_BUILTIN_PMINSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31398 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_psadbw, "__builtin_ia32_psadbw", IX86_BUILTIN_PSADBW, UNKNOWN, (int) V1DI_FTYPE_V8QI_V8QI },
31399 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pmovmskb, "__builtin_ia32_pmovmskb", IX86_BUILTIN_PMOVMSKB, UNKNOWN, (int) INT_FTYPE_V8QI },
31401 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pshufw, "__builtin_ia32_pshufw", IX86_BUILTIN_PSHUFW, UNKNOWN, (int) V4HI_FTYPE_V4HI_INT },
31404 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_shufpd, "__builtin_ia32_shufpd", IX86_BUILTIN_SHUFPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31406 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movmskpd, "__builtin_ia32_movmskpd", IX86_BUILTIN_MOVMSKPD, UNKNOWN, (int) INT_FTYPE_V2DF },
31407 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmovmskb, "__builtin_ia32_pmovmskb128", IX86_BUILTIN_PMOVMSKB128, UNKNOWN, (int) INT_FTYPE_V16QI },
31408 { OPTION_MASK_ISA_SSE2, CODE_FOR_sqrtv2df2, "__builtin_ia32_sqrtpd", IX86_BUILTIN_SQRTPD, UNKNOWN, (int) V2DF_FTYPE_V2DF },
31409 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2pd, "__builtin_ia32_cvtdq2pd", IX86_BUILTIN_CVTDQ2PD, UNKNOWN, (int) V2DF_FTYPE_V4SI },
31410 { OPTION_MASK_ISA_SSE2, CODE_FOR_floatv4siv4sf2, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
31412 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2dq, "__builtin_ia32_cvtpd2dq", IX86_BUILTIN_CVTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
31413 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2pi, "__builtin_ia32_cvtpd2pi", IX86_BUILTIN_CVTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
31414 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2ps, "__builtin_ia32_cvtpd2ps", IX86_BUILTIN_CVTPD2PS, UNKNOWN, (int) V4SF_FTYPE_V2DF },
31415 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2dq, "__builtin_ia32_cvttpd2dq", IX86_BUILTIN_CVTTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
31416 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2pi, "__builtin_ia32_cvttpd2pi", IX86_BUILTIN_CVTTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
31418 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpi2pd, "__builtin_ia32_cvtpi2pd", IX86_BUILTIN_CVTPI2PD, UNKNOWN, (int) V2DF_FTYPE_V2SI },
31420 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2si, "__builtin_ia32_cvtsd2si", IX86_BUILTIN_CVTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
31421 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttsd2si, "__builtin_ia32_cvttsd2si", IX86_BUILTIN_CVTTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
31422 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq, "__builtin_ia32_cvtsd2si64", IX86_BUILTIN_CVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
31423 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq, "__builtin_ia32_cvttsd2si64", IX86_BUILTIN_CVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
31425 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_fix_notruncv4sfv4si, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
31426 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
31427 { OPTION_MASK_ISA_SSE2, CODE_FOR_fix_truncv4sfv4si2, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
31429 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2df3, "__builtin_ia32_addpd", IX86_BUILTIN_ADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31430 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2df3, "__builtin_ia32_subpd", IX86_BUILTIN_SUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31431 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv2df3, "__builtin_ia32_mulpd", IX86_BUILTIN_MULPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31432 { OPTION_MASK_ISA_SSE2, CODE_FOR_divv2df3, "__builtin_ia32_divpd", IX86_BUILTIN_DIVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31433 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmaddv2df3, "__builtin_ia32_addsd", IX86_BUILTIN_ADDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31434 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsubv2df3, "__builtin_ia32_subsd", IX86_BUILTIN_SUBSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31435 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmulv2df3, "__builtin_ia32_mulsd", IX86_BUILTIN_MULSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31436 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmdivv2df3, "__builtin_ia32_divsd", IX86_BUILTIN_DIVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31438 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpeqpd", IX86_BUILTIN_CMPEQPD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
31439 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpltpd", IX86_BUILTIN_CMPLTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
31440 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmplepd", IX86_BUILTIN_CMPLEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
31441 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgtpd", IX86_BUILTIN_CMPGTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
31442 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgepd", IX86_BUILTIN_CMPGEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP},
31443 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpunordpd", IX86_BUILTIN_CMPUNORDPD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
31444 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpneqpd", IX86_BUILTIN_CMPNEQPD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
31445 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnltpd", IX86_BUILTIN_CMPNLTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
31446 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnlepd", IX86_BUILTIN_CMPNLEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
31447 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngtpd", IX86_BUILTIN_CMPNGTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
31448 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngepd", IX86_BUILTIN_CMPNGEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
31449 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpordpd", IX86_BUILTIN_CMPORDPD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
31450 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpeqsd", IX86_BUILTIN_CMPEQSD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
31451 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpltsd", IX86_BUILTIN_CMPLTSD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
31452 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmplesd", IX86_BUILTIN_CMPLESD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
31453 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpunordsd", IX86_BUILTIN_CMPUNORDSD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
31454 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpneqsd", IX86_BUILTIN_CMPNEQSD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
31455 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnltsd", IX86_BUILTIN_CMPNLTSD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
31456 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnlesd", IX86_BUILTIN_CMPNLESD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
31457 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpordsd", IX86_BUILTIN_CMPORDSD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
31459 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv2df3, "__builtin_ia32_minpd", IX86_BUILTIN_MINPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31460 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv2df3, "__builtin_ia32_maxpd", IX86_BUILTIN_MAXPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31461 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsminv2df3, "__builtin_ia32_minsd", IX86_BUILTIN_MINSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31462 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsmaxv2df3, "__builtin_ia32_maxsd", IX86_BUILTIN_MAXSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31464 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2df3, "__builtin_ia32_andpd", IX86_BUILTIN_ANDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31465 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2df3, "__builtin_ia32_andnpd", IX86_BUILTIN_ANDNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31466 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2df3, "__builtin_ia32_orpd", IX86_BUILTIN_ORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31467 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2df3, "__builtin_ia32_xorpd", IX86_BUILTIN_XORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31469 { OPTION_MASK_ISA_SSE2, CODE_FOR_copysignv2df3, "__builtin_ia32_copysignpd", IX86_BUILTIN_CPYSGNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31471 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movsd, "__builtin_ia32_movsd", IX86_BUILTIN_MOVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31472 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2df, "__builtin_ia32_unpckhpd", IX86_BUILTIN_UNPCKHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31473 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2df, "__builtin_ia32_unpcklpd", IX86_BUILTIN_UNPCKLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31475 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_pack_sfix_v2df, "__builtin_ia32_vec_pack_sfix", IX86_BUILTIN_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
31477 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv16qi3, "__builtin_ia32_paddb128", IX86_BUILTIN_PADDB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31478 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv8hi3, "__builtin_ia32_paddw128", IX86_BUILTIN_PADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31479 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv4si3, "__builtin_ia32_paddd128", IX86_BUILTIN_PADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31480 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2di3, "__builtin_ia32_paddq128", IX86_BUILTIN_PADDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31481 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv16qi3, "__builtin_ia32_psubb128", IX86_BUILTIN_PSUBB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31482 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv8hi3, "__builtin_ia32_psubw128", IX86_BUILTIN_PSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31483 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv4si3, "__builtin_ia32_psubd128", IX86_BUILTIN_PSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31484 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2di3, "__builtin_ia32_psubq128", IX86_BUILTIN_PSUBQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31486 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv16qi3, "__builtin_ia32_paddsb128", IX86_BUILTIN_PADDSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31487 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv8hi3, "__builtin_ia32_paddsw128", IX86_BUILTIN_PADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31488 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv16qi3, "__builtin_ia32_psubsb128", IX86_BUILTIN_PSUBSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31489 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv8hi3, "__builtin_ia32_psubsw128", IX86_BUILTIN_PSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31490 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv16qi3, "__builtin_ia32_paddusb128", IX86_BUILTIN_PADDUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31491 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv8hi3, "__builtin_ia32_paddusw128", IX86_BUILTIN_PADDUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31492 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv16qi3, "__builtin_ia32_psubusb128", IX86_BUILTIN_PSUBUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31493 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv8hi3, "__builtin_ia32_psubusw128", IX86_BUILTIN_PSUBUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31495 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv8hi3, "__builtin_ia32_pmullw128", IX86_BUILTIN_PMULLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31496 { OPTION_MASK_ISA_SSE2, CODE_FOR_smulv8hi3_highpart, "__builtin_ia32_pmulhw128", IX86_BUILTIN_PMULHW128, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI },
31498 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2di3, "__builtin_ia32_pand128", IX86_BUILTIN_PAND128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31499 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2di3, "__builtin_ia32_pandn128", IX86_BUILTIN_PANDN128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31500 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2di3, "__builtin_ia32_por128", IX86_BUILTIN_POR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31501 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2di3, "__builtin_ia32_pxor128", IX86_BUILTIN_PXOR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31503 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv16qi3, "__builtin_ia32_pavgb128", IX86_BUILTIN_PAVGB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31504 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv8hi3, "__builtin_ia32_pavgw128", IX86_BUILTIN_PAVGW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31506 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv16qi3, "__builtin_ia32_pcmpeqb128", IX86_BUILTIN_PCMPEQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31507 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv8hi3, "__builtin_ia32_pcmpeqw128", IX86_BUILTIN_PCMPEQW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31508 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv4si3, "__builtin_ia32_pcmpeqd128", IX86_BUILTIN_PCMPEQD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31509 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv16qi3, "__builtin_ia32_pcmpgtb128", IX86_BUILTIN_PCMPGTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31510 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv8hi3, "__builtin_ia32_pcmpgtw128", IX86_BUILTIN_PCMPGTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31511 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv4si3, "__builtin_ia32_pcmpgtd128", IX86_BUILTIN_PCMPGTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31513 { OPTION_MASK_ISA_SSE2, CODE_FOR_umaxv16qi3, "__builtin_ia32_pmaxub128", IX86_BUILTIN_PMAXUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31514 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv8hi3, "__builtin_ia32_pmaxsw128", IX86_BUILTIN_PMAXSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31515 { OPTION_MASK_ISA_SSE2, CODE_FOR_uminv16qi3, "__builtin_ia32_pminub128", IX86_BUILTIN_PMINUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31516 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv8hi3, "__builtin_ia32_pminsw128", IX86_BUILTIN_PMINSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31518 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv16qi, "__builtin_ia32_punpckhbw128", IX86_BUILTIN_PUNPCKHBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31519 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv8hi, "__builtin_ia32_punpckhwd128", IX86_BUILTIN_PUNPCKHWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31520 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv4si, "__builtin_ia32_punpckhdq128", IX86_BUILTIN_PUNPCKHDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31521 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2di, "__builtin_ia32_punpckhqdq128", IX86_BUILTIN_PUNPCKHQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31522 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv16qi, "__builtin_ia32_punpcklbw128", IX86_BUILTIN_PUNPCKLBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31523 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv8hi, "__builtin_ia32_punpcklwd128", IX86_BUILTIN_PUNPCKLWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31524 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv4si, "__builtin_ia32_punpckldq128", IX86_BUILTIN_PUNPCKLDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31525 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2di, "__builtin_ia32_punpcklqdq128", IX86_BUILTIN_PUNPCKLQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31527 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packsswb, "__builtin_ia32_packsswb128", IX86_BUILTIN_PACKSSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
31528 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packssdw, "__builtin_ia32_packssdw128", IX86_BUILTIN_PACKSSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
31529 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packuswb, "__builtin_ia32_packuswb128", IX86_BUILTIN_PACKUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
31531 { OPTION_MASK_ISA_SSE2, CODE_FOR_umulv8hi3_highpart, "__builtin_ia32_pmulhuw128", IX86_BUILTIN_PMULHUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31532 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_psadbw, "__builtin_ia32_psadbw128", IX86_BUILTIN_PSADBW128, UNKNOWN, (int) V2DI_FTYPE_V16QI_V16QI },
31534 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv1siv1di3, "__builtin_ia32_pmuludq", IX86_BUILTIN_PMULUDQ, UNKNOWN, (int) V1DI_FTYPE_V2SI_V2SI },
31535 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_widen_umult_even_v4si, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
31537 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmaddwd, "__builtin_ia32_pmaddwd128", IX86_BUILTIN_PMADDWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI },
31539 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsi2sd, "__builtin_ia32_cvtsi2sd", IX86_BUILTIN_CVTSI2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_SI },
31540 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq, "__builtin_ia32_cvtsi642sd", IX86_BUILTIN_CVTSI642SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_DI },
31541 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2ss, "__builtin_ia32_cvtsd2ss", IX86_BUILTIN_CVTSD2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF },
31542 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtss2sd, "__builtin_ia32_cvtss2sd", IX86_BUILTIN_CVTSS2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF },
31544 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ashlv1ti3, "__builtin_ia32_pslldqi128", IX86_BUILTIN_PSLLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
31545 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllwi128", IX86_BUILTIN_PSLLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
31546 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslldi128", IX86_BUILTIN_PSLLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
31547 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllqi128", IX86_BUILTIN_PSLLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
31548 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllw128", IX86_BUILTIN_PSLLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
31549 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslld128", IX86_BUILTIN_PSLLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
31550 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllq128", IX86_BUILTIN_PSLLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
31552 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lshrv1ti3, "__builtin_ia32_psrldqi128", IX86_BUILTIN_PSRLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
31553 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlwi128", IX86_BUILTIN_PSRLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
31554 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrldi128", IX86_BUILTIN_PSRLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
31555 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlqi128", IX86_BUILTIN_PSRLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
31556 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlw128", IX86_BUILTIN_PSRLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
31557 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrld128", IX86_BUILTIN_PSRLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
31558 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlq128", IX86_BUILTIN_PSRLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
31560 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psrawi128", IX86_BUILTIN_PSRAWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
31561 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psradi128", IX86_BUILTIN_PSRADI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
31562 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psraw128", IX86_BUILTIN_PSRAW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
31563 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psrad128", IX86_BUILTIN_PSRAD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
31565 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufd, "__builtin_ia32_pshufd", IX86_BUILTIN_PSHUFD, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT },
31566 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshuflw, "__builtin_ia32_pshuflw", IX86_BUILTIN_PSHUFLW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
31567 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufhw, "__builtin_ia32_pshufhw", IX86_BUILTIN_PSHUFHW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
31569 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsqrtv2df2, "__builtin_ia32_sqrtsd", IX86_BUILTIN_SQRTSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_VEC_MERGE },
31571 { OPTION_MASK_ISA_SSE, CODE_FOR_sse2_movq128, "__builtin_ia32_movq128", IX86_BUILTIN_MOVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
31574 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_addv1di3, "__builtin_ia32_paddq", IX86_BUILTIN_PADDQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
31575 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_subv1di3, "__builtin_ia32_psubq", IX86_BUILTIN_PSUBQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
31578 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movshdup, "__builtin_ia32_movshdup", IX86_BUILTIN_MOVSHDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF},
31579 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movsldup, "__builtin_ia32_movsldup", IX86_BUILTIN_MOVSLDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31581 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv4sf3, "__builtin_ia32_addsubps", IX86_BUILTIN_ADDSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31582 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv2df3, "__builtin_ia32_addsubpd", IX86_BUILTIN_ADDSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31583 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv4sf3, "__builtin_ia32_haddps", IX86_BUILTIN_HADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31584 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv2df3, "__builtin_ia32_haddpd", IX86_BUILTIN_HADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31585 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv4sf3, "__builtin_ia32_hsubps", IX86_BUILTIN_HSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31586 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv2df3, "__builtin_ia32_hsubpd", IX86_BUILTIN_HSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31589 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv16qi2, "__builtin_ia32_pabsb128", IX86_BUILTIN_PABSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
31590 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8qi2, "__builtin_ia32_pabsb", IX86_BUILTIN_PABSB, UNKNOWN, (int) V8QI_FTYPE_V8QI },
31591 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8hi2, "__builtin_ia32_pabsw128", IX86_BUILTIN_PABSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
31592 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4hi2, "__builtin_ia32_pabsw", IX86_BUILTIN_PABSW, UNKNOWN, (int) V4HI_FTYPE_V4HI },
31593 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4si2, "__builtin_ia32_pabsd128", IX86_BUILTIN_PABSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
31594 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv2si2, "__builtin_ia32_pabsd", IX86_BUILTIN_PABSD, UNKNOWN, (int) V2SI_FTYPE_V2SI },
31596 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv8hi3, "__builtin_ia32_phaddw128", IX86_BUILTIN_PHADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31597 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv4hi3, "__builtin_ia32_phaddw", IX86_BUILTIN_PHADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31598 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv4si3, "__builtin_ia32_phaddd128", IX86_BUILTIN_PHADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31599 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv2si3, "__builtin_ia32_phaddd", IX86_BUILTIN_PHADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31600 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv8hi3, "__builtin_ia32_phaddsw128", IX86_BUILTIN_PHADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31601 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv4hi3, "__builtin_ia32_phaddsw", IX86_BUILTIN_PHADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31602 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv8hi3, "__builtin_ia32_phsubw128", IX86_BUILTIN_PHSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31603 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv4hi3, "__builtin_ia32_phsubw", IX86_BUILTIN_PHSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31604 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv4si3, "__builtin_ia32_phsubd128", IX86_BUILTIN_PHSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31605 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv2si3, "__builtin_ia32_phsubd", IX86_BUILTIN_PHSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31606 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv8hi3, "__builtin_ia32_phsubsw128", IX86_BUILTIN_PHSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31607 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv4hi3, "__builtin_ia32_phsubsw", IX86_BUILTIN_PHSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31608 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw128, "__builtin_ia32_pmaddubsw128", IX86_BUILTIN_PMADDUBSW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI },
31609 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw, "__builtin_ia32_pmaddubsw", IX86_BUILTIN_PMADDUBSW, UNKNOWN, (int) V4HI_FTYPE_V8QI_V8QI },
31610 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv8hi3, "__builtin_ia32_pmulhrsw128", IX86_BUILTIN_PMULHRSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31611 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv4hi3, "__builtin_ia32_pmulhrsw", IX86_BUILTIN_PMULHRSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31612 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv16qi3, "__builtin_ia32_pshufb128", IX86_BUILTIN_PSHUFB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31613 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv8qi3, "__builtin_ia32_pshufb", IX86_BUILTIN_PSHUFB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31614 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv16qi3, "__builtin_ia32_psignb128", IX86_BUILTIN_PSIGNB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31615 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8qi3, "__builtin_ia32_psignb", IX86_BUILTIN_PSIGNB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31616 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8hi3, "__builtin_ia32_psignw128", IX86_BUILTIN_PSIGNW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31617 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4hi3, "__builtin_ia32_psignw", IX86_BUILTIN_PSIGNW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31618 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4si3, "__builtin_ia32_psignd128", IX86_BUILTIN_PSIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31619 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv2si3, "__builtin_ia32_psignd", IX86_BUILTIN_PSIGND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31622 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrti, "__builtin_ia32_palignr128", IX86_BUILTIN_PALIGNR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_CONVERT },
31623 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrdi, "__builtin_ia32_palignr", IX86_BUILTIN_PALIGNR, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_INT_CONVERT },
31626 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendpd, "__builtin_ia32_blendpd", IX86_BUILTIN_BLENDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31627 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendps, "__builtin_ia32_blendps", IX86_BUILTIN_BLENDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31628 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvpd, "__builtin_ia32_blendvpd", IX86_BUILTIN_BLENDVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF },
31629 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvps, "__builtin_ia32_blendvps", IX86_BUILTIN_BLENDVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF },
31630 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dppd, "__builtin_ia32_dppd", IX86_BUILTIN_DPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31631 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dpps, "__builtin_ia32_dpps", IX86_BUILTIN_DPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31632 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_insertps, "__builtin_ia32_insertps128", IX86_BUILTIN_INSERTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31633 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mpsadbw, "__builtin_ia32_mpsadbw128", IX86_BUILTIN_MPSADBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT },
31634 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendvb, "__builtin_ia32_pblendvb128", IX86_BUILTIN_PBLENDVB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI },
31635 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendw, "__builtin_ia32_pblendw128", IX86_BUILTIN_PBLENDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_INT },
31637 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
31638 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
31639 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
31640 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
31641 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
31642 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
31643 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv8qiv8hi2, "__builtin_ia32_pmovzxbw128", IX86_BUILTIN_PMOVZXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
31644 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4qiv4si2, "__builtin_ia32_pmovzxbd128", IX86_BUILTIN_PMOVZXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
31645 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2qiv2di2, "__builtin_ia32_pmovzxbq128", IX86_BUILTIN_PMOVZXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
31646 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4hiv4si2, "__builtin_ia32_pmovzxwd128", IX86_BUILTIN_PMOVZXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
31647 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2hiv2di2, "__builtin_ia32_pmovzxwq128", IX86_BUILTIN_PMOVZXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
31648 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2siv2di2, "__builtin_ia32_pmovzxdq128", IX86_BUILTIN_PMOVZXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
31649 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_phminposuw, "__builtin_ia32_phminposuw128", IX86_BUILTIN_PHMINPOSUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
31651 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_packusdw, "__builtin_ia32_packusdw128", IX86_BUILTIN_PACKUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
31652 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_eqv2di3, "__builtin_ia32_pcmpeqq", IX86_BUILTIN_PCMPEQQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31653 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv16qi3, "__builtin_ia32_pmaxsb128", IX86_BUILTIN_PMAXSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31654 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv4si3, "__builtin_ia32_pmaxsd128", IX86_BUILTIN_PMAXSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31655 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv4si3, "__builtin_ia32_pmaxud128", IX86_BUILTIN_PMAXUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31656 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv8hi3, "__builtin_ia32_pmaxuw128", IX86_BUILTIN_PMAXUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31657 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv16qi3, "__builtin_ia32_pminsb128", IX86_BUILTIN_PMINSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31658 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv4si3, "__builtin_ia32_pminsd128", IX86_BUILTIN_PMINSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31659 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv4si3, "__builtin_ia32_pminud128", IX86_BUILTIN_PMINUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31660 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv8hi3, "__builtin_ia32_pminuw128", IX86_BUILTIN_PMINUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31661 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mulv2siv2di3, "__builtin_ia32_pmuldq128", IX86_BUILTIN_PMULDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
31662 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_mulv4si3, "__builtin_ia32_pmulld128", IX86_BUILTIN_PMULLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31665 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_roundpd", IX86_BUILTIN_ROUNDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
31666 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_roundps", IX86_BUILTIN_ROUNDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
31667 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundsd, "__builtin_ia32_roundsd", IX86_BUILTIN_ROUNDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31668 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundss, "__builtin_ia32_roundss", IX86_BUILTIN_ROUNDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31670 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_floorpd", IX86_BUILTIN_FLOORPD, (enum rtx_code) ROUND_FLOOR, (int) V2DF_FTYPE_V2DF_ROUND },
31671 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_ceilpd", IX86_BUILTIN_CEILPD, (enum rtx_code) ROUND_CEIL, (int) V2DF_FTYPE_V2DF_ROUND },
31672 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_truncpd", IX86_BUILTIN_TRUNCPD, (enum rtx_code) ROUND_TRUNC, (int) V2DF_FTYPE_V2DF_ROUND },
31673 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_rintpd", IX86_BUILTIN_RINTPD, (enum rtx_code) ROUND_MXCSR, (int) V2DF_FTYPE_V2DF_ROUND },
31675 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_floorpd_vec_pack_sfix", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
31676 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_ceilpd_vec_pack_sfix", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
31678 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2, "__builtin_ia32_roundpd_az", IX86_BUILTIN_ROUNDPD_AZ, UNKNOWN, (int) V2DF_FTYPE_V2DF },
31679 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
31681 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_floorps", IX86_BUILTIN_FLOORPS, (enum rtx_code) ROUND_FLOOR, (int) V4SF_FTYPE_V4SF_ROUND },
31682 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_ceilps", IX86_BUILTIN_CEILPS, (enum rtx_code) ROUND_CEIL, (int) V4SF_FTYPE_V4SF_ROUND },
31683 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_truncps", IX86_BUILTIN_TRUNCPS, (enum rtx_code) ROUND_TRUNC, (int) V4SF_FTYPE_V4SF_ROUND },
31684 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_rintps", IX86_BUILTIN_RINTPS, (enum rtx_code) ROUND_MXCSR, (int) V4SF_FTYPE_V4SF_ROUND },
31686 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_floorps_sfix", IX86_BUILTIN_FLOORPS_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V4SF_ROUND },
31687 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_ceilps_sfix", IX86_BUILTIN_CEILPS_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V4SF_ROUND },
31689 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2, "__builtin_ia32_roundps_az", IX86_BUILTIN_ROUNDPS_AZ, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31690 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2_sfix, "__builtin_ia32_roundps_az_sfix", IX86_BUILTIN_ROUNDPS_AZ_SFIX, UNKNOWN, (int) V4SI_FTYPE_V4SF },
31692 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestz128", IX86_BUILTIN_PTESTZ, EQ, (int) INT_FTYPE_V2DI_V2DI_PTEST },
31693 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestc128", IX86_BUILTIN_PTESTC, LTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
31694 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestnzc128", IX86_BUILTIN_PTESTNZC, GTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
31697 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_gtv2di3, "__builtin_ia32_pcmpgtq", IX86_BUILTIN_PCMPGTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31698 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32qi, "__builtin_ia32_crc32qi", IX86_BUILTIN_CRC32QI, UNKNOWN, (int) UINT_FTYPE_UINT_UCHAR },
31699 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32hi, "__builtin_ia32_crc32hi", IX86_BUILTIN_CRC32HI, UNKNOWN, (int) UINT_FTYPE_UINT_USHORT },
31700 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32si, "__builtin_ia32_crc32si", IX86_BUILTIN_CRC32SI, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
31701 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse4_2_crc32di, "__builtin_ia32_crc32di", IX86_BUILTIN_CRC32DI, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
31704 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrqi, "__builtin_ia32_extrqi", IX86_BUILTIN_EXTRQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_UINT_UINT },
31705 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrq, "__builtin_ia32_extrq", IX86_BUILTIN_EXTRQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V16QI },
31706 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertqi, "__builtin_ia32_insertqi", IX86_BUILTIN_INSERTQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_UINT_UINT },
31707 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertq, "__builtin_ia32_insertq", IX86_BUILTIN_INSERTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31710 { OPTION_MASK_ISA_SSE2, CODE_FOR_aeskeygenassist, 0, IX86_BUILTIN_AESKEYGENASSIST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT },
31711 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesimc, 0, IX86_BUILTIN_AESIMC128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
31713 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenc, 0, IX86_BUILTIN_AESENC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31714 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenclast, 0, IX86_BUILTIN_AESENCLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31715 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdec, 0, IX86_BUILTIN_AESDEC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31716 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdeclast, 0, IX86_BUILTIN_AESDECLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31719 { OPTION_MASK_ISA_SSE2, CODE_FOR_pclmulqdq, 0, IX86_BUILTIN_PCLMULQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT },
31722 { OPTION_MASK_ISA_AVX, CODE_FOR_addv4df3, "__builtin_ia32_addpd256", IX86_BUILTIN_ADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31723 { OPTION_MASK_ISA_AVX, CODE_FOR_addv8sf3, "__builtin_ia32_addps256", IX86_BUILTIN_ADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31724 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv4df3, "__builtin_ia32_addsubpd256", IX86_BUILTIN_ADDSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31725 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv8sf3, "__builtin_ia32_addsubps256", IX86_BUILTIN_ADDSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31726 { OPTION_MASK_ISA_AVX, CODE_FOR_andv4df3, "__builtin_ia32_andpd256", IX86_BUILTIN_ANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31727 { OPTION_MASK_ISA_AVX, CODE_FOR_andv8sf3, "__builtin_ia32_andps256", IX86_BUILTIN_ANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31728 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv4df3, "__builtin_ia32_andnpd256", IX86_BUILTIN_ANDNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31729 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv8sf3, "__builtin_ia32_andnps256", IX86_BUILTIN_ANDNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31730 { OPTION_MASK_ISA_AVX, CODE_FOR_divv4df3, "__builtin_ia32_divpd256", IX86_BUILTIN_DIVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31731 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_divv8sf3, "__builtin_ia32_divps256", IX86_BUILTIN_DIVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31732 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv4df3, "__builtin_ia32_haddpd256", IX86_BUILTIN_HADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31733 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv8sf3, "__builtin_ia32_hsubps256", IX86_BUILTIN_HSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31734 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv4df3, "__builtin_ia32_hsubpd256", IX86_BUILTIN_HSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31735 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv8sf3, "__builtin_ia32_haddps256", IX86_BUILTIN_HADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31736 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv4df3, "__builtin_ia32_maxpd256", IX86_BUILTIN_MAXPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31737 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv8sf3, "__builtin_ia32_maxps256", IX86_BUILTIN_MAXPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31738 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv4df3, "__builtin_ia32_minpd256", IX86_BUILTIN_MINPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31739 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv8sf3, "__builtin_ia32_minps256", IX86_BUILTIN_MINPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31740 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv4df3, "__builtin_ia32_mulpd256", IX86_BUILTIN_MULPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31741 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv8sf3, "__builtin_ia32_mulps256", IX86_BUILTIN_MULPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31742 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv4df3, "__builtin_ia32_orpd256", IX86_BUILTIN_ORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31743 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv8sf3, "__builtin_ia32_orps256", IX86_BUILTIN_ORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31744 { OPTION_MASK_ISA_AVX, CODE_FOR_subv4df3, "__builtin_ia32_subpd256", IX86_BUILTIN_SUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31745 { OPTION_MASK_ISA_AVX, CODE_FOR_subv8sf3, "__builtin_ia32_subps256", IX86_BUILTIN_SUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31746 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv4df3, "__builtin_ia32_xorpd256", IX86_BUILTIN_XORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31747 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv8sf3, "__builtin_ia32_xorps256", IX86_BUILTIN_XORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31749 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv2df3, "__builtin_ia32_vpermilvarpd", IX86_BUILTIN_VPERMILVARPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI },
31750 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4sf3, "__builtin_ia32_vpermilvarps", IX86_BUILTIN_VPERMILVARPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI },
31751 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4df3, "__builtin_ia32_vpermilvarpd256", IX86_BUILTIN_VPERMILVARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI },
31752 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv8sf3, "__builtin_ia32_vpermilvarps256", IX86_BUILTIN_VPERMILVARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
31754 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendpd256, "__builtin_ia32_blendpd256", IX86_BUILTIN_BLENDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
31755 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendps256, "__builtin_ia32_blendps256", IX86_BUILTIN_BLENDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
31756 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvpd256, "__builtin_ia32_blendvpd256", IX86_BUILTIN_BLENDVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF },
31757 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvps256, "__builtin_ia32_blendvps256", IX86_BUILTIN_BLENDVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF },
31758 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_dpps256, "__builtin_ia32_dpps256", IX86_BUILTIN_DPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
31759 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufpd256, "__builtin_ia32_shufpd256", IX86_BUILTIN_SHUFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
31760 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufps256, "__builtin_ia32_shufps256", IX86_BUILTIN_SHUFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
31761 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv2df3, "__builtin_ia32_cmpsd", IX86_BUILTIN_CMPSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31762 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv4sf3, "__builtin_ia32_cmpss", IX86_BUILTIN_CMPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31763 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv2df3, "__builtin_ia32_cmppd", IX86_BUILTIN_CMPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31764 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4sf3, "__builtin_ia32_cmpps", IX86_BUILTIN_CMPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31765 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4df3, "__builtin_ia32_cmppd256", IX86_BUILTIN_CMPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
31766 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv8sf3, "__builtin_ia32_cmpps256", IX86_BUILTIN_CMPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
31767 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v4df, "__builtin_ia32_vextractf128_pd256", IX86_BUILTIN_EXTRACTF128PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT },
31768 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8sf, "__builtin_ia32_vextractf128_ps256", IX86_BUILTIN_EXTRACTF128PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT },
31769 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8si, "__builtin_ia32_vextractf128_si256", IX86_BUILTIN_EXTRACTF128SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT },
31770 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv4siv4df2, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
31771 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv8siv8sf2, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
31772 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
31773 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_fix_notruncv8sfv8si, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
31774 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
31775 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv4dfv4si2, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
31776 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
31777 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv8sfv8si2, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
31778 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v4df3, "__builtin_ia32_vperm2f128_pd256", IX86_BUILTIN_VPERM2F128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
31779 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8sf3, "__builtin_ia32_vperm2f128_ps256", IX86_BUILTIN_VPERM2F128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
31780 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8si3, "__builtin_ia32_vperm2f128_si256", IX86_BUILTIN_VPERM2F128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
31781 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv2df, "__builtin_ia32_vpermilpd", IX86_BUILTIN_VPERMILPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
31782 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4sf, "__builtin_ia32_vpermilps", IX86_BUILTIN_VPERMILPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
31783 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4df, "__builtin_ia32_vpermilpd256", IX86_BUILTIN_VPERMILPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
31784 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv8sf, "__builtin_ia32_vpermilps256", IX86_BUILTIN_VPERMILPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
31785 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v4df, "__builtin_ia32_vinsertf128_pd256", IX86_BUILTIN_VINSERTF128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT },
31786 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8sf, "__builtin_ia32_vinsertf128_ps256", IX86_BUILTIN_VINSERTF128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT },
31787 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8si, "__builtin_ia32_vinsertf128_si256", IX86_BUILTIN_VINSERTF128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT },
31789 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movshdup256, "__builtin_ia32_movshdup256", IX86_BUILTIN_MOVSHDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31790 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movsldup256, "__builtin_ia32_movsldup256", IX86_BUILTIN_MOVSLDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31791 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movddup256, "__builtin_ia32_movddup256", IX86_BUILTIN_MOVDDUP256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
31793 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv4df2, "__builtin_ia32_sqrtpd256", IX86_BUILTIN_SQRTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
31794 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_sqrtv8sf2, "__builtin_ia32_sqrtps256", IX86_BUILTIN_SQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31795 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv8sf2, "__builtin_ia32_sqrtps_nr256", IX86_BUILTIN_SQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31796 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rsqrtv8sf2, "__builtin_ia32_rsqrtps256", IX86_BUILTIN_RSQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31797 { OPTION_MASK_ISA_AVX, CODE_FOR_rsqrtv8sf2, "__builtin_ia32_rsqrtps_nr256", IX86_BUILTIN_RSQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31799 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rcpv8sf2, "__builtin_ia32_rcpps256", IX86_BUILTIN_RCPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31801 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_roundpd256", IX86_BUILTIN_ROUNDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
31802 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_roundps256", IX86_BUILTIN_ROUNDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
31804 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_floorpd256", IX86_BUILTIN_FLOORPD256, (enum rtx_code) ROUND_FLOOR, (int) V4DF_FTYPE_V4DF_ROUND },
31805 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_ceilpd256", IX86_BUILTIN_CEILPD256, (enum rtx_code) ROUND_CEIL, (int) V4DF_FTYPE_V4DF_ROUND },
31806 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_truncpd256", IX86_BUILTIN_TRUNCPD256, (enum rtx_code) ROUND_TRUNC, (int) V4DF_FTYPE_V4DF_ROUND },
31807 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_rintpd256", IX86_BUILTIN_RINTPD256, (enum rtx_code) ROUND_MXCSR, (int) V4DF_FTYPE_V4DF_ROUND },
31809 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2, "__builtin_ia32_roundpd_az256", IX86_BUILTIN_ROUNDPD_AZ256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
31810 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix256", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
31812 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_floorpd_vec_pack_sfix256", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
31813 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_ceilpd_vec_pack_sfix256", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
31815 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_floorps256", IX86_BUILTIN_FLOORPS256, (enum rtx_code) ROUND_FLOOR, (int) V8SF_FTYPE_V8SF_ROUND },
31816 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_ceilps256", IX86_BUILTIN_CEILPS256, (enum rtx_code) ROUND_CEIL, (int) V8SF_FTYPE_V8SF_ROUND },
31817 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_truncps256", IX86_BUILTIN_TRUNCPS256, (enum rtx_code) ROUND_TRUNC, (int) V8SF_FTYPE_V8SF_ROUND },
31818 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_rintps256", IX86_BUILTIN_RINTPS256, (enum rtx_code) ROUND_MXCSR, (int) V8SF_FTYPE_V8SF_ROUND },
31820 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_floorps_sfix256", IX86_BUILTIN_FLOORPS_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V8SF_ROUND },
31821 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_ceilps_sfix256", IX86_BUILTIN_CEILPS_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V8SF_ROUND },
31823 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2, "__builtin_ia32_roundps_az256", IX86_BUILTIN_ROUNDPS_AZ256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31824 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2_sfix, "__builtin_ia32_roundps_az_sfix256", IX86_BUILTIN_ROUNDPS_AZ_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
31826 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhpd256, "__builtin_ia32_unpckhpd256", IX86_BUILTIN_UNPCKHPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31827 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklpd256, "__builtin_ia32_unpcklpd256", IX86_BUILTIN_UNPCKLPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31828 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhps256, "__builtin_ia32_unpckhps256", IX86_BUILTIN_UNPCKHPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31829 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklps256, "__builtin_ia32_unpcklps256", IX86_BUILTIN_UNPCKLPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31831 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si256_si, "__builtin_ia32_si256_si", IX86_BUILTIN_SI256_SI, UNKNOWN, (int) V8SI_FTYPE_V4SI },
31832 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps256_ps, "__builtin_ia32_ps256_ps", IX86_BUILTIN_PS256_PS, UNKNOWN, (int) V8SF_FTYPE_V4SF },
31833 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd256_pd, "__builtin_ia32_pd256_pd", IX86_BUILTIN_PD256_PD, UNKNOWN, (int) V4DF_FTYPE_V2DF },
31834 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8si, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI },
31835 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8sf, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF },
31836 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v4df, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF },
31838 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestzpd", IX86_BUILTIN_VTESTZPD, EQ, (int) INT_FTYPE_V2DF_V2DF_PTEST },
31839 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestcpd", IX86_BUILTIN_VTESTCPD, LTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
31840 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestnzcpd", IX86_BUILTIN_VTESTNZCPD, GTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
31841 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestzps", IX86_BUILTIN_VTESTZPS, EQ, (int) INT_FTYPE_V4SF_V4SF_PTEST },
31842 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestcps", IX86_BUILTIN_VTESTCPS, LTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
31843 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestnzcps", IX86_BUILTIN_VTESTNZCPS, GTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
31844 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestzpd256", IX86_BUILTIN_VTESTZPD256, EQ, (int) INT_FTYPE_V4DF_V4DF_PTEST },
31845 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestcpd256", IX86_BUILTIN_VTESTCPD256, LTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
31846 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestnzcpd256", IX86_BUILTIN_VTESTNZCPD256, GTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
31847 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestzps256", IX86_BUILTIN_VTESTZPS256, EQ, (int) INT_FTYPE_V8SF_V8SF_PTEST },
31848 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestcps256", IX86_BUILTIN_VTESTCPS256, LTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
31849 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestnzcps256", IX86_BUILTIN_VTESTNZCPS256, GTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
31850 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestz256", IX86_BUILTIN_PTESTZ256, EQ, (int) INT_FTYPE_V4DI_V4DI_PTEST },
31851 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestc256", IX86_BUILTIN_PTESTC256, LTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
31852 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestnzc256", IX86_BUILTIN_PTESTNZC256, GTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
31854 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF },
31855 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF },
31857 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv8sf3, "__builtin_ia32_copysignps256", IX86_BUILTIN_CPYSGNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31858 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv4df3, "__builtin_ia32_copysignpd256", IX86_BUILTIN_CPYSGNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31860 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_pack_sfix_v4df, "__builtin_ia32_vec_pack_sfix256 ", IX86_BUILTIN_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
31863 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_mpsadbw, "__builtin_ia32_mpsadbw256", IX86_BUILTIN_MPSADBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_INT },
31864 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv32qi2, "__builtin_ia32_pabsb256", IX86_BUILTIN_PABSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI },
31865 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv16hi2, "__builtin_ia32_pabsw256", IX86_BUILTIN_PABSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI },
31866 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv8si2, "__builtin_ia32_pabsd256", IX86_BUILTIN_PABSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI },
31867 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packssdw, "__builtin_ia32_packssdw256", IX86_BUILTIN_PACKSSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
31868 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packsswb, "__builtin_ia32_packsswb256", IX86_BUILTIN_PACKSSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
31869 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packusdw, "__builtin_ia32_packusdw256", IX86_BUILTIN_PACKUSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
31870 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packuswb, "__builtin_ia32_packuswb256", IX86_BUILTIN_PACKUSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
31871 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv32qi3, "__builtin_ia32_paddb256", IX86_BUILTIN_PADDB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31872 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv16hi3, "__builtin_ia32_paddw256", IX86_BUILTIN_PADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31873 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv8si3, "__builtin_ia32_paddd256", IX86_BUILTIN_PADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31874 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv4di3, "__builtin_ia32_paddq256", IX86_BUILTIN_PADDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31875 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv32qi3, "__builtin_ia32_paddsb256", IX86_BUILTIN_PADDSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31876 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv16hi3, "__builtin_ia32_paddsw256", IX86_BUILTIN_PADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31877 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv32qi3, "__builtin_ia32_paddusb256", IX86_BUILTIN_PADDUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31878 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv16hi3, "__builtin_ia32_paddusw256", IX86_BUILTIN_PADDUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31879 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_palignrv2ti, "__builtin_ia32_palignr256", IX86_BUILTIN_PALIGNR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_CONVERT },
31880 { OPTION_MASK_ISA_AVX2, CODE_FOR_andv4di3, "__builtin_ia32_andsi256", IX86_BUILTIN_AND256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31881 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_andnotv4di3, "__builtin_ia32_andnotsi256", IX86_BUILTIN_ANDNOT256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31882 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv32qi3, "__builtin_ia32_pavgb256", IX86_BUILTIN_PAVGB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31883 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv16hi3, "__builtin_ia32_pavgw256", IX86_BUILTIN_PAVGW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31884 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendvb, "__builtin_ia32_pblendvb256", IX86_BUILTIN_PBLENDVB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI },
31885 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendw, "__builtin_ia32_pblendw256", IX86_BUILTIN_PBLENDVW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_INT },
31886 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv32qi3, "__builtin_ia32_pcmpeqb256", IX86_BUILTIN_PCMPEQB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31887 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv16hi3, "__builtin_ia32_pcmpeqw256", IX86_BUILTIN_PCMPEQW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31888 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv8si3, "__builtin_ia32_pcmpeqd256", IX86_BUILTIN_PCMPEQD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31889 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv4di3, "__builtin_ia32_pcmpeqq256", IX86_BUILTIN_PCMPEQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31890 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv32qi3, "__builtin_ia32_pcmpgtb256", IX86_BUILTIN_PCMPGTB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31891 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv16hi3, "__builtin_ia32_pcmpgtw256", IX86_BUILTIN_PCMPGTW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31892 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv8si3, "__builtin_ia32_pcmpgtd256", IX86_BUILTIN_PCMPGTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31893 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv4di3, "__builtin_ia32_pcmpgtq256", IX86_BUILTIN_PCMPGTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31894 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddwv16hi3, "__builtin_ia32_phaddw256", IX86_BUILTIN_PHADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31895 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phadddv8si3, "__builtin_ia32_phaddd256", IX86_BUILTIN_PHADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31896 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddswv16hi3, "__builtin_ia32_phaddsw256", IX86_BUILTIN_PHADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31897 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubwv16hi3, "__builtin_ia32_phsubw256", IX86_BUILTIN_PHSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31898 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubdv8si3, "__builtin_ia32_phsubd256", IX86_BUILTIN_PHSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31899 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubswv16hi3, "__builtin_ia32_phsubsw256", IX86_BUILTIN_PHSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31900 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddubsw256, "__builtin_ia32_pmaddubsw256", IX86_BUILTIN_PMADDUBSW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
31901 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddwd, "__builtin_ia32_pmaddwd256", IX86_BUILTIN_PMADDWD256, UNKNOWN, (int) V8SI_FTYPE_V16HI_V16HI },
31902 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv32qi3, "__builtin_ia32_pmaxsb256", IX86_BUILTIN_PMAXSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31903 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv16hi3, "__builtin_ia32_pmaxsw256", IX86_BUILTIN_PMAXSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31904 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv8si3 , "__builtin_ia32_pmaxsd256", IX86_BUILTIN_PMAXSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31905 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv32qi3, "__builtin_ia32_pmaxub256", IX86_BUILTIN_PMAXUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31906 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv16hi3, "__builtin_ia32_pmaxuw256", IX86_BUILTIN_PMAXUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31907 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv8si3 , "__builtin_ia32_pmaxud256", IX86_BUILTIN_PMAXUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31908 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv32qi3, "__builtin_ia32_pminsb256", IX86_BUILTIN_PMINSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31909 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv16hi3, "__builtin_ia32_pminsw256", IX86_BUILTIN_PMINSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31910 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv8si3 , "__builtin_ia32_pminsd256", IX86_BUILTIN_PMINSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31911 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv32qi3, "__builtin_ia32_pminub256", IX86_BUILTIN_PMINUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31912 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv16hi3, "__builtin_ia32_pminuw256", IX86_BUILTIN_PMINUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31913 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv8si3 , "__builtin_ia32_pminud256", IX86_BUILTIN_PMINUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31914 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmovmskb, "__builtin_ia32_pmovmskb256", IX86_BUILTIN_PMOVMSKB256, UNKNOWN, (int) INT_FTYPE_V32QI },
31915 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv16qiv16hi2, "__builtin_ia32_pmovsxbw256", IX86_BUILTIN_PMOVSXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
31916 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8qiv8si2 , "__builtin_ia32_pmovsxbd256", IX86_BUILTIN_PMOVSXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
31917 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4qiv4di2 , "__builtin_ia32_pmovsxbq256", IX86_BUILTIN_PMOVSXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
31918 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8hiv8si2 , "__builtin_ia32_pmovsxwd256", IX86_BUILTIN_PMOVSXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
31919 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4hiv4di2 , "__builtin_ia32_pmovsxwq256", IX86_BUILTIN_PMOVSXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
31920 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4siv4di2 , "__builtin_ia32_pmovsxdq256", IX86_BUILTIN_PMOVSXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
31921 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv16qiv16hi2, "__builtin_ia32_pmovzxbw256", IX86_BUILTIN_PMOVZXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
31922 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8qiv8si2 , "__builtin_ia32_pmovzxbd256", IX86_BUILTIN_PMOVZXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
31923 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4qiv4di2 , "__builtin_ia32_pmovzxbq256", IX86_BUILTIN_PMOVZXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
31924 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8hiv8si2 , "__builtin_ia32_pmovzxwd256", IX86_BUILTIN_PMOVZXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
31925 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4hiv4di2 , "__builtin_ia32_pmovzxwq256", IX86_BUILTIN_PMOVZXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
31926 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4siv4di2 , "__builtin_ia32_pmovzxdq256", IX86_BUILTIN_PMOVZXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
31927 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_smult_even_v8si, "__builtin_ia32_pmuldq256", IX86_BUILTIN_PMULDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
31928 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmulhrswv16hi3 , "__builtin_ia32_pmulhrsw256", IX86_BUILTIN_PMULHRSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31929 { OPTION_MASK_ISA_AVX2, CODE_FOR_umulv16hi3_highpart, "__builtin_ia32_pmulhuw256" , IX86_BUILTIN_PMULHUW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31930 { OPTION_MASK_ISA_AVX2, CODE_FOR_smulv16hi3_highpart, "__builtin_ia32_pmulhw256" , IX86_BUILTIN_PMULHW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31931 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv16hi3, "__builtin_ia32_pmullw256" , IX86_BUILTIN_PMULLW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31932 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv8si3, "__builtin_ia32_pmulld256" , IX86_BUILTIN_PMULLD256 , UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31933 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_umult_even_v8si, "__builtin_ia32_pmuludq256", IX86_BUILTIN_PMULUDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
31934 { OPTION_MASK_ISA_AVX2, CODE_FOR_iorv4di3, "__builtin_ia32_por256", IX86_BUILTIN_POR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31935 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psadbw, "__builtin_ia32_psadbw256", IX86_BUILTIN_PSADBW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
31936 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufbv32qi3, "__builtin_ia32_pshufb256", IX86_BUILTIN_PSHUFB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31937 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufdv3, "__builtin_ia32_pshufd256", IX86_BUILTIN_PSHUFD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT },
31938 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufhwv3, "__builtin_ia32_pshufhw256", IX86_BUILTIN_PSHUFHW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
31939 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshuflwv3, "__builtin_ia32_pshuflw256", IX86_BUILTIN_PSHUFLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
31940 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv32qi3, "__builtin_ia32_psignb256", IX86_BUILTIN_PSIGNB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31941 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv16hi3, "__builtin_ia32_psignw256", IX86_BUILTIN_PSIGNW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31942 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv8si3 , "__builtin_ia32_psignd256", IX86_BUILTIN_PSIGND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31943 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlv2ti3, "__builtin_ia32_pslldqi256", IX86_BUILTIN_PSLLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
31944 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllwi256", IX86_BUILTIN_PSLLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
31945 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllw256", IX86_BUILTIN_PSLLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
31946 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslldi256", IX86_BUILTIN_PSLLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
31947 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslld256", IX86_BUILTIN_PSLLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
31948 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllqi256", IX86_BUILTIN_PSLLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
31949 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllq256", IX86_BUILTIN_PSLLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
31950 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psrawi256", IX86_BUILTIN_PSRAWI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
31951 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psraw256", IX86_BUILTIN_PSRAW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
31952 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psradi256", IX86_BUILTIN_PSRADI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
31953 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psrad256", IX86_BUILTIN_PSRAD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
31954 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrv2ti3, "__builtin_ia32_psrldqi256", IX86_BUILTIN_PSRLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
31955 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlwi256", IX86_BUILTIN_PSRLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
31956 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlw256", IX86_BUILTIN_PSRLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
31957 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrldi256", IX86_BUILTIN_PSRLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
31958 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrld256", IX86_BUILTIN_PSRLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
31959 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlqi256", IX86_BUILTIN_PSRLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
31960 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlq256", IX86_BUILTIN_PSRLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
31961 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv32qi3, "__builtin_ia32_psubb256", IX86_BUILTIN_PSUBB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31962 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv16hi3, "__builtin_ia32_psubw256", IX86_BUILTIN_PSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31963 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv8si3, "__builtin_ia32_psubd256", IX86_BUILTIN_PSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31964 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv4di3, "__builtin_ia32_psubq256", IX86_BUILTIN_PSUBQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31965 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv32qi3, "__builtin_ia32_psubsb256", IX86_BUILTIN_PSUBSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31966 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv16hi3, "__builtin_ia32_psubsw256", IX86_BUILTIN_PSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31967 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv32qi3, "__builtin_ia32_psubusb256", IX86_BUILTIN_PSUBUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31968 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv16hi3, "__builtin_ia32_psubusw256", IX86_BUILTIN_PSUBUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31969 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv32qi, "__builtin_ia32_punpckhbw256", IX86_BUILTIN_PUNPCKHBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31970 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv16hi, "__builtin_ia32_punpckhwd256", IX86_BUILTIN_PUNPCKHWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31971 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv8si, "__builtin_ia32_punpckhdq256", IX86_BUILTIN_PUNPCKHDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31972 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv4di, "__builtin_ia32_punpckhqdq256", IX86_BUILTIN_PUNPCKHQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31973 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv32qi, "__builtin_ia32_punpcklbw256", IX86_BUILTIN_PUNPCKLBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31974 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv16hi, "__builtin_ia32_punpcklwd256", IX86_BUILTIN_PUNPCKLWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31975 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv8si, "__builtin_ia32_punpckldq256", IX86_BUILTIN_PUNPCKLDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31976 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv4di, "__builtin_ia32_punpcklqdq256", IX86_BUILTIN_PUNPCKLQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31977 { OPTION_MASK_ISA_AVX2, CODE_FOR_xorv4di3, "__builtin_ia32_pxor256", IX86_BUILTIN_PXOR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31978 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4sf, "__builtin_ia32_vbroadcastss_ps", IX86_BUILTIN_VBROADCASTSS_PS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31979 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv8sf, "__builtin_ia32_vbroadcastss_ps256", IX86_BUILTIN_VBROADCASTSS_PS256, UNKNOWN, (int) V8SF_FTYPE_V4SF },
31980 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4df, "__builtin_ia32_vbroadcastsd_pd256", IX86_BUILTIN_VBROADCASTSD_PD256, UNKNOWN, (int) V4DF_FTYPE_V2DF },
31981 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vbroadcasti128_v4di, "__builtin_ia32_vbroadcastsi256", IX86_BUILTIN_VBROADCASTSI256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
31982 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv4si, "__builtin_ia32_pblendd128", IX86_BUILTIN_PBLENDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT },
31983 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv8si, "__builtin_ia32_pblendd256", IX86_BUILTIN_PBLENDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
31984 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv32qi, "__builtin_ia32_pbroadcastb256", IX86_BUILTIN_PBROADCASTB256, UNKNOWN, (int) V32QI_FTYPE_V16QI },
31985 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16hi, "__builtin_ia32_pbroadcastw256", IX86_BUILTIN_PBROADCASTW256, UNKNOWN, (int) V16HI_FTYPE_V8HI },
31986 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8si, "__builtin_ia32_pbroadcastd256", IX86_BUILTIN_PBROADCASTD256, UNKNOWN, (int) V8SI_FTYPE_V4SI },
31987 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4di, "__builtin_ia32_pbroadcastq256", IX86_BUILTIN_PBROADCASTQ256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
31988 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16qi, "__builtin_ia32_pbroadcastb128", IX86_BUILTIN_PBROADCASTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
31989 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8hi, "__builtin_ia32_pbroadcastw128", IX86_BUILTIN_PBROADCASTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
31990 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4si, "__builtin_ia32_pbroadcastd128", IX86_BUILTIN_PBROADCASTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
31991 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv2di, "__builtin_ia32_pbroadcastq128", IX86_BUILTIN_PBROADCASTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
31992 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8si, "__builtin_ia32_permvarsi256", IX86_BUILTIN_VPERMVARSI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31993 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8sf, "__builtin_ia32_permvarsf256", IX86_BUILTIN_VPERMVARSF256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
31994 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4df, "__builtin_ia32_permdf256", IX86_BUILTIN_VPERMDF256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
31995 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4di, "__builtin_ia32_permdi256", IX86_BUILTIN_VPERMDI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT },
31996 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv2ti, "__builtin_ia32_permti256", IX86_BUILTIN_VPERMTI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT },
31997 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx_vextractf128v4di, "__builtin_ia32_extract128i256", IX86_BUILTIN_VEXTRACT128I256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT },
31998 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx_vinsertf128v4di, "__builtin_ia32_insert128i256", IX86_BUILTIN_VINSERT128I256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_INT },
31999 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4di, "__builtin_ia32_psllv4di", IX86_BUILTIN_PSLLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
32000 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv2di, "__builtin_ia32_psllv2di", IX86_BUILTIN_PSLLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
32001 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv8si, "__builtin_ia32_psllv8si", IX86_BUILTIN_PSLLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
32002 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4si, "__builtin_ia32_psllv4si", IX86_BUILTIN_PSLLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32003 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv8si, "__builtin_ia32_psrav8si", IX86_BUILTIN_PSRAVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
32004 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv4si, "__builtin_ia32_psrav4si", IX86_BUILTIN_PSRAVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32005 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4di, "__builtin_ia32_psrlv4di", IX86_BUILTIN_PSRLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
32006 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv2di, "__builtin_ia32_psrlv2di", IX86_BUILTIN_PSRLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
32007 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv8si, "__builtin_ia32_psrlv8si", IX86_BUILTIN_PSRLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
32008 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4si, "__builtin_ia32_psrlv4si", IX86_BUILTIN_PSRLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32010 { OPTION_MASK_ISA_LZCNT, CODE_FOR_clzhi2_lzcnt, "__builtin_clzs", IX86_BUILTIN_CLZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
32013 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_si, "__builtin_ia32_bextr_u32", IX86_BUILTIN_BEXTR32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
32014 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_di, "__builtin_ia32_bextr_u64", IX86_BUILTIN_BEXTR64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
32015 { OPTION_MASK_ISA_BMI, CODE_FOR_ctzhi2, "__builtin_ctzs", IX86_BUILTIN_CTZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
32018 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_si, "__builtin_ia32_bextri_u32", IX86_BUILTIN_BEXTRI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
32019 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_di, "__builtin_ia32_bextri_u64", IX86_BUILTIN_BEXTRI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
32022 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps, "__builtin_ia32_vcvtph2ps", IX86_BUILTIN_CVTPH2PS, UNKNOWN, (int) V4SF_FTYPE_V8HI },
32023 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps256, "__builtin_ia32_vcvtph2ps256", IX86_BUILTIN_CVTPH2PS256, UNKNOWN, (int) V8SF_FTYPE_V8HI },
32024 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph, "__builtin_ia32_vcvtps2ph", IX86_BUILTIN_CVTPS2PH, UNKNOWN, (int) V8HI_FTYPE_V4SF_INT },
32025 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph256, "__builtin_ia32_vcvtps2ph256", IX86_BUILTIN_CVTPS2PH256, UNKNOWN, (int) V8HI_FTYPE_V8SF_INT },
32028 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_si3, "__builtin_ia32_bzhi_si", IX86_BUILTIN_BZHI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
32029 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_di3, "__builtin_ia32_bzhi_di", IX86_BUILTIN_BZHI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
32030 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_si3, "__builtin_ia32_pdep_si", IX86_BUILTIN_PDEP32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
32031 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_di3, "__builtin_ia32_pdep_di", IX86_BUILTIN_PDEP64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
32032 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_si3, "__builtin_ia32_pext_si", IX86_BUILTIN_PEXT32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
32033 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_di3, "__builtin_ia32_pext_di", IX86_BUILTIN_PEXT64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
32036 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_si512_256si, "__builtin_ia32_si512_256si", IX86_BUILTIN_SI512_SI256, UNKNOWN, (int) V16SI_FTYPE_V8SI },
32037 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ps512_256ps, "__builtin_ia32_ps512_256ps", IX86_BUILTIN_PS512_PS256, UNKNOWN, (int) V16SF_FTYPE_V8SF },
32038 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_pd512_256pd, "__builtin_ia32_pd512_256pd", IX86_BUILTIN_PD512_PD256, UNKNOWN, (int) V8DF_FTYPE_V4DF },
32039 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_si512_si, "__builtin_ia32_si512_si", IX86_BUILTIN_SI512_SI, UNKNOWN, (int) V16SI_FTYPE_V4SI },
32040 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ps512_ps, "__builtin_ia32_ps512_ps", IX86_BUILTIN_PS512_PS, UNKNOWN, (int) V16SF_FTYPE_V4SF },
32041 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_pd512_pd, "__builtin_ia32_pd512_pd", IX86_BUILTIN_PD512_PD, UNKNOWN, (int) V8DF_FTYPE_V2DF },
32042 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_alignv16si_mask, "__builtin_ia32_alignd512_mask", IX86_BUILTIN_ALIGND512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_INT_V16SI_HI },
32043 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_alignv8di_mask, "__builtin_ia32_alignq512_mask", IX86_BUILTIN_ALIGNQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_INT_V8DI_QI },
32044 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_blendmv16si, "__builtin_ia32_blendmd_512_mask", IX86_BUILTIN_BLENDMD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
32045 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_blendmv8df, "__builtin_ia32_blendmpd_512_mask", IX86_BUILTIN_BLENDMPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
32046 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_blendmv16sf, "__builtin_ia32_blendmps_512_mask", IX86_BUILTIN_BLENDMPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
32047 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_blendmv8di, "__builtin_ia32_blendmq_512_mask", IX86_BUILTIN_BLENDMQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32048 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_broadcastv16sf_mask, "__builtin_ia32_broadcastf32x4_512", IX86_BUILTIN_BROADCASTF32X4_512, UNKNOWN, (int) V16SF_FTYPE_V4SF_V16SF_HI },
32049 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_broadcastv8df_mask, "__builtin_ia32_broadcastf64x4_512", IX86_BUILTIN_BROADCASTF64X4_512, UNKNOWN, (int) V8DF_FTYPE_V4DF_V8DF_QI },
32050 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_broadcastv16si_mask, "__builtin_ia32_broadcasti32x4_512", IX86_BUILTIN_BROADCASTI32X4_512, UNKNOWN, (int) V16SI_FTYPE_V4SI_V16SI_HI },
32051 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_broadcastv8di_mask, "__builtin_ia32_broadcasti64x4_512", IX86_BUILTIN_BROADCASTI64X4_512, UNKNOWN, (int) V8DI_FTYPE_V4DI_V8DI_QI },
32052 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dupv8df_mask, "__builtin_ia32_broadcastsd512", IX86_BUILTIN_BROADCASTSD512, UNKNOWN, (int) V8DF_FTYPE_V2DF_V8DF_QI },
32053 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dupv16sf_mask, "__builtin_ia32_broadcastss512", IX86_BUILTIN_BROADCASTSS512, UNKNOWN, (int) V16SF_FTYPE_V4SF_V16SF_HI },
32054 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cmpv16si3_mask, "__builtin_ia32_cmpd512_mask", IX86_BUILTIN_CMPD512, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_INT_HI },
32055 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cmpv8di3_mask, "__builtin_ia32_cmpq512_mask", IX86_BUILTIN_CMPQ512, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_INT_QI },
32056 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressv8df_mask, "__builtin_ia32_compressdf512_mask", IX86_BUILTIN_COMPRESSPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
32057 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressv16sf_mask, "__builtin_ia32_compresssf512_mask", IX86_BUILTIN_COMPRESSPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
32058 { OPTION_MASK_ISA_AVX512F, CODE_FOR_floatv8siv8df2_mask, "__builtin_ia32_cvtdq2pd512_mask", IX86_BUILTIN_CVTDQ2PD512, UNKNOWN, (int) V8DF_FTYPE_V8SI_V8DF_QI },
32059 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvtps2ph512_mask, "__builtin_ia32_vcvtps2ph512_mask", IX86_BUILTIN_CVTPS2PH512, UNKNOWN, (int) V16HI_FTYPE_V16SF_INT_V16HI_HI },
32060 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ufloatv8siv8df2_mask, "__builtin_ia32_cvtudq2pd512_mask", IX86_BUILTIN_CVTUDQ2PD512, UNKNOWN, (int) V8DF_FTYPE_V8SI_V8DF_QI },
32061 { OPTION_MASK_ISA_AVX512F, CODE_FOR_cvtusi2sd32, "__builtin_ia32_cvtusi2sd32", IX86_BUILTIN_CVTUSI2SD32, UNKNOWN, (int) V2DF_FTYPE_V2DF_UINT },
32062 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8df_mask, "__builtin_ia32_expanddf512_mask", IX86_BUILTIN_EXPANDPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
32063 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8df_maskz, "__builtin_ia32_expanddf512_maskz", IX86_BUILTIN_EXPANDPD512Z, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
32064 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16sf_mask, "__builtin_ia32_expandsf512_mask", IX86_BUILTIN_EXPANDPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
32065 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16sf_maskz, "__builtin_ia32_expandsf512_maskz", IX86_BUILTIN_EXPANDPS512Z, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
32066 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vextractf32x4_mask, "__builtin_ia32_extractf32x4_mask", IX86_BUILTIN_EXTRACTF32X4, UNKNOWN, (int) V4SF_FTYPE_V16SF_INT_V4SF_QI },
32067 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vextractf64x4_mask, "__builtin_ia32_extractf64x4_mask", IX86_BUILTIN_EXTRACTF64X4, UNKNOWN, (int) V4DF_FTYPE_V8DF_INT_V4DF_QI },
32068 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vextracti32x4_mask, "__builtin_ia32_extracti32x4_mask", IX86_BUILTIN_EXTRACTI32X4, UNKNOWN, (int) V4SI_FTYPE_V16SI_INT_V4SI_QI },
32069 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vextracti64x4_mask, "__builtin_ia32_extracti64x4_mask", IX86_BUILTIN_EXTRACTI64X4, UNKNOWN, (int) V4DI_FTYPE_V8DI_INT_V4DI_QI },
32070 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vinsertf32x4_mask, "__builtin_ia32_insertf32x4_mask", IX86_BUILTIN_INSERTF32X4, UNKNOWN, (int) V16SF_FTYPE_V16SF_V4SF_INT_V16SF_HI },
32071 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vinsertf64x4_mask, "__builtin_ia32_insertf64x4_mask", IX86_BUILTIN_INSERTF64X4, UNKNOWN, (int) V8DF_FTYPE_V8DF_V4DF_INT_V8DF_QI },
32072 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vinserti32x4_mask, "__builtin_ia32_inserti32x4_mask", IX86_BUILTIN_INSERTI32X4, UNKNOWN, (int) V16SI_FTYPE_V16SI_V4SI_INT_V16SI_HI },
32073 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vinserti64x4_mask, "__builtin_ia32_inserti64x4_mask", IX86_BUILTIN_INSERTI64X4, UNKNOWN, (int) V8DI_FTYPE_V8DI_V4DI_INT_V8DI_QI },
32074 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv8df_mask, "__builtin_ia32_movapd512_mask", IX86_BUILTIN_MOVAPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
32075 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv16sf_mask, "__builtin_ia32_movaps512_mask", IX86_BUILTIN_MOVAPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
32076 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movddup512_mask, "__builtin_ia32_movddup512_mask", IX86_BUILTIN_MOVDDUP512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
32077 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv16si_mask, "__builtin_ia32_movdqa32_512_mask", IX86_BUILTIN_MOVDQA32_512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
32078 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv8di_mask, "__builtin_ia32_movdqa64_512_mask", IX86_BUILTIN_MOVDQA64_512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32079 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movshdup512_mask, "__builtin_ia32_movshdup512_mask", IX86_BUILTIN_MOVSHDUP512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
32080 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movsldup512_mask, "__builtin_ia32_movsldup512_mask", IX86_BUILTIN_MOVSLDUP512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
32081 { OPTION_MASK_ISA_AVX512F, CODE_FOR_absv16si2_mask, "__builtin_ia32_pabsd512_mask", IX86_BUILTIN_PABSD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
32082 { OPTION_MASK_ISA_AVX512F, CODE_FOR_absv8di2_mask, "__builtin_ia32_pabsq512_mask", IX86_BUILTIN_PABSQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32083 { OPTION_MASK_ISA_AVX512F, CODE_FOR_addv16si3_mask, "__builtin_ia32_paddd512_mask", IX86_BUILTIN_PADDD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32084 { OPTION_MASK_ISA_AVX512F, CODE_FOR_addv8di3_mask, "__builtin_ia32_paddq512_mask", IX86_BUILTIN_PADDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32085 { OPTION_MASK_ISA_AVX512F, CODE_FOR_andv16si3_mask, "__builtin_ia32_pandd512_mask", IX86_BUILTIN_PANDD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32086 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_andnotv16si3_mask, "__builtin_ia32_pandnd512_mask", IX86_BUILTIN_PANDND512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32087 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_andnotv8di3_mask, "__builtin_ia32_pandnq512_mask", IX86_BUILTIN_PANDNQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32088 { OPTION_MASK_ISA_AVX512F, CODE_FOR_andv8di3_mask, "__builtin_ia32_pandq512_mask", IX86_BUILTIN_PANDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32089 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dupv16si_mask, "__builtin_ia32_pbroadcastd512", IX86_BUILTIN_PBROADCASTD512, UNKNOWN, (int) V16SI_FTYPE_V4SI_V16SI_HI },
32090 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dup_gprv16si_mask, "__builtin_ia32_pbroadcastd512_gpr_mask", IX86_BUILTIN_PBROADCASTD512_GPR, UNKNOWN, (int) V16SI_FTYPE_SI_V16SI_HI },
32091 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskb_vec_dupv8di, "__builtin_ia32_broadcastmb512", IX86_BUILTIN_PBROADCASTMB512, UNKNOWN, (int) V8DI_FTYPE_QI },
32092 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskw_vec_dupv16si, "__builtin_ia32_broadcastmw512", IX86_BUILTIN_PBROADCASTMW512, UNKNOWN, (int) V16SI_FTYPE_HI },
32093 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dupv8di_mask, "__builtin_ia32_pbroadcastq512", IX86_BUILTIN_PBROADCASTQ512, UNKNOWN, (int) V8DI_FTYPE_V2DI_V8DI_QI },
32094 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dup_gprv8di_mask, "__builtin_ia32_pbroadcastq512_gpr_mask", IX86_BUILTIN_PBROADCASTQ512_GPR, UNKNOWN, (int) V8DI_FTYPE_DI_V8DI_QI },
32095 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_eqv16si3_mask, "__builtin_ia32_pcmpeqd512_mask", IX86_BUILTIN_PCMPEQD512_MASK, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_HI },
32096 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_eqv8di3_mask, "__builtin_ia32_pcmpeqq512_mask", IX86_BUILTIN_PCMPEQQ512_MASK, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_QI },
32097 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_gtv16si3_mask, "__builtin_ia32_pcmpgtd512_mask", IX86_BUILTIN_PCMPGTD512_MASK, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_HI },
32098 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_gtv8di3_mask, "__builtin_ia32_pcmpgtq512_mask", IX86_BUILTIN_PCMPGTQ512_MASK, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_QI },
32099 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressv16si_mask, "__builtin_ia32_compresssi512_mask", IX86_BUILTIN_PCOMPRESSD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
32100 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressv8di_mask, "__builtin_ia32_compressdi512_mask", IX86_BUILTIN_PCOMPRESSQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32101 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16si_mask, "__builtin_ia32_expandsi512_mask", IX86_BUILTIN_PEXPANDD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
32102 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16si_maskz, "__builtin_ia32_expandsi512_maskz", IX86_BUILTIN_PEXPANDD512Z, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
32103 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8di_mask, "__builtin_ia32_expanddi512_mask", IX86_BUILTIN_PEXPANDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32104 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8di_maskz, "__builtin_ia32_expanddi512_maskz", IX86_BUILTIN_PEXPANDQ512Z, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32105 { OPTION_MASK_ISA_AVX512F, CODE_FOR_smaxv16si3_mask, "__builtin_ia32_pmaxsd512_mask", IX86_BUILTIN_PMAXSD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32106 { OPTION_MASK_ISA_AVX512F, CODE_FOR_smaxv8di3_mask, "__builtin_ia32_pmaxsq512_mask", IX86_BUILTIN_PMAXSQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32107 { OPTION_MASK_ISA_AVX512F, CODE_FOR_umaxv16si3_mask, "__builtin_ia32_pmaxud512_mask", IX86_BUILTIN_PMAXUD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32108 { OPTION_MASK_ISA_AVX512F, CODE_FOR_umaxv8di3_mask, "__builtin_ia32_pmaxuq512_mask", IX86_BUILTIN_PMAXUQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32109 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sminv16si3_mask, "__builtin_ia32_pminsd512_mask", IX86_BUILTIN_PMINSD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32110 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sminv8di3_mask, "__builtin_ia32_pminsq512_mask", IX86_BUILTIN_PMINSQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32111 { OPTION_MASK_ISA_AVX512F, CODE_FOR_uminv16si3_mask, "__builtin_ia32_pminud512_mask", IX86_BUILTIN_PMINUD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32112 { OPTION_MASK_ISA_AVX512F, CODE_FOR_uminv8di3_mask, "__builtin_ia32_pminuq512_mask", IX86_BUILTIN_PMINUQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32113 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev16siv16qi2_mask, "__builtin_ia32_pmovdb512_mask", IX86_BUILTIN_PMOVDB512, UNKNOWN, (int) V16QI_FTYPE_V16SI_V16QI_HI },
32114 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev16siv16hi2_mask, "__builtin_ia32_pmovdw512_mask", IX86_BUILTIN_PMOVDW512, UNKNOWN, (int) V16HI_FTYPE_V16SI_V16HI_HI },
32115 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div16qi2_mask, "__builtin_ia32_pmovqb512_mask", IX86_BUILTIN_PMOVQB512, UNKNOWN, (int) V16QI_FTYPE_V8DI_V16QI_QI },
32116 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div8si2_mask, "__builtin_ia32_pmovqd512_mask", IX86_BUILTIN_PMOVQD512, UNKNOWN, (int) V8SI_FTYPE_V8DI_V8SI_QI },
32117 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div8hi2_mask, "__builtin_ia32_pmovqw512_mask", IX86_BUILTIN_PMOVQW512, UNKNOWN, (int) V8HI_FTYPE_V8DI_V8HI_QI },
32118 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev16siv16qi2_mask, "__builtin_ia32_pmovsdb512_mask", IX86_BUILTIN_PMOVSDB512, UNKNOWN, (int) V16QI_FTYPE_V16SI_V16QI_HI },
32119 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev16siv16hi2_mask, "__builtin_ia32_pmovsdw512_mask", IX86_BUILTIN_PMOVSDW512, UNKNOWN, (int) V16HI_FTYPE_V16SI_V16HI_HI },
32120 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div16qi2_mask, "__builtin_ia32_pmovsqb512_mask", IX86_BUILTIN_PMOVSQB512, UNKNOWN, (int) V16QI_FTYPE_V8DI_V16QI_QI },
32121 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div8si2_mask, "__builtin_ia32_pmovsqd512_mask", IX86_BUILTIN_PMOVSQD512, UNKNOWN, (int) V8SI_FTYPE_V8DI_V8SI_QI },
32122 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div8hi2_mask, "__builtin_ia32_pmovsqw512_mask", IX86_BUILTIN_PMOVSQW512, UNKNOWN, (int) V8HI_FTYPE_V8DI_V8HI_QI },
32123 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sign_extendv16qiv16si2_mask, "__builtin_ia32_pmovsxbd512_mask", IX86_BUILTIN_PMOVSXBD512, UNKNOWN, (int) V16SI_FTYPE_V16QI_V16SI_HI },
32124 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sign_extendv8qiv8di2_mask, "__builtin_ia32_pmovsxbq512_mask", IX86_BUILTIN_PMOVSXBQ512, UNKNOWN, (int) V8DI_FTYPE_V16QI_V8DI_QI },
32125 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sign_extendv8siv8di2_mask, "__builtin_ia32_pmovsxdq512_mask", IX86_BUILTIN_PMOVSXDQ512, UNKNOWN, (int) V8DI_FTYPE_V8SI_V8DI_QI },
32126 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sign_extendv16hiv16si2_mask, "__builtin_ia32_pmovsxwd512_mask", IX86_BUILTIN_PMOVSXWD512, UNKNOWN, (int) V16SI_FTYPE_V16HI_V16SI_HI },
32127 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sign_extendv8hiv8di2_mask, "__builtin_ia32_pmovsxwq512_mask", IX86_BUILTIN_PMOVSXWQ512, UNKNOWN, (int) V8DI_FTYPE_V8HI_V8DI_QI },
32128 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev16siv16qi2_mask, "__builtin_ia32_pmovusdb512_mask", IX86_BUILTIN_PMOVUSDB512, UNKNOWN, (int) V16QI_FTYPE_V16SI_V16QI_HI },
32129 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev16siv16hi2_mask, "__builtin_ia32_pmovusdw512_mask", IX86_BUILTIN_PMOVUSDW512, UNKNOWN, (int) V16HI_FTYPE_V16SI_V16HI_HI },
32130 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div16qi2_mask, "__builtin_ia32_pmovusqb512_mask", IX86_BUILTIN_PMOVUSQB512, UNKNOWN, (int) V16QI_FTYPE_V8DI_V16QI_QI },
32131 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div8si2_mask, "__builtin_ia32_pmovusqd512_mask", IX86_BUILTIN_PMOVUSQD512, UNKNOWN, (int) V8SI_FTYPE_V8DI_V8SI_QI },
32132 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div8hi2_mask, "__builtin_ia32_pmovusqw512_mask", IX86_BUILTIN_PMOVUSQW512, UNKNOWN, (int) V8HI_FTYPE_V8DI_V8HI_QI },
32133 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_zero_extendv16qiv16si2_mask, "__builtin_ia32_pmovzxbd512_mask", IX86_BUILTIN_PMOVZXBD512, UNKNOWN, (int) V16SI_FTYPE_V16QI_V16SI_HI },
32134 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_zero_extendv8qiv8di2_mask, "__builtin_ia32_pmovzxbq512_mask", IX86_BUILTIN_PMOVZXBQ512, UNKNOWN, (int) V8DI_FTYPE_V16QI_V8DI_QI },
32135 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_zero_extendv8siv8di2_mask, "__builtin_ia32_pmovzxdq512_mask", IX86_BUILTIN_PMOVZXDQ512, UNKNOWN, (int) V8DI_FTYPE_V8SI_V8DI_QI },
32136 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_zero_extendv16hiv16si2_mask, "__builtin_ia32_pmovzxwd512_mask", IX86_BUILTIN_PMOVZXWD512, UNKNOWN, (int) V16SI_FTYPE_V16HI_V16SI_HI },
32137 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_zero_extendv8hiv8di2_mask, "__builtin_ia32_pmovzxwq512_mask", IX86_BUILTIN_PMOVZXWQ512, UNKNOWN, (int) V8DI_FTYPE_V8HI_V8DI_QI },
32138 { OPTION_MASK_ISA_AVX512F, CODE_FOR_vec_widen_smult_even_v16si_mask, "__builtin_ia32_pmuldq512_mask", IX86_BUILTIN_PMULDQ512, UNKNOWN, (int) V8DI_FTYPE_V16SI_V16SI_V8DI_QI },
32139 { OPTION_MASK_ISA_AVX512F, CODE_FOR_mulv16si3_mask, "__builtin_ia32_pmulld512_mask" , IX86_BUILTIN_PMULLD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32140 { OPTION_MASK_ISA_AVX512F, CODE_FOR_vec_widen_umult_even_v16si_mask, "__builtin_ia32_pmuludq512_mask", IX86_BUILTIN_PMULUDQ512, UNKNOWN, (int) V8DI_FTYPE_V16SI_V16SI_V8DI_QI },
32141 { OPTION_MASK_ISA_AVX512F, CODE_FOR_iorv16si3_mask, "__builtin_ia32_pord512_mask", IX86_BUILTIN_PORD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32142 { OPTION_MASK_ISA_AVX512F, CODE_FOR_iorv8di3_mask, "__builtin_ia32_porq512_mask", IX86_BUILTIN_PORQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32143 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rolv16si_mask, "__builtin_ia32_prold512_mask", IX86_BUILTIN_PROLD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32144 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rolv8di_mask, "__builtin_ia32_prolq512_mask", IX86_BUILTIN_PROLQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32145 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rolvv16si_mask, "__builtin_ia32_prolvd512_mask", IX86_BUILTIN_PROLVD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32146 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rolvv8di_mask, "__builtin_ia32_prolvq512_mask", IX86_BUILTIN_PROLVQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32147 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rorv16si_mask, "__builtin_ia32_prord512_mask", IX86_BUILTIN_PRORD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32148 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rorv8di_mask, "__builtin_ia32_prorq512_mask", IX86_BUILTIN_PRORQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32149 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rorvv16si_mask, "__builtin_ia32_prorvd512_mask", IX86_BUILTIN_PRORVD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32150 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rorvv8di_mask, "__builtin_ia32_prorvq512_mask", IX86_BUILTIN_PRORVQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32151 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_pshufdv3_mask, "__builtin_ia32_pshufd512_mask", IX86_BUILTIN_PSHUFD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32152 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashlv16si3_mask, "__builtin_ia32_pslld512_mask", IX86_BUILTIN_PSLLD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V4SI_V16SI_HI },
32153 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashlv16si3_mask, "__builtin_ia32_pslldi512_mask", IX86_BUILTIN_PSLLDI512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32154 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashlv8di3_mask, "__builtin_ia32_psllq512_mask", IX86_BUILTIN_PSLLQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V2DI_V8DI_QI },
32155 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashlv8di3_mask, "__builtin_ia32_psllqi512_mask", IX86_BUILTIN_PSLLQI512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32156 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ashlvv16si_mask, "__builtin_ia32_psllv16si_mask", IX86_BUILTIN_PSLLVV16SI, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32157 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ashlvv8di_mask, "__builtin_ia32_psllv8di_mask", IX86_BUILTIN_PSLLVV8DI, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32158 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashrv16si3_mask, "__builtin_ia32_psrad512_mask", IX86_BUILTIN_PSRAD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V4SI_V16SI_HI },
32159 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashrv16si3_mask, "__builtin_ia32_psradi512_mask", IX86_BUILTIN_PSRADI512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32160 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashrv8di3_mask, "__builtin_ia32_psraq512_mask", IX86_BUILTIN_PSRAQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V2DI_V8DI_QI },
32161 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashrv8di3_mask, "__builtin_ia32_psraqi512_mask", IX86_BUILTIN_PSRAQI512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32162 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ashrvv16si_mask, "__builtin_ia32_psrav16si_mask", IX86_BUILTIN_PSRAVV16SI, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32163 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ashrvv8di_mask, "__builtin_ia32_psrav8di_mask", IX86_BUILTIN_PSRAVV8DI, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32164 { OPTION_MASK_ISA_AVX512F, CODE_FOR_lshrv16si3_mask, "__builtin_ia32_psrld512_mask", IX86_BUILTIN_PSRLD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V4SI_V16SI_HI },
32165 { OPTION_MASK_ISA_AVX512F, CODE_FOR_lshrv16si3_mask, "__builtin_ia32_psrldi512_mask", IX86_BUILTIN_PSRLDI512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32166 { OPTION_MASK_ISA_AVX512F, CODE_FOR_lshrv8di3_mask, "__builtin_ia32_psrlq512_mask", IX86_BUILTIN_PSRLQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V2DI_V8DI_QI },
32167 { OPTION_MASK_ISA_AVX512F, CODE_FOR_lshrv8di3_mask, "__builtin_ia32_psrlqi512_mask", IX86_BUILTIN_PSRLQI512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32168 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_lshrvv16si_mask, "__builtin_ia32_psrlv16si_mask", IX86_BUILTIN_PSRLVV16SI, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32169 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_lshrvv8di_mask, "__builtin_ia32_psrlv8di_mask", IX86_BUILTIN_PSRLVV8DI, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32170 { OPTION_MASK_ISA_AVX512F, CODE_FOR_subv16si3_mask, "__builtin_ia32_psubd512_mask", IX86_BUILTIN_PSUBD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32171 { OPTION_MASK_ISA_AVX512F, CODE_FOR_subv8di3_mask, "__builtin_ia32_psubq512_mask", IX86_BUILTIN_PSUBQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32172 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_testmv16si3_mask, "__builtin_ia32_ptestmd512", IX86_BUILTIN_PTESTMD512, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_HI },
32173 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_testmv8di3_mask, "__builtin_ia32_ptestmq512", IX86_BUILTIN_PTESTMQ512, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_QI },
32174 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_testnmv16si3_mask, "__builtin_ia32_ptestnmd512", IX86_BUILTIN_PTESTNMD512, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_HI },
32175 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_testnmv8di3_mask, "__builtin_ia32_ptestnmq512", IX86_BUILTIN_PTESTNMQ512, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_QI },
32176 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_interleave_highv16si_mask, "__builtin_ia32_punpckhdq512_mask", IX86_BUILTIN_PUNPCKHDQ512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32177 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_interleave_highv8di_mask, "__builtin_ia32_punpckhqdq512_mask", IX86_BUILTIN_PUNPCKHQDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32178 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_interleave_lowv16si_mask, "__builtin_ia32_punpckldq512_mask", IX86_BUILTIN_PUNPCKLDQ512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32179 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_interleave_lowv8di_mask, "__builtin_ia32_punpcklqdq512_mask", IX86_BUILTIN_PUNPCKLQDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32180 { OPTION_MASK_ISA_AVX512F, CODE_FOR_xorv16si3_mask, "__builtin_ia32_pxord512_mask", IX86_BUILTIN_PXORD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32181 { OPTION_MASK_ISA_AVX512F, CODE_FOR_xorv8di3_mask, "__builtin_ia32_pxorq512_mask", IX86_BUILTIN_PXORQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32182 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rcp14v8df_mask, "__builtin_ia32_rcp14pd512_mask", IX86_BUILTIN_RCP14PD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
32183 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rcp14v16sf_mask, "__builtin_ia32_rcp14ps512_mask", IX86_BUILTIN_RCP14PS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
32184 { OPTION_MASK_ISA_AVX512F, CODE_FOR_srcp14v2df, "__builtin_ia32_rcp14sd", IX86_BUILTIN_RCP14SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
32185 { OPTION_MASK_ISA_AVX512F, CODE_FOR_srcp14v4sf, "__builtin_ia32_rcp14ss", IX86_BUILTIN_RCP14SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
32186 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rsqrt14v8df_mask, "__builtin_ia32_rsqrt14pd512_mask", IX86_BUILTIN_RSQRT14PD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
32187 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rsqrt14v16sf_mask, "__builtin_ia32_rsqrt14ps512_mask", IX86_BUILTIN_RSQRT14PS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
32188 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rsqrt14v2df, "__builtin_ia32_rsqrt14sd", IX86_BUILTIN_RSQRT14SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
32189 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rsqrt14v4sf, "__builtin_ia32_rsqrt14ss", IX86_BUILTIN_RSQRT14SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
32190 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shufpd512_mask, "__builtin_ia32_shufpd512_mask", IX86_BUILTIN_SHUFPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI },
32191 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shufps512_mask, "__builtin_ia32_shufps512_mask", IX86_BUILTIN_SHUFPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI },
32192 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shuf_f32x4_mask, "__builtin_ia32_shuf_f32x4_mask", IX86_BUILTIN_SHUF_F32x4, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI },
32193 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shuf_f64x2_mask, "__builtin_ia32_shuf_f64x2_mask", IX86_BUILTIN_SHUF_F64x2, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI },
32194 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shuf_i32x4_mask, "__builtin_ia32_shuf_i32x4_mask", IX86_BUILTIN_SHUF_I32x4, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_INT_V16SI_HI },
32195 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shuf_i64x2_mask, "__builtin_ia32_shuf_i64x2_mask", IX86_BUILTIN_SHUF_I64x2, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_INT_V8DI_QI },
32196 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ucmpv16si3_mask, "__builtin_ia32_ucmpd512_mask", IX86_BUILTIN_UCMPD512, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_INT_HI },
32197 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ucmpv8di3_mask, "__builtin_ia32_ucmpq512_mask", IX86_BUILTIN_UCMPQ512, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_INT_QI },
32198 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_unpckhpd512_mask, "__builtin_ia32_unpckhpd512_mask", IX86_BUILTIN_UNPCKHPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI },
32199 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_unpckhps512_mask, "__builtin_ia32_unpckhps512_mask", IX86_BUILTIN_UNPCKHPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32200 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_unpcklpd512_mask, "__builtin_ia32_unpcklpd512_mask", IX86_BUILTIN_UNPCKLPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI },
32201 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_unpcklps512_mask, "__builtin_ia32_unpcklps512_mask", IX86_BUILTIN_UNPCKLPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32202 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_clzv16si2_mask, "__builtin_ia32_vplzcntd_512_mask", IX86_BUILTIN_VPCLZCNTD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
32203 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_clzv8di2_mask, "__builtin_ia32_vplzcntq_512_mask", IX86_BUILTIN_VPCLZCNTQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32204 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_conflictv16si_mask, "__builtin_ia32_vpconflictsi_512_mask", IX86_BUILTIN_VPCONFLICTD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
32205 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_conflictv8di_mask, "__builtin_ia32_vpconflictdi_512_mask", IX86_BUILTIN_VPCONFLICTQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32206 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permv8df_mask, "__builtin_ia32_permdf512_mask", IX86_BUILTIN_VPERMDF512, UNKNOWN, (int) V8DF_FTYPE_V8DF_INT_V8DF_QI },
32207 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permv8di_mask, "__builtin_ia32_permdi512_mask", IX86_BUILTIN_VPERMDI512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32208 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermi2varv16si3_mask, "__builtin_ia32_vpermi2vard512_mask", IX86_BUILTIN_VPERMI2VARD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32209 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermi2varv8df3_mask, "__builtin_ia32_vpermi2varpd512_mask", IX86_BUILTIN_VPERMI2VARPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DI_V8DF_QI },
32210 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermi2varv16sf3_mask, "__builtin_ia32_vpermi2varps512_mask", IX86_BUILTIN_VPERMI2VARPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SI_V16SF_HI },
32211 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermi2varv8di3_mask, "__builtin_ia32_vpermi2varq512_mask", IX86_BUILTIN_VPERMI2VARQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32212 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermilv8df_mask, "__builtin_ia32_vpermilpd512_mask", IX86_BUILTIN_VPERMILPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_INT_V8DF_QI },
32213 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermilv16sf_mask, "__builtin_ia32_vpermilps512_mask", IX86_BUILTIN_VPERMILPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_INT_V16SF_HI },
32214 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermilvarv8df3_mask, "__builtin_ia32_vpermilvarpd512_mask", IX86_BUILTIN_VPERMILVARPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DI_V8DF_QI },
32215 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermilvarv16sf3_mask, "__builtin_ia32_vpermilvarps512_mask", IX86_BUILTIN_VPERMILVARPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SI_V16SF_HI },
32216 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv16si3_mask, "__builtin_ia32_vpermt2vard512_mask", IX86_BUILTIN_VPERMT2VARD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32217 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv16si3_maskz, "__builtin_ia32_vpermt2vard512_maskz", IX86_BUILTIN_VPERMT2VARD512_MASKZ, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32218 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv8df3_mask, "__builtin_ia32_vpermt2varpd512_mask", IX86_BUILTIN_VPERMT2VARPD512, UNKNOWN, (int) V8DF_FTYPE_V8DI_V8DF_V8DF_QI },
32219 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv8df3_maskz, "__builtin_ia32_vpermt2varpd512_maskz", IX86_BUILTIN_VPERMT2VARPD512_MASKZ, UNKNOWN, (int) V8DF_FTYPE_V8DI_V8DF_V8DF_QI },
32220 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv16sf3_mask, "__builtin_ia32_vpermt2varps512_mask", IX86_BUILTIN_VPERMT2VARPS512, UNKNOWN, (int) V16SF_FTYPE_V16SI_V16SF_V16SF_HI },
32221 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv16sf3_maskz, "__builtin_ia32_vpermt2varps512_maskz", IX86_BUILTIN_VPERMT2VARPS512_MASKZ, UNKNOWN, (int) V16SF_FTYPE_V16SI_V16SF_V16SF_HI },
32222 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv8di3_mask, "__builtin_ia32_vpermt2varq512_mask", IX86_BUILTIN_VPERMT2VARQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32223 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv8di3_maskz, "__builtin_ia32_vpermt2varq512_maskz", IX86_BUILTIN_VPERMT2VARQ512_MASKZ, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32224 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permvarv8df_mask, "__builtin_ia32_permvardf512_mask", IX86_BUILTIN_VPERMVARDF512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DI_V8DF_QI },
32225 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permvarv8di_mask, "__builtin_ia32_permvardi512_mask", IX86_BUILTIN_VPERMVARDI512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32226 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permvarv16sf_mask, "__builtin_ia32_permvarsf512_mask", IX86_BUILTIN_VPERMVARSF512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SI_V16SF_HI },
32227 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permvarv16si_mask, "__builtin_ia32_permvarsi512_mask", IX86_BUILTIN_VPERMVARSI512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32228 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vternlogv16si_mask, "__builtin_ia32_pternlogd512_mask", IX86_BUILTIN_VTERNLOGD512_MASK, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_INT_HI },
32229 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vternlogv16si_maskz, "__builtin_ia32_pternlogd512_maskz", IX86_BUILTIN_VTERNLOGD512_MASKZ, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_INT_HI },
32230 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vternlogv8di_mask, "__builtin_ia32_pternlogq512_mask", IX86_BUILTIN_VTERNLOGQ512_MASK, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_INT_QI },
32231 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vternlogv8di_maskz, "__builtin_ia32_pternlogq512_maskz", IX86_BUILTIN_VTERNLOGQ512_MASKZ, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_INT_QI },
32233 { OPTION_MASK_ISA_AVX512F, CODE_FOR_copysignv16sf3, "__builtin_ia32_copysignps512", IX86_BUILTIN_CPYSGNPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF },
32234 { OPTION_MASK_ISA_AVX512F, CODE_FOR_copysignv8df3, "__builtin_ia32_copysignpd512", IX86_BUILTIN_CPYSGNPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF },
32235 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sqrtv8df2, "__builtin_ia32_sqrtpd512", IX86_BUILTIN_SQRTPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF },
32236 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sqrtv16sf2, "__builtin_ia32_sqrtps512", IX86_BUILTIN_SQRTPS_NR512, UNKNOWN, (int) V16SF_FTYPE_V16SF },
32237 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_exp2v16sf, "__builtin_ia32_exp2ps", IX86_BUILTIN_EXP2PS, UNKNOWN, (int) V16SF_FTYPE_V16SF },
32238 { OPTION_MASK_ISA_AVX512F, CODE_FOR_roundv8df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix512", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX512, UNKNOWN, (int) V16SI_FTYPE_V8DF_V8DF },
32239 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_roundpd_vec_pack_sfix512, "__builtin_ia32_floorpd_vec_pack_sfix512", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX512, (enum rtx_code) ROUND_FLOOR, (int) V16SI_FTYPE_V8DF_V8DF_ROUND },
32240 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_roundpd_vec_pack_sfix512, "__builtin_ia32_ceilpd_vec_pack_sfix512", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX512, (enum rtx_code) ROUND_CEIL, (int) V16SI_FTYPE_V8DF_V8DF_ROUND },
32242 /* Mask arithmetic operations */
32243 { OPTION_MASK_ISA_AVX512F, CODE_FOR_andhi3, "__builtin_ia32_kandhi", IX86_BUILTIN_KAND16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32244 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kandnhi, "__builtin_ia32_kandnhi", IX86_BUILTIN_KANDN16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32245 { OPTION_MASK_ISA_AVX512F, CODE_FOR_one_cmplhi2, "__builtin_ia32_knothi", IX86_BUILTIN_KNOT16, UNKNOWN, (int) HI_FTYPE_HI },
32246 { OPTION_MASK_ISA_AVX512F, CODE_FOR_iorhi3, "__builtin_ia32_korhi", IX86_BUILTIN_KOR16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32247 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kortestchi, "__builtin_ia32_kortestchi", IX86_BUILTIN_KORTESTC16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32248 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kortestzhi, "__builtin_ia32_kortestzhi", IX86_BUILTIN_KORTESTZ16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32249 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kunpckhi, "__builtin_ia32_kunpckhi", IX86_BUILTIN_KUNPCKBW, UNKNOWN, (int) HI_FTYPE_HI_HI },
32250 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kxnorhi, "__builtin_ia32_kxnorhi", IX86_BUILTIN_KXNOR16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32251 { OPTION_MASK_ISA_AVX512F, CODE_FOR_xorhi3, "__builtin_ia32_kxorhi", IX86_BUILTIN_KXOR16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32252 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kmovw, "__builtin_ia32_kmov16", IX86_BUILTIN_KMOV16, UNKNOWN, (int) HI_FTYPE_HI },
32255 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha1msg1, 0, IX86_BUILTIN_SHA1MSG1, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32256 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha1msg2, 0, IX86_BUILTIN_SHA1MSG2, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32257 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha1nexte, 0, IX86_BUILTIN_SHA1NEXTE, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32258 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha1rnds4, 0, IX86_BUILTIN_SHA1RNDS4, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT },
32259 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha256msg1, 0, IX86_BUILTIN_SHA256MSG1, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32260 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha256msg2, 0, IX86_BUILTIN_SHA256MSG2, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32261 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha256rnds2, 0, IX86_BUILTIN_SHA256RNDS2, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI },
32264 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_palignrv32qi_mask, "__builtin_ia32_palignr256_mask", IX86_BUILTIN_PALIGNR256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_V4DI_SI_CONVERT },
32265 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ssse3_palignrv16qi_mask, "__builtin_ia32_palignr128_mask", IX86_BUILTIN_PALIGNR128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_V2DI_HI_CONVERT },
32266 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4di_mask, "__builtin_ia32_movdqa64_256_mask", IX86_BUILTIN_MOVDQA64_256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32267 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv2di_mask, "__builtin_ia32_movdqa64_128_mask", IX86_BUILTIN_MOVDQA64_128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32268 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv8si_mask, "__builtin_ia32_movdqa32_256_mask", IX86_BUILTIN_MOVDQA32_256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32269 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4si_mask, "__builtin_ia32_movdqa32_128_mask", IX86_BUILTIN_MOVDQA32_128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32270 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4df_mask, "__builtin_ia32_movapd256_mask", IX86_BUILTIN_MOVAPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32271 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv2df_mask, "__builtin_ia32_movapd128_mask", IX86_BUILTIN_MOVAPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32272 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv8sf_mask, "__builtin_ia32_movaps256_mask", IX86_BUILTIN_MOVAPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32273 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4sf_mask, "__builtin_ia32_movaps128_mask", IX86_BUILTIN_MOVAPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32274 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv16hi_mask, "__builtin_ia32_movdquhi256_mask", IX86_BUILTIN_MOVDQUHI256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_HI },
32275 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv8hi_mask, "__builtin_ia32_movdquhi128_mask", IX86_BUILTIN_MOVDQUHI128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_QI },
32276 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_loaddquv32qi_mask, "__builtin_ia32_movdquqi256_mask", IX86_BUILTIN_MOVDQUQI256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_SI },
32277 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_loaddquv16qi_mask, "__builtin_ia32_movdquqi128_mask", IX86_BUILTIN_MOVDQUQI128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_HI },
32278 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv4sf3_mask, "__builtin_ia32_minps_mask", IX86_BUILTIN_MINPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32279 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv4sf3_mask, "__builtin_ia32_maxps_mask", IX86_BUILTIN_MAXPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32280 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv2df3_mask, "__builtin_ia32_minpd_mask", IX86_BUILTIN_MINPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32281 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv2df3_mask, "__builtin_ia32_maxpd_mask", IX86_BUILTIN_MAXPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32282 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv4df3_mask, "__builtin_ia32_maxpd256_mask", IX86_BUILTIN_MAXPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32283 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv8sf3_mask, "__builtin_ia32_maxps256_mask", IX86_BUILTIN_MAXPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32284 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv4df3_mask, "__builtin_ia32_minpd256_mask", IX86_BUILTIN_MINPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32285 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv8sf3_mask, "__builtin_ia32_minps256_mask", IX86_BUILTIN_MINPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32286 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv4sf3_mask, "__builtin_ia32_mulps_mask", IX86_BUILTIN_MULPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32287 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse_divv4sf3_mask, "__builtin_ia32_divps_mask", IX86_BUILTIN_DIVPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32288 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv2df3_mask, "__builtin_ia32_mulpd_mask", IX86_BUILTIN_MULPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32289 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_divv2df3_mask, "__builtin_ia32_divpd_mask", IX86_BUILTIN_DIVPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32290 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_divv4df3_mask, "__builtin_ia32_divpd256_mask", IX86_BUILTIN_DIVPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32291 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_divv8sf3_mask, "__builtin_ia32_divps256_mask", IX86_BUILTIN_DIVPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32292 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv4df3_mask, "__builtin_ia32_mulpd256_mask", IX86_BUILTIN_MULPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32293 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv8sf3_mask, "__builtin_ia32_mulps256_mask", IX86_BUILTIN_MULPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32294 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv2df3_mask, "__builtin_ia32_addpd128_mask", IX86_BUILTIN_ADDPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32295 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv4df3_mask, "__builtin_ia32_addpd256_mask", IX86_BUILTIN_ADDPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32296 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv4sf3_mask, "__builtin_ia32_addps128_mask", IX86_BUILTIN_ADDPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32297 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv8sf3_mask, "__builtin_ia32_addps256_mask", IX86_BUILTIN_ADDPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32298 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv2df3_mask, "__builtin_ia32_subpd128_mask", IX86_BUILTIN_SUBPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32299 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv4df3_mask, "__builtin_ia32_subpd256_mask", IX86_BUILTIN_SUBPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32300 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv4sf3_mask, "__builtin_ia32_subps128_mask", IX86_BUILTIN_SUBPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32301 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv8sf3_mask, "__builtin_ia32_subps256_mask", IX86_BUILTIN_SUBPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32302 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv4df3_mask, "__builtin_ia32_xorpd256_mask", IX86_BUILTIN_XORPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32303 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv2df3_mask, "__builtin_ia32_xorpd128_mask", IX86_BUILTIN_XORPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32304 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv8sf3_mask, "__builtin_ia32_xorps256_mask", IX86_BUILTIN_XORPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32305 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv4sf3_mask, "__builtin_ia32_xorps128_mask", IX86_BUILTIN_XORPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32306 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv4df3_mask, "__builtin_ia32_orpd256_mask", IX86_BUILTIN_ORPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32307 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv2df3_mask, "__builtin_ia32_orpd128_mask", IX86_BUILTIN_ORPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32308 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv8sf3_mask, "__builtin_ia32_orps256_mask", IX86_BUILTIN_ORPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32309 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv4sf3_mask, "__builtin_ia32_orps128_mask", IX86_BUILTIN_ORPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32310 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_broadcastv8sf_mask, "__builtin_ia32_broadcastf32x2_256_mask", IX86_BUILTIN_BROADCASTF32x2_256, UNKNOWN, (int) V8SF_FTYPE_V4SF_V8SF_QI },
32311 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_broadcastv8si_mask, "__builtin_ia32_broadcasti32x2_256_mask", IX86_BUILTIN_BROADCASTI32x2_256, UNKNOWN, (int) V8SI_FTYPE_V4SI_V8SI_QI },
32312 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_broadcastv4si_mask, "__builtin_ia32_broadcasti32x2_128_mask", IX86_BUILTIN_BROADCASTI32x2_128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32313 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_broadcastv4df_mask_1, "__builtin_ia32_broadcastf64x2_256_mask", IX86_BUILTIN_BROADCASTF64X2_256, UNKNOWN, (int) V4DF_FTYPE_V2DF_V4DF_QI },
32314 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_broadcastv4di_mask_1, "__builtin_ia32_broadcasti64x2_256_mask", IX86_BUILTIN_BROADCASTI64X2_256, UNKNOWN, (int) V4DI_FTYPE_V2DI_V4DI_QI },
32315 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_broadcastv8sf_mask_1, "__builtin_ia32_broadcastf32x4_256_mask", IX86_BUILTIN_BROADCASTF32X4_256, UNKNOWN, (int) V8SF_FTYPE_V4SF_V8SF_QI },
32316 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_broadcastv8si_mask_1, "__builtin_ia32_broadcasti32x4_256_mask", IX86_BUILTIN_BROADCASTI32X4_256, UNKNOWN, (int) V8SI_FTYPE_V4SI_V8SI_QI },
32317 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vextractf128v8sf, "__builtin_ia32_extractf32x4_256_mask", IX86_BUILTIN_EXTRACTF32X4_256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT_V4SF_QI },
32318 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vextractf128v8si, "__builtin_ia32_extracti32x4_256_mask", IX86_BUILTIN_EXTRACTI32X4_256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT_V4SI_QI },
32319 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_dbpsadbwv16hi_mask, "__builtin_ia32_dbpsadbw256_mask", IX86_BUILTIN_DBPSADBW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI_INT_V16HI_HI },
32320 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_dbpsadbwv8hi_mask, "__builtin_ia32_dbpsadbw128_mask", IX86_BUILTIN_DBPSADBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI_INT_V8HI_QI },
32321 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv4dfv4di2_mask, "__builtin_ia32_cvttpd2qq256_mask", IX86_BUILTIN_CVTTPD2QQ256, UNKNOWN, (int) V4DI_FTYPE_V4DF_V4DI_QI },
32322 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv2dfv2di2_mask, "__builtin_ia32_cvttpd2qq128_mask", IX86_BUILTIN_CVTTPD2QQ128, UNKNOWN, (int) V2DI_FTYPE_V2DF_V2DI_QI },
32323 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv4dfv4di2_mask, "__builtin_ia32_cvttpd2uqq256_mask", IX86_BUILTIN_CVTTPD2UQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DF_V4DI_QI },
32324 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv2dfv2di2_mask, "__builtin_ia32_cvttpd2uqq128_mask", IX86_BUILTIN_CVTTPD2UQQ128, UNKNOWN, (int) V2DI_FTYPE_V2DF_V2DI_QI },
32325 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_notruncv4dfv4di2_mask, "__builtin_ia32_cvtpd2qq256_mask", IX86_BUILTIN_CVTPD2QQ256, UNKNOWN, (int) V4DI_FTYPE_V4DF_V4DI_QI },
32326 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_notruncv2dfv2di2_mask, "__builtin_ia32_cvtpd2qq128_mask", IX86_BUILTIN_CVTPD2QQ128, UNKNOWN, (int) V2DI_FTYPE_V2DF_V2DI_QI },
32327 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_notruncv4dfv4di2_mask, "__builtin_ia32_cvtpd2uqq256_mask", IX86_BUILTIN_CVTPD2UQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DF_V4DI_QI },
32328 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_notruncv2dfv2di2_mask, "__builtin_ia32_cvtpd2uqq128_mask", IX86_BUILTIN_CVTPD2UQQ128, UNKNOWN, (int) V2DI_FTYPE_V2DF_V2DI_QI },
32329 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_notruncv4dfv4si2_mask, "__builtin_ia32_cvtpd2udq256_mask", IX86_BUILTIN_CVTPD2UDQ256_MASK, UNKNOWN, (int) V4SI_FTYPE_V4DF_V4SI_QI },
32330 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_notruncv2dfv2si2_mask, "__builtin_ia32_cvtpd2udq128_mask", IX86_BUILTIN_CVTPD2UDQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V2DF_V4SI_QI },
32331 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv4sfv4di2_mask, "__builtin_ia32_cvttps2qq256_mask", IX86_BUILTIN_CVTTPS2QQ256, UNKNOWN, (int) V4DI_FTYPE_V4SF_V4DI_QI },
32332 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv2sfv2di2_mask, "__builtin_ia32_cvttps2qq128_mask", IX86_BUILTIN_CVTTPS2QQ128, UNKNOWN, (int) V2DI_FTYPE_V4SF_V2DI_QI },
32333 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv4sfv4di2_mask, "__builtin_ia32_cvttps2uqq256_mask", IX86_BUILTIN_CVTTPS2UQQ256, UNKNOWN, (int) V4DI_FTYPE_V4SF_V4DI_QI },
32334 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv2sfv2di2_mask, "__builtin_ia32_cvttps2uqq128_mask", IX86_BUILTIN_CVTTPS2UQQ128, UNKNOWN, (int) V2DI_FTYPE_V4SF_V2DI_QI },
32335 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv8sfv8si2_mask, "__builtin_ia32_cvttps2dq256_mask", IX86_BUILTIN_CVTTPS2DQ256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SF_V8SI_QI },
32336 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv4sfv4si2_mask, "__builtin_ia32_cvttps2dq128_mask", IX86_BUILTIN_CVTTPS2DQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SF_V4SI_QI },
32337 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv8sfv8si2_mask, "__builtin_ia32_cvttps2udq256_mask", IX86_BUILTIN_CVTTPS2UDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF_V8SI_QI },
32338 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv4sfv4si2_mask, "__builtin_ia32_cvttps2udq128_mask", IX86_BUILTIN_CVTTPS2UDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SF_V4SI_QI },
32339 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv4dfv4si2_mask, "__builtin_ia32_cvttpd2dq256_mask", IX86_BUILTIN_CVTTPD2DQ256_MASK, UNKNOWN, (int) V4SI_FTYPE_V4DF_V4SI_QI },
32340 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_cvttpd2dq_mask, "__builtin_ia32_cvttpd2dq128_mask", IX86_BUILTIN_CVTTPD2DQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V2DF_V4SI_QI },
32341 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv4dfv4si2_mask, "__builtin_ia32_cvttpd2udq256_mask", IX86_BUILTIN_CVTTPD2UDQ256_MASK, UNKNOWN, (int) V4SI_FTYPE_V4DF_V4SI_QI },
32342 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv2dfv2si2_mask, "__builtin_ia32_cvttpd2udq128_mask", IX86_BUILTIN_CVTTPD2UDQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V2DF_V4SI_QI },
32343 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_cvtpd2dq256_mask, "__builtin_ia32_cvtpd2dq256_mask", IX86_BUILTIN_CVTPD2DQ256_MASK, UNKNOWN, (int) V4SI_FTYPE_V4DF_V4SI_QI },
32344 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_cvtpd2dq_mask, "__builtin_ia32_cvtpd2dq128_mask", IX86_BUILTIN_CVTPD2DQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V2DF_V4SI_QI },
32345 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv4siv4df2_mask, "__builtin_ia32_cvtdq2pd256_mask", IX86_BUILTIN_CVTDQ2PD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4SI_V4DF_QI },
32346 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_cvtdq2pd_mask, "__builtin_ia32_cvtdq2pd128_mask", IX86_BUILTIN_CVTDQ2PD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V4SI_V2DF_QI },
32347 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv4siv4df2_mask, "__builtin_ia32_cvtudq2pd256_mask", IX86_BUILTIN_CVTUDQ2PD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4SI_V4DF_QI },
32348 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv2siv2df2_mask, "__builtin_ia32_cvtudq2pd128_mask", IX86_BUILTIN_CVTUDQ2PD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V4SI_V2DF_QI },
32349 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv8siv8sf2_mask, "__builtin_ia32_cvtdq2ps256_mask", IX86_BUILTIN_CVTDQ2PS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SI_V8SF_QI },
32350 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv4siv4sf2_mask, "__builtin_ia32_cvtdq2ps128_mask", IX86_BUILTIN_CVTDQ2PS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SI_V4SF_QI },
32351 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv8siv8sf2_mask, "__builtin_ia32_cvtudq2ps256_mask", IX86_BUILTIN_CVTUDQ2PS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SI_V8SF_QI },
32352 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv4siv4sf2_mask, "__builtin_ia32_cvtudq2ps128_mask", IX86_BUILTIN_CVTUDQ2PS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SI_V4SF_QI },
32353 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_cvtps2pd256_mask, "__builtin_ia32_cvtps2pd256_mask", IX86_BUILTIN_CVTPS2PD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4SF_V4DF_QI },
32354 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_cvtps2pd_mask, "__builtin_ia32_cvtps2pd128_mask", IX86_BUILTIN_CVTPS2PD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V4SF_V2DF_QI },
32355 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv32qi_mask, "__builtin_ia32_pbroadcastb256_mask", IX86_BUILTIN_PBROADCASTB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V16QI_V32QI_SI },
32356 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv32qi_mask, "__builtin_ia32_pbroadcastb256_gpr_mask", IX86_BUILTIN_PBROADCASTB256_GPR_MASK, UNKNOWN, (int) V32QI_FTYPE_QI_V32QI_SI },
32357 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv16qi_mask, "__builtin_ia32_pbroadcastb128_mask", IX86_BUILTIN_PBROADCASTB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_HI },
32358 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv16qi_mask, "__builtin_ia32_pbroadcastb128_gpr_mask", IX86_BUILTIN_PBROADCASTB128_GPR_MASK, UNKNOWN, (int) V16QI_FTYPE_QI_V16QI_HI },
32359 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv16hi_mask, "__builtin_ia32_pbroadcastw256_mask", IX86_BUILTIN_PBROADCASTW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V8HI_V16HI_HI },
32360 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv16hi_mask, "__builtin_ia32_pbroadcastw256_gpr_mask", IX86_BUILTIN_PBROADCASTW256_GPR_MASK, UNKNOWN, (int) V16HI_FTYPE_HI_V16HI_HI },
32361 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv8hi_mask, "__builtin_ia32_pbroadcastw128_mask", IX86_BUILTIN_PBROADCASTW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_QI },
32362 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv8hi_mask, "__builtin_ia32_pbroadcastw128_gpr_mask", IX86_BUILTIN_PBROADCASTW128_GPR_MASK, UNKNOWN, (int) V8HI_FTYPE_HI_V8HI_QI },
32363 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv8si_mask, "__builtin_ia32_pbroadcastd256_mask", IX86_BUILTIN_PBROADCASTD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V4SI_V8SI_QI },
32364 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv8si_mask, "__builtin_ia32_pbroadcastd256_gpr_mask", IX86_BUILTIN_PBROADCASTD256_GPR_MASK, UNKNOWN, (int) V8SI_FTYPE_SI_V8SI_QI },
32365 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv4si_mask, "__builtin_ia32_pbroadcastd128_mask", IX86_BUILTIN_PBROADCASTD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32366 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv4si_mask, "__builtin_ia32_pbroadcastd128_gpr_mask", IX86_BUILTIN_PBROADCASTD128_GPR_MASK, UNKNOWN, (int) V4SI_FTYPE_SI_V4SI_QI },
32367 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv4di_mask, "__builtin_ia32_pbroadcastq256_mask", IX86_BUILTIN_PBROADCASTQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V2DI_V4DI_QI },
32368 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv4di_mask, "__builtin_ia32_pbroadcastq256_gpr_mask", IX86_BUILTIN_PBROADCASTQ256_GPR_MASK, UNKNOWN, (int) V4DI_FTYPE_DI_V4DI_QI },
32369 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv2di_mask, "__builtin_ia32_pbroadcastq128_mask", IX86_BUILTIN_PBROADCASTQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32370 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv2di_mask, "__builtin_ia32_pbroadcastq128_gpr_mask", IX86_BUILTIN_PBROADCASTQ128_GPR_MASK, UNKNOWN, (int) V2DI_FTYPE_DI_V2DI_QI },
32371 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv8sf_mask, "__builtin_ia32_broadcastss256_mask", IX86_BUILTIN_BROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_V4SF_V8SF_QI },
32372 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv4sf_mask, "__builtin_ia32_broadcastss128_mask", IX86_BUILTIN_BROADCASTSS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32373 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv4df_mask, "__builtin_ia32_broadcastsd256_mask", IX86_BUILTIN_BROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_V2DF_V4DF_QI },
32374 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vextractf128v4df, "__builtin_ia32_extractf64x2_256_mask", IX86_BUILTIN_EXTRACTF64X2_256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT_V2DF_QI },
32375 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vextractf128v4di, "__builtin_ia32_extracti64x2_256_mask", IX86_BUILTIN_EXTRACTI64X2_256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT_V2DI_QI },
32376 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vinsertv8sf, "__builtin_ia32_insertf32x4_256_mask", IX86_BUILTIN_INSERTF32X4_256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT_V8SF_QI },
32377 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vinsertv8si, "__builtin_ia32_inserti32x4_256_mask", IX86_BUILTIN_INSERTI32X4_256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT_V8SI_QI },
32378 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv16qiv16hi2_mask, "__builtin_ia32_pmovsxbw256_mask", IX86_BUILTIN_PMOVSXBW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16QI_V16HI_HI },
32379 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv8qiv8hi2_mask, "__builtin_ia32_pmovsxbw128_mask", IX86_BUILTIN_PMOVSXBW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V16QI_V8HI_QI },
32380 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv8qiv8si2_mask, "__builtin_ia32_pmovsxbd256_mask", IX86_BUILTIN_PMOVSXBD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V16QI_V8SI_QI },
32381 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv4qiv4si2_mask, "__builtin_ia32_pmovsxbd128_mask", IX86_BUILTIN_PMOVSXBD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V16QI_V4SI_QI },
32382 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv4qiv4di2_mask, "__builtin_ia32_pmovsxbq256_mask", IX86_BUILTIN_PMOVSXBQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V16QI_V4DI_QI },
32383 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv2qiv2di2_mask, "__builtin_ia32_pmovsxbq128_mask", IX86_BUILTIN_PMOVSXBQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V16QI_V2DI_QI },
32384 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv8hiv8si2_mask, "__builtin_ia32_pmovsxwd256_mask", IX86_BUILTIN_PMOVSXWD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8HI_V8SI_QI },
32385 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv4hiv4si2_mask, "__builtin_ia32_pmovsxwd128_mask", IX86_BUILTIN_PMOVSXWD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V8HI_V4SI_QI },
32386 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv4hiv4di2_mask, "__builtin_ia32_pmovsxwq256_mask", IX86_BUILTIN_PMOVSXWQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V8HI_V4DI_QI },
32387 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv2hiv2di2_mask, "__builtin_ia32_pmovsxwq128_mask", IX86_BUILTIN_PMOVSXWQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V8HI_V2DI_QI },
32388 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv4siv4di2_mask, "__builtin_ia32_pmovsxdq256_mask", IX86_BUILTIN_PMOVSXDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4SI_V4DI_QI },
32389 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv2siv2di2_mask, "__builtin_ia32_pmovsxdq128_mask", IX86_BUILTIN_PMOVSXDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V4SI_V2DI_QI },
32390 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv16qiv16hi2_mask, "__builtin_ia32_pmovzxbw256_mask", IX86_BUILTIN_PMOVZXBW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16QI_V16HI_HI },
32391 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv8qiv8hi2_mask, "__builtin_ia32_pmovzxbw128_mask", IX86_BUILTIN_PMOVZXBW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V16QI_V8HI_QI },
32392 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv8qiv8si2_mask, "__builtin_ia32_pmovzxbd256_mask", IX86_BUILTIN_PMOVZXBD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V16QI_V8SI_QI },
32393 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv4qiv4si2_mask, "__builtin_ia32_pmovzxbd128_mask", IX86_BUILTIN_PMOVZXBD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V16QI_V4SI_QI },
32394 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv4qiv4di2_mask, "__builtin_ia32_pmovzxbq256_mask", IX86_BUILTIN_PMOVZXBQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V16QI_V4DI_QI },
32395 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv2qiv2di2_mask, "__builtin_ia32_pmovzxbq128_mask", IX86_BUILTIN_PMOVZXBQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V16QI_V2DI_QI },
32396 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv8hiv8si2_mask, "__builtin_ia32_pmovzxwd256_mask", IX86_BUILTIN_PMOVZXWD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8HI_V8SI_QI },
32397 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv4hiv4si2_mask, "__builtin_ia32_pmovzxwd128_mask", IX86_BUILTIN_PMOVZXWD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V8HI_V4SI_QI },
32398 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv4hiv4di2_mask, "__builtin_ia32_pmovzxwq256_mask", IX86_BUILTIN_PMOVZXWQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V8HI_V4DI_QI },
32399 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv2hiv2di2_mask, "__builtin_ia32_pmovzxwq128_mask", IX86_BUILTIN_PMOVZXWQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V8HI_V2DI_QI },
32400 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv4siv4di2_mask, "__builtin_ia32_pmovzxdq256_mask", IX86_BUILTIN_PMOVZXDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4SI_V4DI_QI },
32401 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv2siv2di2_mask, "__builtin_ia32_pmovzxdq128_mask", IX86_BUILTIN_PMOVZXDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V4SI_V2DI_QI },
32402 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv4df_mask, "__builtin_ia32_reducepd256_mask", IX86_BUILTIN_REDUCEPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT_V4DF_QI },
32403 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv2df_mask, "__builtin_ia32_reducepd128_mask", IX86_BUILTIN_REDUCEPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT_V2DF_QI },
32404 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv8sf_mask, "__builtin_ia32_reduceps256_mask", IX86_BUILTIN_REDUCEPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT_V8SF_QI },
32405 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv4sf_mask, "__builtin_ia32_reduceps128_mask", IX86_BUILTIN_REDUCEPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT_V4SF_QI },
32406 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducesv2df, "__builtin_ia32_reducesd", IX86_BUILTIN_REDUCESD_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
32407 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducesv4sf, "__builtin_ia32_reducess", IX86_BUILTIN_REDUCESS_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
32408 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permvarv16hi_mask, "__builtin_ia32_permvarhi256_mask", IX86_BUILTIN_VPERMVARHI256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32409 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permvarv8hi_mask, "__builtin_ia32_permvarhi128_mask", IX86_BUILTIN_VPERMVARHI128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32410 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv16hi3_mask, "__builtin_ia32_vpermt2varhi256_mask", IX86_BUILTIN_VPERMT2VARHI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32411 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv16hi3_maskz, "__builtin_ia32_vpermt2varhi256_maskz", IX86_BUILTIN_VPERMT2VARHI256_MASKZ, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32412 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8hi3_mask, "__builtin_ia32_vpermt2varhi128_mask", IX86_BUILTIN_VPERMT2VARHI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32413 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8hi3_maskz, "__builtin_ia32_vpermt2varhi128_maskz", IX86_BUILTIN_VPERMT2VARHI128_MASKZ, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32414 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv16hi3_mask, "__builtin_ia32_vpermi2varhi256_mask", IX86_BUILTIN_VPERMI2VARHI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32415 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv8hi3_mask, "__builtin_ia32_vpermi2varhi128_mask", IX86_BUILTIN_VPERMI2VARHI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32416 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rcp14v4df_mask, "__builtin_ia32_rcp14pd256_mask", IX86_BUILTIN_RCP14PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32417 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rcp14v2df_mask, "__builtin_ia32_rcp14pd128_mask", IX86_BUILTIN_RCP14PD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32418 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rcp14v8sf_mask, "__builtin_ia32_rcp14ps256_mask", IX86_BUILTIN_RCP14PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32419 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rcp14v4sf_mask, "__builtin_ia32_rcp14ps128_mask", IX86_BUILTIN_RCP14PS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32420 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rsqrt14v4df_mask, "__builtin_ia32_rsqrt14pd256_mask", IX86_BUILTIN_RSQRT14PD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32421 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rsqrt14v2df_mask, "__builtin_ia32_rsqrt14pd128_mask", IX86_BUILTIN_RSQRT14PD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32422 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rsqrt14v8sf_mask, "__builtin_ia32_rsqrt14ps256_mask", IX86_BUILTIN_RSQRT14PS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32423 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rsqrt14v4sf_mask, "__builtin_ia32_rsqrt14ps128_mask", IX86_BUILTIN_RSQRT14PS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32424 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_sqrtv4df2_mask, "__builtin_ia32_sqrtpd256_mask", IX86_BUILTIN_SQRTPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32425 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_sqrtv2df2_mask, "__builtin_ia32_sqrtpd128_mask", IX86_BUILTIN_SQRTPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32426 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_sqrtv8sf2_mask, "__builtin_ia32_sqrtps256_mask", IX86_BUILTIN_SQRTPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32427 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse_sqrtv4sf2_mask, "__builtin_ia32_sqrtps128_mask", IX86_BUILTIN_SQRTPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32428 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv16qi3_mask, "__builtin_ia32_paddb128_mask", IX86_BUILTIN_PADDB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32429 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv8hi3_mask, "__builtin_ia32_paddw128_mask", IX86_BUILTIN_PADDW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32430 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv4si3_mask, "__builtin_ia32_paddd128_mask", IX86_BUILTIN_PADDD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32431 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv2di3_mask, "__builtin_ia32_paddq128_mask", IX86_BUILTIN_PADDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32432 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv16qi3_mask, "__builtin_ia32_psubb128_mask", IX86_BUILTIN_PSUBB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32433 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv8hi3_mask, "__builtin_ia32_psubw128_mask", IX86_BUILTIN_PSUBW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32434 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv4si3_mask, "__builtin_ia32_psubd128_mask", IX86_BUILTIN_PSUBD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32435 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv2di3_mask, "__builtin_ia32_psubq128_mask", IX86_BUILTIN_PSUBQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32436 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_ssaddv16qi3_mask, "__builtin_ia32_paddsb128_mask", IX86_BUILTIN_PADDSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32437 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_ssaddv8hi3_mask, "__builtin_ia32_paddsw128_mask", IX86_BUILTIN_PADDSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32438 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_sssubv16qi3_mask, "__builtin_ia32_psubsb128_mask", IX86_BUILTIN_PSUBSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32439 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_sssubv8hi3_mask, "__builtin_ia32_psubsw128_mask", IX86_BUILTIN_PSUBSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32440 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_usaddv16qi3_mask, "__builtin_ia32_paddusb128_mask", IX86_BUILTIN_PADDUSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32441 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_usaddv8hi3_mask, "__builtin_ia32_paddusw128_mask", IX86_BUILTIN_PADDUSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32442 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_ussubv16qi3_mask, "__builtin_ia32_psubusb128_mask", IX86_BUILTIN_PSUBUSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32443 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_ussubv8hi3_mask, "__builtin_ia32_psubusw128_mask", IX86_BUILTIN_PSUBUSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32444 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv32qi3_mask, "__builtin_ia32_paddb256_mask", IX86_BUILTIN_PADDB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32445 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv16hi3_mask, "__builtin_ia32_paddw256_mask", IX86_BUILTIN_PADDW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32446 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv8si3_mask, "__builtin_ia32_paddd256_mask", IX86_BUILTIN_PADDD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32447 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv4di3_mask, "__builtin_ia32_paddq256_mask", IX86_BUILTIN_PADDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32448 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ssaddv32qi3_mask, "__builtin_ia32_paddsb256_mask", IX86_BUILTIN_PADDSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32449 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ssaddv16hi3_mask, "__builtin_ia32_paddsw256_mask", IX86_BUILTIN_PADDSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32450 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_usaddv32qi3_mask, "__builtin_ia32_paddusb256_mask", IX86_BUILTIN_PADDUSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32451 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_usaddv16hi3_mask, "__builtin_ia32_paddusw256_mask", IX86_BUILTIN_PADDUSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32452 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv32qi3_mask, "__builtin_ia32_psubb256_mask", IX86_BUILTIN_PSUBB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32453 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv16hi3_mask, "__builtin_ia32_psubw256_mask", IX86_BUILTIN_PSUBW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32454 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv8si3_mask, "__builtin_ia32_psubd256_mask", IX86_BUILTIN_PSUBD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32455 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv4di3_mask, "__builtin_ia32_psubq256_mask", IX86_BUILTIN_PSUBQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32456 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sssubv32qi3_mask, "__builtin_ia32_psubsb256_mask", IX86_BUILTIN_PSUBSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32457 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sssubv16hi3_mask, "__builtin_ia32_psubsw256_mask", IX86_BUILTIN_PSUBSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32458 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ussubv32qi3_mask, "__builtin_ia32_psubusb256_mask", IX86_BUILTIN_PSUBUSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32459 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ussubv16hi3_mask, "__builtin_ia32_psubusw256_mask", IX86_BUILTIN_PSUBUSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32460 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_shuf_f64x2_mask, "__builtin_ia32_shuf_f64x2_256_mask", IX86_BUILTIN_SHUF_F64x2_256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT_V4DF_QI },
32461 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_shuf_i64x2_mask, "__builtin_ia32_shuf_i64x2_256_mask", IX86_BUILTIN_SHUF_I64x2_256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_V4DI_QI },
32462 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_shuf_i32x4_mask, "__builtin_ia32_shuf_i32x4_256_mask", IX86_BUILTIN_SHUF_I32x4_256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT_V8SI_QI },
32463 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_shuf_f32x4_mask, "__builtin_ia32_shuf_f32x4_256_mask", IX86_BUILTIN_SHUF_F32x4_256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT_V8SF_QI },
32464 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev8hiv8qi2_mask, "__builtin_ia32_pmovwb128_mask", IX86_BUILTIN_PMOVWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V16QI_QI },
32465 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev16hiv16qi2_mask, "__builtin_ia32_pmovwb256_mask", IX86_BUILTIN_PMOVWB256, UNKNOWN, (int) V16QI_FTYPE_V16HI_V16QI_HI },
32466 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev8hiv8qi2_mask, "__builtin_ia32_pmovswb128_mask", IX86_BUILTIN_PMOVSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V16QI_QI },
32467 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev16hiv16qi2_mask, "__builtin_ia32_pmovswb256_mask", IX86_BUILTIN_PMOVSWB256, UNKNOWN, (int) V16QI_FTYPE_V16HI_V16QI_HI },
32468 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev8hiv8qi2_mask, "__builtin_ia32_pmovuswb128_mask", IX86_BUILTIN_PMOVUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V16QI_QI },
32469 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev16hiv16qi2_mask, "__builtin_ia32_pmovuswb256_mask", IX86_BUILTIN_PMOVUSWB256, UNKNOWN, (int) V16QI_FTYPE_V16HI_V16QI_HI },
32470 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4siv4qi2_mask, "__builtin_ia32_pmovdb128_mask", IX86_BUILTIN_PMOVDB128, UNKNOWN, (int) V16QI_FTYPE_V4SI_V16QI_QI },
32471 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev8siv8qi2_mask, "__builtin_ia32_pmovdb256_mask", IX86_BUILTIN_PMOVDB256, UNKNOWN, (int) V16QI_FTYPE_V8SI_V16QI_QI },
32472 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4siv4qi2_mask, "__builtin_ia32_pmovsdb128_mask", IX86_BUILTIN_PMOVSDB128, UNKNOWN, (int) V16QI_FTYPE_V4SI_V16QI_QI },
32473 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev8siv8qi2_mask, "__builtin_ia32_pmovsdb256_mask", IX86_BUILTIN_PMOVSDB256, UNKNOWN, (int) V16QI_FTYPE_V8SI_V16QI_QI },
32474 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4siv4qi2_mask, "__builtin_ia32_pmovusdb128_mask", IX86_BUILTIN_PMOVUSDB128, UNKNOWN, (int) V16QI_FTYPE_V4SI_V16QI_QI },
32475 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev8siv8qi2_mask, "__builtin_ia32_pmovusdb256_mask", IX86_BUILTIN_PMOVUSDB256, UNKNOWN, (int) V16QI_FTYPE_V8SI_V16QI_QI },
32476 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4siv4hi2_mask, "__builtin_ia32_pmovdw128_mask", IX86_BUILTIN_PMOVDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V8HI_QI },
32477 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev8siv8hi2_mask, "__builtin_ia32_pmovdw256_mask", IX86_BUILTIN_PMOVDW256, UNKNOWN, (int) V8HI_FTYPE_V8SI_V8HI_QI },
32478 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4siv4hi2_mask, "__builtin_ia32_pmovsdw128_mask", IX86_BUILTIN_PMOVSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V8HI_QI },
32479 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev8siv8hi2_mask, "__builtin_ia32_pmovsdw256_mask", IX86_BUILTIN_PMOVSDW256, UNKNOWN, (int) V8HI_FTYPE_V8SI_V8HI_QI },
32480 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4siv4hi2_mask, "__builtin_ia32_pmovusdw128_mask", IX86_BUILTIN_PMOVUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V8HI_QI },
32481 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev8siv8hi2_mask, "__builtin_ia32_pmovusdw256_mask", IX86_BUILTIN_PMOVUSDW256, UNKNOWN, (int) V8HI_FTYPE_V8SI_V8HI_QI },
32482 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2qi2_mask, "__builtin_ia32_pmovqb128_mask", IX86_BUILTIN_PMOVQB128, UNKNOWN, (int) V16QI_FTYPE_V2DI_V16QI_QI },
32483 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4qi2_mask, "__builtin_ia32_pmovqb256_mask", IX86_BUILTIN_PMOVQB256, UNKNOWN, (int) V16QI_FTYPE_V4DI_V16QI_QI },
32484 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2qi2_mask, "__builtin_ia32_pmovsqb128_mask", IX86_BUILTIN_PMOVSQB128, UNKNOWN, (int) V16QI_FTYPE_V2DI_V16QI_QI },
32485 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4qi2_mask, "__builtin_ia32_pmovsqb256_mask", IX86_BUILTIN_PMOVSQB256, UNKNOWN, (int) V16QI_FTYPE_V4DI_V16QI_QI },
32486 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2qi2_mask, "__builtin_ia32_pmovusqb128_mask", IX86_BUILTIN_PMOVUSQB128, UNKNOWN, (int) V16QI_FTYPE_V2DI_V16QI_QI },
32487 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4qi2_mask, "__builtin_ia32_pmovusqb256_mask", IX86_BUILTIN_PMOVUSQB256, UNKNOWN, (int) V16QI_FTYPE_V4DI_V16QI_QI },
32488 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2hi2_mask, "__builtin_ia32_pmovqw128_mask", IX86_BUILTIN_PMOVQW128, UNKNOWN, (int) V8HI_FTYPE_V2DI_V8HI_QI },
32489 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4hi2_mask, "__builtin_ia32_pmovqw256_mask", IX86_BUILTIN_PMOVQW256, UNKNOWN, (int) V8HI_FTYPE_V4DI_V8HI_QI },
32490 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2hi2_mask, "__builtin_ia32_pmovsqw128_mask", IX86_BUILTIN_PMOVSQW128, UNKNOWN, (int) V8HI_FTYPE_V2DI_V8HI_QI },
32491 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4hi2_mask, "__builtin_ia32_pmovsqw256_mask", IX86_BUILTIN_PMOVSQW256, UNKNOWN, (int) V8HI_FTYPE_V4DI_V8HI_QI },
32492 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2hi2_mask, "__builtin_ia32_pmovusqw128_mask", IX86_BUILTIN_PMOVUSQW128, UNKNOWN, (int) V8HI_FTYPE_V2DI_V8HI_QI },
32493 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4hi2_mask, "__builtin_ia32_pmovusqw256_mask", IX86_BUILTIN_PMOVUSQW256, UNKNOWN, (int) V8HI_FTYPE_V4DI_V8HI_QI },
32494 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2si2_mask, "__builtin_ia32_pmovqd128_mask", IX86_BUILTIN_PMOVQD128, UNKNOWN, (int) V4SI_FTYPE_V2DI_V4SI_QI },
32495 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4si2_mask, "__builtin_ia32_pmovqd256_mask", IX86_BUILTIN_PMOVQD256, UNKNOWN, (int) V4SI_FTYPE_V4DI_V4SI_QI },
32496 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2si2_mask, "__builtin_ia32_pmovsqd128_mask", IX86_BUILTIN_PMOVSQD128, UNKNOWN, (int) V4SI_FTYPE_V2DI_V4SI_QI },
32497 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4si2_mask, "__builtin_ia32_pmovsqd256_mask", IX86_BUILTIN_PMOVSQD256, UNKNOWN, (int) V4SI_FTYPE_V4DI_V4SI_QI },
32498 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2si2_mask, "__builtin_ia32_pmovusqd128_mask", IX86_BUILTIN_PMOVUSQD128, UNKNOWN, (int) V4SI_FTYPE_V2DI_V4SI_QI },
32499 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4si2_mask, "__builtin_ia32_pmovusqd256_mask", IX86_BUILTIN_PMOVUSQD256, UNKNOWN, (int) V4SI_FTYPE_V4DI_V4SI_QI },
32500 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_rangepv4df_mask, "__builtin_ia32_rangepd256_mask", IX86_BUILTIN_RANGEPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT_V4DF_QI },
32501 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_rangepv2df_mask, "__builtin_ia32_rangepd128_mask", IX86_BUILTIN_RANGEPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI },
32502 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_rangepv8sf_mask, "__builtin_ia32_rangeps256_mask", IX86_BUILTIN_RANGEPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT_V8SF_QI },
32503 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_rangepv4sf_mask, "__builtin_ia32_rangeps128_mask", IX86_BUILTIN_RANGEPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI },
32504 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getexpv8sf_mask, "__builtin_ia32_getexpps256_mask", IX86_BUILTIN_GETEXPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32505 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getexpv4df_mask, "__builtin_ia32_getexppd256_mask", IX86_BUILTIN_GETEXPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32506 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getexpv4sf_mask, "__builtin_ia32_getexpps128_mask", IX86_BUILTIN_GETEXPPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32507 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getexpv2df_mask, "__builtin_ia32_getexppd128_mask", IX86_BUILTIN_GETEXPPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32508 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv4df_mask, "__builtin_ia32_fixupimmpd256_mask", IX86_BUILTIN_FIXUPIMMPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DI_INT_QI },
32509 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv4df_maskz, "__builtin_ia32_fixupimmpd256_maskz", IX86_BUILTIN_FIXUPIMMPD256_MASKZ, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DI_INT_QI },
32510 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv8sf_mask, "__builtin_ia32_fixupimmps256_mask", IX86_BUILTIN_FIXUPIMMPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SI_INT_QI },
32511 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv8sf_maskz, "__builtin_ia32_fixupimmps256_maskz", IX86_BUILTIN_FIXUPIMMPS256_MASKZ, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SI_INT_QI },
32512 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv2df_mask, "__builtin_ia32_fixupimmpd128_mask", IX86_BUILTIN_FIXUPIMMPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI },
32513 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv2df_maskz, "__builtin_ia32_fixupimmpd128_maskz", IX86_BUILTIN_FIXUPIMMPD128_MASKZ, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI },
32514 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv4sf_mask, "__builtin_ia32_fixupimmps128_mask", IX86_BUILTIN_FIXUPIMMPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI },
32515 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv4sf_maskz, "__builtin_ia32_fixupimmps128_maskz", IX86_BUILTIN_FIXUPIMMPS128_MASKZ, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI },
32516 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv4di2_mask, "__builtin_ia32_pabsq256_mask", IX86_BUILTIN_PABSQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32517 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv2di2_mask, "__builtin_ia32_pabsq128_mask", IX86_BUILTIN_PABSQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32518 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv8si2_mask, "__builtin_ia32_pabsd256_mask", IX86_BUILTIN_PABSD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32519 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv4si2_mask, "__builtin_ia32_pabsd128_mask", IX86_BUILTIN_PABSD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32520 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_pmulhrswv16hi3_mask , "__builtin_ia32_pmulhrsw256_mask", IX86_BUILTIN_PMULHRSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32521 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ssse3_pmulhrswv8hi3_mask, "__builtin_ia32_pmulhrsw128_mask", IX86_BUILTIN_PMULHRSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32522 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umulv8hi3_highpart_mask, "__builtin_ia32_pmulhuw128_mask", IX86_BUILTIN_PMULHUW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32523 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umulv16hi3_highpart_mask, "__builtin_ia32_pmulhuw256_mask" , IX86_BUILTIN_PMULHUW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32524 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smulv16hi3_highpart_mask, "__builtin_ia32_pmulhw256_mask" , IX86_BUILTIN_PMULHW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32525 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smulv8hi3_highpart_mask, "__builtin_ia32_pmulhw128_mask", IX86_BUILTIN_PMULHW128_MASK, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32526 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv16hi3_mask, "__builtin_ia32_pmullw256_mask" , IX86_BUILTIN_PMULLW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32527 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv8hi3_mask, "__builtin_ia32_pmullw128_mask", IX86_BUILTIN_PMULLW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32528 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_mulv4di3_mask, "__builtin_ia32_pmullq256_mask", IX86_BUILTIN_PMULLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32529 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_mulv2di3_mask, "__builtin_ia32_pmullq128_mask", IX86_BUILTIN_PMULLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32530 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv4df3_mask, "__builtin_ia32_andpd256_mask", IX86_BUILTIN_ANDPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32531 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv2df3_mask, "__builtin_ia32_andpd128_mask", IX86_BUILTIN_ANDPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32532 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv8sf3_mask, "__builtin_ia32_andps256_mask", IX86_BUILTIN_ANDPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32533 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv4sf3_mask, "__builtin_ia32_andps128_mask", IX86_BUILTIN_ANDPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32534 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_andnotv4df3_mask, "__builtin_ia32_andnpd256_mask", IX86_BUILTIN_ANDNPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32535 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_andnotv2df3_mask, "__builtin_ia32_andnpd128_mask", IX86_BUILTIN_ANDNPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32536 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_andnotv8sf3_mask, "__builtin_ia32_andnps256_mask", IX86_BUILTIN_ANDNPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32537 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse_andnotv4sf3_mask, "__builtin_ia32_andnps128_mask", IX86_BUILTIN_ANDNPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32538 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv8hi3_mask, "__builtin_ia32_psllwi128_mask", IX86_BUILTIN_PSLLWI128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT_V8HI_QI },
32539 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv4si3_mask, "__builtin_ia32_pslldi128_mask", IX86_BUILTIN_PSLLDI128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32540 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv2di3_mask, "__builtin_ia32_psllqi128_mask", IX86_BUILTIN_PSLLQI128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_V2DI_QI },
32541 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv8hi3_mask, "__builtin_ia32_psllw128_mask", IX86_BUILTIN_PSLLW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32542 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv4si3_mask, "__builtin_ia32_pslld128_mask", IX86_BUILTIN_PSLLD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32543 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv2di3_mask, "__builtin_ia32_psllq128_mask", IX86_BUILTIN_PSLLQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32544 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv16hi3_mask, "__builtin_ia32_psllwi256_mask", IX86_BUILTIN_PSLLWI256_MASK , UNKNOWN, (int) V16HI_FTYPE_V16HI_INT_V16HI_HI },
32545 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv16hi3_mask, "__builtin_ia32_psllw256_mask", IX86_BUILTIN_PSLLW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_V16HI_HI },
32546 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv8si3_mask, "__builtin_ia32_pslldi256_mask", IX86_BUILTIN_PSLLDI256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32547 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv8si3_mask, "__builtin_ia32_pslld256_mask", IX86_BUILTIN_PSLLD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_V8SI_QI },
32548 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv4di3_mask, "__builtin_ia32_psllqi256_mask", IX86_BUILTIN_PSLLQI256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32549 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv4di3_mask, "__builtin_ia32_psllq256_mask", IX86_BUILTIN_PSLLQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_V4DI_QI },
32550 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv4si3_mask, "__builtin_ia32_psradi128_mask", IX86_BUILTIN_PSRADI128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32551 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv4si3_mask, "__builtin_ia32_psrad128_mask", IX86_BUILTIN_PSRAD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32552 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv8si3_mask, "__builtin_ia32_psradi256_mask", IX86_BUILTIN_PSRADI256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32553 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv8si3_mask, "__builtin_ia32_psrad256_mask", IX86_BUILTIN_PSRAD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_V8SI_QI },
32554 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv2di3_mask, "__builtin_ia32_psraqi128_mask", IX86_BUILTIN_PSRAQI128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_V2DI_QI },
32555 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv2di3_mask, "__builtin_ia32_psraq128_mask", IX86_BUILTIN_PSRAQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32556 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv4di3_mask, "__builtin_ia32_psraqi256_mask", IX86_BUILTIN_PSRAQI256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32557 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv4di3_mask, "__builtin_ia32_psraq256_mask", IX86_BUILTIN_PSRAQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_V4DI_QI },
32558 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv8si3_mask, "__builtin_ia32_pandd256_mask", IX86_BUILTIN_PANDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32559 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv4si3_mask, "__builtin_ia32_pandd128_mask", IX86_BUILTIN_PANDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32560 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv4si3_mask, "__builtin_ia32_psrldi128_mask", IX86_BUILTIN_PSRLDI128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32561 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv4si3_mask, "__builtin_ia32_psrld128_mask", IX86_BUILTIN_PSRLD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32562 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv8si3_mask, "__builtin_ia32_psrldi256_mask", IX86_BUILTIN_PSRLDI256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32563 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv8si3_mask, "__builtin_ia32_psrld256_mask", IX86_BUILTIN_PSRLD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_V8SI_QI },
32564 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv2di3_mask, "__builtin_ia32_psrlqi128_mask", IX86_BUILTIN_PSRLQI128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_V2DI_QI },
32565 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv2di3_mask, "__builtin_ia32_psrlq128_mask", IX86_BUILTIN_PSRLQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32566 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv4di3_mask, "__builtin_ia32_psrlqi256_mask", IX86_BUILTIN_PSRLQI256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32567 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv4di3_mask, "__builtin_ia32_psrlq256_mask", IX86_BUILTIN_PSRLQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_V4DI_QI },
32568 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv4di3_mask, "__builtin_ia32_pandq256_mask", IX86_BUILTIN_PANDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32569 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv2di3_mask, "__builtin_ia32_pandq128_mask", IX86_BUILTIN_PANDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32570 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_andnotv8si3_mask, "__builtin_ia32_pandnd256_mask", IX86_BUILTIN_PANDND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32571 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_andnotv4si3_mask, "__builtin_ia32_pandnd128_mask", IX86_BUILTIN_PANDND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32572 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_andnotv4di3_mask, "__builtin_ia32_pandnq256_mask", IX86_BUILTIN_PANDNQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32573 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_andnotv2di3_mask, "__builtin_ia32_pandnq128_mask", IX86_BUILTIN_PANDNQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32574 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv8si3_mask, "__builtin_ia32_pord256_mask", IX86_BUILTIN_PORD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32575 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv4si3_mask, "__builtin_ia32_pord128_mask", IX86_BUILTIN_PORD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32576 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv4di3_mask, "__builtin_ia32_porq256_mask", IX86_BUILTIN_PORQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32577 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv2di3_mask, "__builtin_ia32_porq128_mask", IX86_BUILTIN_PORQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32578 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv8si3_mask, "__builtin_ia32_pxord256_mask", IX86_BUILTIN_PXORD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32579 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv4si3_mask, "__builtin_ia32_pxord128_mask", IX86_BUILTIN_PXORD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32580 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv4di3_mask, "__builtin_ia32_pxorq256_mask", IX86_BUILTIN_PXORQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32581 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv2di3_mask, "__builtin_ia32_pxorq128_mask", IX86_BUILTIN_PXORQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32582 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_packsswb_mask, "__builtin_ia32_packsswb256_mask", IX86_BUILTIN_PACKSSWB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI_V32QI_SI },
32583 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_packsswb_mask, "__builtin_ia32_packsswb128_mask", IX86_BUILTIN_PACKSSWB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI_V16QI_HI },
32584 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_packuswb_mask, "__builtin_ia32_packuswb256_mask", IX86_BUILTIN_PACKUSWB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI_V32QI_SI },
32585 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_packuswb_mask, "__builtin_ia32_packuswb128_mask", IX86_BUILTIN_PACKUSWB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI_V16QI_HI },
32586 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rndscalev8sf_mask, "__builtin_ia32_rndscaleps_256_mask", IX86_BUILTIN_RNDSCALEPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT_V8SF_QI },
32587 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rndscalev4df_mask, "__builtin_ia32_rndscalepd_256_mask", IX86_BUILTIN_RNDSCALEPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT_V4DF_QI },
32588 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rndscalev4sf_mask, "__builtin_ia32_rndscaleps_128_mask", IX86_BUILTIN_RNDSCALEPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT_V4SF_QI },
32589 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rndscalev2df_mask, "__builtin_ia32_rndscalepd_128_mask", IX86_BUILTIN_RNDSCALEPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT_V2DF_QI },
32590 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv4di_mask, "__builtin_ia32_pternlogq256_mask", IX86_BUILTIN_VTERNLOGQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_INT_QI },
32591 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv4di_maskz, "__builtin_ia32_pternlogq256_maskz", IX86_BUILTIN_VTERNLOGQ256_MASKZ, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_INT_QI },
32592 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv8si_mask, "__builtin_ia32_pternlogd256_mask", IX86_BUILTIN_VTERNLOGD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_INT_QI },
32593 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv8si_maskz, "__builtin_ia32_pternlogd256_maskz", IX86_BUILTIN_VTERNLOGD256_MASKZ, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_INT_QI },
32594 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv2di_mask, "__builtin_ia32_pternlogq128_mask", IX86_BUILTIN_VTERNLOGQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_INT_QI },
32595 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv2di_maskz, "__builtin_ia32_pternlogq128_maskz", IX86_BUILTIN_VTERNLOGQ128_MASKZ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_INT_QI },
32596 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv4si_mask, "__builtin_ia32_pternlogd128_mask", IX86_BUILTIN_VTERNLOGD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_INT_QI },
32597 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv4si_maskz, "__builtin_ia32_pternlogd128_maskz", IX86_BUILTIN_VTERNLOGD128_MASKZ, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_INT_QI },
32598 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_scalefv4df_mask, "__builtin_ia32_scalefpd256_mask", IX86_BUILTIN_SCALEFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32599 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_scalefv8sf_mask, "__builtin_ia32_scalefps256_mask", IX86_BUILTIN_SCALEFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32600 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_scalefv2df_mask, "__builtin_ia32_scalefpd128_mask", IX86_BUILTIN_SCALEFPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32601 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_scalefv4sf_mask, "__builtin_ia32_scalefps128_mask", IX86_BUILTIN_SCALEFPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32602 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4df_mask, "__builtin_ia32_vfmaddpd256_mask", IX86_BUILTIN_VFMADDPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32603 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4df_mask3, "__builtin_ia32_vfmaddpd256_mask3", IX86_BUILTIN_VFMADDPD256_MASK3, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32604 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4df_maskz, "__builtin_ia32_vfmaddpd256_maskz", IX86_BUILTIN_VFMADDPD256_MASKZ, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32605 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v2df_mask, "__builtin_ia32_vfmaddpd128_mask", IX86_BUILTIN_VFMADDPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32606 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v2df_mask3, "__builtin_ia32_vfmaddpd128_mask3", IX86_BUILTIN_VFMADDPD128_MASK3, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32607 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v2df_maskz, "__builtin_ia32_vfmaddpd128_maskz", IX86_BUILTIN_VFMADDPD128_MASKZ, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32608 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v8sf_mask, "__builtin_ia32_vfmaddps256_mask", IX86_BUILTIN_VFMADDPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32609 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v8sf_mask3, "__builtin_ia32_vfmaddps256_mask3", IX86_BUILTIN_VFMADDPS256_MASK3, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32610 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v8sf_maskz, "__builtin_ia32_vfmaddps256_maskz", IX86_BUILTIN_VFMADDPS256_MASKZ, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32611 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4sf_mask, "__builtin_ia32_vfmaddps128_mask", IX86_BUILTIN_VFMADDPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32612 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4sf_mask3, "__builtin_ia32_vfmaddps128_mask3", IX86_BUILTIN_VFMADDPS128_MASK3, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32613 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4sf_maskz, "__builtin_ia32_vfmaddps128_maskz", IX86_BUILTIN_VFMADDPS128_MASKZ, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32614 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsub_v4df_mask3, "__builtin_ia32_vfmsubpd256_mask3", IX86_BUILTIN_VFMSUBPD256_MASK3, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32615 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsub_v2df_mask3, "__builtin_ia32_vfmsubpd128_mask3", IX86_BUILTIN_VFMSUBPD128_MASK3, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32616 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsub_v8sf_mask3, "__builtin_ia32_vfmsubps256_mask3", IX86_BUILTIN_VFMSUBPS256_MASK3, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32617 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsub_v4sf_mask3, "__builtin_ia32_vfmsubps128_mask3", IX86_BUILTIN_VFMSUBPS128_MASK3, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32618 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmadd_v4df_mask, "__builtin_ia32_vfnmaddpd256_mask", IX86_BUILTIN_VFNMADDPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32619 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmadd_v2df_mask, "__builtin_ia32_vfnmaddpd128_mask", IX86_BUILTIN_VFNMADDPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32620 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmadd_v8sf_mask, "__builtin_ia32_vfnmaddps256_mask", IX86_BUILTIN_VFNMADDPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32621 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmadd_v4sf_mask, "__builtin_ia32_vfnmaddps128_mask", IX86_BUILTIN_VFNMADDPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32622 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v4df_mask, "__builtin_ia32_vfnmsubpd256_mask", IX86_BUILTIN_VFNMSUBPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32623 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v4df_mask3, "__builtin_ia32_vfnmsubpd256_mask3", IX86_BUILTIN_VFNMSUBPD256_MASK3, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32624 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v2df_mask, "__builtin_ia32_vfnmsubpd128_mask", IX86_BUILTIN_VFNMSUBPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32625 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v2df_mask3, "__builtin_ia32_vfnmsubpd128_mask3", IX86_BUILTIN_VFNMSUBPD128_MASK3, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32626 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v8sf_mask, "__builtin_ia32_vfnmsubps256_mask", IX86_BUILTIN_VFNMSUBPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32627 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v8sf_mask3, "__builtin_ia32_vfnmsubps256_mask3", IX86_BUILTIN_VFNMSUBPS256_MASK3, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32628 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v4sf_mask, "__builtin_ia32_vfnmsubps128_mask", IX86_BUILTIN_VFNMSUBPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32629 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v4sf_mask3, "__builtin_ia32_vfnmsubps128_mask3", IX86_BUILTIN_VFNMSUBPS128_MASK3, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32630 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4df_mask, "__builtin_ia32_vfmaddsubpd256_mask", IX86_BUILTIN_VFMADDSUBPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32631 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4df_mask3, "__builtin_ia32_vfmaddsubpd256_mask3", IX86_BUILTIN_VFMADDSUBPD256_MASK3, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32632 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4df_maskz, "__builtin_ia32_vfmaddsubpd256_maskz", IX86_BUILTIN_VFMADDSUBPD256_MASKZ, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32633 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v2df_mask, "__builtin_ia32_vfmaddsubpd128_mask", IX86_BUILTIN_VFMADDSUBPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32634 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v2df_mask3, "__builtin_ia32_vfmaddsubpd128_mask3", IX86_BUILTIN_VFMADDSUBPD128_MASK3, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32635 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v2df_maskz, "__builtin_ia32_vfmaddsubpd128_maskz", IX86_BUILTIN_VFMADDSUBPD128_MASKZ, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32636 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v8sf_mask, "__builtin_ia32_vfmaddsubps256_mask", IX86_BUILTIN_VFMADDSUBPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32637 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v8sf_mask3, "__builtin_ia32_vfmaddsubps256_mask3", IX86_BUILTIN_VFMADDSUBPS256_MASK3, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32638 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v8sf_maskz, "__builtin_ia32_vfmaddsubps256_maskz", IX86_BUILTIN_VFMADDSUBPS256_MASKZ, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32639 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4sf_mask, "__builtin_ia32_vfmaddsubps128_mask", IX86_BUILTIN_VFMADDSUBPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32640 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4sf_mask3, "__builtin_ia32_vfmaddsubps128_mask3", IX86_BUILTIN_VFMADDSUBPS128_MASK3, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32641 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4sf_maskz, "__builtin_ia32_vfmaddsubps128_maskz", IX86_BUILTIN_VFMADDSUBPS128_MASKZ, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32642 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsubadd_v4df_mask3, "__builtin_ia32_vfmsubaddpd256_mask3", IX86_BUILTIN_VFMSUBADDPD256_MASK3, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32643 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsubadd_v2df_mask3, "__builtin_ia32_vfmsubaddpd128_mask3", IX86_BUILTIN_VFMSUBADDPD128_MASK3, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32644 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsubadd_v8sf_mask3, "__builtin_ia32_vfmsubaddps256_mask3", IX86_BUILTIN_VFMSUBADDPS256_MASK3, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32645 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsubadd_v4sf_mask3, "__builtin_ia32_vfmsubaddps128_mask3", IX86_BUILTIN_VFMSUBADDPS128_MASK3, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32646 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vinsertv4df, "__builtin_ia32_insertf64x2_256_mask", IX86_BUILTIN_INSERTF64X2_256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT_V4DF_QI },
32647 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vinsertv4di, "__builtin_ia32_inserti64x2_256_mask", IX86_BUILTIN_INSERTI64X2_256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_INT_V4DI_QI },
32648 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ashrvv16hi_mask, "__builtin_ia32_psrav16hi_mask", IX86_BUILTIN_PSRAVV16HI, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32649 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ashrvv8hi_mask, "__builtin_ia32_psrav8hi_mask", IX86_BUILTIN_PSRAVV8HI, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32650 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_pmaddubsw512v16hi_mask, "__builtin_ia32_pmaddubsw256_mask", IX86_BUILTIN_PMADDUBSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI_V16HI_HI },
32651 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_pmaddubsw512v8hi_mask, "__builtin_ia32_pmaddubsw128_mask", IX86_BUILTIN_PMADDUBSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI_V8HI_QI },
32652 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_pmaddwd512v16hi_mask, "__builtin_ia32_pmaddwd256_mask", IX86_BUILTIN_PMADDWD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V16HI_V16HI_V8SI_QI },
32653 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_pmaddwd512v8hi_mask, "__builtin_ia32_pmaddwd128_mask", IX86_BUILTIN_PMADDWD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI_V4SI_QI },
32654 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_lshrvv16hi_mask, "__builtin_ia32_psrlv16hi_mask", IX86_BUILTIN_PSRLVV16HI, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32655 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_lshrvv8hi_mask, "__builtin_ia32_psrlv8hi_mask", IX86_BUILTIN_PSRLVV8HI, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32656 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_fix_notruncv8sfv8si_mask, "__builtin_ia32_cvtps2dq256_mask", IX86_BUILTIN_CVTPS2DQ256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SF_V8SI_QI },
32657 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_fix_notruncv4sfv4si_mask, "__builtin_ia32_cvtps2dq128_mask", IX86_BUILTIN_CVTPS2DQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SF_V4SI_QI },
32658 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ufix_notruncv8sfv8si_mask, "__builtin_ia32_cvtps2udq256_mask", IX86_BUILTIN_CVTPS2UDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF_V8SI_QI },
32659 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ufix_notruncv4sfv4si_mask, "__builtin_ia32_cvtps2udq128_mask", IX86_BUILTIN_CVTPS2UDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SF_V4SI_QI },
32660 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_cvtps2qqv4di_mask, "__builtin_ia32_cvtps2qq256_mask", IX86_BUILTIN_CVTPS2QQ256, UNKNOWN, (int) V4DI_FTYPE_V4SF_V4DI_QI },
32661 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_cvtps2qqv2di_mask, "__builtin_ia32_cvtps2qq128_mask", IX86_BUILTIN_CVTPS2QQ128, UNKNOWN, (int) V2DI_FTYPE_V4SF_V2DI_QI },
32662 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_cvtps2uqqv4di_mask, "__builtin_ia32_cvtps2uqq256_mask", IX86_BUILTIN_CVTPS2UQQ256, UNKNOWN, (int) V4DI_FTYPE_V4SF_V4DI_QI },
32663 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_cvtps2uqqv2di_mask, "__builtin_ia32_cvtps2uqq128_mask", IX86_BUILTIN_CVTPS2UQQ128, UNKNOWN, (int) V2DI_FTYPE_V4SF_V2DI_QI },
32664 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getmantv8sf_mask, "__builtin_ia32_getmantps256_mask", IX86_BUILTIN_GETMANTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT_V8SF_QI },
32665 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getmantv4sf_mask, "__builtin_ia32_getmantps128_mask", IX86_BUILTIN_GETMANTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT_V4SF_QI },
32666 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getmantv4df_mask, "__builtin_ia32_getmantpd256_mask", IX86_BUILTIN_GETMANTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT_V4DF_QI },
32667 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getmantv2df_mask, "__builtin_ia32_getmantpd128_mask", IX86_BUILTIN_GETMANTPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT_V2DF_QI },
32668 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_movddup256_mask, "__builtin_ia32_movddup256_mask", IX86_BUILTIN_MOVDDUP256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32669 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_dupv2df_mask, "__builtin_ia32_movddup128_mask", IX86_BUILTIN_MOVDDUP128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32670 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_movshdup256_mask, "__builtin_ia32_movshdup256_mask", IX86_BUILTIN_MOVSHDUP256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32671 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse3_movshdup_mask, "__builtin_ia32_movshdup128_mask", IX86_BUILTIN_MOVSHDUP128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32672 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_movsldup256_mask, "__builtin_ia32_movsldup256_mask", IX86_BUILTIN_MOVSLDUP256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32673 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse3_movsldup_mask, "__builtin_ia32_movsldup128_mask", IX86_BUILTIN_MOVSLDUP128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32674 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv4div4sf2_mask, "__builtin_ia32_cvtqq2ps256_mask", IX86_BUILTIN_CVTQQ2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DI_V4SF_QI },
32675 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv2div2sf2_mask, "__builtin_ia32_cvtqq2ps128_mask", IX86_BUILTIN_CVTQQ2PS128, UNKNOWN, (int) V4SF_FTYPE_V2DI_V4SF_QI },
32676 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv4div4sf2_mask, "__builtin_ia32_cvtuqq2ps256_mask", IX86_BUILTIN_CVTUQQ2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DI_V4SF_QI },
32677 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv2div2sf2_mask, "__builtin_ia32_cvtuqq2ps128_mask", IX86_BUILTIN_CVTUQQ2PS128, UNKNOWN, (int) V4SF_FTYPE_V2DI_V4SF_QI },
32678 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv4div4df2_mask, "__builtin_ia32_cvtqq2pd256_mask", IX86_BUILTIN_CVTQQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4DI_V4DF_QI },
32679 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv2div2df2_mask, "__builtin_ia32_cvtqq2pd128_mask", IX86_BUILTIN_CVTQQ2PD128, UNKNOWN, (int) V2DF_FTYPE_V2DI_V2DF_QI },
32680 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv4div4df2_mask, "__builtin_ia32_cvtuqq2pd256_mask", IX86_BUILTIN_CVTUQQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4DI_V4DF_QI },
32681 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv2div2df2_mask, "__builtin_ia32_cvtuqq2pd128_mask", IX86_BUILTIN_CVTUQQ2PD128, UNKNOWN, (int) V2DF_FTYPE_V2DI_V2DF_QI },
32682 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4di3_mask, "__builtin_ia32_vpermt2varq256_mask", IX86_BUILTIN_VPERMT2VARQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32683 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4di3_maskz, "__builtin_ia32_vpermt2varq256_maskz", IX86_BUILTIN_VPERMT2VARQ256_MASKZ, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32684 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8si3_mask, "__builtin_ia32_vpermt2vard256_mask", IX86_BUILTIN_VPERMT2VARD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32685 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8si3_maskz, "__builtin_ia32_vpermt2vard256_maskz", IX86_BUILTIN_VPERMT2VARD256_MASKZ, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32686 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv4di3_mask, "__builtin_ia32_vpermi2varq256_mask", IX86_BUILTIN_VPERMI2VARQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32687 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv8si3_mask, "__builtin_ia32_vpermi2vard256_mask", IX86_BUILTIN_VPERMI2VARD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32688 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4df3_mask, "__builtin_ia32_vpermt2varpd256_mask", IX86_BUILTIN_VPERMT2VARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DI_V4DF_V4DF_QI },
32689 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4df3_maskz, "__builtin_ia32_vpermt2varpd256_maskz", IX86_BUILTIN_VPERMT2VARPD256_MASKZ, UNKNOWN, (int) V4DF_FTYPE_V4DI_V4DF_V4DF_QI },
32690 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8sf3_mask, "__builtin_ia32_vpermt2varps256_mask", IX86_BUILTIN_VPERMT2VARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SI_V8SF_V8SF_QI },
32691 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8sf3_maskz, "__builtin_ia32_vpermt2varps256_maskz", IX86_BUILTIN_VPERMT2VARPS256_MASKZ, UNKNOWN, (int) V8SF_FTYPE_V8SI_V8SF_V8SF_QI },
32692 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv4df3_mask, "__builtin_ia32_vpermi2varpd256_mask", IX86_BUILTIN_VPERMI2VARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI_V4DF_QI },
32693 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv8sf3_mask, "__builtin_ia32_vpermi2varps256_mask", IX86_BUILTIN_VPERMI2VARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI_V8SF_QI },
32694 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv2di3_mask, "__builtin_ia32_vpermt2varq128_mask", IX86_BUILTIN_VPERMT2VARQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32695 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv2di3_maskz, "__builtin_ia32_vpermt2varq128_maskz", IX86_BUILTIN_VPERMT2VARQ128_MASKZ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32696 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4si3_mask, "__builtin_ia32_vpermt2vard128_mask", IX86_BUILTIN_VPERMT2VARD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32697 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4si3_maskz, "__builtin_ia32_vpermt2vard128_maskz", IX86_BUILTIN_VPERMT2VARD128_MASKZ, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32698 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv2di3_mask, "__builtin_ia32_vpermi2varq128_mask", IX86_BUILTIN_VPERMI2VARQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32699 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv4si3_mask, "__builtin_ia32_vpermi2vard128_mask", IX86_BUILTIN_VPERMI2VARD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32700 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv2df3_mask, "__builtin_ia32_vpermt2varpd128_mask", IX86_BUILTIN_VPERMT2VARPD128, UNKNOWN, (int) V2DF_FTYPE_V2DI_V2DF_V2DF_QI },
32701 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv2df3_maskz, "__builtin_ia32_vpermt2varpd128_maskz", IX86_BUILTIN_VPERMT2VARPD128_MASKZ, UNKNOWN, (int) V2DF_FTYPE_V2DI_V2DF_V2DF_QI },
32702 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4sf3_mask, "__builtin_ia32_vpermt2varps128_mask", IX86_BUILTIN_VPERMT2VARPS128, UNKNOWN, (int) V4SF_FTYPE_V4SI_V4SF_V4SF_QI },
32703 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4sf3_maskz, "__builtin_ia32_vpermt2varps128_maskz", IX86_BUILTIN_VPERMT2VARPS128_MASKZ, UNKNOWN, (int) V4SF_FTYPE_V4SI_V4SF_V4SF_QI },
32704 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv2df3_mask, "__builtin_ia32_vpermi2varpd128_mask", IX86_BUILTIN_VPERMI2VARPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI_V2DF_QI },
32705 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv4sf3_mask, "__builtin_ia32_vpermi2varps128_mask", IX86_BUILTIN_VPERMI2VARPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI_V4SF_QI },
32706 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_pshufbv32qi3_mask, "__builtin_ia32_pshufb256_mask", IX86_BUILTIN_PSHUFB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32707 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ssse3_pshufbv16qi3_mask, "__builtin_ia32_pshufb128_mask", IX86_BUILTIN_PSHUFB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32708 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshufhwv3_mask, "__builtin_ia32_pshufhw256_mask", IX86_BUILTIN_PSHUFHW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT_V16HI_HI },
32709 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshufhw_mask, "__builtin_ia32_pshufhw128_mask", IX86_BUILTIN_PSHUFHW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT_V8HI_QI },
32710 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshuflwv3_mask, "__builtin_ia32_pshuflw256_mask", IX86_BUILTIN_PSHUFLW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT_V16HI_HI },
32711 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshuflw_mask, "__builtin_ia32_pshuflw128_mask", IX86_BUILTIN_PSHUFLW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT_V8HI_QI },
32712 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshufdv3_mask, "__builtin_ia32_pshufd256_mask", IX86_BUILTIN_PSHUFD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32713 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshufd_mask, "__builtin_ia32_pshufd128_mask", IX86_BUILTIN_PSHUFD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32714 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_shufpd256_mask, "__builtin_ia32_shufpd256_mask", IX86_BUILTIN_SHUFPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT_V4DF_QI },
32715 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_shufpd_mask, "__builtin_ia32_shufpd128_mask", IX86_BUILTIN_SHUFPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI },
32716 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_shufps256_mask, "__builtin_ia32_shufps256_mask", IX86_BUILTIN_SHUFPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT_V8SF_QI },
32717 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse_shufps_mask, "__builtin_ia32_shufps128_mask", IX86_BUILTIN_SHUFPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI },
32718 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolvv4di_mask, "__builtin_ia32_prolvq256_mask", IX86_BUILTIN_PROLVQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32719 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolvv2di_mask, "__builtin_ia32_prolvq128_mask", IX86_BUILTIN_PROLVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32720 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolv4di_mask, "__builtin_ia32_prolq256_mask", IX86_BUILTIN_PROLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32721 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolv2di_mask, "__builtin_ia32_prolq128_mask", IX86_BUILTIN_PROLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_V2DI_QI },
32722 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorvv4di_mask, "__builtin_ia32_prorvq256_mask", IX86_BUILTIN_PRORVQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32723 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorvv2di_mask, "__builtin_ia32_prorvq128_mask", IX86_BUILTIN_PRORVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32724 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorv4di_mask, "__builtin_ia32_prorq256_mask", IX86_BUILTIN_PRORQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32725 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorv2di_mask, "__builtin_ia32_prorq128_mask", IX86_BUILTIN_PRORQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_V2DI_QI },
32726 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashrvv2di_mask, "__builtin_ia32_psravq128_mask", IX86_BUILTIN_PSRAVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32727 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashrvv4di_mask, "__builtin_ia32_psravq256_mask", IX86_BUILTIN_PSRAVQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32728 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashlvv4di_mask, "__builtin_ia32_psllv4di_mask", IX86_BUILTIN_PSLLVV4DI_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32729 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashlvv2di_mask, "__builtin_ia32_psllv2di_mask", IX86_BUILTIN_PSLLVV2DI_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32730 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashlvv8si_mask, "__builtin_ia32_psllv8si_mask", IX86_BUILTIN_PSLLVV8SI_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32731 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashlvv4si_mask, "__builtin_ia32_psllv4si_mask", IX86_BUILTIN_PSLLVV4SI_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32732 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashrvv8si_mask, "__builtin_ia32_psrav8si_mask", IX86_BUILTIN_PSRAVV8SI_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32733 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashrvv4si_mask, "__builtin_ia32_psrav4si_mask", IX86_BUILTIN_PSRAVV4SI_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32734 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_lshrvv4di_mask, "__builtin_ia32_psrlv4di_mask", IX86_BUILTIN_PSRLVV4DI_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32735 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_lshrvv2di_mask, "__builtin_ia32_psrlv2di_mask", IX86_BUILTIN_PSRLVV2DI_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32736 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_lshrvv8si_mask, "__builtin_ia32_psrlv8si_mask", IX86_BUILTIN_PSRLVV8SI_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32737 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_lshrvv4si_mask, "__builtin_ia32_psrlv4si_mask", IX86_BUILTIN_PSRLVV4SI_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32738 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv16hi3_mask, "__builtin_ia32_psrawi256_mask", IX86_BUILTIN_PSRAWI256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT_V16HI_HI },
32739 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv16hi3_mask, "__builtin_ia32_psraw256_mask", IX86_BUILTIN_PSRAW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_V16HI_HI },
32740 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv8hi3_mask, "__builtin_ia32_psrawi128_mask", IX86_BUILTIN_PSRAWI128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT_V8HI_QI },
32741 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv8hi3_mask, "__builtin_ia32_psraw128_mask", IX86_BUILTIN_PSRAW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32742 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv16hi3_mask, "__builtin_ia32_psrlwi256_mask", IX86_BUILTIN_PSRLWI256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT_V16HI_HI },
32743 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv16hi3_mask, "__builtin_ia32_psrlw256_mask", IX86_BUILTIN_PSRLW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_V16HI_HI },
32744 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv8hi3_mask, "__builtin_ia32_psrlwi128_mask", IX86_BUILTIN_PSRLWI128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT_V8HI_QI },
32745 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv8hi3_mask, "__builtin_ia32_psrlw128_mask", IX86_BUILTIN_PSRLW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32746 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorvv8si_mask, "__builtin_ia32_prorvd256_mask", IX86_BUILTIN_PRORVD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32747 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolvv8si_mask, "__builtin_ia32_prolvd256_mask", IX86_BUILTIN_PROLVD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32748 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorv8si_mask, "__builtin_ia32_prord256_mask", IX86_BUILTIN_PRORD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32749 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolv8si_mask, "__builtin_ia32_prold256_mask", IX86_BUILTIN_PROLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32750 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorvv4si_mask, "__builtin_ia32_prorvd128_mask", IX86_BUILTIN_PRORVD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32751 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolvv4si_mask, "__builtin_ia32_prolvd128_mask", IX86_BUILTIN_PROLVD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32752 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorv4si_mask, "__builtin_ia32_prord128_mask", IX86_BUILTIN_PRORD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32753 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolv4si_mask, "__builtin_ia32_prold128_mask", IX86_BUILTIN_PROLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32754 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_fpclassv4df_mask, "__builtin_ia32_fpclasspd256_mask", IX86_BUILTIN_FPCLASSPD256, UNKNOWN, (int) QI_FTYPE_V4DF_INT_QI },
32755 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_fpclassv2df_mask, "__builtin_ia32_fpclasspd128_mask", IX86_BUILTIN_FPCLASSPD128, UNKNOWN, (int) QI_FTYPE_V2DF_INT_QI },
32756 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vmfpclassv2df, "__builtin_ia32_fpclasssd", IX86_BUILTIN_FPCLASSSD, UNKNOWN, (int) QI_FTYPE_V2DF_INT },
32757 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_fpclassv8sf_mask, "__builtin_ia32_fpclassps256_mask", IX86_BUILTIN_FPCLASSPS256, UNKNOWN, (int) QI_FTYPE_V8SF_INT_QI },
32758 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_fpclassv4sf_mask, "__builtin_ia32_fpclassps128_mask", IX86_BUILTIN_FPCLASSPS128, UNKNOWN, (int) QI_FTYPE_V4SF_INT_QI },
32759 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vmfpclassv4sf, "__builtin_ia32_fpclassss", IX86_BUILTIN_FPCLASSSS, UNKNOWN, (int) QI_FTYPE_V4SF_INT },
32760 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtb2maskv16qi, "__builtin_ia32_cvtb2mask128", IX86_BUILTIN_CVTB2MASK128, UNKNOWN, (int) HI_FTYPE_V16QI },
32761 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtb2maskv32qi, "__builtin_ia32_cvtb2mask256", IX86_BUILTIN_CVTB2MASK256, UNKNOWN, (int) SI_FTYPE_V32QI },
32762 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtw2maskv8hi, "__builtin_ia32_cvtw2mask128", IX86_BUILTIN_CVTW2MASK128, UNKNOWN, (int) QI_FTYPE_V8HI },
32763 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtw2maskv16hi, "__builtin_ia32_cvtw2mask256", IX86_BUILTIN_CVTW2MASK256, UNKNOWN, (int) HI_FTYPE_V16HI },
32764 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtd2maskv4si, "__builtin_ia32_cvtd2mask128", IX86_BUILTIN_CVTD2MASK128, UNKNOWN, (int) QI_FTYPE_V4SI },
32765 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtd2maskv8si, "__builtin_ia32_cvtd2mask256", IX86_BUILTIN_CVTD2MASK256, UNKNOWN, (int) QI_FTYPE_V8SI },
32766 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtq2maskv2di, "__builtin_ia32_cvtq2mask128", IX86_BUILTIN_CVTQ2MASK128, UNKNOWN, (int) QI_FTYPE_V2DI },
32767 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtq2maskv4di, "__builtin_ia32_cvtq2mask256", IX86_BUILTIN_CVTQ2MASK256, UNKNOWN, (int) QI_FTYPE_V4DI },
32768 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2bv16qi, "__builtin_ia32_cvtmask2b128", IX86_BUILTIN_CVTMASK2B128, UNKNOWN, (int) V16QI_FTYPE_HI },
32769 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2bv32qi, "__builtin_ia32_cvtmask2b256", IX86_BUILTIN_CVTMASK2B256, UNKNOWN, (int) V32QI_FTYPE_SI },
32770 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2wv8hi, "__builtin_ia32_cvtmask2w128", IX86_BUILTIN_CVTMASK2W128, UNKNOWN, (int) V8HI_FTYPE_QI },
32771 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2wv16hi, "__builtin_ia32_cvtmask2w256", IX86_BUILTIN_CVTMASK2W256, UNKNOWN, (int) V16HI_FTYPE_HI },
32772 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2dv4si, "__builtin_ia32_cvtmask2d128", IX86_BUILTIN_CVTMASK2D128, UNKNOWN, (int) V4SI_FTYPE_QI },
32773 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2dv8si, "__builtin_ia32_cvtmask2d256", IX86_BUILTIN_CVTMASK2D256, UNKNOWN, (int) V8SI_FTYPE_QI },
32774 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2qv2di, "__builtin_ia32_cvtmask2q128", IX86_BUILTIN_CVTMASK2Q128, UNKNOWN, (int) V2DI_FTYPE_QI },
32775 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2qv4di, "__builtin_ia32_cvtmask2q256", IX86_BUILTIN_CVTMASK2Q256, UNKNOWN, (int) V4DI_FTYPE_QI },
32776 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv16qi3_mask, "__builtin_ia32_pcmpeqb128_mask", IX86_BUILTIN_PCMPEQB128_MASK, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_HI },
32777 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv32qi3_mask, "__builtin_ia32_pcmpeqb256_mask", IX86_BUILTIN_PCMPEQB256_MASK, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_SI },
32778 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv8hi3_mask, "__builtin_ia32_pcmpeqw128_mask", IX86_BUILTIN_PCMPEQW128_MASK, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_QI },
32779 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv16hi3_mask, "__builtin_ia32_pcmpeqw256_mask", IX86_BUILTIN_PCMPEQW256_MASK, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_HI },
32780 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv4si3_mask, "__builtin_ia32_pcmpeqd128_mask", IX86_BUILTIN_PCMPEQD128_MASK, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_QI },
32781 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv8si3_mask, "__builtin_ia32_pcmpeqd256_mask", IX86_BUILTIN_PCMPEQD256_MASK, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_QI },
32782 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv2di3_mask, "__builtin_ia32_pcmpeqq128_mask", IX86_BUILTIN_PCMPEQQ128_MASK, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_QI },
32783 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv4di3_mask, "__builtin_ia32_pcmpeqq256_mask", IX86_BUILTIN_PCMPEQQ256_MASK, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_QI },
32784 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv16qi3_mask, "__builtin_ia32_pcmpgtb128_mask", IX86_BUILTIN_PCMPGTB128_MASK, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_HI },
32785 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv32qi3_mask, "__builtin_ia32_pcmpgtb256_mask", IX86_BUILTIN_PCMPGTB256_MASK, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_SI },
32786 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv8hi3_mask, "__builtin_ia32_pcmpgtw128_mask", IX86_BUILTIN_PCMPGTW128_MASK, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_QI },
32787 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv16hi3_mask, "__builtin_ia32_pcmpgtw256_mask", IX86_BUILTIN_PCMPGTW256_MASK, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_HI },
32788 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv4si3_mask, "__builtin_ia32_pcmpgtd128_mask", IX86_BUILTIN_PCMPGTD128_MASK, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_QI },
32789 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv8si3_mask, "__builtin_ia32_pcmpgtd256_mask", IX86_BUILTIN_PCMPGTD256_MASK, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_QI },
32790 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv2di3_mask, "__builtin_ia32_pcmpgtq128_mask", IX86_BUILTIN_PCMPGTQ128_MASK, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_QI },
32791 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv4di3_mask, "__builtin_ia32_pcmpgtq256_mask", IX86_BUILTIN_PCMPGTQ256_MASK, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_QI },
32792 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv16qi3_mask, "__builtin_ia32_ptestmb128", IX86_BUILTIN_PTESTMB128, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_HI },
32793 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv32qi3_mask, "__builtin_ia32_ptestmb256", IX86_BUILTIN_PTESTMB256, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_SI },
32794 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv8hi3_mask, "__builtin_ia32_ptestmw128", IX86_BUILTIN_PTESTMW128, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_QI },
32795 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv16hi3_mask, "__builtin_ia32_ptestmw256", IX86_BUILTIN_PTESTMW256, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_HI },
32796 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv4si3_mask, "__builtin_ia32_ptestmd128", IX86_BUILTIN_PTESTMD128, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_QI },
32797 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv8si3_mask, "__builtin_ia32_ptestmd256", IX86_BUILTIN_PTESTMD256, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_QI },
32798 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv2di3_mask, "__builtin_ia32_ptestmq128", IX86_BUILTIN_PTESTMQ128, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_QI },
32799 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv4di3_mask, "__builtin_ia32_ptestmq256", IX86_BUILTIN_PTESTMQ256, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_QI },
32800 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv16qi3_mask, "__builtin_ia32_ptestnmb128", IX86_BUILTIN_PTESTNMB128, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_HI },
32801 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv32qi3_mask, "__builtin_ia32_ptestnmb256", IX86_BUILTIN_PTESTNMB256, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_SI },
32802 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv8hi3_mask, "__builtin_ia32_ptestnmw128", IX86_BUILTIN_PTESTNMW128, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_QI },
32803 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv16hi3_mask, "__builtin_ia32_ptestnmw256", IX86_BUILTIN_PTESTNMW256, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_HI },
32804 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv4si3_mask, "__builtin_ia32_ptestnmd128", IX86_BUILTIN_PTESTNMD128, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_QI },
32805 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv8si3_mask, "__builtin_ia32_ptestnmd256", IX86_BUILTIN_PTESTNMD256, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_QI },
32806 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv2di3_mask, "__builtin_ia32_ptestnmq128", IX86_BUILTIN_PTESTNMQ128, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_QI },
32807 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv4di3_mask, "__builtin_ia32_ptestnmq256", IX86_BUILTIN_PTESTNMQ256, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_QI },
32808 { OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskb_vec_dupv2di, "__builtin_ia32_broadcastmb128", IX86_BUILTIN_PBROADCASTMB128, UNKNOWN, (int) V2DI_FTYPE_QI },
32809 { OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskb_vec_dupv4di, "__builtin_ia32_broadcastmb256", IX86_BUILTIN_PBROADCASTMB256, UNKNOWN, (int) V4DI_FTYPE_QI },
32810 { OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskw_vec_dupv4si, "__builtin_ia32_broadcastmw128", IX86_BUILTIN_PBROADCASTMW128, UNKNOWN, (int) V4SI_FTYPE_HI },
32811 { OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskw_vec_dupv8si, "__builtin_ia32_broadcastmw256", IX86_BUILTIN_PBROADCASTMW256, UNKNOWN, (int) V8SI_FTYPE_HI },
32812 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv4df_mask, "__builtin_ia32_compressdf256_mask", IX86_BUILTIN_COMPRESSPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32813 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv2df_mask, "__builtin_ia32_compressdf128_mask", IX86_BUILTIN_COMPRESSPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32814 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv8sf_mask, "__builtin_ia32_compresssf256_mask", IX86_BUILTIN_COMPRESSPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32815 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv4sf_mask, "__builtin_ia32_compresssf128_mask", IX86_BUILTIN_COMPRESSPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32816 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv4di_mask, "__builtin_ia32_compressdi256_mask", IX86_BUILTIN_PCOMPRESSQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32817 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv2di_mask, "__builtin_ia32_compressdi128_mask", IX86_BUILTIN_PCOMPRESSQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32818 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv8si_mask, "__builtin_ia32_compresssi256_mask", IX86_BUILTIN_PCOMPRESSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32819 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv4si_mask, "__builtin_ia32_compresssi128_mask", IX86_BUILTIN_PCOMPRESSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32820 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4df_mask, "__builtin_ia32_expanddf256_mask", IX86_BUILTIN_EXPANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32821 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2df_mask, "__builtin_ia32_expanddf128_mask", IX86_BUILTIN_EXPANDPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32822 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8sf_mask, "__builtin_ia32_expandsf256_mask", IX86_BUILTIN_EXPANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32823 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4sf_mask, "__builtin_ia32_expandsf128_mask", IX86_BUILTIN_EXPANDPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32824 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4di_mask, "__builtin_ia32_expanddi256_mask", IX86_BUILTIN_PEXPANDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32825 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2di_mask, "__builtin_ia32_expanddi128_mask", IX86_BUILTIN_PEXPANDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32826 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8si_mask, "__builtin_ia32_expandsi256_mask", IX86_BUILTIN_PEXPANDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32827 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4si_mask, "__builtin_ia32_expandsi128_mask", IX86_BUILTIN_PEXPANDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32828 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4df_maskz, "__builtin_ia32_expanddf256_maskz", IX86_BUILTIN_EXPANDPD256Z, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32829 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2df_maskz, "__builtin_ia32_expanddf128_maskz", IX86_BUILTIN_EXPANDPD128Z, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32830 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8sf_maskz, "__builtin_ia32_expandsf256_maskz", IX86_BUILTIN_EXPANDPS256Z, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32831 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4sf_maskz, "__builtin_ia32_expandsf128_maskz", IX86_BUILTIN_EXPANDPS128Z, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32832 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4di_maskz, "__builtin_ia32_expanddi256_maskz", IX86_BUILTIN_PEXPANDQ256Z, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32833 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2di_maskz, "__builtin_ia32_expanddi128_maskz", IX86_BUILTIN_PEXPANDQ128Z, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32834 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8si_maskz, "__builtin_ia32_expandsi256_maskz", IX86_BUILTIN_PEXPANDD256Z, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32835 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4si_maskz, "__builtin_ia32_expandsi128_maskz", IX86_BUILTIN_PEXPANDD128Z, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32836 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv8si3_mask, "__builtin_ia32_pmaxsd256_mask", IX86_BUILTIN_PMAXSD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32837 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv8si3_mask, "__builtin_ia32_pminsd256_mask", IX86_BUILTIN_PMINSD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32838 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv8si3_mask, "__builtin_ia32_pmaxud256_mask", IX86_BUILTIN_PMAXUD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32839 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv8si3_mask, "__builtin_ia32_pminud256_mask", IX86_BUILTIN_PMINUD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32840 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv4si3_mask, "__builtin_ia32_pmaxsd128_mask", IX86_BUILTIN_PMAXSD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32841 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv4si3_mask, "__builtin_ia32_pminsd128_mask", IX86_BUILTIN_PMINSD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32842 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv4si3_mask, "__builtin_ia32_pmaxud128_mask", IX86_BUILTIN_PMAXUD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32843 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv4si3_mask, "__builtin_ia32_pminud128_mask", IX86_BUILTIN_PMINUD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32844 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv4di3_mask, "__builtin_ia32_pmaxsq256_mask", IX86_BUILTIN_PMAXSQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32845 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv4di3_mask, "__builtin_ia32_pminsq256_mask", IX86_BUILTIN_PMINSQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32846 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv4di3_mask, "__builtin_ia32_pmaxuq256_mask", IX86_BUILTIN_PMAXUQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32847 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv4di3_mask, "__builtin_ia32_pminuq256_mask", IX86_BUILTIN_PMINUQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32848 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv2di3_mask, "__builtin_ia32_pmaxsq128_mask", IX86_BUILTIN_PMAXSQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32849 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv2di3_mask, "__builtin_ia32_pminsq128_mask", IX86_BUILTIN_PMINSQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32850 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv2di3_mask, "__builtin_ia32_pmaxuq128_mask", IX86_BUILTIN_PMAXUQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32851 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv2di3_mask, "__builtin_ia32_pminuq128_mask", IX86_BUILTIN_PMINUQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32852 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv32qi3_mask, "__builtin_ia32_pminsb256_mask", IX86_BUILTIN_PMINSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32853 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv32qi3_mask, "__builtin_ia32_pminub256_mask", IX86_BUILTIN_PMINUB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32854 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv32qi3_mask, "__builtin_ia32_pmaxsb256_mask", IX86_BUILTIN_PMAXSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32855 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv32qi3_mask, "__builtin_ia32_pmaxub256_mask", IX86_BUILTIN_PMAXUB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32856 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv16qi3_mask, "__builtin_ia32_pminsb128_mask", IX86_BUILTIN_PMINSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32857 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv16qi3_mask, "__builtin_ia32_pminub128_mask", IX86_BUILTIN_PMINUB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32858 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv16qi3_mask, "__builtin_ia32_pmaxsb128_mask", IX86_BUILTIN_PMAXSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32859 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv16qi3_mask, "__builtin_ia32_pmaxub128_mask", IX86_BUILTIN_PMAXUB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32860 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv16hi3_mask, "__builtin_ia32_pminsw256_mask", IX86_BUILTIN_PMINSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32861 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv16hi3_mask, "__builtin_ia32_pminuw256_mask", IX86_BUILTIN_PMINUW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32862 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv16hi3_mask, "__builtin_ia32_pmaxsw256_mask", IX86_BUILTIN_PMAXSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32863 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv16hi3_mask, "__builtin_ia32_pmaxuw256_mask", IX86_BUILTIN_PMAXUW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32864 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv8hi3_mask, "__builtin_ia32_pminsw128_mask", IX86_BUILTIN_PMINSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32865 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv8hi3_mask, "__builtin_ia32_pminuw128_mask", IX86_BUILTIN_PMINUW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32866 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv8hi3_mask, "__builtin_ia32_pmaxsw128_mask", IX86_BUILTIN_PMAXSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32867 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv8hi3_mask, "__builtin_ia32_pmaxuw128_mask", IX86_BUILTIN_PMAXUW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32868 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_conflictv4di_mask, "__builtin_ia32_vpconflictdi_256_mask", IX86_BUILTIN_VPCONFLICTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32869 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_conflictv8si_mask, "__builtin_ia32_vpconflictsi_256_mask", IX86_BUILTIN_VPCONFLICTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32870 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_clzv4di2_mask, "__builtin_ia32_vplzcntq_256_mask", IX86_BUILTIN_VPCLZCNTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32871 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_clzv8si2_mask, "__builtin_ia32_vplzcntd_256_mask", IX86_BUILTIN_VPCLZCNTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32872 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_unpckhpd256_mask, "__builtin_ia32_unpckhpd256_mask", IX86_BUILTIN_UNPCKHPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32873 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_unpckhpd128_mask, "__builtin_ia32_unpckhpd128_mask", IX86_BUILTIN_UNPCKHPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32874 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_unpckhps256_mask, "__builtin_ia32_unpckhps256_mask", IX86_BUILTIN_UNPCKHPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32875 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_highv4sf_mask, "__builtin_ia32_unpckhps128_mask", IX86_BUILTIN_UNPCKHPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32876 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_unpcklpd256_mask, "__builtin_ia32_unpcklpd256_mask", IX86_BUILTIN_UNPCKLPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32877 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_unpcklpd128_mask, "__builtin_ia32_unpcklpd128_mask", IX86_BUILTIN_UNPCKLPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32878 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_unpcklps256_mask, "__builtin_ia32_unpcklps256_mask", IX86_BUILTIN_UNPCKLPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32879 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_conflictv2di_mask, "__builtin_ia32_vpconflictdi_128_mask", IX86_BUILTIN_VPCONFLICTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32880 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_conflictv4si_mask, "__builtin_ia32_vpconflictsi_128_mask", IX86_BUILTIN_VPCONFLICTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32881 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_clzv2di2_mask, "__builtin_ia32_vplzcntq_128_mask", IX86_BUILTIN_VPCLZCNTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32882 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_clzv4si2_mask, "__builtin_ia32_vplzcntd_128_mask", IX86_BUILTIN_VPCLZCNTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32883 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_unpcklps128_mask, "__builtin_ia32_unpcklps128_mask", IX86_BUILTIN_UNPCKLPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32884 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_alignv8si_mask, "__builtin_ia32_alignd256_mask", IX86_BUILTIN_ALIGND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT_V8SI_QI },
32885 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_alignv4di_mask, "__builtin_ia32_alignq256_mask", IX86_BUILTIN_ALIGNQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_V4DI_QI },
32886 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_alignv4si_mask, "__builtin_ia32_alignd128_mask", IX86_BUILTIN_ALIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT_V4SI_QI },
32887 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_alignv2di_mask, "__builtin_ia32_alignq128_mask", IX86_BUILTIN_ALIGNQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_V2DI_QI },
32888 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vcvtps2ph256_mask, "__builtin_ia32_vcvtps2ph256_mask", IX86_BUILTIN_CVTPS2PH256_MASK, UNKNOWN, (int) V8HI_FTYPE_V8SF_INT_V8HI_QI },
32889 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vcvtps2ph_mask, "__builtin_ia32_vcvtps2ph_mask", IX86_BUILTIN_CVTPS2PH_MASK, UNKNOWN, (int) V8HI_FTYPE_V4SF_INT_V8HI_QI },
32890 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vcvtph2ps_mask, "__builtin_ia32_vcvtph2ps_mask", IX86_BUILTIN_CVTPH2PS_MASK, UNKNOWN, (int) V4SF_FTYPE_V8HI_V4SF_QI },
32891 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vcvtph2ps256_mask, "__builtin_ia32_vcvtph2ps256_mask", IX86_BUILTIN_CVTPH2PS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8HI_V8SF_QI },
32892 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_highv4si_mask, "__builtin_ia32_punpckhdq128_mask", IX86_BUILTIN_PUNPCKHDQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32893 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_highv8si_mask, "__builtin_ia32_punpckhdq256_mask", IX86_BUILTIN_PUNPCKHDQ256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32894 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_highv2di_mask, "__builtin_ia32_punpckhqdq128_mask", IX86_BUILTIN_PUNPCKHQDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32895 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_highv4di_mask, "__builtin_ia32_punpckhqdq256_mask", IX86_BUILTIN_PUNPCKHQDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32896 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_lowv4si_mask, "__builtin_ia32_punpckldq128_mask", IX86_BUILTIN_PUNPCKLDQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32897 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_lowv8si_mask, "__builtin_ia32_punpckldq256_mask", IX86_BUILTIN_PUNPCKLDQ256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32898 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_lowv2di_mask, "__builtin_ia32_punpcklqdq128_mask", IX86_BUILTIN_PUNPCKLQDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32899 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_lowv4di_mask, "__builtin_ia32_punpcklqdq256_mask", IX86_BUILTIN_PUNPCKLQDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32900 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_highv16qi_mask, "__builtin_ia32_punpckhbw128_mask", IX86_BUILTIN_PUNPCKHBW128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32901 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_highv32qi_mask, "__builtin_ia32_punpckhbw256_mask", IX86_BUILTIN_PUNPCKHBW256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32902 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_highv8hi_mask, "__builtin_ia32_punpckhwd128_mask", IX86_BUILTIN_PUNPCKHWD128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32903 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_highv16hi_mask, "__builtin_ia32_punpckhwd256_mask", IX86_BUILTIN_PUNPCKHWD256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32904 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_lowv16qi_mask, "__builtin_ia32_punpcklbw128_mask", IX86_BUILTIN_PUNPCKLBW128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32905 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_lowv32qi_mask, "__builtin_ia32_punpcklbw256_mask", IX86_BUILTIN_PUNPCKLBW256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32906 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_lowv8hi_mask, "__builtin_ia32_punpcklwd128_mask", IX86_BUILTIN_PUNPCKLWD128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32907 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_lowv16hi_mask, "__builtin_ia32_punpcklwd256_mask", IX86_BUILTIN_PUNPCKLWD256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32908 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ashlvv16hi_mask, "__builtin_ia32_psllv16hi_mask", IX86_BUILTIN_PSLLVV16HI, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32909 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ashlvv8hi_mask, "__builtin_ia32_psllv8hi_mask", IX86_BUILTIN_PSLLVV8HI, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32910 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_packssdw_mask, "__builtin_ia32_packssdw256_mask", IX86_BUILTIN_PACKSSDW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI_V16HI_HI },
32911 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_packssdw_mask, "__builtin_ia32_packssdw128_mask", IX86_BUILTIN_PACKSSDW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI_V8HI_QI },
32912 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_packusdw_mask, "__builtin_ia32_packusdw256_mask", IX86_BUILTIN_PACKUSDW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI_V16HI_HI },
32913 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_packusdw_mask, "__builtin_ia32_packusdw128_mask", IX86_BUILTIN_PACKUSDW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI_V8HI_QI },
32914 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_uavgv32qi3_mask, "__builtin_ia32_pavgb256_mask", IX86_BUILTIN_PAVGB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32915 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_uavgv16hi3_mask, "__builtin_ia32_pavgw256_mask", IX86_BUILTIN_PAVGW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32916 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_uavgv16qi3_mask, "__builtin_ia32_pavgb128_mask", IX86_BUILTIN_PAVGB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32917 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_uavgv8hi3_mask, "__builtin_ia32_pavgw128_mask", IX86_BUILTIN_PAVGW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32918 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_permvarv8sf_mask, "__builtin_ia32_permvarsf256_mask", IX86_BUILTIN_VPERMVARSF256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI_V8SF_QI },
32919 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_permvarv4df_mask, "__builtin_ia32_permvardf256_mask", IX86_BUILTIN_VPERMVARDF256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI_V4DF_QI },
32920 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permv4df_mask, "__builtin_ia32_permdf256_mask", IX86_BUILTIN_VPERMDF256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT_V4DF_QI },
32921 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv32qi2_mask, "__builtin_ia32_pabsb256_mask", IX86_BUILTIN_PABSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_SI },
32922 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv16qi2_mask, "__builtin_ia32_pabsb128_mask", IX86_BUILTIN_PABSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_HI },
32923 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv16hi2_mask, "__builtin_ia32_pabsw256_mask", IX86_BUILTIN_PABSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_HI },
32924 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv8hi2_mask, "__builtin_ia32_pabsw128_mask", IX86_BUILTIN_PABSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_QI },
32925 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilvarv2df3_mask, "__builtin_ia32_vpermilvarpd_mask", IX86_BUILTIN_VPERMILVARPD_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI_V2DF_QI },
32926 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilvarv4sf3_mask, "__builtin_ia32_vpermilvarps_mask", IX86_BUILTIN_VPERMILVARPS_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI_V4SF_QI },
32927 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilvarv4df3_mask, "__builtin_ia32_vpermilvarpd256_mask", IX86_BUILTIN_VPERMILVARPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI_V4DF_QI },
32928 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilvarv8sf3_mask, "__builtin_ia32_vpermilvarps256_mask", IX86_BUILTIN_VPERMILVARPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI_V8SF_QI },
32929 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilv2df_mask, "__builtin_ia32_vpermilpd_mask", IX86_BUILTIN_VPERMILPD_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT_V2DF_QI },
32930 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilv4sf_mask, "__builtin_ia32_vpermilps_mask", IX86_BUILTIN_VPERMILPS_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT_V4SF_QI },
32931 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilv4df_mask, "__builtin_ia32_vpermilpd256_mask", IX86_BUILTIN_VPERMILPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT_V4DF_QI },
32932 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilv8sf_mask, "__builtin_ia32_vpermilps256_mask", IX86_BUILTIN_VPERMILPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT_V8SF_QI },
32933 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv4di, "__builtin_ia32_blendmq_256_mask", IX86_BUILTIN_BLENDMQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32934 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv8si, "__builtin_ia32_blendmd_256_mask", IX86_BUILTIN_BLENDMD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32935 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv4df, "__builtin_ia32_blendmpd_256_mask", IX86_BUILTIN_BLENDMPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32936 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv8sf, "__builtin_ia32_blendmps_256_mask", IX86_BUILTIN_BLENDMPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32937 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv2di, "__builtin_ia32_blendmq_128_mask", IX86_BUILTIN_BLENDMQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32938 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv4si, "__builtin_ia32_blendmd_128_mask", IX86_BUILTIN_BLENDMD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32939 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv2df, "__builtin_ia32_blendmpd_128_mask", IX86_BUILTIN_BLENDMPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32940 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv4sf, "__builtin_ia32_blendmps_128_mask", IX86_BUILTIN_BLENDMPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32941 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv16hi, "__builtin_ia32_blendmw_256_mask", IX86_BUILTIN_BLENDMW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_HI },
32942 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv32qi, "__builtin_ia32_blendmb_256_mask", IX86_BUILTIN_BLENDMB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_SI },
32943 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv8hi, "__builtin_ia32_blendmw_128_mask", IX86_BUILTIN_BLENDMW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_QI },
32944 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv16qi, "__builtin_ia32_blendmb_128_mask", IX86_BUILTIN_BLENDMB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_HI },
32945 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv8si3_mask, "__builtin_ia32_pmulld256_mask", IX86_BUILTIN_PMULLD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32946 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv4si3_mask, "__builtin_ia32_pmulld128_mask", IX86_BUILTIN_PMULLD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32947 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_widen_umult_even_v8si_mask, "__builtin_ia32_pmuludq256_mask", IX86_BUILTIN_PMULUDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI_V4DI_QI },
32948 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_widen_smult_even_v8si_mask, "__builtin_ia32_pmuldq256_mask", IX86_BUILTIN_PMULDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI_V4DI_QI },
32949 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_mulv2siv2di3_mask, "__builtin_ia32_pmuldq128_mask", IX86_BUILTIN_PMULDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI_V2DI_QI },
32950 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_widen_umult_even_v4si_mask, "__builtin_ia32_pmuludq128_mask", IX86_BUILTIN_PMULUDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI_V2DI_QI },
32951 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_cvtpd2ps256_mask, "__builtin_ia32_cvtpd2ps256_mask", IX86_BUILTIN_CVTPD2PS256_MASK, UNKNOWN, (int) V4SF_FTYPE_V4DF_V4SF_QI },
32952 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_cvtpd2ps_mask, "__builtin_ia32_cvtpd2ps_mask", IX86_BUILTIN_CVTPD2PS_MASK, UNKNOWN, (int) V4SF_FTYPE_V2DF_V4SF_QI },
32953 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_permvarv8si_mask, "__builtin_ia32_permvarsi256_mask", IX86_BUILTIN_VPERMVARSI256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32954 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_permvarv4di_mask, "__builtin_ia32_permvardi256_mask", IX86_BUILTIN_VPERMVARDI256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32955 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permv4di_mask, "__builtin_ia32_permdi256_mask", IX86_BUILTIN_VPERMDI256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32956 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv4di3_mask, "__builtin_ia32_cmpq256_mask", IX86_BUILTIN_CMPQ256, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_INT_QI },
32957 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv8si3_mask, "__builtin_ia32_cmpd256_mask", IX86_BUILTIN_CMPD256, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_INT_QI },
32958 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv4di3_mask, "__builtin_ia32_ucmpq256_mask", IX86_BUILTIN_UCMPQ256, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_INT_QI },
32959 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv8si3_mask, "__builtin_ia32_ucmpd256_mask", IX86_BUILTIN_UCMPD256, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_INT_QI },
32960 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv32qi3_mask, "__builtin_ia32_cmpb256_mask", IX86_BUILTIN_CMPB256, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_INT_SI },
32961 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv16hi3_mask, "__builtin_ia32_cmpw256_mask", IX86_BUILTIN_CMPW256, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_INT_HI },
32962 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv32qi3_mask, "__builtin_ia32_ucmpb256_mask", IX86_BUILTIN_UCMPB256, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_INT_SI },
32963 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv16hi3_mask, "__builtin_ia32_ucmpw256_mask", IX86_BUILTIN_UCMPW256, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_INT_HI },
32964 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv4df3_mask, "__builtin_ia32_cmppd256_mask", IX86_BUILTIN_CMPPD256_MASK, UNKNOWN, (int) QI_FTYPE_V4DF_V4DF_INT_QI },
32965 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv8sf3_mask, "__builtin_ia32_cmpps256_mask", IX86_BUILTIN_CMPPS256_MASK, UNKNOWN, (int) QI_FTYPE_V8SF_V8SF_INT_QI },
32966 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv2di3_mask, "__builtin_ia32_cmpq128_mask", IX86_BUILTIN_CMPQ128, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_INT_QI },
32967 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv4si3_mask, "__builtin_ia32_cmpd128_mask", IX86_BUILTIN_CMPD128, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_INT_QI },
32968 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv2di3_mask, "__builtin_ia32_ucmpq128_mask", IX86_BUILTIN_UCMPQ128, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_INT_QI },
32969 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv4si3_mask, "__builtin_ia32_ucmpd128_mask", IX86_BUILTIN_UCMPD128, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_INT_QI },
32970 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv16qi3_mask, "__builtin_ia32_cmpb128_mask", IX86_BUILTIN_CMPB128, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_INT_HI },
32971 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv8hi3_mask, "__builtin_ia32_cmpw128_mask", IX86_BUILTIN_CMPW128, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_INT_QI },
32972 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv16qi3_mask, "__builtin_ia32_ucmpb128_mask", IX86_BUILTIN_UCMPB128, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_INT_HI },
32973 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv8hi3_mask, "__builtin_ia32_ucmpw128_mask", IX86_BUILTIN_UCMPW128, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_INT_QI },
32974 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv2df3_mask, "__builtin_ia32_cmppd128_mask", IX86_BUILTIN_CMPPD128_MASK, UNKNOWN, (int) QI_FTYPE_V2DF_V2DF_INT_QI },
32975 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv4sf3_mask, "__builtin_ia32_cmpps128_mask", IX86_BUILTIN_CMPPS128_MASK, UNKNOWN, (int) QI_FTYPE_V4SF_V4SF_INT_QI },
32978 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv16sf_mask, "__builtin_ia32_broadcastf32x2_512_mask", IX86_BUILTIN_BROADCASTF32x2_512, UNKNOWN, (int) V16SF_FTYPE_V4SF_V16SF_HI },
32979 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv16si_mask, "__builtin_ia32_broadcasti32x2_512_mask", IX86_BUILTIN_BROADCASTI32x2_512, UNKNOWN, (int) V16SI_FTYPE_V4SI_V16SI_HI },
32980 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv8df_mask_1, "__builtin_ia32_broadcastf64x2_512_mask", IX86_BUILTIN_BROADCASTF64X2_512, UNKNOWN, (int) V8DF_FTYPE_V2DF_V8DF_QI },
32981 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv8di_mask_1, "__builtin_ia32_broadcasti64x2_512_mask", IX86_BUILTIN_BROADCASTI64X2_512, UNKNOWN, (int) V8DI_FTYPE_V2DI_V8DI_QI },
32982 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv16sf_mask_1, "__builtin_ia32_broadcastf32x8_512_mask", IX86_BUILTIN_BROADCASTF32X8_512, UNKNOWN, (int) V16SF_FTYPE_V8SF_V16SF_HI },
32983 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv16si_mask_1, "__builtin_ia32_broadcasti32x8_512_mask", IX86_BUILTIN_BROADCASTI32X8_512, UNKNOWN, (int) V16SI_FTYPE_V8SI_V16SI_HI },
32984 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vextractf64x2_mask, "__builtin_ia32_extractf64x2_512_mask", IX86_BUILTIN_EXTRACTF64X2_512, UNKNOWN, (int) V2DF_FTYPE_V8DF_INT_V2DF_QI },
32985 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vextractf32x8_mask, "__builtin_ia32_extractf32x8_mask", IX86_BUILTIN_EXTRACTF32X8, UNKNOWN, (int) V8SF_FTYPE_V16SF_INT_V8SF_QI },
32986 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vextracti64x2_mask, "__builtin_ia32_extracti64x2_512_mask", IX86_BUILTIN_EXTRACTI64X2_512, UNKNOWN, (int) V2DI_FTYPE_V8DI_INT_V2DI_QI },
32987 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vextracti32x8_mask, "__builtin_ia32_extracti32x8_mask", IX86_BUILTIN_EXTRACTI32X8, UNKNOWN, (int) V8SI_FTYPE_V16SI_INT_V8SI_QI },
32988 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducepv8df_mask, "__builtin_ia32_reducepd512_mask", IX86_BUILTIN_REDUCEPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_INT_V8DF_QI },
32989 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducepv16sf_mask, "__builtin_ia32_reduceps512_mask", IX86_BUILTIN_REDUCEPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_INT_V16SF_HI },
32990 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_mulv8di3_mask, "__builtin_ia32_pmullq512_mask", IX86_BUILTIN_PMULLQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32991 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_xorv8df3_mask, "__builtin_ia32_xorpd512_mask", IX86_BUILTIN_XORPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI },
32992 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_xorv16sf3_mask, "__builtin_ia32_xorps512_mask", IX86_BUILTIN_XORPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32993 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_iorv8df3_mask, "__builtin_ia32_orpd512_mask", IX86_BUILTIN_ORPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI },
32994 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_iorv16sf3_mask, "__builtin_ia32_orps512_mask", IX86_BUILTIN_ORPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32995 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_andv8df3_mask, "__builtin_ia32_andpd512_mask", IX86_BUILTIN_ANDPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI },
32996 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_andv16sf3_mask, "__builtin_ia32_andps512_mask", IX86_BUILTIN_ANDPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32997 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_andnotv8df3_mask, "__builtin_ia32_andnpd512_mask", IX86_BUILTIN_ANDNPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI},
32998 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_andnotv16sf3_mask, "__builtin_ia32_andnps512_mask", IX86_BUILTIN_ANDNPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32999 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vinsertf32x8_mask, "__builtin_ia32_insertf32x8_mask", IX86_BUILTIN_INSERTF32X8, UNKNOWN, (int) V16SF_FTYPE_V16SF_V8SF_INT_V16SF_HI },
33000 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vinserti32x8_mask, "__builtin_ia32_inserti32x8_mask", IX86_BUILTIN_INSERTI32X8, UNKNOWN, (int) V16SI_FTYPE_V16SI_V8SI_INT_V16SI_HI },
33001 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vinsertf64x2_mask, "__builtin_ia32_insertf64x2_512_mask", IX86_BUILTIN_INSERTF64X2_512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V2DF_INT_V8DF_QI },
33002 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vinserti64x2_mask, "__builtin_ia32_inserti64x2_512_mask", IX86_BUILTIN_INSERTI64X2_512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V2DI_INT_V8DI_QI },
33003 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_fpclassv8df_mask, "__builtin_ia32_fpclasspd512_mask", IX86_BUILTIN_FPCLASSPD512, UNKNOWN, (int) QI_FTYPE_V8DF_INT_QI },
33004 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_fpclassv16sf_mask, "__builtin_ia32_fpclassps512_mask", IX86_BUILTIN_FPCLASSPS512, UNKNOWN, (int) HI_FTYPE_V16SF_INT_HI },
33005 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_cvtd2maskv16si, "__builtin_ia32_cvtd2mask512", IX86_BUILTIN_CVTD2MASK512, UNKNOWN, (int) HI_FTYPE_V16SI },
33006 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_cvtq2maskv8di, "__builtin_ia32_cvtq2mask512", IX86_BUILTIN_CVTQ2MASK512, UNKNOWN, (int) QI_FTYPE_V8DI },
33007 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_cvtmask2dv16si, "__builtin_ia32_cvtmask2d512", IX86_BUILTIN_CVTMASK2D512, UNKNOWN, (int) V16SI_FTYPE_HI },
33008 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_cvtmask2qv8di, "__builtin_ia32_cvtmask2q512", IX86_BUILTIN_CVTMASK2Q512, UNKNOWN, (int) V8DI_FTYPE_QI },
33011 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_kunpcksi, "__builtin_ia32_kunpcksi", IX86_BUILTIN_KUNPCKWD, UNKNOWN, (int) SI_FTYPE_SI_SI },
33012 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_kunpckdi, "__builtin_ia32_kunpckdi", IX86_BUILTIN_KUNPCKDQ, UNKNOWN, (int) DI_FTYPE_DI_DI },
33013 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_packusdw_mask, "__builtin_ia32_packusdw512_mask", IX86_BUILTIN_PACKUSDW512, UNKNOWN, (int) V32HI_FTYPE_V16SI_V16SI_V32HI_SI },
33014 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ashlv4ti3, "__builtin_ia32_pslldq512", IX86_BUILTIN_PSLLDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_CONVERT },
33015 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_lshrv4ti3, "__builtin_ia32_psrldq512", IX86_BUILTIN_PSRLDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_CONVERT },
33016 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_packssdw_mask, "__builtin_ia32_packssdw512_mask", IX86_BUILTIN_PACKSSDW512, UNKNOWN, (int) V32HI_FTYPE_V16SI_V16SI_V32HI_SI },
33017 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_palignrv4ti, "__builtin_ia32_palignr512", IX86_BUILTIN_PALIGNR512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_INT_CONVERT },
33018 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_palignrv64qi_mask, "__builtin_ia32_palignr512_mask", IX86_BUILTIN_PALIGNR512_MASK, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_INT_V8DI_DI_CONVERT },
33019 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_loaddquv32hi_mask, "__builtin_ia32_movdquhi512_mask", IX86_BUILTIN_MOVDQUHI512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_SI },
33020 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512f_loaddquv64qi_mask, "__builtin_ia32_movdquqi512_mask", IX86_BUILTIN_MOVDQUQI512_MASK, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_DI },
33021 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512f_psadbw, "__builtin_ia32_psadbw512", IX86_BUILTIN_PSADBW512, UNKNOWN, (int) V8DI_FTYPE_V64QI_V64QI },
33022 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_dbpsadbwv32hi_mask, "__builtin_ia32_dbpsadbw512_mask", IX86_BUILTIN_DBPSADBW512, UNKNOWN, (int) V32HI_FTYPE_V64QI_V64QI_INT_V32HI_SI },
33023 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vec_dupv64qi_mask, "__builtin_ia32_pbroadcastb512_mask", IX86_BUILTIN_PBROADCASTB512, UNKNOWN, (int) V64QI_FTYPE_V16QI_V64QI_DI },
33024 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vec_dup_gprv64qi_mask, "__builtin_ia32_pbroadcastb512_gpr_mask", IX86_BUILTIN_PBROADCASTB512_GPR, UNKNOWN, (int) V64QI_FTYPE_QI_V64QI_DI },
33025 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vec_dupv32hi_mask, "__builtin_ia32_pbroadcastw512_mask", IX86_BUILTIN_PBROADCASTW512, UNKNOWN, (int) V32HI_FTYPE_V8HI_V32HI_SI },
33026 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vec_dup_gprv32hi_mask, "__builtin_ia32_pbroadcastw512_gpr_mask", IX86_BUILTIN_PBROADCASTW512_GPR, UNKNOWN, (int) V32HI_FTYPE_HI_V32HI_SI },
33027 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_sign_extendv32qiv32hi2_mask, "__builtin_ia32_pmovsxbw512_mask", IX86_BUILTIN_PMOVSXBW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32QI_V32HI_SI },
33028 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_zero_extendv32qiv32hi2_mask, "__builtin_ia32_pmovzxbw512_mask", IX86_BUILTIN_PMOVZXBW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32QI_V32HI_SI },
33029 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_permvarv32hi_mask, "__builtin_ia32_permvarhi512_mask", IX86_BUILTIN_VPERMVARHI512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33030 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vpermt2varv32hi3_mask, "__builtin_ia32_vpermt2varhi512_mask", IX86_BUILTIN_VPERMT2VARHI512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33031 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vpermt2varv32hi3_maskz, "__builtin_ia32_vpermt2varhi512_maskz", IX86_BUILTIN_VPERMT2VARHI512_MASKZ, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33032 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vpermi2varv32hi3_mask, "__builtin_ia32_vpermi2varhi512_mask", IX86_BUILTIN_VPERMI2VARHI512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33033 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_uavgv64qi3_mask, "__builtin_ia32_pavgb512_mask", IX86_BUILTIN_PAVGB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33034 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_uavgv32hi3_mask, "__builtin_ia32_pavgw512_mask", IX86_BUILTIN_PAVGW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33035 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_addv64qi3_mask, "__builtin_ia32_paddb512_mask", IX86_BUILTIN_PADDB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33036 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_subv64qi3_mask, "__builtin_ia32_psubb512_mask", IX86_BUILTIN_PSUBB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33037 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_sssubv64qi3_mask, "__builtin_ia32_psubsb512_mask", IX86_BUILTIN_PSUBSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33038 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ssaddv64qi3_mask, "__builtin_ia32_paddsb512_mask", IX86_BUILTIN_PADDSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33039 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ussubv64qi3_mask, "__builtin_ia32_psubusb512_mask", IX86_BUILTIN_PSUBUSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33040 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_usaddv64qi3_mask, "__builtin_ia32_paddusb512_mask", IX86_BUILTIN_PADDUSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33041 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_subv32hi3_mask, "__builtin_ia32_psubw512_mask", IX86_BUILTIN_PSUBW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33042 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_addv32hi3_mask, "__builtin_ia32_paddw512_mask", IX86_BUILTIN_PADDW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33043 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_sssubv32hi3_mask, "__builtin_ia32_psubsw512_mask", IX86_BUILTIN_PSUBSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33044 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ssaddv32hi3_mask, "__builtin_ia32_paddsw512_mask", IX86_BUILTIN_PADDSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33045 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ussubv32hi3_mask, "__builtin_ia32_psubusw512_mask", IX86_BUILTIN_PSUBUSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33046 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_usaddv32hi3_mask, "__builtin_ia32_paddusw512_mask", IX86_BUILTIN_PADDUSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33047 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_umaxv32hi3_mask, "__builtin_ia32_pmaxuw512_mask", IX86_BUILTIN_PMAXUW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33048 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_smaxv32hi3_mask, "__builtin_ia32_pmaxsw512_mask", IX86_BUILTIN_PMAXSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33049 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_uminv32hi3_mask, "__builtin_ia32_pminuw512_mask", IX86_BUILTIN_PMINUW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33050 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_sminv32hi3_mask, "__builtin_ia32_pminsw512_mask", IX86_BUILTIN_PMINSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33051 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_umaxv64qi3_mask, "__builtin_ia32_pmaxub512_mask", IX86_BUILTIN_PMAXUB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33052 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_smaxv64qi3_mask, "__builtin_ia32_pmaxsb512_mask", IX86_BUILTIN_PMAXSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33053 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_uminv64qi3_mask, "__builtin_ia32_pminub512_mask", IX86_BUILTIN_PMINUB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33054 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_sminv64qi3_mask, "__builtin_ia32_pminsb512_mask", IX86_BUILTIN_PMINSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33055 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_truncatev32hiv32qi2_mask, "__builtin_ia32_pmovwb512_mask", IX86_BUILTIN_PMOVWB512, UNKNOWN, (int) V32QI_FTYPE_V32HI_V32QI_SI },
33056 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ss_truncatev32hiv32qi2_mask, "__builtin_ia32_pmovswb512_mask", IX86_BUILTIN_PMOVSWB512, UNKNOWN, (int) V32QI_FTYPE_V32HI_V32QI_SI },
33057 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_us_truncatev32hiv32qi2_mask, "__builtin_ia32_pmovuswb512_mask", IX86_BUILTIN_PMOVUSWB512, UNKNOWN, (int) V32QI_FTYPE_V32HI_V32QI_SI },
33058 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_umulhrswv32hi3_mask, "__builtin_ia32_pmulhrsw512_mask", IX86_BUILTIN_PMULHRSW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33059 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_umulv32hi3_highpart_mask, "__builtin_ia32_pmulhuw512_mask" , IX86_BUILTIN_PMULHUW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33060 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_smulv32hi3_highpart_mask, "__builtin_ia32_pmulhw512_mask" , IX86_BUILTIN_PMULHW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33061 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_mulv32hi3_mask, "__builtin_ia32_pmullw512_mask", IX86_BUILTIN_PMULLW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33062 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_ashlv32hi3_mask, "__builtin_ia32_psllwi512_mask", IX86_BUILTIN_PSLLWI512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_INT_V32HI_SI },
33063 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_ashlv32hi3_mask, "__builtin_ia32_psllw512_mask", IX86_BUILTIN_PSLLW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V8HI_V32HI_SI },
33064 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_packsswb_mask, "__builtin_ia32_packsswb512_mask", IX86_BUILTIN_PACKSSWB512, UNKNOWN, (int) V64QI_FTYPE_V32HI_V32HI_V64QI_DI },
33065 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_packuswb_mask, "__builtin_ia32_packuswb512_mask", IX86_BUILTIN_PACKUSWB512, UNKNOWN, (int) V64QI_FTYPE_V32HI_V32HI_V64QI_DI },
33066 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ashrvv32hi_mask, "__builtin_ia32_psrav32hi_mask", IX86_BUILTIN_PSRAVV32HI, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33067 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_pmaddubsw512v32hi_mask, "__builtin_ia32_pmaddubsw512_mask", IX86_BUILTIN_PMADDUBSW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V64QI_V64QI_V32HI_SI },
33068 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_pmaddwd512v32hi_mask, "__builtin_ia32_pmaddwd512_mask", IX86_BUILTIN_PMADDWD512_MASK, UNKNOWN, (int) V16SI_FTYPE_V32HI_V32HI_V16SI_HI },
33069 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_lshrvv32hi_mask, "__builtin_ia32_psrlv32hi_mask", IX86_BUILTIN_PSRLVV32HI, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33070 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_interleave_highv64qi_mask, "__builtin_ia32_punpckhbw512_mask", IX86_BUILTIN_PUNPCKHBW512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33071 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_interleave_highv32hi_mask, "__builtin_ia32_punpckhwd512_mask", IX86_BUILTIN_PUNPCKHWD512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33072 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_interleave_lowv64qi_mask, "__builtin_ia32_punpcklbw512_mask", IX86_BUILTIN_PUNPCKLBW512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33073 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_interleave_lowv32hi_mask, "__builtin_ia32_punpcklwd512_mask", IX86_BUILTIN_PUNPCKLWD512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33074 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_pshufbv64qi3_mask, "__builtin_ia32_pshufb512_mask", IX86_BUILTIN_PSHUFB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33075 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_pshufhwv32hi_mask, "__builtin_ia32_pshufhw512_mask", IX86_BUILTIN_PSHUFHW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_INT_V32HI_SI },
33076 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_pshuflwv32hi_mask, "__builtin_ia32_pshuflw512_mask", IX86_BUILTIN_PSHUFLW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_INT_V32HI_SI },
33077 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_ashrv32hi3_mask, "__builtin_ia32_psrawi512_mask", IX86_BUILTIN_PSRAWI512, UNKNOWN, (int) V32HI_FTYPE_V32HI_INT_V32HI_SI },
33078 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_ashrv32hi3_mask, "__builtin_ia32_psraw512_mask", IX86_BUILTIN_PSRAW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V8HI_V32HI_SI },
33079 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_lshrv32hi3_mask, "__builtin_ia32_psrlwi512_mask", IX86_BUILTIN_PSRLWI512, UNKNOWN, (int) V32HI_FTYPE_V32HI_INT_V32HI_SI },
33080 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_lshrv32hi3_mask, "__builtin_ia32_psrlw512_mask", IX86_BUILTIN_PSRLW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V8HI_V32HI_SI },
33081 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cvtb2maskv64qi, "__builtin_ia32_cvtb2mask512", IX86_BUILTIN_CVTB2MASK512, UNKNOWN, (int) DI_FTYPE_V64QI },
33082 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cvtw2maskv32hi, "__builtin_ia32_cvtw2mask512", IX86_BUILTIN_CVTW2MASK512, UNKNOWN, (int) SI_FTYPE_V32HI },
33083 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cvtmask2bv64qi, "__builtin_ia32_cvtmask2b512", IX86_BUILTIN_CVTMASK2B512, UNKNOWN, (int) V64QI_FTYPE_DI },
33084 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cvtmask2wv32hi, "__builtin_ia32_cvtmask2w512", IX86_BUILTIN_CVTMASK2W512, UNKNOWN, (int) V32HI_FTYPE_SI },
33085 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_eqv64qi3_mask, "__builtin_ia32_pcmpeqb512_mask", IX86_BUILTIN_PCMPEQB512_MASK, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_DI },
33086 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_eqv32hi3_mask, "__builtin_ia32_pcmpeqw512_mask", IX86_BUILTIN_PCMPEQW512_MASK, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_SI },
33087 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_gtv64qi3_mask, "__builtin_ia32_pcmpgtb512_mask", IX86_BUILTIN_PCMPGTB512_MASK, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_DI },
33088 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_gtv32hi3_mask, "__builtin_ia32_pcmpgtw512_mask", IX86_BUILTIN_PCMPGTW512_MASK, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_SI },
33089 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_testmv64qi3_mask, "__builtin_ia32_ptestmb512", IX86_BUILTIN_PTESTMB512, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_DI },
33090 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_testmv32hi3_mask, "__builtin_ia32_ptestmw512", IX86_BUILTIN_PTESTMW512, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_SI },
33091 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_testnmv64qi3_mask, "__builtin_ia32_ptestnmb512", IX86_BUILTIN_PTESTNMB512, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_DI },
33092 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_testnmv32hi3_mask, "__builtin_ia32_ptestnmw512", IX86_BUILTIN_PTESTNMW512, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_SI },
33093 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ashlvv32hi_mask, "__builtin_ia32_psllv32hi_mask", IX86_BUILTIN_PSLLVV32HI, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
33094 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_absv64qi2_mask, "__builtin_ia32_pabsb512_mask", IX86_BUILTIN_PABSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_DI },
33095 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_absv32hi2_mask, "__builtin_ia32_pabsw512_mask", IX86_BUILTIN_PABSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_SI },
33096 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_blendmv32hi, "__builtin_ia32_blendmw_512_mask", IX86_BUILTIN_BLENDMW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_SI },
33097 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_blendmv64qi, "__builtin_ia32_blendmb_512_mask", IX86_BUILTIN_BLENDMB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_DI },
33098 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cmpv64qi3_mask, "__builtin_ia32_cmpb512_mask", IX86_BUILTIN_CMPB512, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_INT_DI },
33099 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cmpv32hi3_mask, "__builtin_ia32_cmpw512_mask", IX86_BUILTIN_CMPW512, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_INT_SI },
33100 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ucmpv64qi3_mask, "__builtin_ia32_ucmpb512_mask", IX86_BUILTIN_UCMPB512, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_INT_DI },
33101 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ucmpv32hi3_mask, "__builtin_ia32_ucmpw512_mask", IX86_BUILTIN_UCMPW512, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_INT_SI },
33104 { OPTION_MASK_ISA_AVX512IFMA, CODE_FOR_vpamdd52luqv8di_mask, "__builtin_ia32_vpmadd52luq512_mask", IX86_BUILTIN_VPMADD52LUQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
33105 { OPTION_MASK_ISA_AVX512IFMA, CODE_FOR_vpamdd52luqv8di_maskz, "__builtin_ia32_vpmadd52luq512_maskz", IX86_BUILTIN_VPMADD52LUQ512_MASKZ, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
33106 { OPTION_MASK_ISA_AVX512IFMA, CODE_FOR_vpamdd52huqv8di_mask, "__builtin_ia32_vpmadd52huq512_mask", IX86_BUILTIN_VPMADD52HUQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
33107 { OPTION_MASK_ISA_AVX512IFMA, CODE_FOR_vpamdd52huqv8di_maskz, "__builtin_ia32_vpmadd52huq512_maskz", IX86_BUILTIN_VPMADD52HUQ512_MASKZ, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
33108 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52luqv4di_mask, "__builtin_ia32_vpmadd52luq256_mask", IX86_BUILTIN_VPMADD52LUQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
33109 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52luqv4di_maskz, "__builtin_ia32_vpmadd52luq256_maskz", IX86_BUILTIN_VPMADD52LUQ256_MASKZ, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
33110 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52huqv4di_mask, "__builtin_ia32_vpmadd52huq256_mask", IX86_BUILTIN_VPMADD52HUQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
33111 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52huqv4di_maskz, "__builtin_ia32_vpmadd52huq256_maskz", IX86_BUILTIN_VPMADD52HUQ256_MASKZ, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
33112 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52luqv2di_mask, "__builtin_ia32_vpmadd52luq128_mask", IX86_BUILTIN_VPMADD52LUQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
33113 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52luqv2di_maskz, "__builtin_ia32_vpmadd52luq128_maskz", IX86_BUILTIN_VPMADD52LUQ128_MASKZ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
33114 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52huqv2di_mask, "__builtin_ia32_vpmadd52huq128_mask", IX86_BUILTIN_VPMADD52HUQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
33115 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52huqv2di_maskz, "__builtin_ia32_vpmadd52huq128_maskz", IX86_BUILTIN_VPMADD52HUQ128_MASKZ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
33118 { OPTION_MASK_ISA_AVX512VBMI, CODE_FOR_vpmultishiftqbv64qi_mask, "__builtin_ia32_vpmultishiftqb512_mask", IX86_BUILTIN_VPMULTISHIFTQB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33119 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpmultishiftqbv32qi_mask, "__builtin_ia32_vpmultishiftqb256_mask", IX86_BUILTIN_VPMULTISHIFTQB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
33120 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpmultishiftqbv16qi_mask, "__builtin_ia32_vpmultishiftqb128_mask", IX86_BUILTIN_VPMULTISHIFTQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
33121 { OPTION_MASK_ISA_AVX512VBMI, CODE_FOR_avx512bw_permvarv64qi_mask, "__builtin_ia32_permvarqi512_mask", IX86_BUILTIN_VPERMVARQI512_MASK, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33122 { OPTION_MASK_ISA_AVX512VBMI, CODE_FOR_avx512bw_vpermt2varv64qi3_mask, "__builtin_ia32_vpermt2varqi512_mask", IX86_BUILTIN_VPERMT2VARQI512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33123 { OPTION_MASK_ISA_AVX512VBMI, CODE_FOR_avx512bw_vpermt2varv64qi3_maskz, "__builtin_ia32_vpermt2varqi512_maskz", IX86_BUILTIN_VPERMT2VARQI512_MASKZ, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33124 { OPTION_MASK_ISA_AVX512VBMI, CODE_FOR_avx512bw_vpermi2varv64qi3_mask, "__builtin_ia32_vpermi2varqi512_mask", IX86_BUILTIN_VPERMI2VARQI512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33125 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permvarv32qi_mask, "__builtin_ia32_permvarqi256_mask", IX86_BUILTIN_VPERMVARQI256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
33126 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permvarv16qi_mask, "__builtin_ia32_permvarqi128_mask", IX86_BUILTIN_VPERMVARQI128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
33127 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv32qi3_mask, "__builtin_ia32_vpermt2varqi256_mask", IX86_BUILTIN_VPERMT2VARQI256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
33128 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv32qi3_maskz, "__builtin_ia32_vpermt2varqi256_maskz", IX86_BUILTIN_VPERMT2VARQI256_MASKZ, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
33129 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv16qi3_mask, "__builtin_ia32_vpermt2varqi128_mask", IX86_BUILTIN_VPERMT2VARQI128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
33130 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv16qi3_maskz, "__builtin_ia32_vpermt2varqi128_maskz", IX86_BUILTIN_VPERMT2VARQI128_MASKZ, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
33131 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv32qi3_mask, "__builtin_ia32_vpermi2varqi256_mask", IX86_BUILTIN_VPERMI2VARQI256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
33132 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv16qi3_mask, "__builtin_ia32_vpermi2varqi128_mask", IX86_BUILTIN_VPERMI2VARQI128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
33135 /* Builtins with rounding support. */
33136 static const struct builtin_description bdesc_round_args[] =
33139 { OPTION_MASK_ISA_AVX512F, CODE_FOR_addv8df3_mask_round, "__builtin_ia32_addpd512_mask", IX86_BUILTIN_ADDPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33140 { OPTION_MASK_ISA_AVX512F, CODE_FOR_addv16sf3_mask_round, "__builtin_ia32_addps512_mask", IX86_BUILTIN_ADDPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33141 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmaddv2df3_round, "__builtin_ia32_addsd_round", IX86_BUILTIN_ADDSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33142 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmaddv4sf3_round, "__builtin_ia32_addss_round", IX86_BUILTIN_ADDSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33143 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cmpv8df3_mask_round, "__builtin_ia32_cmppd512_mask", IX86_BUILTIN_CMPPD512, UNKNOWN, (int) QI_FTYPE_V8DF_V8DF_INT_QI_INT },
33144 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cmpv16sf3_mask_round, "__builtin_ia32_cmpps512_mask", IX86_BUILTIN_CMPPS512, UNKNOWN, (int) HI_FTYPE_V16SF_V16SF_INT_HI_INT },
33145 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vmcmpv2df3_mask_round, "__builtin_ia32_cmpsd_mask", IX86_BUILTIN_CMPSD_MASK, UNKNOWN, (int) QI_FTYPE_V2DF_V2DF_INT_QI_INT },
33146 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vmcmpv4sf3_mask_round, "__builtin_ia32_cmpss_mask", IX86_BUILTIN_CMPSS_MASK, UNKNOWN, (int) QI_FTYPE_V4SF_V4SF_INT_QI_INT },
33147 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_comi_round, "__builtin_ia32_vcomisd", IX86_BUILTIN_COMIDF, UNKNOWN, (int) INT_FTYPE_V2DF_V2DF_INT_INT },
33148 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_comi_round, "__builtin_ia32_vcomiss", IX86_BUILTIN_COMISF, UNKNOWN, (int) INT_FTYPE_V4SF_V4SF_INT_INT },
33149 { OPTION_MASK_ISA_AVX512F, CODE_FOR_floatv16siv16sf2_mask_round, "__builtin_ia32_cvtdq2ps512_mask", IX86_BUILTIN_CVTDQ2PS512, UNKNOWN, (int) V16SF_FTYPE_V16SI_V16SF_HI_INT },
33150 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cvtpd2dq512_mask_round, "__builtin_ia32_cvtpd2dq512_mask", IX86_BUILTIN_CVTPD2DQ512, UNKNOWN, (int) V8SI_FTYPE_V8DF_V8SI_QI_INT },
33151 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cvtpd2ps512_mask_round, "__builtin_ia32_cvtpd2ps512_mask", IX86_BUILTIN_CVTPD2PS512, UNKNOWN, (int) V8SF_FTYPE_V8DF_V8SF_QI_INT },
33152 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ufix_notruncv8dfv8si2_mask_round, "__builtin_ia32_cvtpd2udq512_mask", IX86_BUILTIN_CVTPD2UDQ512, UNKNOWN, (int) V8SI_FTYPE_V8DF_V8SI_QI_INT },
33153 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvtph2ps512_mask_round, "__builtin_ia32_vcvtph2ps512_mask", IX86_BUILTIN_CVTPH2PS512, UNKNOWN, (int) V16SF_FTYPE_V16HI_V16SF_HI_INT },
33154 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fix_notruncv16sfv16si_mask_round, "__builtin_ia32_cvtps2dq512_mask", IX86_BUILTIN_CVTPS2DQ512, UNKNOWN, (int) V16SI_FTYPE_V16SF_V16SI_HI_INT },
33155 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cvtps2pd512_mask_round, "__builtin_ia32_cvtps2pd512_mask", IX86_BUILTIN_CVTPS2PD512, UNKNOWN, (int) V8DF_FTYPE_V8SF_V8DF_QI_INT },
33156 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ufix_notruncv16sfv16si_mask_round, "__builtin_ia32_cvtps2udq512_mask", IX86_BUILTIN_CVTPS2UDQ512, UNKNOWN, (int) V16SI_FTYPE_V16SF_V16SI_HI_INT },
33157 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_cvtsd2ss_round, "__builtin_ia32_cvtsd2ss_round", IX86_BUILTIN_CVTSD2SS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF_INT },
33158 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq_round, "__builtin_ia32_cvtsi2sd64", IX86_BUILTIN_CVTSI2SD64, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT64_INT },
33159 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_cvtsi2ss_round, "__builtin_ia32_cvtsi2ss32", IX86_BUILTIN_CVTSI2SS32, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT_INT },
33160 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq_round, "__builtin_ia32_cvtsi2ss64", IX86_BUILTIN_CVTSI2SS64, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT64_INT },
33161 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_cvtss2sd_round, "__builtin_ia32_cvtss2sd_round", IX86_BUILTIN_CVTSS2SD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF_INT },
33162 { OPTION_MASK_ISA_AVX512F, CODE_FOR_fix_truncv8dfv8si2_mask_round, "__builtin_ia32_cvttpd2dq512_mask", IX86_BUILTIN_CVTTPD2DQ512, UNKNOWN, (int) V8SI_FTYPE_V8DF_V8SI_QI_INT },
33163 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ufix_truncv8dfv8si2_mask_round, "__builtin_ia32_cvttpd2udq512_mask", IX86_BUILTIN_CVTTPD2UDQ512, UNKNOWN, (int) V8SI_FTYPE_V8DF_V8SI_QI_INT },
33164 { OPTION_MASK_ISA_AVX512F, CODE_FOR_fix_truncv16sfv16si2_mask_round, "__builtin_ia32_cvttps2dq512_mask", IX86_BUILTIN_CVTTPS2DQ512, UNKNOWN, (int) V16SI_FTYPE_V16SF_V16SI_HI_INT },
33165 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ufix_truncv16sfv16si2_mask_round, "__builtin_ia32_cvttps2udq512_mask", IX86_BUILTIN_CVTTPS2UDQ512, UNKNOWN, (int) V16SI_FTYPE_V16SF_V16SI_HI_INT },
33166 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ufloatv16siv16sf2_mask_round, "__builtin_ia32_cvtudq2ps512_mask", IX86_BUILTIN_CVTUDQ2PS512, UNKNOWN, (int) V16SF_FTYPE_V16SI_V16SF_HI_INT },
33167 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_cvtusi2sd64_round, "__builtin_ia32_cvtusi2sd64", IX86_BUILTIN_CVTUSI2SD64, UNKNOWN, (int) V2DF_FTYPE_V2DF_UINT64_INT },
33168 { OPTION_MASK_ISA_AVX512F, CODE_FOR_cvtusi2ss32_round, "__builtin_ia32_cvtusi2ss32", IX86_BUILTIN_CVTUSI2SS32, UNKNOWN, (int) V4SF_FTYPE_V4SF_UINT_INT },
33169 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_cvtusi2ss64_round, "__builtin_ia32_cvtusi2ss64", IX86_BUILTIN_CVTUSI2SS64, UNKNOWN, (int) V4SF_FTYPE_V4SF_UINT64_INT },
33170 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_divv8df3_mask_round, "__builtin_ia32_divpd512_mask", IX86_BUILTIN_DIVPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33171 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_divv16sf3_mask_round, "__builtin_ia32_divps512_mask", IX86_BUILTIN_DIVPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33172 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmdivv2df3_round, "__builtin_ia32_divsd_round", IX86_BUILTIN_DIVSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33173 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmdivv4sf3_round, "__builtin_ia32_divss_round", IX86_BUILTIN_DIVSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33174 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fixupimmv8df_mask_round, "__builtin_ia32_fixupimmpd512_mask", IX86_BUILTIN_FIXUPIMMPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DI_INT_QI_INT },
33175 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fixupimmv8df_maskz_round, "__builtin_ia32_fixupimmpd512_maskz", IX86_BUILTIN_FIXUPIMMPD512_MASKZ, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DI_INT_QI_INT },
33176 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fixupimmv16sf_mask_round, "__builtin_ia32_fixupimmps512_mask", IX86_BUILTIN_FIXUPIMMPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI_INT },
33177 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fixupimmv16sf_maskz_round, "__builtin_ia32_fixupimmps512_maskz", IX86_BUILTIN_FIXUPIMMPS512_MASKZ, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI_INT },
33178 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sfixupimmv2df_mask_round, "__builtin_ia32_fixupimmsd_mask", IX86_BUILTIN_FIXUPIMMSD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI_INT },
33179 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sfixupimmv2df_maskz_round, "__builtin_ia32_fixupimmsd_maskz", IX86_BUILTIN_FIXUPIMMSD128_MASKZ, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI_INT },
33180 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sfixupimmv4sf_mask_round, "__builtin_ia32_fixupimmss_mask", IX86_BUILTIN_FIXUPIMMSS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI_INT },
33181 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sfixupimmv4sf_maskz_round, "__builtin_ia32_fixupimmss_maskz", IX86_BUILTIN_FIXUPIMMSS128_MASKZ, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI_INT },
33182 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_getexpv8df_mask_round, "__builtin_ia32_getexppd512_mask", IX86_BUILTIN_GETEXPPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI_INT },
33183 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_getexpv16sf_mask_round, "__builtin_ia32_getexpps512_mask", IX86_BUILTIN_GETEXPPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI_INT },
33184 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sgetexpv2df_round, "__builtin_ia32_getexpsd128_round", IX86_BUILTIN_GETEXPSD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33185 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sgetexpv4sf_round, "__builtin_ia32_getexpss128_round", IX86_BUILTIN_GETEXPSS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33186 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_getmantv8df_mask_round, "__builtin_ia32_getmantpd512_mask", IX86_BUILTIN_GETMANTPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_INT_V8DF_QI_INT },
33187 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_getmantv16sf_mask_round, "__builtin_ia32_getmantps512_mask", IX86_BUILTIN_GETMANTPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_INT_V16SF_HI_INT },
33188 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vgetmantv2df_round, "__builtin_ia32_getmantsd_round", IX86_BUILTIN_GETMANTSD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT_INT },
33189 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vgetmantv4sf_round, "__builtin_ia32_getmantss_round", IX86_BUILTIN_GETMANTSS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT_INT },
33190 { OPTION_MASK_ISA_AVX512F, CODE_FOR_smaxv8df3_mask_round, "__builtin_ia32_maxpd512_mask", IX86_BUILTIN_MAXPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33191 { OPTION_MASK_ISA_AVX512F, CODE_FOR_smaxv16sf3_mask_round, "__builtin_ia32_maxps512_mask", IX86_BUILTIN_MAXPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33192 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmsmaxv2df3_round, "__builtin_ia32_maxsd_round", IX86_BUILTIN_MAXSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33193 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmsmaxv4sf3_round, "__builtin_ia32_maxss_round", IX86_BUILTIN_MAXSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33194 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sminv8df3_mask_round, "__builtin_ia32_minpd512_mask", IX86_BUILTIN_MINPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33195 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sminv16sf3_mask_round, "__builtin_ia32_minps512_mask", IX86_BUILTIN_MINPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33196 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmsminv2df3_round, "__builtin_ia32_minsd_round", IX86_BUILTIN_MINSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33197 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmsminv4sf3_round, "__builtin_ia32_minss_round", IX86_BUILTIN_MINSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33198 { OPTION_MASK_ISA_AVX512F, CODE_FOR_mulv8df3_mask_round, "__builtin_ia32_mulpd512_mask", IX86_BUILTIN_MULPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33199 { OPTION_MASK_ISA_AVX512F, CODE_FOR_mulv16sf3_mask_round, "__builtin_ia32_mulps512_mask", IX86_BUILTIN_MULPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33200 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmmulv2df3_round, "__builtin_ia32_mulsd_round", IX86_BUILTIN_MULSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33201 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmmulv4sf3_round, "__builtin_ia32_mulss_round", IX86_BUILTIN_MULSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33202 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rndscalev8df_mask_round, "__builtin_ia32_rndscalepd_mask", IX86_BUILTIN_RNDSCALEPD, UNKNOWN, (int) V8DF_FTYPE_V8DF_INT_V8DF_QI_INT },
33203 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rndscalev16sf_mask_round, "__builtin_ia32_rndscaleps_mask", IX86_BUILTIN_RNDSCALEPS, UNKNOWN, (int) V16SF_FTYPE_V16SF_INT_V16SF_HI_INT },
33204 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rndscalev2df_round, "__builtin_ia32_rndscalesd_round", IX86_BUILTIN_RNDSCALESD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT_INT },
33205 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rndscalev4sf_round, "__builtin_ia32_rndscaless_round", IX86_BUILTIN_RNDSCALESS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT_INT },
33206 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_scalefv8df_mask_round, "__builtin_ia32_scalefpd512_mask", IX86_BUILTIN_SCALEFPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33207 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_scalefv16sf_mask_round, "__builtin_ia32_scalefps512_mask", IX86_BUILTIN_SCALEFPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33208 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vmscalefv2df_round, "__builtin_ia32_scalefsd_round", IX86_BUILTIN_SCALEFSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33209 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vmscalefv4sf_round, "__builtin_ia32_scalefss_round", IX86_BUILTIN_SCALEFSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33210 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sqrtv8df2_mask_round, "__builtin_ia32_sqrtpd512_mask", IX86_BUILTIN_SQRTPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI_INT },
33211 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sqrtv16sf2_mask_round, "__builtin_ia32_sqrtps512_mask", IX86_BUILTIN_SQRTPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI_INT },
33212 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmsqrtv2df2_round, "__builtin_ia32_sqrtsd_round", IX86_BUILTIN_SQRTSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33213 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmsqrtv4sf2_round, "__builtin_ia32_sqrtss_round", IX86_BUILTIN_SQRTSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33214 { OPTION_MASK_ISA_AVX512F, CODE_FOR_subv8df3_mask_round, "__builtin_ia32_subpd512_mask", IX86_BUILTIN_SUBPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33215 { OPTION_MASK_ISA_AVX512F, CODE_FOR_subv16sf3_mask_round, "__builtin_ia32_subps512_mask", IX86_BUILTIN_SUBPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33216 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmsubv2df3_round, "__builtin_ia32_subsd_round", IX86_BUILTIN_SUBSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33217 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmsubv4sf3_round, "__builtin_ia32_subss_round", IX86_BUILTIN_SUBSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33218 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_cvtsd2si_round, "__builtin_ia32_vcvtsd2si32", IX86_BUILTIN_VCVTSD2SI32, UNKNOWN, (int) INT_FTYPE_V2DF_INT },
33219 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq_round, "__builtin_ia32_vcvtsd2si64", IX86_BUILTIN_VCVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF_INT },
33220 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvtsd2usi_round, "__builtin_ia32_vcvtsd2usi32", IX86_BUILTIN_VCVTSD2USI32, UNKNOWN, (int) UINT_FTYPE_V2DF_INT },
33221 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_avx512f_vcvtsd2usiq_round, "__builtin_ia32_vcvtsd2usi64", IX86_BUILTIN_VCVTSD2USI64, UNKNOWN, (int) UINT64_FTYPE_V2DF_INT },
33222 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_cvtss2si_round, "__builtin_ia32_vcvtss2si32", IX86_BUILTIN_VCVTSS2SI32, UNKNOWN, (int) INT_FTYPE_V4SF_INT },
33223 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq_round, "__builtin_ia32_vcvtss2si64", IX86_BUILTIN_VCVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF_INT },
33224 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvtss2usi_round, "__builtin_ia32_vcvtss2usi32", IX86_BUILTIN_VCVTSS2USI32, UNKNOWN, (int) UINT_FTYPE_V4SF_INT },
33225 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_avx512f_vcvtss2usiq_round, "__builtin_ia32_vcvtss2usi64", IX86_BUILTIN_VCVTSS2USI64, UNKNOWN, (int) UINT64_FTYPE_V4SF_INT },
33226 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_cvttsd2si_round, "__builtin_ia32_vcvttsd2si32", IX86_BUILTIN_VCVTTSD2SI32, UNKNOWN, (int) INT_FTYPE_V2DF_INT },
33227 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq_round, "__builtin_ia32_vcvttsd2si64", IX86_BUILTIN_VCVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF_INT },
33228 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvttsd2usi_round, "__builtin_ia32_vcvttsd2usi32", IX86_BUILTIN_VCVTTSD2USI32, UNKNOWN, (int) UINT_FTYPE_V2DF_INT },
33229 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_avx512f_vcvttsd2usiq_round, "__builtin_ia32_vcvttsd2usi64", IX86_BUILTIN_VCVTTSD2USI64, UNKNOWN, (int) UINT64_FTYPE_V2DF_INT },
33230 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_cvttss2si_round, "__builtin_ia32_vcvttss2si32", IX86_BUILTIN_VCVTTSS2SI32, UNKNOWN, (int) INT_FTYPE_V4SF_INT },
33231 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq_round, "__builtin_ia32_vcvttss2si64", IX86_BUILTIN_VCVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF_INT },
33232 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvttss2usi_round, "__builtin_ia32_vcvttss2usi32", IX86_BUILTIN_VCVTTSS2USI32, UNKNOWN, (int) UINT_FTYPE_V4SF_INT },
33233 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_avx512f_vcvttss2usiq_round, "__builtin_ia32_vcvttss2usi64", IX86_BUILTIN_VCVTTSS2USI64, UNKNOWN, (int) UINT64_FTYPE_V4SF_INT },
33234 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v8df_mask_round, "__builtin_ia32_vfmaddpd512_mask", IX86_BUILTIN_VFMADDPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33235 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v8df_mask3_round, "__builtin_ia32_vfmaddpd512_mask3", IX86_BUILTIN_VFMADDPD512_MASK3, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33236 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v8df_maskz_round, "__builtin_ia32_vfmaddpd512_maskz", IX86_BUILTIN_VFMADDPD512_MASKZ, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33237 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v16sf_mask_round, "__builtin_ia32_vfmaddps512_mask", IX86_BUILTIN_VFMADDPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33238 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v16sf_mask3_round, "__builtin_ia32_vfmaddps512_mask3", IX86_BUILTIN_VFMADDPS512_MASK3, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33239 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v16sf_maskz_round, "__builtin_ia32_vfmaddps512_maskz", IX86_BUILTIN_VFMADDPS512_MASKZ, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33240 { OPTION_MASK_ISA_AVX512F, CODE_FOR_fmai_vmfmadd_v2df_round, "__builtin_ia32_vfmaddsd3_round", IX86_BUILTIN_VFMADDSD3_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_INT },
33241 { OPTION_MASK_ISA_AVX512F, CODE_FOR_fmai_vmfmadd_v4sf_round, "__builtin_ia32_vfmaddss3_round", IX86_BUILTIN_VFMADDSS3_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_INT },
33242 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v8df_mask_round, "__builtin_ia32_vfmaddsubpd512_mask", IX86_BUILTIN_VFMADDSUBPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33243 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v8df_mask3_round, "__builtin_ia32_vfmaddsubpd512_mask3", IX86_BUILTIN_VFMADDSUBPD512_MASK3, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33244 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v8df_maskz_round, "__builtin_ia32_vfmaddsubpd512_maskz", IX86_BUILTIN_VFMADDSUBPD512_MASKZ, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33245 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v16sf_mask_round, "__builtin_ia32_vfmaddsubps512_mask", IX86_BUILTIN_VFMADDSUBPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33246 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v16sf_mask3_round, "__builtin_ia32_vfmaddsubps512_mask3", IX86_BUILTIN_VFMADDSUBPS512_MASK3, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33247 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v16sf_maskz_round, "__builtin_ia32_vfmaddsubps512_maskz", IX86_BUILTIN_VFMADDSUBPS512_MASKZ, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33248 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmsubadd_v8df_mask3_round, "__builtin_ia32_vfmsubaddpd512_mask3", IX86_BUILTIN_VFMSUBADDPD512_MASK3, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33249 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmsubadd_v16sf_mask3_round, "__builtin_ia32_vfmsubaddps512_mask3", IX86_BUILTIN_VFMSUBADDPS512_MASK3, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33250 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmsub_v8df_mask3_round, "__builtin_ia32_vfmsubpd512_mask3", IX86_BUILTIN_VFMSUBPD512_MASK3, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33251 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmsub_v16sf_mask3_round, "__builtin_ia32_vfmsubps512_mask3", IX86_BUILTIN_VFMSUBPS512_MASK3, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33252 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmadd_v8df_mask_round, "__builtin_ia32_vfnmaddpd512_mask", IX86_BUILTIN_VFNMADDPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33253 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmadd_v16sf_mask_round, "__builtin_ia32_vfnmaddps512_mask", IX86_BUILTIN_VFNMADDPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33254 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmsub_v8df_mask_round, "__builtin_ia32_vfnmsubpd512_mask", IX86_BUILTIN_VFNMSUBPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33255 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmsub_v8df_mask3_round, "__builtin_ia32_vfnmsubpd512_mask3", IX86_BUILTIN_VFNMSUBPD512_MASK3, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33256 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmsub_v16sf_mask_round, "__builtin_ia32_vfnmsubps512_mask", IX86_BUILTIN_VFNMSUBPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33257 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmsub_v16sf_mask3_round, "__builtin_ia32_vfnmsubps512_mask3", IX86_BUILTIN_VFNMSUBPS512_MASK3, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33260 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_exp2v8df_mask_round, "__builtin_ia32_exp2pd_mask", IX86_BUILTIN_EXP2PD_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI_INT },
33261 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_exp2v16sf_mask_round, "__builtin_ia32_exp2ps_mask", IX86_BUILTIN_EXP2PS_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI_INT },
33262 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_rcp28v8df_mask_round, "__builtin_ia32_rcp28pd_mask", IX86_BUILTIN_RCP28PD, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI_INT },
33263 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_rcp28v16sf_mask_round, "__builtin_ia32_rcp28ps_mask", IX86_BUILTIN_RCP28PS, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI_INT },
33264 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_vmrcp28v2df_round, "__builtin_ia32_rcp28sd_round", IX86_BUILTIN_RCP28SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33265 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_vmrcp28v4sf_round, "__builtin_ia32_rcp28ss_round", IX86_BUILTIN_RCP28SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33266 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_rsqrt28v8df_mask_round, "__builtin_ia32_rsqrt28pd_mask", IX86_BUILTIN_RSQRT28PD, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI_INT },
33267 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_rsqrt28v16sf_mask_round, "__builtin_ia32_rsqrt28ps_mask", IX86_BUILTIN_RSQRT28PS, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI_INT },
33268 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_vmrsqrt28v2df_round, "__builtin_ia32_rsqrt28sd_round", IX86_BUILTIN_RSQRT28SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33269 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_vmrsqrt28v4sf_round, "__builtin_ia32_rsqrt28ss_round", IX86_BUILTIN_RSQRT28SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33272 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_rangesv2df_round, "__builtin_ia32_rangesd128_round", IX86_BUILTIN_RANGESD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT_INT },
33273 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_rangesv4sf_round, "__builtin_ia32_rangess128_round", IX86_BUILTIN_RANGESS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT_INT },
33274 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_fix_notruncv8dfv8di2_mask_round, "__builtin_ia32_cvtpd2qq512_mask", IX86_BUILTIN_CVTPD2QQ512, UNKNOWN, (int) V8DI_FTYPE_V8DF_V8DI_QI_INT },
33275 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_cvtps2qqv8di_mask_round, "__builtin_ia32_cvtps2qq512_mask", IX86_BUILTIN_CVTPS2QQ512, UNKNOWN, (int) V8DI_FTYPE_V8SF_V8DI_QI_INT },
33276 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_ufix_notruncv8dfv8di2_mask_round, "__builtin_ia32_cvtpd2uqq512_mask", IX86_BUILTIN_CVTPD2UQQ512, UNKNOWN, (int) V8DI_FTYPE_V8DF_V8DI_QI_INT },
33277 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_cvtps2uqqv8di_mask_round, "__builtin_ia32_cvtps2uqq512_mask", IX86_BUILTIN_CVTPS2UQQ512, UNKNOWN, (int) V8DI_FTYPE_V8SF_V8DI_QI_INT },
33278 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_floatv8div8sf2_mask_round, "__builtin_ia32_cvtqq2ps512_mask", IX86_BUILTIN_CVTQQ2PS512, UNKNOWN, (int) V8SF_FTYPE_V8DI_V8SF_QI_INT },
33279 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_ufloatv8div8sf2_mask_round, "__builtin_ia32_cvtuqq2ps512_mask", IX86_BUILTIN_CVTUQQ2PS512, UNKNOWN, (int) V8SF_FTYPE_V8DI_V8SF_QI_INT },
33280 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_floatv8div8df2_mask_round, "__builtin_ia32_cvtqq2pd512_mask", IX86_BUILTIN_CVTQQ2PD512, UNKNOWN, (int) V8DF_FTYPE_V8DI_V8DF_QI_INT },
33281 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_ufloatv8div8df2_mask_round, "__builtin_ia32_cvtuqq2pd512_mask", IX86_BUILTIN_CVTUQQ2PD512, UNKNOWN, (int) V8DF_FTYPE_V8DI_V8DF_QI_INT },
33282 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_fix_truncv8sfv8di2_mask_round, "__builtin_ia32_cvttps2qq512_mask", IX86_BUILTIN_CVTTPS2QQ512, UNKNOWN, (int) V8DI_FTYPE_V8SF_V8DI_QI_INT },
33283 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_ufix_truncv8sfv8di2_mask_round, "__builtin_ia32_cvttps2uqq512_mask", IX86_BUILTIN_CVTTPS2UQQ512, UNKNOWN, (int) V8DI_FTYPE_V8SF_V8DI_QI_INT },
33284 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_fix_truncv8dfv8di2_mask_round, "__builtin_ia32_cvttpd2qq512_mask", IX86_BUILTIN_CVTTPD2QQ512, UNKNOWN, (int) V8DI_FTYPE_V8DF_V8DI_QI_INT },
33285 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_ufix_truncv8dfv8di2_mask_round, "__builtin_ia32_cvttpd2uqq512_mask", IX86_BUILTIN_CVTTPD2UQQ512, UNKNOWN, (int) V8DI_FTYPE_V8DF_V8DI_QI_INT },
33286 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_rangepv16sf_mask_round, "__builtin_ia32_rangeps512_mask", IX86_BUILTIN_RANGEPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI_INT },
33287 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_rangepv8df_mask_round, "__builtin_ia32_rangepd512_mask", IX86_BUILTIN_RANGEPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI_INT },
33290 /* Bultins for MPX. */
33291 static const struct builtin_description bdesc_mpx[] =
33293 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndstx", IX86_BUILTIN_BNDSTX, UNKNOWN, (int) VOID_FTYPE_PCVOID_BND_PCVOID },
33294 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndcl", IX86_BUILTIN_BNDCL, UNKNOWN, (int) VOID_FTYPE_PCVOID_BND },
33295 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndcu", IX86_BUILTIN_BNDCU, UNKNOWN, (int) VOID_FTYPE_PCVOID_BND },
33298 /* Const builtins for MPX. */
33299 static const struct builtin_description bdesc_mpx_const[] =
33301 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndmk", IX86_BUILTIN_BNDMK, UNKNOWN, (int) BND_FTYPE_PCVOID_ULONG },
33302 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndldx", IX86_BUILTIN_BNDLDX, UNKNOWN, (int) BND_FTYPE_PCVOID_PCVOID },
33303 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_narrow_bounds", IX86_BUILTIN_BNDNARROW, UNKNOWN, (int) PVOID_FTYPE_PCVOID_BND_ULONG },
33304 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndint", IX86_BUILTIN_BNDINT, UNKNOWN, (int) BND_FTYPE_BND_BND },
33305 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_sizeof", IX86_BUILTIN_SIZEOF, UNKNOWN, (int) ULONG_FTYPE_VOID },
33306 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndlower", IX86_BUILTIN_BNDLOWER, UNKNOWN, (int) PVOID_FTYPE_BND },
33307 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndupper", IX86_BUILTIN_BNDUPPER, UNKNOWN, (int) PVOID_FTYPE_BND },
33308 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndret", IX86_BUILTIN_BNDRET, UNKNOWN, (int) BND_FTYPE_PCVOID },
33311 /* FMA4 and XOP. */
33312 #define MULTI_ARG_4_DF2_DI_I V2DF_FTYPE_V2DF_V2DF_V2DI_INT
33313 #define MULTI_ARG_4_DF2_DI_I1 V4DF_FTYPE_V4DF_V4DF_V4DI_INT
33314 #define MULTI_ARG_4_SF2_SI_I V4SF_FTYPE_V4SF_V4SF_V4SI_INT
33315 #define MULTI_ARG_4_SF2_SI_I1 V8SF_FTYPE_V8SF_V8SF_V8SI_INT
33316 #define MULTI_ARG_3_SF V4SF_FTYPE_V4SF_V4SF_V4SF
33317 #define MULTI_ARG_3_DF V2DF_FTYPE_V2DF_V2DF_V2DF
33318 #define MULTI_ARG_3_SF2 V8SF_FTYPE_V8SF_V8SF_V8SF
33319 #define MULTI_ARG_3_DF2 V4DF_FTYPE_V4DF_V4DF_V4DF
33320 #define MULTI_ARG_3_DI V2DI_FTYPE_V2DI_V2DI_V2DI
33321 #define MULTI_ARG_3_SI V4SI_FTYPE_V4SI_V4SI_V4SI
33322 #define MULTI_ARG_3_SI_DI V4SI_FTYPE_V4SI_V4SI_V2DI
33323 #define MULTI_ARG_3_HI V8HI_FTYPE_V8HI_V8HI_V8HI
33324 #define MULTI_ARG_3_HI_SI V8HI_FTYPE_V8HI_V8HI_V4SI
33325 #define MULTI_ARG_3_QI V16QI_FTYPE_V16QI_V16QI_V16QI
33326 #define MULTI_ARG_3_DI2 V4DI_FTYPE_V4DI_V4DI_V4DI
33327 #define MULTI_ARG_3_SI2 V8SI_FTYPE_V8SI_V8SI_V8SI
33328 #define MULTI_ARG_3_HI2 V16HI_FTYPE_V16HI_V16HI_V16HI
33329 #define MULTI_ARG_3_QI2 V32QI_FTYPE_V32QI_V32QI_V32QI
33330 #define MULTI_ARG_2_SF V4SF_FTYPE_V4SF_V4SF
33331 #define MULTI_ARG_2_DF V2DF_FTYPE_V2DF_V2DF
33332 #define MULTI_ARG_2_DI V2DI_FTYPE_V2DI_V2DI
33333 #define MULTI_ARG_2_SI V4SI_FTYPE_V4SI_V4SI
33334 #define MULTI_ARG_2_HI V8HI_FTYPE_V8HI_V8HI
33335 #define MULTI_ARG_2_QI V16QI_FTYPE_V16QI_V16QI
33336 #define MULTI_ARG_2_DI_IMM V2DI_FTYPE_V2DI_SI
33337 #define MULTI_ARG_2_SI_IMM V4SI_FTYPE_V4SI_SI
33338 #define MULTI_ARG_2_HI_IMM V8HI_FTYPE_V8HI_SI
33339 #define MULTI_ARG_2_QI_IMM V16QI_FTYPE_V16QI_SI
33340 #define MULTI_ARG_2_DI_CMP V2DI_FTYPE_V2DI_V2DI_CMP
33341 #define MULTI_ARG_2_SI_CMP V4SI_FTYPE_V4SI_V4SI_CMP
33342 #define MULTI_ARG_2_HI_CMP V8HI_FTYPE_V8HI_V8HI_CMP
33343 #define MULTI_ARG_2_QI_CMP V16QI_FTYPE_V16QI_V16QI_CMP
33344 #define MULTI_ARG_2_SF_TF V4SF_FTYPE_V4SF_V4SF_TF
33345 #define MULTI_ARG_2_DF_TF V2DF_FTYPE_V2DF_V2DF_TF
33346 #define MULTI_ARG_2_DI_TF V2DI_FTYPE_V2DI_V2DI_TF
33347 #define MULTI_ARG_2_SI_TF V4SI_FTYPE_V4SI_V4SI_TF
33348 #define MULTI_ARG_2_HI_TF V8HI_FTYPE_V8HI_V8HI_TF
33349 #define MULTI_ARG_2_QI_TF V16QI_FTYPE_V16QI_V16QI_TF
33350 #define MULTI_ARG_1_SF V4SF_FTYPE_V4SF
33351 #define MULTI_ARG_1_DF V2DF_FTYPE_V2DF
33352 #define MULTI_ARG_1_SF2 V8SF_FTYPE_V8SF
33353 #define MULTI_ARG_1_DF2 V4DF_FTYPE_V4DF
33354 #define MULTI_ARG_1_DI V2DI_FTYPE_V2DI
33355 #define MULTI_ARG_1_SI V4SI_FTYPE_V4SI
33356 #define MULTI_ARG_1_HI V8HI_FTYPE_V8HI
33357 #define MULTI_ARG_1_QI V16QI_FTYPE_V16QI
33358 #define MULTI_ARG_1_SI_DI V2DI_FTYPE_V4SI
33359 #define MULTI_ARG_1_HI_DI V2DI_FTYPE_V8HI
33360 #define MULTI_ARG_1_HI_SI V4SI_FTYPE_V8HI
33361 #define MULTI_ARG_1_QI_DI V2DI_FTYPE_V16QI
33362 #define MULTI_ARG_1_QI_SI V4SI_FTYPE_V16QI
33363 #define MULTI_ARG_1_QI_HI V8HI_FTYPE_V16QI
33365 static const struct builtin_description bdesc_multi_arg[] =
33367 { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmadd_v4sf,
33368 "__builtin_ia32_vfmaddss", IX86_BUILTIN_VFMADDSS,
33369 UNKNOWN, (int)MULTI_ARG_3_SF },
33370 { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmadd_v2df,
33371 "__builtin_ia32_vfmaddsd", IX86_BUILTIN_VFMADDSD,
33372 UNKNOWN, (int)MULTI_ARG_3_DF },
33374 { OPTION_MASK_ISA_FMA, CODE_FOR_fmai_vmfmadd_v4sf,
33375 "__builtin_ia32_vfmaddss3", IX86_BUILTIN_VFMADDSS3,
33376 UNKNOWN, (int)MULTI_ARG_3_SF },
33377 { OPTION_MASK_ISA_FMA, CODE_FOR_fmai_vmfmadd_v2df,
33378 "__builtin_ia32_vfmaddsd3", IX86_BUILTIN_VFMADDSD3,
33379 UNKNOWN, (int)MULTI_ARG_3_DF },
33381 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v4sf,
33382 "__builtin_ia32_vfmaddps", IX86_BUILTIN_VFMADDPS,
33383 UNKNOWN, (int)MULTI_ARG_3_SF },
33384 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v2df,
33385 "__builtin_ia32_vfmaddpd", IX86_BUILTIN_VFMADDPD,
33386 UNKNOWN, (int)MULTI_ARG_3_DF },
33387 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v8sf,
33388 "__builtin_ia32_vfmaddps256", IX86_BUILTIN_VFMADDPS256,
33389 UNKNOWN, (int)MULTI_ARG_3_SF2 },
33390 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v4df,
33391 "__builtin_ia32_vfmaddpd256", IX86_BUILTIN_VFMADDPD256,
33392 UNKNOWN, (int)MULTI_ARG_3_DF2 },
33394 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v4sf,
33395 "__builtin_ia32_vfmaddsubps", IX86_BUILTIN_VFMADDSUBPS,
33396 UNKNOWN, (int)MULTI_ARG_3_SF },
33397 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v2df,
33398 "__builtin_ia32_vfmaddsubpd", IX86_BUILTIN_VFMADDSUBPD,
33399 UNKNOWN, (int)MULTI_ARG_3_DF },
33400 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v8sf,
33401 "__builtin_ia32_vfmaddsubps256", IX86_BUILTIN_VFMADDSUBPS256,
33402 UNKNOWN, (int)MULTI_ARG_3_SF2 },
33403 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v4df,
33404 "__builtin_ia32_vfmaddsubpd256", IX86_BUILTIN_VFMADDSUBPD256,
33405 UNKNOWN, (int)MULTI_ARG_3_DF2 },
33407 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov", IX86_BUILTIN_VPCMOV, UNKNOWN, (int)MULTI_ARG_3_DI },
33408 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov_v2di", IX86_BUILTIN_VPCMOV_V2DI, UNKNOWN, (int)MULTI_ARG_3_DI },
33409 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4si, "__builtin_ia32_vpcmov_v4si", IX86_BUILTIN_VPCMOV_V4SI, UNKNOWN, (int)MULTI_ARG_3_SI },
33410 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8hi, "__builtin_ia32_vpcmov_v8hi", IX86_BUILTIN_VPCMOV_V8HI, UNKNOWN, (int)MULTI_ARG_3_HI },
33411 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16qi, "__builtin_ia32_vpcmov_v16qi",IX86_BUILTIN_VPCMOV_V16QI,UNKNOWN, (int)MULTI_ARG_3_QI },
33412 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2df, "__builtin_ia32_vpcmov_v2df", IX86_BUILTIN_VPCMOV_V2DF, UNKNOWN, (int)MULTI_ARG_3_DF },
33413 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4sf, "__builtin_ia32_vpcmov_v4sf", IX86_BUILTIN_VPCMOV_V4SF, UNKNOWN, (int)MULTI_ARG_3_SF },
33415 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov256", IX86_BUILTIN_VPCMOV256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
33416 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov_v4di256", IX86_BUILTIN_VPCMOV_V4DI256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
33417 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8si256, "__builtin_ia32_vpcmov_v8si256", IX86_BUILTIN_VPCMOV_V8SI256, UNKNOWN, (int)MULTI_ARG_3_SI2 },
33418 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16hi256, "__builtin_ia32_vpcmov_v16hi256", IX86_BUILTIN_VPCMOV_V16HI256, UNKNOWN, (int)MULTI_ARG_3_HI2 },
33419 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v32qi256, "__builtin_ia32_vpcmov_v32qi256", IX86_BUILTIN_VPCMOV_V32QI256, UNKNOWN, (int)MULTI_ARG_3_QI2 },
33420 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4df256, "__builtin_ia32_vpcmov_v4df256", IX86_BUILTIN_VPCMOV_V4DF256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
33421 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8sf256, "__builtin_ia32_vpcmov_v8sf256", IX86_BUILTIN_VPCMOV_V8SF256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
33423 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pperm, "__builtin_ia32_vpperm", IX86_BUILTIN_VPPERM, UNKNOWN, (int)MULTI_ARG_3_QI },
33425 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssww, "__builtin_ia32_vpmacssww", IX86_BUILTIN_VPMACSSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
33426 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsww, "__builtin_ia32_vpmacsww", IX86_BUILTIN_VPMACSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
33427 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsswd, "__builtin_ia32_vpmacsswd", IX86_BUILTIN_VPMACSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
33428 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacswd, "__builtin_ia32_vpmacswd", IX86_BUILTIN_VPMACSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
33429 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdd, "__builtin_ia32_vpmacssdd", IX86_BUILTIN_VPMACSSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
33430 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdd, "__builtin_ia32_vpmacsdd", IX86_BUILTIN_VPMACSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
33431 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdql, "__builtin_ia32_vpmacssdql", IX86_BUILTIN_VPMACSSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
33432 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdqh, "__builtin_ia32_vpmacssdqh", IX86_BUILTIN_VPMACSSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
33433 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdql, "__builtin_ia32_vpmacsdql", IX86_BUILTIN_VPMACSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
33434 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdqh, "__builtin_ia32_vpmacsdqh", IX86_BUILTIN_VPMACSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
33435 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcsswd, "__builtin_ia32_vpmadcsswd", IX86_BUILTIN_VPMADCSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
33436 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcswd, "__builtin_ia32_vpmadcswd", IX86_BUILTIN_VPMADCSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
33438 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv2di3, "__builtin_ia32_vprotq", IX86_BUILTIN_VPROTQ, UNKNOWN, (int)MULTI_ARG_2_DI },
33439 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv4si3, "__builtin_ia32_vprotd", IX86_BUILTIN_VPROTD, UNKNOWN, (int)MULTI_ARG_2_SI },
33440 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv8hi3, "__builtin_ia32_vprotw", IX86_BUILTIN_VPROTW, UNKNOWN, (int)MULTI_ARG_2_HI },
33441 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv16qi3, "__builtin_ia32_vprotb", IX86_BUILTIN_VPROTB, UNKNOWN, (int)MULTI_ARG_2_QI },
33442 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv2di3, "__builtin_ia32_vprotqi", IX86_BUILTIN_VPROTQ_IMM, UNKNOWN, (int)MULTI_ARG_2_DI_IMM },
33443 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv4si3, "__builtin_ia32_vprotdi", IX86_BUILTIN_VPROTD_IMM, UNKNOWN, (int)MULTI_ARG_2_SI_IMM },
33444 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv8hi3, "__builtin_ia32_vprotwi", IX86_BUILTIN_VPROTW_IMM, UNKNOWN, (int)MULTI_ARG_2_HI_IMM },
33445 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv16qi3, "__builtin_ia32_vprotbi", IX86_BUILTIN_VPROTB_IMM, UNKNOWN, (int)MULTI_ARG_2_QI_IMM },
33446 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav2di3, "__builtin_ia32_vpshaq", IX86_BUILTIN_VPSHAQ, UNKNOWN, (int)MULTI_ARG_2_DI },
33447 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav4si3, "__builtin_ia32_vpshad", IX86_BUILTIN_VPSHAD, UNKNOWN, (int)MULTI_ARG_2_SI },
33448 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav8hi3, "__builtin_ia32_vpshaw", IX86_BUILTIN_VPSHAW, UNKNOWN, (int)MULTI_ARG_2_HI },
33449 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav16qi3, "__builtin_ia32_vpshab", IX86_BUILTIN_VPSHAB, UNKNOWN, (int)MULTI_ARG_2_QI },
33450 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv2di3, "__builtin_ia32_vpshlq", IX86_BUILTIN_VPSHLQ, UNKNOWN, (int)MULTI_ARG_2_DI },
33451 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv4si3, "__builtin_ia32_vpshld", IX86_BUILTIN_VPSHLD, UNKNOWN, (int)MULTI_ARG_2_SI },
33452 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv8hi3, "__builtin_ia32_vpshlw", IX86_BUILTIN_VPSHLW, UNKNOWN, (int)MULTI_ARG_2_HI },
33453 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv16qi3, "__builtin_ia32_vpshlb", IX86_BUILTIN_VPSHLB, UNKNOWN, (int)MULTI_ARG_2_QI },
33455 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv4sf2, "__builtin_ia32_vfrczss", IX86_BUILTIN_VFRCZSS, UNKNOWN, (int)MULTI_ARG_1_SF },
33456 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv2df2, "__builtin_ia32_vfrczsd", IX86_BUILTIN_VFRCZSD, UNKNOWN, (int)MULTI_ARG_1_DF },
33457 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4sf2, "__builtin_ia32_vfrczps", IX86_BUILTIN_VFRCZPS, UNKNOWN, (int)MULTI_ARG_1_SF },
33458 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv2df2, "__builtin_ia32_vfrczpd", IX86_BUILTIN_VFRCZPD, UNKNOWN, (int)MULTI_ARG_1_DF },
33459 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 },
33460 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 },
33462 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbw, "__builtin_ia32_vphaddbw", IX86_BUILTIN_VPHADDBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
33463 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbd, "__builtin_ia32_vphaddbd", IX86_BUILTIN_VPHADDBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
33464 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbq, "__builtin_ia32_vphaddbq", IX86_BUILTIN_VPHADDBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
33465 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwd, "__builtin_ia32_vphaddwd", IX86_BUILTIN_VPHADDWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
33466 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwq, "__builtin_ia32_vphaddwq", IX86_BUILTIN_VPHADDWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
33467 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadddq, "__builtin_ia32_vphadddq", IX86_BUILTIN_VPHADDDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
33468 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubw, "__builtin_ia32_vphaddubw", IX86_BUILTIN_VPHADDUBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
33469 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubd, "__builtin_ia32_vphaddubd", IX86_BUILTIN_VPHADDUBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
33470 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubq, "__builtin_ia32_vphaddubq", IX86_BUILTIN_VPHADDUBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
33471 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwd, "__builtin_ia32_vphadduwd", IX86_BUILTIN_VPHADDUWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
33472 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwq, "__builtin_ia32_vphadduwq", IX86_BUILTIN_VPHADDUWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
33473 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddudq, "__builtin_ia32_vphaddudq", IX86_BUILTIN_VPHADDUDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
33474 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubbw, "__builtin_ia32_vphsubbw", IX86_BUILTIN_VPHSUBBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
33475 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubwd, "__builtin_ia32_vphsubwd", IX86_BUILTIN_VPHSUBWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
33476 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubdq, "__builtin_ia32_vphsubdq", IX86_BUILTIN_VPHSUBDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
33478 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomeqb", IX86_BUILTIN_VPCOMEQB, EQ, (int)MULTI_ARG_2_QI_CMP },
33479 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
33480 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneqb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
33481 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomltb", IX86_BUILTIN_VPCOMLTB, LT, (int)MULTI_ARG_2_QI_CMP },
33482 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomleb", IX86_BUILTIN_VPCOMLEB, LE, (int)MULTI_ARG_2_QI_CMP },
33483 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgtb", IX86_BUILTIN_VPCOMGTB, GT, (int)MULTI_ARG_2_QI_CMP },
33484 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgeb", IX86_BUILTIN_VPCOMGEB, GE, (int)MULTI_ARG_2_QI_CMP },
33486 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomeqw", IX86_BUILTIN_VPCOMEQW, EQ, (int)MULTI_ARG_2_HI_CMP },
33487 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomnew", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
33488 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomneqw", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
33489 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomltw", IX86_BUILTIN_VPCOMLTW, LT, (int)MULTI_ARG_2_HI_CMP },
33490 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomlew", IX86_BUILTIN_VPCOMLEW, LE, (int)MULTI_ARG_2_HI_CMP },
33491 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgtw", IX86_BUILTIN_VPCOMGTW, GT, (int)MULTI_ARG_2_HI_CMP },
33492 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgew", IX86_BUILTIN_VPCOMGEW, GE, (int)MULTI_ARG_2_HI_CMP },
33494 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomeqd", IX86_BUILTIN_VPCOMEQD, EQ, (int)MULTI_ARG_2_SI_CMP },
33495 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomned", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
33496 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomneqd", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
33497 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomltd", IX86_BUILTIN_VPCOMLTD, LT, (int)MULTI_ARG_2_SI_CMP },
33498 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomled", IX86_BUILTIN_VPCOMLED, LE, (int)MULTI_ARG_2_SI_CMP },
33499 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomgtd", IX86_BUILTIN_VPCOMGTD, GT, (int)MULTI_ARG_2_SI_CMP },
33500 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomged", IX86_BUILTIN_VPCOMGED, GE, (int)MULTI_ARG_2_SI_CMP },
33502 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomeqq", IX86_BUILTIN_VPCOMEQQ, EQ, (int)MULTI_ARG_2_DI_CMP },
33503 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
33504 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneqq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
33505 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomltq", IX86_BUILTIN_VPCOMLTQ, LT, (int)MULTI_ARG_2_DI_CMP },
33506 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomleq", IX86_BUILTIN_VPCOMLEQ, LE, (int)MULTI_ARG_2_DI_CMP },
33507 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgtq", IX86_BUILTIN_VPCOMGTQ, GT, (int)MULTI_ARG_2_DI_CMP },
33508 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgeq", IX86_BUILTIN_VPCOMGEQ, GE, (int)MULTI_ARG_2_DI_CMP },
33510 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomequb", IX86_BUILTIN_VPCOMEQUB, EQ, (int)MULTI_ARG_2_QI_CMP },
33511 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomneub", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
33512 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomnequb", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
33513 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomltub", IX86_BUILTIN_VPCOMLTUB, LTU, (int)MULTI_ARG_2_QI_CMP },
33514 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomleub", IX86_BUILTIN_VPCOMLEUB, LEU, (int)MULTI_ARG_2_QI_CMP },
33515 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgtub", IX86_BUILTIN_VPCOMGTUB, GTU, (int)MULTI_ARG_2_QI_CMP },
33516 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgeub", IX86_BUILTIN_VPCOMGEUB, GEU, (int)MULTI_ARG_2_QI_CMP },
33518 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomequw", IX86_BUILTIN_VPCOMEQUW, EQ, (int)MULTI_ARG_2_HI_CMP },
33519 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomneuw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
33520 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomnequw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
33521 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomltuw", IX86_BUILTIN_VPCOMLTUW, LTU, (int)MULTI_ARG_2_HI_CMP },
33522 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomleuw", IX86_BUILTIN_VPCOMLEUW, LEU, (int)MULTI_ARG_2_HI_CMP },
33523 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgtuw", IX86_BUILTIN_VPCOMGTUW, GTU, (int)MULTI_ARG_2_HI_CMP },
33524 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgeuw", IX86_BUILTIN_VPCOMGEUW, GEU, (int)MULTI_ARG_2_HI_CMP },
33526 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomequd", IX86_BUILTIN_VPCOMEQUD, EQ, (int)MULTI_ARG_2_SI_CMP },
33527 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomneud", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
33528 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomnequd", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
33529 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomltud", IX86_BUILTIN_VPCOMLTUD, LTU, (int)MULTI_ARG_2_SI_CMP },
33530 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomleud", IX86_BUILTIN_VPCOMLEUD, LEU, (int)MULTI_ARG_2_SI_CMP },
33531 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgtud", IX86_BUILTIN_VPCOMGTUD, GTU, (int)MULTI_ARG_2_SI_CMP },
33532 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgeud", IX86_BUILTIN_VPCOMGEUD, GEU, (int)MULTI_ARG_2_SI_CMP },
33534 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomequq", IX86_BUILTIN_VPCOMEQUQ, EQ, (int)MULTI_ARG_2_DI_CMP },
33535 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomneuq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
33536 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomnequq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
33537 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomltuq", IX86_BUILTIN_VPCOMLTUQ, LTU, (int)MULTI_ARG_2_DI_CMP },
33538 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomleuq", IX86_BUILTIN_VPCOMLEUQ, LEU, (int)MULTI_ARG_2_DI_CMP },
33539 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgtuq", IX86_BUILTIN_VPCOMGTUQ, GTU, (int)MULTI_ARG_2_DI_CMP },
33540 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgeuq", IX86_BUILTIN_VPCOMGEUQ, GEU, (int)MULTI_ARG_2_DI_CMP },
33542 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseb", IX86_BUILTIN_VPCOMFALSEB, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
33543 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalsew", IX86_BUILTIN_VPCOMFALSEW, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
33544 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalsed", IX86_BUILTIN_VPCOMFALSED, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
33545 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseq", IX86_BUILTIN_VPCOMFALSEQ, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
33546 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseub",IX86_BUILTIN_VPCOMFALSEUB,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
33547 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalseuw",IX86_BUILTIN_VPCOMFALSEUW,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
33548 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalseud",IX86_BUILTIN_VPCOMFALSEUD,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
33549 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseuq",IX86_BUILTIN_VPCOMFALSEUQ,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
33551 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueb", IX86_BUILTIN_VPCOMTRUEB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
33552 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtruew", IX86_BUILTIN_VPCOMTRUEW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
33553 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrued", IX86_BUILTIN_VPCOMTRUED, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
33554 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueq", IX86_BUILTIN_VPCOMTRUEQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
33555 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueub", IX86_BUILTIN_VPCOMTRUEUB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
33556 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtrueuw", IX86_BUILTIN_VPCOMTRUEUW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
33557 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrueud", IX86_BUILTIN_VPCOMTRUEUD, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
33558 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueuq", IX86_BUILTIN_VPCOMTRUEUQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
33560 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v2df3, "__builtin_ia32_vpermil2pd", IX86_BUILTIN_VPERMIL2PD, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I },
33561 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4sf3, "__builtin_ia32_vpermil2ps", IX86_BUILTIN_VPERMIL2PS, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I },
33562 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4df3, "__builtin_ia32_vpermil2pd256", IX86_BUILTIN_VPERMIL2PD256, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I1 },
33563 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v8sf3, "__builtin_ia32_vpermil2ps256", IX86_BUILTIN_VPERMIL2PS256, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I1 },
33567 /* TM vector builtins. */
33569 /* Reuse the existing x86-specific `struct builtin_description' cause
33570 we're lazy. Add casts to make them fit. */
33571 static const struct builtin_description bdesc_tm[] =
33573 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WM64", (enum ix86_builtins) BUILT_IN_TM_STORE_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
33574 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaRM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
33575 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaWM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
33576 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
33577 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaRM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
33578 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
33579 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RfWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
33581 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WM128", (enum ix86_builtins) BUILT_IN_TM_STORE_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
33582 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaRM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
33583 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaWM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
33584 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
33585 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaRM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
33586 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
33587 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RfWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
33589 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WM256", (enum ix86_builtins) BUILT_IN_TM_STORE_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
33590 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaRM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
33591 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaWM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
33592 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
33593 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaRM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
33594 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
33595 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RfWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
33597 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_LM64", (enum ix86_builtins) BUILT_IN_TM_LOG_M64, UNKNOWN, VOID_FTYPE_PCVOID },
33598 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_LM128", (enum ix86_builtins) BUILT_IN_TM_LOG_M128, UNKNOWN, VOID_FTYPE_PCVOID },
33599 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_LM256", (enum ix86_builtins) BUILT_IN_TM_LOG_M256, UNKNOWN, VOID_FTYPE_PCVOID },
33602 /* TM callbacks. */
33604 /* Return the builtin decl needed to load a vector of TYPE. */
33607 ix86_builtin_tm_load (tree type)
33609 if (TREE_CODE (type) == VECTOR_TYPE)
33611 switch (tree_to_uhwi (TYPE_SIZE (type)))
33614 return builtin_decl_explicit (BUILT_IN_TM_LOAD_M64);
33616 return builtin_decl_explicit (BUILT_IN_TM_LOAD_M128);
33618 return builtin_decl_explicit (BUILT_IN_TM_LOAD_M256);
33624 /* Return the builtin decl needed to store a vector of TYPE. */
33627 ix86_builtin_tm_store (tree type)
33629 if (TREE_CODE (type) == VECTOR_TYPE)
33631 switch (tree_to_uhwi (TYPE_SIZE (type)))
33634 return builtin_decl_explicit (BUILT_IN_TM_STORE_M64);
33636 return builtin_decl_explicit (BUILT_IN_TM_STORE_M128);
33638 return builtin_decl_explicit (BUILT_IN_TM_STORE_M256);
33644 /* Initialize the transactional memory vector load/store builtins. */
33647 ix86_init_tm_builtins (void)
33649 enum ix86_builtin_func_type ftype;
33650 const struct builtin_description *d;
33653 tree attrs_load, attrs_type_load, attrs_store, attrs_type_store;
33654 tree attrs_log, attrs_type_log;
33659 /* If there are no builtins defined, we must be compiling in a
33660 language without trans-mem support. */
33661 if (!builtin_decl_explicit_p (BUILT_IN_TM_LOAD_1))
33664 /* Use whatever attributes a normal TM load has. */
33665 decl = builtin_decl_explicit (BUILT_IN_TM_LOAD_1);
33666 attrs_load = DECL_ATTRIBUTES (decl);
33667 attrs_type_load = TYPE_ATTRIBUTES (TREE_TYPE (decl));
33668 /* Use whatever attributes a normal TM store has. */
33669 decl = builtin_decl_explicit (BUILT_IN_TM_STORE_1);
33670 attrs_store = DECL_ATTRIBUTES (decl);
33671 attrs_type_store = TYPE_ATTRIBUTES (TREE_TYPE (decl));
33672 /* Use whatever attributes a normal TM log has. */
33673 decl = builtin_decl_explicit (BUILT_IN_TM_LOG);
33674 attrs_log = DECL_ATTRIBUTES (decl);
33675 attrs_type_log = TYPE_ATTRIBUTES (TREE_TYPE (decl));
33677 for (i = 0, d = bdesc_tm;
33678 i < ARRAY_SIZE (bdesc_tm);
33681 if ((d->mask & ix86_isa_flags) != 0
33682 || (lang_hooks.builtin_function
33683 == lang_hooks.builtin_function_ext_scope))
33685 tree type, attrs, attrs_type;
33686 enum built_in_function code = (enum built_in_function) d->code;
33688 ftype = (enum ix86_builtin_func_type) d->flag;
33689 type = ix86_get_builtin_func_type (ftype);
33691 if (BUILTIN_TM_LOAD_P (code))
33693 attrs = attrs_load;
33694 attrs_type = attrs_type_load;
33696 else if (BUILTIN_TM_STORE_P (code))
33698 attrs = attrs_store;
33699 attrs_type = attrs_type_store;
33704 attrs_type = attrs_type_log;
33706 decl = add_builtin_function (d->name, type, code, BUILT_IN_NORMAL,
33707 /* The builtin without the prefix for
33708 calling it directly. */
33709 d->name + strlen ("__builtin_"),
33711 /* add_builtin_function() will set the DECL_ATTRIBUTES, now
33712 set the TYPE_ATTRIBUTES. */
33713 decl_attributes (&TREE_TYPE (decl), attrs_type, ATTR_FLAG_BUILT_IN);
33715 set_builtin_decl (code, decl, false);
33720 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
33721 in the current target ISA to allow the user to compile particular modules
33722 with different target specific options that differ from the command line
33725 ix86_init_mmx_sse_builtins (void)
33727 const struct builtin_description * d;
33728 enum ix86_builtin_func_type ftype;
33731 /* Add all special builtins with variable number of operands. */
33732 for (i = 0, d = bdesc_special_args;
33733 i < ARRAY_SIZE (bdesc_special_args);
33739 ftype = (enum ix86_builtin_func_type) d->flag;
33740 def_builtin (d->mask, d->name, ftype, d->code);
33743 /* Add all builtins with variable number of operands. */
33744 for (i = 0, d = bdesc_args;
33745 i < ARRAY_SIZE (bdesc_args);
33751 ftype = (enum ix86_builtin_func_type) d->flag;
33752 def_builtin_const (d->mask, d->name, ftype, d->code);
33755 /* Add all builtins with rounding. */
33756 for (i = 0, d = bdesc_round_args;
33757 i < ARRAY_SIZE (bdesc_round_args);
33763 ftype = (enum ix86_builtin_func_type) d->flag;
33764 def_builtin_const (d->mask, d->name, ftype, d->code);
33767 /* pcmpestr[im] insns. */
33768 for (i = 0, d = bdesc_pcmpestr;
33769 i < ARRAY_SIZE (bdesc_pcmpestr);
33772 if (d->code == IX86_BUILTIN_PCMPESTRM128)
33773 ftype = V16QI_FTYPE_V16QI_INT_V16QI_INT_INT;
33775 ftype = INT_FTYPE_V16QI_INT_V16QI_INT_INT;
33776 def_builtin_const (d->mask, d->name, ftype, d->code);
33779 /* pcmpistr[im] insns. */
33780 for (i = 0, d = bdesc_pcmpistr;
33781 i < ARRAY_SIZE (bdesc_pcmpistr);
33784 if (d->code == IX86_BUILTIN_PCMPISTRM128)
33785 ftype = V16QI_FTYPE_V16QI_V16QI_INT;
33787 ftype = INT_FTYPE_V16QI_V16QI_INT;
33788 def_builtin_const (d->mask, d->name, ftype, d->code);
33791 /* comi/ucomi insns. */
33792 for (i = 0, d = bdesc_comi; i < ARRAY_SIZE (bdesc_comi); i++, d++)
33794 if (d->mask == OPTION_MASK_ISA_SSE2)
33795 ftype = INT_FTYPE_V2DF_V2DF;
33797 ftype = INT_FTYPE_V4SF_V4SF;
33798 def_builtin_const (d->mask, d->name, ftype, d->code);
33802 def_builtin (OPTION_MASK_ISA_SSE, "__builtin_ia32_ldmxcsr",
33803 VOID_FTYPE_UNSIGNED, IX86_BUILTIN_LDMXCSR);
33804 def_builtin (OPTION_MASK_ISA_SSE, "__builtin_ia32_stmxcsr",
33805 UNSIGNED_FTYPE_VOID, IX86_BUILTIN_STMXCSR);
33807 /* SSE or 3DNow!A */
33808 def_builtin (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A,
33809 "__builtin_ia32_maskmovq", VOID_FTYPE_V8QI_V8QI_PCHAR,
33810 IX86_BUILTIN_MASKMOVQ);
33813 def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_maskmovdqu",
33814 VOID_FTYPE_V16QI_V16QI_PCHAR, IX86_BUILTIN_MASKMOVDQU);
33816 def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_clflush",
33817 VOID_FTYPE_PCVOID, IX86_BUILTIN_CLFLUSH);
33818 x86_mfence = def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_mfence",
33819 VOID_FTYPE_VOID, IX86_BUILTIN_MFENCE);
33822 def_builtin (OPTION_MASK_ISA_SSE3, "__builtin_ia32_monitor",
33823 VOID_FTYPE_PCVOID_UNSIGNED_UNSIGNED, IX86_BUILTIN_MONITOR);
33824 def_builtin (OPTION_MASK_ISA_SSE3, "__builtin_ia32_mwait",
33825 VOID_FTYPE_UNSIGNED_UNSIGNED, IX86_BUILTIN_MWAIT);
33828 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesenc128",
33829 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESENC128);
33830 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesenclast128",
33831 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESENCLAST128);
33832 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesdec128",
33833 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESDEC128);
33834 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesdeclast128",
33835 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESDECLAST128);
33836 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesimc128",
33837 V2DI_FTYPE_V2DI, IX86_BUILTIN_AESIMC128);
33838 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aeskeygenassist128",
33839 V2DI_FTYPE_V2DI_INT, IX86_BUILTIN_AESKEYGENASSIST128);
33842 def_builtin_const (OPTION_MASK_ISA_PCLMUL, "__builtin_ia32_pclmulqdq128",
33843 V2DI_FTYPE_V2DI_V2DI_INT, IX86_BUILTIN_PCLMULQDQ128);
33846 def_builtin (OPTION_MASK_ISA_RDRND, "__builtin_ia32_rdrand16_step",
33847 INT_FTYPE_PUSHORT, IX86_BUILTIN_RDRAND16_STEP);
33848 def_builtin (OPTION_MASK_ISA_RDRND, "__builtin_ia32_rdrand32_step",
33849 INT_FTYPE_PUNSIGNED, IX86_BUILTIN_RDRAND32_STEP);
33850 def_builtin (OPTION_MASK_ISA_RDRND | OPTION_MASK_ISA_64BIT,
33851 "__builtin_ia32_rdrand64_step", INT_FTYPE_PULONGLONG,
33852 IX86_BUILTIN_RDRAND64_STEP);
33855 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv2df",
33856 V2DF_FTYPE_V2DF_PCDOUBLE_V4SI_V2DF_INT,
33857 IX86_BUILTIN_GATHERSIV2DF);
33859 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv4df",
33860 V4DF_FTYPE_V4DF_PCDOUBLE_V4SI_V4DF_INT,
33861 IX86_BUILTIN_GATHERSIV4DF);
33863 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv2df",
33864 V2DF_FTYPE_V2DF_PCDOUBLE_V2DI_V2DF_INT,
33865 IX86_BUILTIN_GATHERDIV2DF);
33867 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4df",
33868 V4DF_FTYPE_V4DF_PCDOUBLE_V4DI_V4DF_INT,
33869 IX86_BUILTIN_GATHERDIV4DF);
33871 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv4sf",
33872 V4SF_FTYPE_V4SF_PCFLOAT_V4SI_V4SF_INT,
33873 IX86_BUILTIN_GATHERSIV4SF);
33875 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv8sf",
33876 V8SF_FTYPE_V8SF_PCFLOAT_V8SI_V8SF_INT,
33877 IX86_BUILTIN_GATHERSIV8SF);
33879 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4sf",
33880 V4SF_FTYPE_V4SF_PCFLOAT_V2DI_V4SF_INT,
33881 IX86_BUILTIN_GATHERDIV4SF);
33883 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4sf256",
33884 V4SF_FTYPE_V4SF_PCFLOAT_V4DI_V4SF_INT,
33885 IX86_BUILTIN_GATHERDIV8SF);
33887 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv2di",
33888 V2DI_FTYPE_V2DI_PCINT64_V4SI_V2DI_INT,
33889 IX86_BUILTIN_GATHERSIV2DI);
33891 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv4di",
33892 V4DI_FTYPE_V4DI_PCINT64_V4SI_V4DI_INT,
33893 IX86_BUILTIN_GATHERSIV4DI);
33895 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv2di",
33896 V2DI_FTYPE_V2DI_PCINT64_V2DI_V2DI_INT,
33897 IX86_BUILTIN_GATHERDIV2DI);
33899 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4di",
33900 V4DI_FTYPE_V4DI_PCINT64_V4DI_V4DI_INT,
33901 IX86_BUILTIN_GATHERDIV4DI);
33903 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv4si",
33904 V4SI_FTYPE_V4SI_PCINT_V4SI_V4SI_INT,
33905 IX86_BUILTIN_GATHERSIV4SI);
33907 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv8si",
33908 V8SI_FTYPE_V8SI_PCINT_V8SI_V8SI_INT,
33909 IX86_BUILTIN_GATHERSIV8SI);
33911 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4si",
33912 V4SI_FTYPE_V4SI_PCINT_V2DI_V4SI_INT,
33913 IX86_BUILTIN_GATHERDIV4SI);
33915 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4si256",
33916 V4SI_FTYPE_V4SI_PCINT_V4DI_V4SI_INT,
33917 IX86_BUILTIN_GATHERDIV8SI);
33919 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltsiv4df ",
33920 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_V4DF_INT,
33921 IX86_BUILTIN_GATHERALTSIV4DF);
33923 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltdiv4sf256 ",
33924 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_V8SF_INT,
33925 IX86_BUILTIN_GATHERALTDIV8SF);
33927 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltsiv4di ",
33928 V4DI_FTYPE_V4DI_PCINT64_V8SI_V4DI_INT,
33929 IX86_BUILTIN_GATHERALTSIV4DI);
33931 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltdiv4si256 ",
33932 V8SI_FTYPE_V8SI_PCINT_V4DI_V8SI_INT,
33933 IX86_BUILTIN_GATHERALTDIV8SI);
33936 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gathersiv16sf",
33937 V16SF_FTYPE_V16SF_PCFLOAT_V16SI_HI_INT,
33938 IX86_BUILTIN_GATHER3SIV16SF);
33940 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gathersiv8df",
33941 V8DF_FTYPE_V8DF_PCDOUBLE_V8SI_QI_INT,
33942 IX86_BUILTIN_GATHER3SIV8DF);
33944 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatherdiv16sf",
33945 V8SF_FTYPE_V8SF_PCFLOAT_V8DI_QI_INT,
33946 IX86_BUILTIN_GATHER3DIV16SF);
33948 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatherdiv8df",
33949 V8DF_FTYPE_V8DF_PCDOUBLE_V8DI_QI_INT,
33950 IX86_BUILTIN_GATHER3DIV8DF);
33952 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gathersiv16si",
33953 V16SI_FTYPE_V16SI_PCINT_V16SI_HI_INT,
33954 IX86_BUILTIN_GATHER3SIV16SI);
33956 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gathersiv8di",
33957 V8DI_FTYPE_V8DI_PCINT64_V8SI_QI_INT,
33958 IX86_BUILTIN_GATHER3SIV8DI);
33960 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatherdiv16si",
33961 V8SI_FTYPE_V8SI_PCINT_V8DI_QI_INT,
33962 IX86_BUILTIN_GATHER3DIV16SI);
33964 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatherdiv8di",
33965 V8DI_FTYPE_V8DI_PCINT64_V8DI_QI_INT,
33966 IX86_BUILTIN_GATHER3DIV8DI);
33968 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatheraltsiv8df ",
33969 V8DF_FTYPE_V8DF_PCDOUBLE_V16SI_QI_INT,
33970 IX86_BUILTIN_GATHER3ALTSIV8DF);
33972 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatheraltdiv8sf ",
33973 V16SF_FTYPE_V16SF_PCFLOAT_V8DI_HI_INT,
33974 IX86_BUILTIN_GATHER3ALTDIV16SF);
33976 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatheraltsiv8di ",
33977 V8DI_FTYPE_V8DI_PCINT64_V16SI_QI_INT,
33978 IX86_BUILTIN_GATHER3ALTSIV8DI);
33980 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatheraltdiv8si ",
33981 V16SI_FTYPE_V16SI_PCINT_V8DI_HI_INT,
33982 IX86_BUILTIN_GATHER3ALTDIV16SI);
33984 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scattersiv16sf",
33985 VOID_FTYPE_PFLOAT_HI_V16SI_V16SF_INT,
33986 IX86_BUILTIN_SCATTERSIV16SF);
33988 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scattersiv8df",
33989 VOID_FTYPE_PDOUBLE_QI_V8SI_V8DF_INT,
33990 IX86_BUILTIN_SCATTERSIV8DF);
33992 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scatterdiv16sf",
33993 VOID_FTYPE_PFLOAT_QI_V8DI_V8SF_INT,
33994 IX86_BUILTIN_SCATTERDIV16SF);
33996 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scatterdiv8df",
33997 VOID_FTYPE_PDOUBLE_QI_V8DI_V8DF_INT,
33998 IX86_BUILTIN_SCATTERDIV8DF);
34000 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scattersiv16si",
34001 VOID_FTYPE_PINT_HI_V16SI_V16SI_INT,
34002 IX86_BUILTIN_SCATTERSIV16SI);
34004 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scattersiv8di",
34005 VOID_FTYPE_PLONGLONG_QI_V8SI_V8DI_INT,
34006 IX86_BUILTIN_SCATTERSIV8DI);
34008 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scatterdiv16si",
34009 VOID_FTYPE_PINT_QI_V8DI_V8SI_INT,
34010 IX86_BUILTIN_SCATTERDIV16SI);
34012 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scatterdiv8di",
34013 VOID_FTYPE_PLONGLONG_QI_V8DI_V8DI_INT,
34014 IX86_BUILTIN_SCATTERDIV8DI);
34017 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv2df",
34018 V2DF_FTYPE_V2DF_PCDOUBLE_V4SI_QI_INT,
34019 IX86_BUILTIN_GATHER3SIV2DF);
34021 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv4df",
34022 V4DF_FTYPE_V4DF_PCDOUBLE_V4SI_QI_INT,
34023 IX86_BUILTIN_GATHER3SIV4DF);
34025 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div2df",
34026 V2DF_FTYPE_V2DF_PCDOUBLE_V2DI_QI_INT,
34027 IX86_BUILTIN_GATHER3DIV2DF);
34029 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div4df",
34030 V4DF_FTYPE_V4DF_PCDOUBLE_V4DI_QI_INT,
34031 IX86_BUILTIN_GATHER3DIV4DF);
34033 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv4sf",
34034 V4SF_FTYPE_V4SF_PCFLOAT_V4SI_QI_INT,
34035 IX86_BUILTIN_GATHER3SIV4SF);
34037 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv8sf",
34038 V8SF_FTYPE_V8SF_PCFLOAT_V8SI_QI_INT,
34039 IX86_BUILTIN_GATHER3SIV8SF);
34041 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div4sf",
34042 V4SF_FTYPE_V4SF_PCFLOAT_V2DI_QI_INT,
34043 IX86_BUILTIN_GATHER3DIV4SF);
34045 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div8sf",
34046 V4SF_FTYPE_V4SF_PCFLOAT_V4DI_QI_INT,
34047 IX86_BUILTIN_GATHER3DIV8SF);
34049 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv2di",
34050 V2DI_FTYPE_V2DI_PCINT64_V4SI_QI_INT,
34051 IX86_BUILTIN_GATHER3SIV2DI);
34053 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv4di",
34054 V4DI_FTYPE_V4DI_PCINT64_V4SI_QI_INT,
34055 IX86_BUILTIN_GATHER3SIV4DI);
34057 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div2di",
34058 V2DI_FTYPE_V2DI_PCINT64_V2DI_QI_INT,
34059 IX86_BUILTIN_GATHER3DIV2DI);
34061 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div4di",
34062 V4DI_FTYPE_V4DI_PCINT64_V4DI_QI_INT,
34063 IX86_BUILTIN_GATHER3DIV4DI);
34065 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv4si",
34066 V4SI_FTYPE_V4SI_PCINT_V4SI_QI_INT,
34067 IX86_BUILTIN_GATHER3SIV4SI);
34069 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv8si",
34070 V8SI_FTYPE_V8SI_PCINT_V8SI_QI_INT,
34071 IX86_BUILTIN_GATHER3SIV8SI);
34073 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div4si",
34074 V4SI_FTYPE_V4SI_PCINT_V2DI_QI_INT,
34075 IX86_BUILTIN_GATHER3DIV4SI);
34077 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div8si",
34078 V4SI_FTYPE_V4SI_PCINT_V4DI_QI_INT,
34079 IX86_BUILTIN_GATHER3DIV8SI);
34081 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3altsiv4df ",
34082 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_QI_INT,
34083 IX86_BUILTIN_GATHER3ALTSIV4DF);
34085 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3altdiv8sf ",
34086 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_QI_INT,
34087 IX86_BUILTIN_GATHER3ALTDIV8SF);
34089 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3altsiv4di ",
34090 V4DI_FTYPE_V4DI_PCINT64_V8SI_QI_INT,
34091 IX86_BUILTIN_GATHER3ALTSIV4DI);
34093 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3altdiv8si ",
34094 V8SI_FTYPE_V8SI_PCINT_V4DI_QI_INT,
34095 IX86_BUILTIN_GATHER3ALTDIV8SI);
34097 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv8sf",
34098 VOID_FTYPE_PFLOAT_QI_V8SI_V8SF_INT,
34099 IX86_BUILTIN_SCATTERSIV8SF);
34101 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv4sf",
34102 VOID_FTYPE_PFLOAT_QI_V4SI_V4SF_INT,
34103 IX86_BUILTIN_SCATTERSIV4SF);
34105 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv4df",
34106 VOID_FTYPE_PDOUBLE_QI_V4SI_V4DF_INT,
34107 IX86_BUILTIN_SCATTERSIV4DF);
34109 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv2df",
34110 VOID_FTYPE_PDOUBLE_QI_V4SI_V2DF_INT,
34111 IX86_BUILTIN_SCATTERSIV2DF);
34113 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv8sf",
34114 VOID_FTYPE_PFLOAT_QI_V4DI_V4SF_INT,
34115 IX86_BUILTIN_SCATTERDIV8SF);
34117 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv4sf",
34118 VOID_FTYPE_PFLOAT_QI_V2DI_V4SF_INT,
34119 IX86_BUILTIN_SCATTERDIV4SF);
34121 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv4df",
34122 VOID_FTYPE_PDOUBLE_QI_V4DI_V4DF_INT,
34123 IX86_BUILTIN_SCATTERDIV4DF);
34125 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv2df",
34126 VOID_FTYPE_PDOUBLE_QI_V2DI_V2DF_INT,
34127 IX86_BUILTIN_SCATTERDIV2DF);
34129 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv8si",
34130 VOID_FTYPE_PINT_QI_V8SI_V8SI_INT,
34131 IX86_BUILTIN_SCATTERSIV8SI);
34133 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv4si",
34134 VOID_FTYPE_PINT_QI_V4SI_V4SI_INT,
34135 IX86_BUILTIN_SCATTERSIV4SI);
34137 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv4di",
34138 VOID_FTYPE_PLONGLONG_QI_V4SI_V4DI_INT,
34139 IX86_BUILTIN_SCATTERSIV4DI);
34141 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv2di",
34142 VOID_FTYPE_PLONGLONG_QI_V4SI_V2DI_INT,
34143 IX86_BUILTIN_SCATTERSIV2DI);
34145 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv8si",
34146 VOID_FTYPE_PINT_QI_V4DI_V4SI_INT,
34147 IX86_BUILTIN_SCATTERDIV8SI);
34149 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv4si",
34150 VOID_FTYPE_PINT_QI_V2DI_V4SI_INT,
34151 IX86_BUILTIN_SCATTERDIV4SI);
34153 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv4di",
34154 VOID_FTYPE_PLONGLONG_QI_V4DI_V4DI_INT,
34155 IX86_BUILTIN_SCATTERDIV4DI);
34157 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv2di",
34158 VOID_FTYPE_PLONGLONG_QI_V2DI_V2DI_INT,
34159 IX86_BUILTIN_SCATTERDIV2DI);
34162 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_gatherpfdpd",
34163 VOID_FTYPE_QI_V8SI_PCINT64_INT_INT,
34164 IX86_BUILTIN_GATHERPFDPD);
34165 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_gatherpfdps",
34166 VOID_FTYPE_HI_V16SI_PCINT_INT_INT,
34167 IX86_BUILTIN_GATHERPFDPS);
34168 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_gatherpfqpd",
34169 VOID_FTYPE_QI_V8DI_PCINT64_INT_INT,
34170 IX86_BUILTIN_GATHERPFQPD);
34171 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_gatherpfqps",
34172 VOID_FTYPE_QI_V8DI_PCINT_INT_INT,
34173 IX86_BUILTIN_GATHERPFQPS);
34174 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_scatterpfdpd",
34175 VOID_FTYPE_QI_V8SI_PCINT64_INT_INT,
34176 IX86_BUILTIN_SCATTERPFDPD);
34177 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_scatterpfdps",
34178 VOID_FTYPE_HI_V16SI_PCINT_INT_INT,
34179 IX86_BUILTIN_SCATTERPFDPS);
34180 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_scatterpfqpd",
34181 VOID_FTYPE_QI_V8DI_PCINT64_INT_INT,
34182 IX86_BUILTIN_SCATTERPFQPD);
34183 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_scatterpfqps",
34184 VOID_FTYPE_QI_V8DI_PCINT_INT_INT,
34185 IX86_BUILTIN_SCATTERPFQPS);
34188 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha1msg1",
34189 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA1MSG1);
34190 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha1msg2",
34191 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA1MSG2);
34192 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha1nexte",
34193 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA1NEXTE);
34194 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha1rnds4",
34195 V4SI_FTYPE_V4SI_V4SI_INT, IX86_BUILTIN_SHA1RNDS4);
34196 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha256msg1",
34197 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA256MSG1);
34198 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha256msg2",
34199 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA256MSG2);
34200 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha256rnds2",
34201 V4SI_FTYPE_V4SI_V4SI_V4SI, IX86_BUILTIN_SHA256RNDS2);
34204 def_builtin (OPTION_MASK_ISA_RTM, "__builtin_ia32_xabort",
34205 VOID_FTYPE_UNSIGNED, IX86_BUILTIN_XABORT);
34207 /* MMX access to the vec_init patterns. */
34208 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v2si",
34209 V2SI_FTYPE_INT_INT, IX86_BUILTIN_VEC_INIT_V2SI);
34211 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v4hi",
34212 V4HI_FTYPE_HI_HI_HI_HI,
34213 IX86_BUILTIN_VEC_INIT_V4HI);
34215 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v8qi",
34216 V8QI_FTYPE_QI_QI_QI_QI_QI_QI_QI_QI,
34217 IX86_BUILTIN_VEC_INIT_V8QI);
34219 /* Access to the vec_extract patterns. */
34220 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v2df",
34221 DOUBLE_FTYPE_V2DF_INT, IX86_BUILTIN_VEC_EXT_V2DF);
34222 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v2di",
34223 DI_FTYPE_V2DI_INT, IX86_BUILTIN_VEC_EXT_V2DI);
34224 def_builtin_const (OPTION_MASK_ISA_SSE, "__builtin_ia32_vec_ext_v4sf",
34225 FLOAT_FTYPE_V4SF_INT, IX86_BUILTIN_VEC_EXT_V4SF);
34226 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v4si",
34227 SI_FTYPE_V4SI_INT, IX86_BUILTIN_VEC_EXT_V4SI);
34228 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v8hi",
34229 HI_FTYPE_V8HI_INT, IX86_BUILTIN_VEC_EXT_V8HI);
34231 def_builtin_const (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A,
34232 "__builtin_ia32_vec_ext_v4hi",
34233 HI_FTYPE_V4HI_INT, IX86_BUILTIN_VEC_EXT_V4HI);
34235 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_ext_v2si",
34236 SI_FTYPE_V2SI_INT, IX86_BUILTIN_VEC_EXT_V2SI);
34238 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v16qi",
34239 QI_FTYPE_V16QI_INT, IX86_BUILTIN_VEC_EXT_V16QI);
34241 /* Access to the vec_set patterns. */
34242 def_builtin_const (OPTION_MASK_ISA_SSE4_1 | OPTION_MASK_ISA_64BIT,
34243 "__builtin_ia32_vec_set_v2di",
34244 V2DI_FTYPE_V2DI_DI_INT, IX86_BUILTIN_VEC_SET_V2DI);
34246 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v4sf",
34247 V4SF_FTYPE_V4SF_FLOAT_INT, IX86_BUILTIN_VEC_SET_V4SF);
34249 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v4si",
34250 V4SI_FTYPE_V4SI_SI_INT, IX86_BUILTIN_VEC_SET_V4SI);
34252 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_set_v8hi",
34253 V8HI_FTYPE_V8HI_HI_INT, IX86_BUILTIN_VEC_SET_V8HI);
34255 def_builtin_const (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A,
34256 "__builtin_ia32_vec_set_v4hi",
34257 V4HI_FTYPE_V4HI_HI_INT, IX86_BUILTIN_VEC_SET_V4HI);
34259 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v16qi",
34260 V16QI_FTYPE_V16QI_QI_INT, IX86_BUILTIN_VEC_SET_V16QI);
34263 def_builtin (OPTION_MASK_ISA_RDSEED, "__builtin_ia32_rdseed_hi_step",
34264 INT_FTYPE_PUSHORT, IX86_BUILTIN_RDSEED16_STEP);
34265 def_builtin (OPTION_MASK_ISA_RDSEED, "__builtin_ia32_rdseed_si_step",
34266 INT_FTYPE_PUNSIGNED, IX86_BUILTIN_RDSEED32_STEP);
34267 def_builtin (OPTION_MASK_ISA_RDSEED | OPTION_MASK_ISA_64BIT,
34268 "__builtin_ia32_rdseed_di_step",
34269 INT_FTYPE_PULONGLONG, IX86_BUILTIN_RDSEED64_STEP);
34272 def_builtin (0, "__builtin_ia32_addcarryx_u32",
34273 UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED, IX86_BUILTIN_ADDCARRYX32);
34274 def_builtin (OPTION_MASK_ISA_64BIT,
34275 "__builtin_ia32_addcarryx_u64",
34276 UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG,
34277 IX86_BUILTIN_ADDCARRYX64);
34280 def_builtin (0, "__builtin_ia32_sbb_u32",
34281 UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED, IX86_BUILTIN_SBB32);
34282 def_builtin (OPTION_MASK_ISA_64BIT,
34283 "__builtin_ia32_sbb_u64",
34284 UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG,
34285 IX86_BUILTIN_SBB64);
34287 /* Read/write FLAGS. */
34288 def_builtin (~OPTION_MASK_ISA_64BIT, "__builtin_ia32_readeflags_u32",
34289 UNSIGNED_FTYPE_VOID, IX86_BUILTIN_READ_FLAGS);
34290 def_builtin (OPTION_MASK_ISA_64BIT, "__builtin_ia32_readeflags_u64",
34291 UINT64_FTYPE_VOID, IX86_BUILTIN_READ_FLAGS);
34292 def_builtin (~OPTION_MASK_ISA_64BIT, "__builtin_ia32_writeeflags_u32",
34293 VOID_FTYPE_UNSIGNED, IX86_BUILTIN_WRITE_FLAGS);
34294 def_builtin (OPTION_MASK_ISA_64BIT, "__builtin_ia32_writeeflags_u64",
34295 VOID_FTYPE_UINT64, IX86_BUILTIN_WRITE_FLAGS);
34298 def_builtin (OPTION_MASK_ISA_CLFLUSHOPT, "__builtin_ia32_clflushopt",
34299 VOID_FTYPE_PCVOID, IX86_BUILTIN_CLFLUSHOPT);
34302 def_builtin (OPTION_MASK_ISA_CLWB, "__builtin_ia32_clwb",
34303 VOID_FTYPE_PCVOID, IX86_BUILTIN_CLWB);
34305 /* MONITORX and MWAITX. */
34306 def_builtin (OPTION_MASK_ISA_MWAITX, "__builtin_ia32_monitorx",
34307 VOID_FTYPE_PCVOID_UNSIGNED_UNSIGNED, IX86_BUILTIN_MONITORX);
34308 def_builtin (OPTION_MASK_ISA_MWAITX, "__builtin_ia32_mwaitx",
34309 VOID_FTYPE_UNSIGNED_UNSIGNED_UNSIGNED, IX86_BUILTIN_MWAITX);
34311 /* Add FMA4 multi-arg argument instructions */
34312 for (i = 0, d = bdesc_multi_arg; i < ARRAY_SIZE (bdesc_multi_arg); i++, d++)
34317 ftype = (enum ix86_builtin_func_type) d->flag;
34318 def_builtin_const (d->mask, d->name, ftype, d->code);
34323 ix86_init_mpx_builtins ()
34325 const struct builtin_description * d;
34326 enum ix86_builtin_func_type ftype;
34330 for (i = 0, d = bdesc_mpx;
34331 i < ARRAY_SIZE (bdesc_mpx);
34337 ftype = (enum ix86_builtin_func_type) d->flag;
34338 decl = def_builtin (d->mask, d->name, ftype, d->code);
34340 /* With no leaf and nothrow flags for MPX builtins
34341 abnormal edges may follow its call when setjmp
34342 presents in the function. Since we may have a lot
34343 of MPX builtins calls it causes lots of useless
34344 edges and enormous PHI nodes. To avoid this we mark
34345 MPX builtins as leaf and nothrow. */
34348 DECL_ATTRIBUTES (decl) = build_tree_list (get_identifier ("leaf"),
34350 TREE_NOTHROW (decl) = 1;
34354 ix86_builtins_isa[(int)d->code].leaf_p = true;
34355 ix86_builtins_isa[(int)d->code].nothrow_p = true;
34359 for (i = 0, d = bdesc_mpx_const;
34360 i < ARRAY_SIZE (bdesc_mpx_const);
34366 ftype = (enum ix86_builtin_func_type) d->flag;
34367 decl = def_builtin_const (d->mask, d->name, ftype, d->code);
34371 DECL_ATTRIBUTES (decl) = build_tree_list (get_identifier ("leaf"),
34373 TREE_NOTHROW (decl) = 1;
34377 ix86_builtins_isa[(int)d->code].leaf_p = true;
34378 ix86_builtins_isa[(int)d->code].nothrow_p = true;
34383 /* This adds a condition to the basic_block NEW_BB in function FUNCTION_DECL
34384 to return a pointer to VERSION_DECL if the outcome of the expression
34385 formed by PREDICATE_CHAIN is true. This function will be called during
34386 version dispatch to decide which function version to execute. It returns
34387 the basic block at the end, to which more conditions can be added. */
34390 add_condition_to_bb (tree function_decl, tree version_decl,
34391 tree predicate_chain, basic_block new_bb)
34393 gimple return_stmt;
34394 tree convert_expr, result_var;
34395 gimple convert_stmt;
34396 gimple call_cond_stmt;
34397 gimple if_else_stmt;
34399 basic_block bb1, bb2, bb3;
34402 tree cond_var, and_expr_var = NULL_TREE;
34405 tree predicate_decl, predicate_arg;
34407 push_cfun (DECL_STRUCT_FUNCTION (function_decl));
34409 gcc_assert (new_bb != NULL);
34410 gseq = bb_seq (new_bb);
34413 convert_expr = build1 (CONVERT_EXPR, ptr_type_node,
34414 build_fold_addr_expr (version_decl));
34415 result_var = create_tmp_var (ptr_type_node);
34416 convert_stmt = gimple_build_assign (result_var, convert_expr);
34417 return_stmt = gimple_build_return (result_var);
34419 if (predicate_chain == NULL_TREE)
34421 gimple_seq_add_stmt (&gseq, convert_stmt);
34422 gimple_seq_add_stmt (&gseq, return_stmt);
34423 set_bb_seq (new_bb, gseq);
34424 gimple_set_bb (convert_stmt, new_bb);
34425 gimple_set_bb (return_stmt, new_bb);
34430 while (predicate_chain != NULL)
34432 cond_var = create_tmp_var (integer_type_node);
34433 predicate_decl = TREE_PURPOSE (predicate_chain);
34434 predicate_arg = TREE_VALUE (predicate_chain);
34435 call_cond_stmt = gimple_build_call (predicate_decl, 1, predicate_arg);
34436 gimple_call_set_lhs (call_cond_stmt, cond_var);
34438 gimple_set_block (call_cond_stmt, DECL_INITIAL (function_decl));
34439 gimple_set_bb (call_cond_stmt, new_bb);
34440 gimple_seq_add_stmt (&gseq, call_cond_stmt);
34442 predicate_chain = TREE_CHAIN (predicate_chain);
34444 if (and_expr_var == NULL)
34445 and_expr_var = cond_var;
34448 gimple assign_stmt;
34449 /* Use MIN_EXPR to check if any integer is zero?.
34450 and_expr_var = min_expr <cond_var, and_expr_var> */
34451 assign_stmt = gimple_build_assign (and_expr_var,
34452 build2 (MIN_EXPR, integer_type_node,
34453 cond_var, and_expr_var));
34455 gimple_set_block (assign_stmt, DECL_INITIAL (function_decl));
34456 gimple_set_bb (assign_stmt, new_bb);
34457 gimple_seq_add_stmt (&gseq, assign_stmt);
34461 if_else_stmt = gimple_build_cond (GT_EXPR, and_expr_var,
34463 NULL_TREE, NULL_TREE);
34464 gimple_set_block (if_else_stmt, DECL_INITIAL (function_decl));
34465 gimple_set_bb (if_else_stmt, new_bb);
34466 gimple_seq_add_stmt (&gseq, if_else_stmt);
34468 gimple_seq_add_stmt (&gseq, convert_stmt);
34469 gimple_seq_add_stmt (&gseq, return_stmt);
34470 set_bb_seq (new_bb, gseq);
34473 e12 = split_block (bb1, if_else_stmt);
34475 e12->flags &= ~EDGE_FALLTHRU;
34476 e12->flags |= EDGE_TRUE_VALUE;
34478 e23 = split_block (bb2, return_stmt);
34480 gimple_set_bb (convert_stmt, bb2);
34481 gimple_set_bb (return_stmt, bb2);
34484 make_edge (bb1, bb3, EDGE_FALSE_VALUE);
34487 make_edge (bb2, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
34494 /* This parses the attribute arguments to target in DECL and determines
34495 the right builtin to use to match the platform specification.
34496 It returns the priority value for this version decl. If PREDICATE_LIST
34497 is not NULL, it stores the list of cpu features that need to be checked
34498 before dispatching this function. */
34500 static unsigned int
34501 get_builtin_code_for_version (tree decl, tree *predicate_list)
34504 struct cl_target_option cur_target;
34506 struct cl_target_option *new_target;
34507 const char *arg_str = NULL;
34508 const char *attrs_str = NULL;
34509 char *tok_str = NULL;
34512 /* Priority of i386 features, greater value is higher priority. This is
34513 used to decide the order in which function dispatch must happen. For
34514 instance, a version specialized for SSE4.2 should be checked for dispatch
34515 before a version for SSE3, as SSE4.2 implies SSE3. */
34516 enum feature_priority
34547 enum feature_priority priority = P_ZERO;
34549 /* These are the target attribute strings for which a dispatcher is
34550 available, from fold_builtin_cpu. */
34552 static struct _feature_list
34554 const char *const name;
34555 const enum feature_priority priority;
34557 const feature_list[] =
34563 {"sse4a", P_SSE4_A},
34564 {"ssse3", P_SSSE3},
34565 {"sse4.1", P_SSE4_1},
34566 {"sse4.2", P_SSE4_2},
34567 {"popcnt", P_POPCNT},
34575 {"avx512f", P_AVX512F}
34579 static unsigned int NUM_FEATURES
34580 = sizeof (feature_list) / sizeof (struct _feature_list);
34584 tree predicate_chain = NULL_TREE;
34585 tree predicate_decl, predicate_arg;
34587 attrs = lookup_attribute ("target", DECL_ATTRIBUTES (decl));
34588 gcc_assert (attrs != NULL);
34590 attrs = TREE_VALUE (TREE_VALUE (attrs));
34592 gcc_assert (TREE_CODE (attrs) == STRING_CST);
34593 attrs_str = TREE_STRING_POINTER (attrs);
34595 /* Return priority zero for default function. */
34596 if (strcmp (attrs_str, "default") == 0)
34599 /* Handle arch= if specified. For priority, set it to be 1 more than
34600 the best instruction set the processor can handle. For instance, if
34601 there is a version for atom and a version for ssse3 (the highest ISA
34602 priority for atom), the atom version must be checked for dispatch
34603 before the ssse3 version. */
34604 if (strstr (attrs_str, "arch=") != NULL)
34606 cl_target_option_save (&cur_target, &global_options);
34607 target_node = ix86_valid_target_attribute_tree (attrs, &global_options,
34608 &global_options_set);
34610 gcc_assert (target_node);
34611 new_target = TREE_TARGET_OPTION (target_node);
34612 gcc_assert (new_target);
34614 if (new_target->arch_specified && new_target->arch > 0)
34616 switch (new_target->arch)
34618 case PROCESSOR_CORE2:
34620 priority = P_PROC_SSSE3;
34622 case PROCESSOR_NEHALEM:
34623 if (new_target->x_ix86_isa_flags & OPTION_MASK_ISA_AES)
34624 arg_str = "westmere";
34626 /* We translate "arch=corei7" and "arch=nehalem" to
34627 "corei7" so that it will be mapped to M_INTEL_COREI7
34628 as cpu type to cover all M_INTEL_COREI7_XXXs. */
34629 arg_str = "corei7";
34630 priority = P_PROC_SSE4_2;
34632 case PROCESSOR_SANDYBRIDGE:
34633 if (new_target->x_ix86_isa_flags & OPTION_MASK_ISA_F16C)
34634 arg_str = "ivybridge";
34636 arg_str = "sandybridge";
34637 priority = P_PROC_AVX;
34639 case PROCESSOR_HASWELL:
34640 if (new_target->x_ix86_isa_flags & OPTION_MASK_ISA_ADX)
34641 arg_str = "broadwell";
34643 arg_str = "haswell";
34644 priority = P_PROC_AVX2;
34646 case PROCESSOR_BONNELL:
34647 arg_str = "bonnell";
34648 priority = P_PROC_SSSE3;
34650 case PROCESSOR_KNL:
34652 priority = P_PROC_AVX512F;
34654 case PROCESSOR_SILVERMONT:
34655 arg_str = "silvermont";
34656 priority = P_PROC_SSE4_2;
34658 case PROCESSOR_AMDFAM10:
34659 arg_str = "amdfam10h";
34660 priority = P_PROC_SSE4_A;
34662 case PROCESSOR_BTVER1:
34663 arg_str = "btver1";
34664 priority = P_PROC_SSE4_A;
34666 case PROCESSOR_BTVER2:
34667 arg_str = "btver2";
34668 priority = P_PROC_BMI;
34670 case PROCESSOR_BDVER1:
34671 arg_str = "bdver1";
34672 priority = P_PROC_XOP;
34674 case PROCESSOR_BDVER2:
34675 arg_str = "bdver2";
34676 priority = P_PROC_FMA;
34678 case PROCESSOR_BDVER3:
34679 arg_str = "bdver3";
34680 priority = P_PROC_FMA;
34682 case PROCESSOR_BDVER4:
34683 arg_str = "bdver4";
34684 priority = P_PROC_AVX2;
34689 cl_target_option_restore (&global_options, &cur_target);
34691 if (predicate_list && arg_str == NULL)
34693 error_at (DECL_SOURCE_LOCATION (decl),
34694 "No dispatcher found for the versioning attributes");
34698 if (predicate_list)
34700 predicate_decl = ix86_builtins [(int) IX86_BUILTIN_CPU_IS];
34701 /* For a C string literal the length includes the trailing NULL. */
34702 predicate_arg = build_string_literal (strlen (arg_str) + 1, arg_str);
34703 predicate_chain = tree_cons (predicate_decl, predicate_arg,
34708 /* Process feature name. */
34709 tok_str = (char *) xmalloc (strlen (attrs_str) + 1);
34710 strcpy (tok_str, attrs_str);
34711 token = strtok (tok_str, ",");
34712 predicate_decl = ix86_builtins [(int) IX86_BUILTIN_CPU_SUPPORTS];
34714 while (token != NULL)
34716 /* Do not process "arch=" */
34717 if (strncmp (token, "arch=", 5) == 0)
34719 token = strtok (NULL, ",");
34722 for (i = 0; i < NUM_FEATURES; ++i)
34724 if (strcmp (token, feature_list[i].name) == 0)
34726 if (predicate_list)
34728 predicate_arg = build_string_literal (
34729 strlen (feature_list[i].name) + 1,
34730 feature_list[i].name);
34731 predicate_chain = tree_cons (predicate_decl, predicate_arg,
34734 /* Find the maximum priority feature. */
34735 if (feature_list[i].priority > priority)
34736 priority = feature_list[i].priority;
34741 if (predicate_list && i == NUM_FEATURES)
34743 error_at (DECL_SOURCE_LOCATION (decl),
34744 "No dispatcher found for %s", token);
34747 token = strtok (NULL, ",");
34751 if (predicate_list && predicate_chain == NULL_TREE)
34753 error_at (DECL_SOURCE_LOCATION (decl),
34754 "No dispatcher found for the versioning attributes : %s",
34758 else if (predicate_list)
34760 predicate_chain = nreverse (predicate_chain);
34761 *predicate_list = predicate_chain;
34767 /* This compares the priority of target features in function DECL1
34768 and DECL2. It returns positive value if DECL1 is higher priority,
34769 negative value if DECL2 is higher priority and 0 if they are the
34773 ix86_compare_version_priority (tree decl1, tree decl2)
34775 unsigned int priority1 = get_builtin_code_for_version (decl1, NULL);
34776 unsigned int priority2 = get_builtin_code_for_version (decl2, NULL);
34778 return (int)priority1 - (int)priority2;
34781 /* V1 and V2 point to function versions with different priorities
34782 based on the target ISA. This function compares their priorities. */
34785 feature_compare (const void *v1, const void *v2)
34787 typedef struct _function_version_info
34790 tree predicate_chain;
34791 unsigned int dispatch_priority;
34792 } function_version_info;
34794 const function_version_info c1 = *(const function_version_info *)v1;
34795 const function_version_info c2 = *(const function_version_info *)v2;
34796 return (c2.dispatch_priority - c1.dispatch_priority);
34799 /* This function generates the dispatch function for
34800 multi-versioned functions. DISPATCH_DECL is the function which will
34801 contain the dispatch logic. FNDECLS are the function choices for
34802 dispatch, and is a tree chain. EMPTY_BB is the basic block pointer
34803 in DISPATCH_DECL in which the dispatch code is generated. */
34806 dispatch_function_versions (tree dispatch_decl,
34808 basic_block *empty_bb)
34811 gimple ifunc_cpu_init_stmt;
34815 vec<tree> *fndecls;
34816 unsigned int num_versions = 0;
34817 unsigned int actual_versions = 0;
34820 struct _function_version_info
34823 tree predicate_chain;
34824 unsigned int dispatch_priority;
34825 }*function_version_info;
34827 gcc_assert (dispatch_decl != NULL
34828 && fndecls_p != NULL
34829 && empty_bb != NULL);
34831 /*fndecls_p is actually a vector. */
34832 fndecls = static_cast<vec<tree> *> (fndecls_p);
34834 /* At least one more version other than the default. */
34835 num_versions = fndecls->length ();
34836 gcc_assert (num_versions >= 2);
34838 function_version_info = (struct _function_version_info *)
34839 XNEWVEC (struct _function_version_info, (num_versions - 1));
34841 /* The first version in the vector is the default decl. */
34842 default_decl = (*fndecls)[0];
34844 push_cfun (DECL_STRUCT_FUNCTION (dispatch_decl));
34846 gseq = bb_seq (*empty_bb);
34847 /* Function version dispatch is via IFUNC. IFUNC resolvers fire before
34848 constructors, so explicity call __builtin_cpu_init here. */
34849 ifunc_cpu_init_stmt = gimple_build_call_vec (
34850 ix86_builtins [(int) IX86_BUILTIN_CPU_INIT], vNULL);
34851 gimple_seq_add_stmt (&gseq, ifunc_cpu_init_stmt);
34852 gimple_set_bb (ifunc_cpu_init_stmt, *empty_bb);
34853 set_bb_seq (*empty_bb, gseq);
34858 for (ix = 1; fndecls->iterate (ix, &ele); ++ix)
34860 tree version_decl = ele;
34861 tree predicate_chain = NULL_TREE;
34862 unsigned int priority;
34863 /* Get attribute string, parse it and find the right predicate decl.
34864 The predicate function could be a lengthy combination of many
34865 features, like arch-type and various isa-variants. */
34866 priority = get_builtin_code_for_version (version_decl,
34869 if (predicate_chain == NULL_TREE)
34872 function_version_info [actual_versions].version_decl = version_decl;
34873 function_version_info [actual_versions].predicate_chain
34875 function_version_info [actual_versions].dispatch_priority = priority;
34879 /* Sort the versions according to descending order of dispatch priority. The
34880 priority is based on the ISA. This is not a perfect solution. There
34881 could still be ambiguity. If more than one function version is suitable
34882 to execute, which one should be dispatched? In future, allow the user
34883 to specify a dispatch priority next to the version. */
34884 qsort (function_version_info, actual_versions,
34885 sizeof (struct _function_version_info), feature_compare);
34887 for (i = 0; i < actual_versions; ++i)
34888 *empty_bb = add_condition_to_bb (dispatch_decl,
34889 function_version_info[i].version_decl,
34890 function_version_info[i].predicate_chain,
34893 /* dispatch default version at the end. */
34894 *empty_bb = add_condition_to_bb (dispatch_decl, default_decl,
34897 free (function_version_info);
34901 /* Comparator function to be used in qsort routine to sort attribute
34902 specification strings to "target". */
34905 attr_strcmp (const void *v1, const void *v2)
34907 const char *c1 = *(char *const*)v1;
34908 const char *c2 = *(char *const*)v2;
34909 return strcmp (c1, c2);
34912 /* ARGLIST is the argument to target attribute. This function tokenizes
34913 the comma separated arguments, sorts them and returns a string which
34914 is a unique identifier for the comma separated arguments. It also
34915 replaces non-identifier characters "=,-" with "_". */
34918 sorted_attr_string (tree arglist)
34921 size_t str_len_sum = 0;
34922 char **args = NULL;
34923 char *attr_str, *ret_str;
34925 unsigned int argnum = 1;
34928 for (arg = arglist; arg; arg = TREE_CHAIN (arg))
34930 const char *str = TREE_STRING_POINTER (TREE_VALUE (arg));
34931 size_t len = strlen (str);
34932 str_len_sum += len + 1;
34933 if (arg != arglist)
34935 for (i = 0; i < strlen (str); i++)
34940 attr_str = XNEWVEC (char, str_len_sum);
34942 for (arg = arglist; arg; arg = TREE_CHAIN (arg))
34944 const char *str = TREE_STRING_POINTER (TREE_VALUE (arg));
34945 size_t len = strlen (str);
34946 memcpy (attr_str + str_len_sum, str, len);
34947 attr_str[str_len_sum + len] = TREE_CHAIN (arg) ? ',' : '\0';
34948 str_len_sum += len + 1;
34951 /* Replace "=,-" with "_". */
34952 for (i = 0; i < strlen (attr_str); i++)
34953 if (attr_str[i] == '=' || attr_str[i]== '-')
34959 args = XNEWVEC (char *, argnum);
34962 attr = strtok (attr_str, ",");
34963 while (attr != NULL)
34967 attr = strtok (NULL, ",");
34970 qsort (args, argnum, sizeof (char *), attr_strcmp);
34972 ret_str = XNEWVEC (char, str_len_sum);
34974 for (i = 0; i < argnum; i++)
34976 size_t len = strlen (args[i]);
34977 memcpy (ret_str + str_len_sum, args[i], len);
34978 ret_str[str_len_sum + len] = i < argnum - 1 ? '_' : '\0';
34979 str_len_sum += len + 1;
34983 XDELETEVEC (attr_str);
34987 /* This function changes the assembler name for functions that are
34988 versions. If DECL is a function version and has a "target"
34989 attribute, it appends the attribute string to its assembler name. */
34992 ix86_mangle_function_version_assembler_name (tree decl, tree id)
34995 const char *orig_name, *version_string;
34996 char *attr_str, *assembler_name;
34998 if (DECL_DECLARED_INLINE_P (decl)
34999 && lookup_attribute ("gnu_inline",
35000 DECL_ATTRIBUTES (decl)))
35001 error_at (DECL_SOURCE_LOCATION (decl),
35002 "Function versions cannot be marked as gnu_inline,"
35003 " bodies have to be generated");
35005 if (DECL_VIRTUAL_P (decl)
35006 || DECL_VINDEX (decl))
35007 sorry ("Virtual function multiversioning not supported");
35009 version_attr = lookup_attribute ("target", DECL_ATTRIBUTES (decl));
35011 /* target attribute string cannot be NULL. */
35012 gcc_assert (version_attr != NULL_TREE);
35014 orig_name = IDENTIFIER_POINTER (id);
35016 = TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (version_attr)));
35018 if (strcmp (version_string, "default") == 0)
35021 attr_str = sorted_attr_string (TREE_VALUE (version_attr));
35022 assembler_name = XNEWVEC (char, strlen (orig_name) + strlen (attr_str) + 2);
35024 sprintf (assembler_name, "%s.%s", orig_name, attr_str);
35026 /* Allow assembler name to be modified if already set. */
35027 if (DECL_ASSEMBLER_NAME_SET_P (decl))
35028 SET_DECL_RTL (decl, NULL);
35030 tree ret = get_identifier (assembler_name);
35031 XDELETEVEC (attr_str);
35032 XDELETEVEC (assembler_name);
35036 /* This function returns true if FN1 and FN2 are versions of the same function,
35037 that is, the target strings of the function decls are different. This assumes
35038 that FN1 and FN2 have the same signature. */
35041 ix86_function_versions (tree fn1, tree fn2)
35044 char *target1, *target2;
35047 if (TREE_CODE (fn1) != FUNCTION_DECL
35048 || TREE_CODE (fn2) != FUNCTION_DECL)
35051 attr1 = lookup_attribute ("target", DECL_ATTRIBUTES (fn1));
35052 attr2 = lookup_attribute ("target", DECL_ATTRIBUTES (fn2));
35054 /* At least one function decl should have the target attribute specified. */
35055 if (attr1 == NULL_TREE && attr2 == NULL_TREE)
35058 /* Diagnose missing target attribute if one of the decls is already
35059 multi-versioned. */
35060 if (attr1 == NULL_TREE || attr2 == NULL_TREE)
35062 if (DECL_FUNCTION_VERSIONED (fn1) || DECL_FUNCTION_VERSIONED (fn2))
35064 if (attr2 != NULL_TREE)
35071 error_at (DECL_SOURCE_LOCATION (fn2),
35072 "missing %<target%> attribute for multi-versioned %D",
35074 inform (DECL_SOURCE_LOCATION (fn1),
35075 "previous declaration of %D", fn1);
35076 /* Prevent diagnosing of the same error multiple times. */
35077 DECL_ATTRIBUTES (fn2)
35078 = tree_cons (get_identifier ("target"),
35079 copy_node (TREE_VALUE (attr1)),
35080 DECL_ATTRIBUTES (fn2));
35085 target1 = sorted_attr_string (TREE_VALUE (attr1));
35086 target2 = sorted_attr_string (TREE_VALUE (attr2));
35088 /* The sorted target strings must be different for fn1 and fn2
35090 if (strcmp (target1, target2) == 0)
35095 XDELETEVEC (target1);
35096 XDELETEVEC (target2);
35102 ix86_mangle_decl_assembler_name (tree decl, tree id)
35104 /* For function version, add the target suffix to the assembler name. */
35105 if (TREE_CODE (decl) == FUNCTION_DECL
35106 && DECL_FUNCTION_VERSIONED (decl))
35107 id = ix86_mangle_function_version_assembler_name (decl, id);
35108 #ifdef SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME
35109 id = SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME (decl, id);
35115 /* Return a new name by appending SUFFIX to the DECL name. If make_unique
35116 is true, append the full path name of the source file. */
35119 make_name (tree decl, const char *suffix, bool make_unique)
35121 char *global_var_name;
35124 const char *unique_name = NULL;
35126 name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
35128 /* Get a unique name that can be used globally without any chances
35129 of collision at link time. */
35131 unique_name = IDENTIFIER_POINTER (get_file_function_name ("\0"));
35133 name_len = strlen (name) + strlen (suffix) + 2;
35136 name_len += strlen (unique_name) + 1;
35137 global_var_name = XNEWVEC (char, name_len);
35139 /* Use '.' to concatenate names as it is demangler friendly. */
35141 snprintf (global_var_name, name_len, "%s.%s.%s", name, unique_name,
35144 snprintf (global_var_name, name_len, "%s.%s", name, suffix);
35146 return global_var_name;
35149 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
35151 /* Make a dispatcher declaration for the multi-versioned function DECL.
35152 Calls to DECL function will be replaced with calls to the dispatcher
35153 by the front-end. Return the decl created. */
35156 make_dispatcher_decl (const tree decl)
35160 tree fn_type, func_type;
35161 bool is_uniq = false;
35163 if (TREE_PUBLIC (decl) == 0)
35166 func_name = make_name (decl, "ifunc", is_uniq);
35168 fn_type = TREE_TYPE (decl);
35169 func_type = build_function_type (TREE_TYPE (fn_type),
35170 TYPE_ARG_TYPES (fn_type));
35172 func_decl = build_fn_decl (func_name, func_type);
35173 XDELETEVEC (func_name);
35174 TREE_USED (func_decl) = 1;
35175 DECL_CONTEXT (func_decl) = NULL_TREE;
35176 DECL_INITIAL (func_decl) = error_mark_node;
35177 DECL_ARTIFICIAL (func_decl) = 1;
35178 /* Mark this func as external, the resolver will flip it again if
35179 it gets generated. */
35180 DECL_EXTERNAL (func_decl) = 1;
35181 /* This will be of type IFUNCs have to be externally visible. */
35182 TREE_PUBLIC (func_decl) = 1;
35189 /* Returns true if decl is multi-versioned and DECL is the default function,
35190 that is it is not tagged with target specific optimization. */
35193 is_function_default_version (const tree decl)
35195 if (TREE_CODE (decl) != FUNCTION_DECL
35196 || !DECL_FUNCTION_VERSIONED (decl))
35198 tree attr = lookup_attribute ("target", DECL_ATTRIBUTES (decl));
35200 attr = TREE_VALUE (TREE_VALUE (attr));
35201 return (TREE_CODE (attr) == STRING_CST
35202 && strcmp (TREE_STRING_POINTER (attr), "default") == 0);
35205 /* Make a dispatcher declaration for the multi-versioned function DECL.
35206 Calls to DECL function will be replaced with calls to the dispatcher
35207 by the front-end. Returns the decl of the dispatcher function. */
35210 ix86_get_function_versions_dispatcher (void *decl)
35212 tree fn = (tree) decl;
35213 struct cgraph_node *node = NULL;
35214 struct cgraph_node *default_node = NULL;
35215 struct cgraph_function_version_info *node_v = NULL;
35216 struct cgraph_function_version_info *first_v = NULL;
35218 tree dispatch_decl = NULL;
35220 struct cgraph_function_version_info *default_version_info = NULL;
35222 gcc_assert (fn != NULL && DECL_FUNCTION_VERSIONED (fn));
35224 node = cgraph_node::get (fn);
35225 gcc_assert (node != NULL);
35227 node_v = node->function_version ();
35228 gcc_assert (node_v != NULL);
35230 if (node_v->dispatcher_resolver != NULL)
35231 return node_v->dispatcher_resolver;
35233 /* Find the default version and make it the first node. */
35235 /* Go to the beginning of the chain. */
35236 while (first_v->prev != NULL)
35237 first_v = first_v->prev;
35238 default_version_info = first_v;
35239 while (default_version_info != NULL)
35241 if (is_function_default_version
35242 (default_version_info->this_node->decl))
35244 default_version_info = default_version_info->next;
35247 /* If there is no default node, just return NULL. */
35248 if (default_version_info == NULL)
35251 /* Make default info the first node. */
35252 if (first_v != default_version_info)
35254 default_version_info->prev->next = default_version_info->next;
35255 if (default_version_info->next)
35256 default_version_info->next->prev = default_version_info->prev;
35257 first_v->prev = default_version_info;
35258 default_version_info->next = first_v;
35259 default_version_info->prev = NULL;
35262 default_node = default_version_info->this_node;
35264 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
35265 if (targetm.has_ifunc_p ())
35267 struct cgraph_function_version_info *it_v = NULL;
35268 struct cgraph_node *dispatcher_node = NULL;
35269 struct cgraph_function_version_info *dispatcher_version_info = NULL;
35271 /* Right now, the dispatching is done via ifunc. */
35272 dispatch_decl = make_dispatcher_decl (default_node->decl);
35274 dispatcher_node = cgraph_node::get_create (dispatch_decl);
35275 gcc_assert (dispatcher_node != NULL);
35276 dispatcher_node->dispatcher_function = 1;
35277 dispatcher_version_info
35278 = dispatcher_node->insert_new_function_version ();
35279 dispatcher_version_info->next = default_version_info;
35280 dispatcher_node->definition = 1;
35282 /* Set the dispatcher for all the versions. */
35283 it_v = default_version_info;
35284 while (it_v != NULL)
35286 it_v->dispatcher_resolver = dispatch_decl;
35293 error_at (DECL_SOURCE_LOCATION (default_node->decl),
35294 "multiversioning needs ifunc which is not supported "
35298 return dispatch_decl;
35301 /* Makes a function attribute of the form NAME(ARG_NAME) and chains
35305 make_attribute (const char *name, const char *arg_name, tree chain)
35308 tree attr_arg_name;
35312 attr_name = get_identifier (name);
35313 attr_arg_name = build_string (strlen (arg_name), arg_name);
35314 attr_args = tree_cons (NULL_TREE, attr_arg_name, NULL_TREE);
35315 attr = tree_cons (attr_name, attr_args, chain);
35319 /* Make the resolver function decl to dispatch the versions of
35320 a multi-versioned function, DEFAULT_DECL. Create an
35321 empty basic block in the resolver and store the pointer in
35322 EMPTY_BB. Return the decl of the resolver function. */
35325 make_resolver_func (const tree default_decl,
35326 const tree dispatch_decl,
35327 basic_block *empty_bb)
35329 char *resolver_name;
35330 tree decl, type, decl_name, t;
35331 bool is_uniq = false;
35333 /* IFUNC's have to be globally visible. So, if the default_decl is
35334 not, then the name of the IFUNC should be made unique. */
35335 if (TREE_PUBLIC (default_decl) == 0)
35338 /* Append the filename to the resolver function if the versions are
35339 not externally visible. This is because the resolver function has
35340 to be externally visible for the loader to find it. So, appending
35341 the filename will prevent conflicts with a resolver function from
35342 another module which is based on the same version name. */
35343 resolver_name = make_name (default_decl, "resolver", is_uniq);
35345 /* The resolver function should return a (void *). */
35346 type = build_function_type_list (ptr_type_node, NULL_TREE);
35348 decl = build_fn_decl (resolver_name, type);
35349 decl_name = get_identifier (resolver_name);
35350 SET_DECL_ASSEMBLER_NAME (decl, decl_name);
35352 DECL_NAME (decl) = decl_name;
35353 TREE_USED (decl) = 1;
35354 DECL_ARTIFICIAL (decl) = 1;
35355 DECL_IGNORED_P (decl) = 0;
35356 /* IFUNC resolvers have to be externally visible. */
35357 TREE_PUBLIC (decl) = 1;
35358 DECL_UNINLINABLE (decl) = 1;
35360 /* Resolver is not external, body is generated. */
35361 DECL_EXTERNAL (decl) = 0;
35362 DECL_EXTERNAL (dispatch_decl) = 0;
35364 DECL_CONTEXT (decl) = NULL_TREE;
35365 DECL_INITIAL (decl) = make_node (BLOCK);
35366 DECL_STATIC_CONSTRUCTOR (decl) = 0;
35368 if (DECL_COMDAT_GROUP (default_decl)
35369 || TREE_PUBLIC (default_decl))
35371 /* In this case, each translation unit with a call to this
35372 versioned function will put out a resolver. Ensure it
35373 is comdat to keep just one copy. */
35374 DECL_COMDAT (decl) = 1;
35375 make_decl_one_only (decl, DECL_ASSEMBLER_NAME (decl));
35377 /* Build result decl and add to function_decl. */
35378 t = build_decl (UNKNOWN_LOCATION, RESULT_DECL, NULL_TREE, ptr_type_node);
35379 DECL_ARTIFICIAL (t) = 1;
35380 DECL_IGNORED_P (t) = 1;
35381 DECL_RESULT (decl) = t;
35383 gimplify_function_tree (decl);
35384 push_cfun (DECL_STRUCT_FUNCTION (decl));
35385 *empty_bb = init_lowered_empty_function (decl, false, 0);
35387 cgraph_node::add_new_function (decl, true);
35388 symtab->call_cgraph_insertion_hooks (cgraph_node::get_create (decl));
35392 gcc_assert (dispatch_decl != NULL);
35393 /* Mark dispatch_decl as "ifunc" with resolver as resolver_name. */
35394 DECL_ATTRIBUTES (dispatch_decl)
35395 = make_attribute ("ifunc", resolver_name, DECL_ATTRIBUTES (dispatch_decl));
35397 /* Create the alias for dispatch to resolver here. */
35398 /*cgraph_create_function_alias (dispatch_decl, decl);*/
35399 cgraph_node::create_same_body_alias (dispatch_decl, decl);
35400 XDELETEVEC (resolver_name);
35404 /* Generate the dispatching code body to dispatch multi-versioned function
35405 DECL. The target hook is called to process the "target" attributes and
35406 provide the code to dispatch the right function at run-time. NODE points
35407 to the dispatcher decl whose body will be created. */
35410 ix86_generate_version_dispatcher_body (void *node_p)
35412 tree resolver_decl;
35413 basic_block empty_bb;
35414 tree default_ver_decl;
35415 struct cgraph_node *versn;
35416 struct cgraph_node *node;
35418 struct cgraph_function_version_info *node_version_info = NULL;
35419 struct cgraph_function_version_info *versn_info = NULL;
35421 node = (cgraph_node *)node_p;
35423 node_version_info = node->function_version ();
35424 gcc_assert (node->dispatcher_function
35425 && node_version_info != NULL);
35427 if (node_version_info->dispatcher_resolver)
35428 return node_version_info->dispatcher_resolver;
35430 /* The first version in the chain corresponds to the default version. */
35431 default_ver_decl = node_version_info->next->this_node->decl;
35433 /* node is going to be an alias, so remove the finalized bit. */
35434 node->definition = false;
35436 resolver_decl = make_resolver_func (default_ver_decl,
35437 node->decl, &empty_bb);
35439 node_version_info->dispatcher_resolver = resolver_decl;
35441 push_cfun (DECL_STRUCT_FUNCTION (resolver_decl));
35443 auto_vec<tree, 2> fn_ver_vec;
35445 for (versn_info = node_version_info->next; versn_info;
35446 versn_info = versn_info->next)
35448 versn = versn_info->this_node;
35449 /* Check for virtual functions here again, as by this time it should
35450 have been determined if this function needs a vtable index or
35451 not. This happens for methods in derived classes that override
35452 virtual methods in base classes but are not explicitly marked as
35454 if (DECL_VINDEX (versn->decl))
35455 sorry ("Virtual function multiversioning not supported");
35457 fn_ver_vec.safe_push (versn->decl);
35460 dispatch_function_versions (resolver_decl, &fn_ver_vec, &empty_bb);
35461 cgraph_edge::rebuild_edges ();
35463 return resolver_decl;
35465 /* This builds the processor_model struct type defined in
35466 libgcc/config/i386/cpuinfo.c */
35469 build_processor_model_struct (void)
35471 const char *field_name[] = {"__cpu_vendor", "__cpu_type", "__cpu_subtype",
35473 tree field = NULL_TREE, field_chain = NULL_TREE;
35475 tree type = make_node (RECORD_TYPE);
35477 /* The first 3 fields are unsigned int. */
35478 for (i = 0; i < 3; ++i)
35480 field = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
35481 get_identifier (field_name[i]), unsigned_type_node);
35482 if (field_chain != NULL_TREE)
35483 DECL_CHAIN (field) = field_chain;
35484 field_chain = field;
35487 /* The last field is an array of unsigned integers of size one. */
35488 field = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
35489 get_identifier (field_name[3]),
35490 build_array_type (unsigned_type_node,
35491 build_index_type (size_one_node)));
35492 if (field_chain != NULL_TREE)
35493 DECL_CHAIN (field) = field_chain;
35494 field_chain = field;
35496 finish_builtin_struct (type, "__processor_model", field_chain, NULL_TREE);
35500 /* Returns a extern, comdat VAR_DECL of type TYPE and name NAME. */
35503 make_var_decl (tree type, const char *name)
35507 new_decl = build_decl (UNKNOWN_LOCATION,
35509 get_identifier(name),
35512 DECL_EXTERNAL (new_decl) = 1;
35513 TREE_STATIC (new_decl) = 1;
35514 TREE_PUBLIC (new_decl) = 1;
35515 DECL_INITIAL (new_decl) = 0;
35516 DECL_ARTIFICIAL (new_decl) = 0;
35517 DECL_PRESERVE_P (new_decl) = 1;
35519 make_decl_one_only (new_decl, DECL_ASSEMBLER_NAME (new_decl));
35520 assemble_variable (new_decl, 0, 0, 0);
35525 /* FNDECL is a __builtin_cpu_is or a __builtin_cpu_supports call that is folded
35526 into an integer defined in libgcc/config/i386/cpuinfo.c */
35529 fold_builtin_cpu (tree fndecl, tree *args)
35532 enum ix86_builtins fn_code = (enum ix86_builtins)
35533 DECL_FUNCTION_CODE (fndecl);
35534 tree param_string_cst = NULL;
35536 /* This is the order of bit-fields in __processor_features in cpuinfo.c */
35537 enum processor_features
35560 /* These are the values for vendor types and cpu types and subtypes
35561 in cpuinfo.c. Cpu types and subtypes should be subtracted by
35562 the corresponding start value. */
35563 enum processor_model
35573 M_INTEL_SILVERMONT,
35577 M_CPU_SUBTYPE_START,
35578 M_INTEL_COREI7_NEHALEM,
35579 M_INTEL_COREI7_WESTMERE,
35580 M_INTEL_COREI7_SANDYBRIDGE,
35581 M_AMDFAM10H_BARCELONA,
35582 M_AMDFAM10H_SHANGHAI,
35583 M_AMDFAM10H_ISTANBUL,
35584 M_AMDFAM15H_BDVER1,
35585 M_AMDFAM15H_BDVER2,
35586 M_AMDFAM15H_BDVER3,
35587 M_AMDFAM15H_BDVER4,
35588 M_INTEL_COREI7_IVYBRIDGE,
35589 M_INTEL_COREI7_HASWELL,
35590 M_INTEL_COREI7_BROADWELL
35593 static struct _arch_names_table
35595 const char *const name;
35596 const enum processor_model model;
35598 const arch_names_table[] =
35601 {"intel", M_INTEL},
35602 {"atom", M_INTEL_BONNELL},
35603 {"slm", M_INTEL_SILVERMONT},
35604 {"core2", M_INTEL_CORE2},
35605 {"corei7", M_INTEL_COREI7},
35606 {"nehalem", M_INTEL_COREI7_NEHALEM},
35607 {"westmere", M_INTEL_COREI7_WESTMERE},
35608 {"sandybridge", M_INTEL_COREI7_SANDYBRIDGE},
35609 {"ivybridge", M_INTEL_COREI7_IVYBRIDGE},
35610 {"haswell", M_INTEL_COREI7_HASWELL},
35611 {"broadwell", M_INTEL_COREI7_BROADWELL},
35612 {"bonnell", M_INTEL_BONNELL},
35613 {"silvermont", M_INTEL_SILVERMONT},
35614 {"knl", M_INTEL_KNL},
35615 {"amdfam10h", M_AMDFAM10H},
35616 {"barcelona", M_AMDFAM10H_BARCELONA},
35617 {"shanghai", M_AMDFAM10H_SHANGHAI},
35618 {"istanbul", M_AMDFAM10H_ISTANBUL},
35619 {"btver1", M_AMD_BTVER1},
35620 {"amdfam15h", M_AMDFAM15H},
35621 {"bdver1", M_AMDFAM15H_BDVER1},
35622 {"bdver2", M_AMDFAM15H_BDVER2},
35623 {"bdver3", M_AMDFAM15H_BDVER3},
35624 {"bdver4", M_AMDFAM15H_BDVER4},
35625 {"btver2", M_AMD_BTVER2},
35628 static struct _isa_names_table
35630 const char *const name;
35631 const enum processor_features feature;
35633 const isa_names_table[] =
35637 {"popcnt", F_POPCNT},
35641 {"ssse3", F_SSSE3},
35642 {"sse4a", F_SSE4_A},
35643 {"sse4.1", F_SSE4_1},
35644 {"sse4.2", F_SSE4_2},
35650 {"avx512f",F_AVX512F},
35655 tree __processor_model_type = build_processor_model_struct ();
35656 tree __cpu_model_var = make_var_decl (__processor_model_type,
35660 varpool_node::add (__cpu_model_var);
35662 gcc_assert ((args != NULL) && (*args != NULL));
35664 param_string_cst = *args;
35665 while (param_string_cst
35666 && TREE_CODE (param_string_cst) != STRING_CST)
35668 /* *args must be a expr that can contain other EXPRS leading to a
35670 if (!EXPR_P (param_string_cst))
35672 error ("Parameter to builtin must be a string constant or literal");
35673 return integer_zero_node;
35675 param_string_cst = TREE_OPERAND (EXPR_CHECK (param_string_cst), 0);
35678 gcc_assert (param_string_cst);
35680 if (fn_code == IX86_BUILTIN_CPU_IS)
35686 unsigned int field_val = 0;
35687 unsigned int NUM_ARCH_NAMES
35688 = sizeof (arch_names_table) / sizeof (struct _arch_names_table);
35690 for (i = 0; i < NUM_ARCH_NAMES; i++)
35691 if (strcmp (arch_names_table[i].name,
35692 TREE_STRING_POINTER (param_string_cst)) == 0)
35695 if (i == NUM_ARCH_NAMES)
35697 error ("Parameter to builtin not valid: %s",
35698 TREE_STRING_POINTER (param_string_cst));
35699 return integer_zero_node;
35702 field = TYPE_FIELDS (__processor_model_type);
35703 field_val = arch_names_table[i].model;
35705 /* CPU types are stored in the next field. */
35706 if (field_val > M_CPU_TYPE_START
35707 && field_val < M_CPU_SUBTYPE_START)
35709 field = DECL_CHAIN (field);
35710 field_val -= M_CPU_TYPE_START;
35713 /* CPU subtypes are stored in the next field. */
35714 if (field_val > M_CPU_SUBTYPE_START)
35716 field = DECL_CHAIN ( DECL_CHAIN (field));
35717 field_val -= M_CPU_SUBTYPE_START;
35720 /* Get the appropriate field in __cpu_model. */
35721 ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var,
35724 /* Check the value. */
35725 final = build2 (EQ_EXPR, unsigned_type_node, ref,
35726 build_int_cstu (unsigned_type_node, field_val));
35727 return build1 (CONVERT_EXPR, integer_type_node, final);
35729 else if (fn_code == IX86_BUILTIN_CPU_SUPPORTS)
35736 unsigned int field_val = 0;
35737 unsigned int NUM_ISA_NAMES
35738 = sizeof (isa_names_table) / sizeof (struct _isa_names_table);
35740 for (i = 0; i < NUM_ISA_NAMES; i++)
35741 if (strcmp (isa_names_table[i].name,
35742 TREE_STRING_POINTER (param_string_cst)) == 0)
35745 if (i == NUM_ISA_NAMES)
35747 error ("Parameter to builtin not valid: %s",
35748 TREE_STRING_POINTER (param_string_cst));
35749 return integer_zero_node;
35752 field = TYPE_FIELDS (__processor_model_type);
35753 /* Get the last field, which is __cpu_features. */
35754 while (DECL_CHAIN (field))
35755 field = DECL_CHAIN (field);
35757 /* Get the appropriate field: __cpu_model.__cpu_features */
35758 ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var,
35761 /* Access the 0th element of __cpu_features array. */
35762 array_elt = build4 (ARRAY_REF, unsigned_type_node, ref,
35763 integer_zero_node, NULL_TREE, NULL_TREE);
35765 field_val = (1 << isa_names_table[i].feature);
35766 /* Return __cpu_model.__cpu_features[0] & field_val */
35767 final = build2 (BIT_AND_EXPR, unsigned_type_node, array_elt,
35768 build_int_cstu (unsigned_type_node, field_val));
35769 return build1 (CONVERT_EXPR, integer_type_node, final);
35771 gcc_unreachable ();
35775 ix86_fold_builtin (tree fndecl, int n_args,
35776 tree *args, bool ignore ATTRIBUTE_UNUSED)
35778 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
35780 enum ix86_builtins fn_code = (enum ix86_builtins)
35781 DECL_FUNCTION_CODE (fndecl);
35782 if (fn_code == IX86_BUILTIN_CPU_IS
35783 || fn_code == IX86_BUILTIN_CPU_SUPPORTS)
35785 gcc_assert (n_args == 1);
35786 return fold_builtin_cpu (fndecl, args);
35790 #ifdef SUBTARGET_FOLD_BUILTIN
35791 return SUBTARGET_FOLD_BUILTIN (fndecl, n_args, args, ignore);
35797 /* Make builtins to detect cpu type and features supported. NAME is
35798 the builtin name, CODE is the builtin code, and FTYPE is the function
35799 type of the builtin. */
35802 make_cpu_type_builtin (const char* name, int code,
35803 enum ix86_builtin_func_type ftype, bool is_const)
35808 type = ix86_get_builtin_func_type (ftype);
35809 decl = add_builtin_function (name, type, code, BUILT_IN_MD,
35811 gcc_assert (decl != NULL_TREE);
35812 ix86_builtins[(int) code] = decl;
35813 TREE_READONLY (decl) = is_const;
35816 /* Make builtins to get CPU type and features supported. The created
35819 __builtin_cpu_init (), to detect cpu type and features,
35820 __builtin_cpu_is ("<CPUNAME>"), to check if cpu is of type <CPUNAME>,
35821 __builtin_cpu_supports ("<FEATURE>"), to check if cpu supports <FEATURE>
35825 ix86_init_platform_type_builtins (void)
35827 make_cpu_type_builtin ("__builtin_cpu_init", IX86_BUILTIN_CPU_INIT,
35828 INT_FTYPE_VOID, false);
35829 make_cpu_type_builtin ("__builtin_cpu_is", IX86_BUILTIN_CPU_IS,
35830 INT_FTYPE_PCCHAR, true);
35831 make_cpu_type_builtin ("__builtin_cpu_supports", IX86_BUILTIN_CPU_SUPPORTS,
35832 INT_FTYPE_PCCHAR, true);
35835 /* Internal method for ix86_init_builtins. */
35838 ix86_init_builtins_va_builtins_abi (void)
35840 tree ms_va_ref, sysv_va_ref;
35841 tree fnvoid_va_end_ms, fnvoid_va_end_sysv;
35842 tree fnvoid_va_start_ms, fnvoid_va_start_sysv;
35843 tree fnvoid_va_copy_ms, fnvoid_va_copy_sysv;
35844 tree fnattr_ms = NULL_TREE, fnattr_sysv = NULL_TREE;
35848 fnattr_ms = build_tree_list (get_identifier ("ms_abi"), NULL_TREE);
35849 fnattr_sysv = build_tree_list (get_identifier ("sysv_abi"), NULL_TREE);
35850 ms_va_ref = build_reference_type (ms_va_list_type_node);
35852 build_pointer_type (TREE_TYPE (sysv_va_list_type_node));
35855 build_function_type_list (void_type_node, ms_va_ref, NULL_TREE);
35856 fnvoid_va_start_ms =
35857 build_varargs_function_type_list (void_type_node, ms_va_ref, NULL_TREE);
35858 fnvoid_va_end_sysv =
35859 build_function_type_list (void_type_node, sysv_va_ref, NULL_TREE);
35860 fnvoid_va_start_sysv =
35861 build_varargs_function_type_list (void_type_node, sysv_va_ref,
35863 fnvoid_va_copy_ms =
35864 build_function_type_list (void_type_node, ms_va_ref, ms_va_list_type_node,
35866 fnvoid_va_copy_sysv =
35867 build_function_type_list (void_type_node, sysv_va_ref,
35868 sysv_va_ref, NULL_TREE);
35870 add_builtin_function ("__builtin_ms_va_start", fnvoid_va_start_ms,
35871 BUILT_IN_VA_START, BUILT_IN_NORMAL, NULL, fnattr_ms);
35872 add_builtin_function ("__builtin_ms_va_end", fnvoid_va_end_ms,
35873 BUILT_IN_VA_END, BUILT_IN_NORMAL, NULL, fnattr_ms);
35874 add_builtin_function ("__builtin_ms_va_copy", fnvoid_va_copy_ms,
35875 BUILT_IN_VA_COPY, BUILT_IN_NORMAL, NULL, fnattr_ms);
35876 add_builtin_function ("__builtin_sysv_va_start", fnvoid_va_start_sysv,
35877 BUILT_IN_VA_START, BUILT_IN_NORMAL, NULL, fnattr_sysv);
35878 add_builtin_function ("__builtin_sysv_va_end", fnvoid_va_end_sysv,
35879 BUILT_IN_VA_END, BUILT_IN_NORMAL, NULL, fnattr_sysv);
35880 add_builtin_function ("__builtin_sysv_va_copy", fnvoid_va_copy_sysv,
35881 BUILT_IN_VA_COPY, BUILT_IN_NORMAL, NULL, fnattr_sysv);
35885 ix86_init_builtin_types (void)
35887 tree float128_type_node, float80_type_node;
35889 /* The __float80 type. */
35890 float80_type_node = long_double_type_node;
35891 if (TYPE_MODE (float80_type_node) != XFmode)
35893 /* The __float80 type. */
35894 float80_type_node = make_node (REAL_TYPE);
35896 TYPE_PRECISION (float80_type_node) = 80;
35897 layout_type (float80_type_node);
35899 lang_hooks.types.register_builtin_type (float80_type_node, "__float80");
35901 /* The __float128 type. */
35902 float128_type_node = make_node (REAL_TYPE);
35903 TYPE_PRECISION (float128_type_node) = 128;
35904 layout_type (float128_type_node);
35905 lang_hooks.types.register_builtin_type (float128_type_node, "__float128");
35907 /* This macro is built by i386-builtin-types.awk. */
35908 DEFINE_BUILTIN_PRIMITIVE_TYPES;
35912 ix86_init_builtins (void)
35916 ix86_init_builtin_types ();
35918 /* Builtins to get CPU type and features. */
35919 ix86_init_platform_type_builtins ();
35921 /* TFmode support builtins. */
35922 def_builtin_const (0, "__builtin_infq",
35923 FLOAT128_FTYPE_VOID, IX86_BUILTIN_INFQ);
35924 def_builtin_const (0, "__builtin_huge_valq",
35925 FLOAT128_FTYPE_VOID, IX86_BUILTIN_HUGE_VALQ);
35927 /* We will expand them to normal call if SSE isn't available since
35928 they are used by libgcc. */
35929 t = ix86_get_builtin_func_type (FLOAT128_FTYPE_FLOAT128);
35930 t = add_builtin_function ("__builtin_fabsq", t, IX86_BUILTIN_FABSQ,
35931 BUILT_IN_MD, "__fabstf2", NULL_TREE);
35932 TREE_READONLY (t) = 1;
35933 ix86_builtins[(int) IX86_BUILTIN_FABSQ] = t;
35935 t = ix86_get_builtin_func_type (FLOAT128_FTYPE_FLOAT128_FLOAT128);
35936 t = add_builtin_function ("__builtin_copysignq", t, IX86_BUILTIN_COPYSIGNQ,
35937 BUILT_IN_MD, "__copysigntf3", NULL_TREE);
35938 TREE_READONLY (t) = 1;
35939 ix86_builtins[(int) IX86_BUILTIN_COPYSIGNQ] = t;
35941 ix86_init_tm_builtins ();
35942 ix86_init_mmx_sse_builtins ();
35943 ix86_init_mpx_builtins ();
35946 ix86_init_builtins_va_builtins_abi ();
35948 #ifdef SUBTARGET_INIT_BUILTINS
35949 SUBTARGET_INIT_BUILTINS;
35953 /* Return the ix86 builtin for CODE. */
35956 ix86_builtin_decl (unsigned code, bool)
35958 if (code >= IX86_BUILTIN_MAX)
35959 return error_mark_node;
35961 return ix86_builtins[code];
35964 /* Errors in the source file can cause expand_expr to return const0_rtx
35965 where we expect a vector. To avoid crashing, use one of the vector
35966 clear instructions. */
35968 safe_vector_operand (rtx x, machine_mode mode)
35970 if (x == const0_rtx)
35971 x = CONST0_RTX (mode);
35975 /* Fixup modeless constants to fit required mode. */
35977 fixup_modeless_constant (rtx x, machine_mode mode)
35979 if (GET_MODE (x) == VOIDmode)
35980 x = convert_to_mode (mode, x, 1);
35984 /* Subroutine of ix86_expand_builtin to take care of binop insns. */
35987 ix86_expand_binop_builtin (enum insn_code icode, tree exp, rtx target)
35990 tree arg0 = CALL_EXPR_ARG (exp, 0);
35991 tree arg1 = CALL_EXPR_ARG (exp, 1);
35992 rtx op0 = expand_normal (arg0);
35993 rtx op1 = expand_normal (arg1);
35994 machine_mode tmode = insn_data[icode].operand[0].mode;
35995 machine_mode mode0 = insn_data[icode].operand[1].mode;
35996 machine_mode mode1 = insn_data[icode].operand[2].mode;
35998 if (VECTOR_MODE_P (mode0))
35999 op0 = safe_vector_operand (op0, mode0);
36000 if (VECTOR_MODE_P (mode1))
36001 op1 = safe_vector_operand (op1, mode1);
36003 if (optimize || !target
36004 || GET_MODE (target) != tmode
36005 || !insn_data[icode].operand[0].predicate (target, tmode))
36006 target = gen_reg_rtx (tmode);
36008 if (GET_MODE (op1) == SImode && mode1 == TImode)
36010 rtx x = gen_reg_rtx (V4SImode);
36011 emit_insn (gen_sse2_loadd (x, op1));
36012 op1 = gen_lowpart (TImode, x);
36015 if (!insn_data[icode].operand[1].predicate (op0, mode0))
36016 op0 = copy_to_mode_reg (mode0, op0);
36017 if (!insn_data[icode].operand[2].predicate (op1, mode1))
36018 op1 = copy_to_mode_reg (mode1, op1);
36020 pat = GEN_FCN (icode) (target, op0, op1);
36029 /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
36032 ix86_expand_multi_arg_builtin (enum insn_code icode, tree exp, rtx target,
36033 enum ix86_builtin_func_type m_type,
36034 enum rtx_code sub_code)
36039 bool comparison_p = false;
36041 bool last_arg_constant = false;
36042 int num_memory = 0;
36048 machine_mode tmode = insn_data[icode].operand[0].mode;
36052 case MULTI_ARG_4_DF2_DI_I:
36053 case MULTI_ARG_4_DF2_DI_I1:
36054 case MULTI_ARG_4_SF2_SI_I:
36055 case MULTI_ARG_4_SF2_SI_I1:
36057 last_arg_constant = true;
36060 case MULTI_ARG_3_SF:
36061 case MULTI_ARG_3_DF:
36062 case MULTI_ARG_3_SF2:
36063 case MULTI_ARG_3_DF2:
36064 case MULTI_ARG_3_DI:
36065 case MULTI_ARG_3_SI:
36066 case MULTI_ARG_3_SI_DI:
36067 case MULTI_ARG_3_HI:
36068 case MULTI_ARG_3_HI_SI:
36069 case MULTI_ARG_3_QI:
36070 case MULTI_ARG_3_DI2:
36071 case MULTI_ARG_3_SI2:
36072 case MULTI_ARG_3_HI2:
36073 case MULTI_ARG_3_QI2:
36077 case MULTI_ARG_2_SF:
36078 case MULTI_ARG_2_DF:
36079 case MULTI_ARG_2_DI:
36080 case MULTI_ARG_2_SI:
36081 case MULTI_ARG_2_HI:
36082 case MULTI_ARG_2_QI:
36086 case MULTI_ARG_2_DI_IMM:
36087 case MULTI_ARG_2_SI_IMM:
36088 case MULTI_ARG_2_HI_IMM:
36089 case MULTI_ARG_2_QI_IMM:
36091 last_arg_constant = true;
36094 case MULTI_ARG_1_SF:
36095 case MULTI_ARG_1_DF:
36096 case MULTI_ARG_1_SF2:
36097 case MULTI_ARG_1_DF2:
36098 case MULTI_ARG_1_DI:
36099 case MULTI_ARG_1_SI:
36100 case MULTI_ARG_1_HI:
36101 case MULTI_ARG_1_QI:
36102 case MULTI_ARG_1_SI_DI:
36103 case MULTI_ARG_1_HI_DI:
36104 case MULTI_ARG_1_HI_SI:
36105 case MULTI_ARG_1_QI_DI:
36106 case MULTI_ARG_1_QI_SI:
36107 case MULTI_ARG_1_QI_HI:
36111 case MULTI_ARG_2_DI_CMP:
36112 case MULTI_ARG_2_SI_CMP:
36113 case MULTI_ARG_2_HI_CMP:
36114 case MULTI_ARG_2_QI_CMP:
36116 comparison_p = true;
36119 case MULTI_ARG_2_SF_TF:
36120 case MULTI_ARG_2_DF_TF:
36121 case MULTI_ARG_2_DI_TF:
36122 case MULTI_ARG_2_SI_TF:
36123 case MULTI_ARG_2_HI_TF:
36124 case MULTI_ARG_2_QI_TF:
36130 gcc_unreachable ();
36133 if (optimize || !target
36134 || GET_MODE (target) != tmode
36135 || !insn_data[icode].operand[0].predicate (target, tmode))
36136 target = gen_reg_rtx (tmode);
36138 gcc_assert (nargs <= 4);
36140 for (i = 0; i < nargs; i++)
36142 tree arg = CALL_EXPR_ARG (exp, i);
36143 rtx op = expand_normal (arg);
36144 int adjust = (comparison_p) ? 1 : 0;
36145 machine_mode mode = insn_data[icode].operand[i+adjust+1].mode;
36147 if (last_arg_constant && i == nargs - 1)
36149 if (!insn_data[icode].operand[i + 1].predicate (op, mode))
36151 enum insn_code new_icode = icode;
36154 case CODE_FOR_xop_vpermil2v2df3:
36155 case CODE_FOR_xop_vpermil2v4sf3:
36156 case CODE_FOR_xop_vpermil2v4df3:
36157 case CODE_FOR_xop_vpermil2v8sf3:
36158 error ("the last argument must be a 2-bit immediate");
36159 return gen_reg_rtx (tmode);
36160 case CODE_FOR_xop_rotlv2di3:
36161 new_icode = CODE_FOR_rotlv2di3;
36163 case CODE_FOR_xop_rotlv4si3:
36164 new_icode = CODE_FOR_rotlv4si3;
36166 case CODE_FOR_xop_rotlv8hi3:
36167 new_icode = CODE_FOR_rotlv8hi3;
36169 case CODE_FOR_xop_rotlv16qi3:
36170 new_icode = CODE_FOR_rotlv16qi3;
36172 if (CONST_INT_P (op))
36174 int mask = GET_MODE_BITSIZE (GET_MODE_INNER (tmode)) - 1;
36175 op = GEN_INT (INTVAL (op) & mask);
36176 gcc_checking_assert
36177 (insn_data[icode].operand[i + 1].predicate (op, mode));
36181 gcc_checking_assert
36183 && insn_data[new_icode].operand[0].mode == tmode
36184 && insn_data[new_icode].operand[1].mode == tmode
36185 && insn_data[new_icode].operand[2].mode == mode
36186 && insn_data[new_icode].operand[0].predicate
36187 == insn_data[icode].operand[0].predicate
36188 && insn_data[new_icode].operand[1].predicate
36189 == insn_data[icode].operand[1].predicate);
36195 gcc_unreachable ();
36202 if (VECTOR_MODE_P (mode))
36203 op = safe_vector_operand (op, mode);
36205 /* If we aren't optimizing, only allow one memory operand to be
36207 if (memory_operand (op, mode))
36210 gcc_assert (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode);
36213 || !insn_data[icode].operand[i+adjust+1].predicate (op, mode)
36215 op = force_reg (mode, op);
36219 args[i].mode = mode;
36225 pat = GEN_FCN (icode) (target, args[0].op);
36230 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
36231 GEN_INT ((int)sub_code));
36232 else if (! comparison_p)
36233 pat = GEN_FCN (icode) (target, args[0].op, args[1].op);
36236 rtx cmp_op = gen_rtx_fmt_ee (sub_code, GET_MODE (target),
36240 pat = GEN_FCN (icode) (target, cmp_op, args[0].op, args[1].op);
36245 pat = GEN_FCN (icode) (target, args[0].op, args[1].op, args[2].op);
36249 pat = GEN_FCN (icode) (target, args[0].op, args[1].op, args[2].op, args[3].op);
36253 gcc_unreachable ();
36263 /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
36264 insns with vec_merge. */
36267 ix86_expand_unop_vec_merge_builtin (enum insn_code icode, tree exp,
36271 tree arg0 = CALL_EXPR_ARG (exp, 0);
36272 rtx op1, op0 = expand_normal (arg0);
36273 machine_mode tmode = insn_data[icode].operand[0].mode;
36274 machine_mode mode0 = insn_data[icode].operand[1].mode;
36276 if (optimize || !target
36277 || GET_MODE (target) != tmode
36278 || !insn_data[icode].operand[0].predicate (target, tmode))
36279 target = gen_reg_rtx (tmode);
36281 if (VECTOR_MODE_P (mode0))
36282 op0 = safe_vector_operand (op0, mode0);
36284 if ((optimize && !register_operand (op0, mode0))
36285 || !insn_data[icode].operand[1].predicate (op0, mode0))
36286 op0 = copy_to_mode_reg (mode0, op0);
36289 if (!insn_data[icode].operand[2].predicate (op1, mode0))
36290 op1 = copy_to_mode_reg (mode0, op1);
36292 pat = GEN_FCN (icode) (target, op0, op1);
36299 /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
36302 ix86_expand_sse_compare (const struct builtin_description *d,
36303 tree exp, rtx target, bool swap)
36306 tree arg0 = CALL_EXPR_ARG (exp, 0);
36307 tree arg1 = CALL_EXPR_ARG (exp, 1);
36308 rtx op0 = expand_normal (arg0);
36309 rtx op1 = expand_normal (arg1);
36311 machine_mode tmode = insn_data[d->icode].operand[0].mode;
36312 machine_mode mode0 = insn_data[d->icode].operand[1].mode;
36313 machine_mode mode1 = insn_data[d->icode].operand[2].mode;
36314 enum rtx_code comparison = d->comparison;
36316 if (VECTOR_MODE_P (mode0))
36317 op0 = safe_vector_operand (op0, mode0);
36318 if (VECTOR_MODE_P (mode1))
36319 op1 = safe_vector_operand (op1, mode1);
36321 /* Swap operands if we have a comparison that isn't available in
36324 std::swap (op0, op1);
36326 if (optimize || !target
36327 || GET_MODE (target) != tmode
36328 || !insn_data[d->icode].operand[0].predicate (target, tmode))
36329 target = gen_reg_rtx (tmode);
36331 if ((optimize && !register_operand (op0, mode0))
36332 || !insn_data[d->icode].operand[1].predicate (op0, mode0))
36333 op0 = copy_to_mode_reg (mode0, op0);
36334 if ((optimize && !register_operand (op1, mode1))
36335 || !insn_data[d->icode].operand[2].predicate (op1, mode1))
36336 op1 = copy_to_mode_reg (mode1, op1);
36338 op2 = gen_rtx_fmt_ee (comparison, mode0, op0, op1);
36339 pat = GEN_FCN (d->icode) (target, op0, op1, op2);
36346 /* Subroutine of ix86_expand_builtin to take care of comi insns. */
36349 ix86_expand_sse_comi (const struct builtin_description *d, tree exp,
36353 tree arg0 = CALL_EXPR_ARG (exp, 0);
36354 tree arg1 = CALL_EXPR_ARG (exp, 1);
36355 rtx op0 = expand_normal (arg0);
36356 rtx op1 = expand_normal (arg1);
36357 machine_mode mode0 = insn_data[d->icode].operand[0].mode;
36358 machine_mode mode1 = insn_data[d->icode].operand[1].mode;
36359 enum rtx_code comparison = d->comparison;
36361 if (VECTOR_MODE_P (mode0))
36362 op0 = safe_vector_operand (op0, mode0);
36363 if (VECTOR_MODE_P (mode1))
36364 op1 = safe_vector_operand (op1, mode1);
36366 /* Swap operands if we have a comparison that isn't available in
36368 if (d->flag & BUILTIN_DESC_SWAP_OPERANDS)
36369 std::swap (op0, op1);
36371 target = gen_reg_rtx (SImode);
36372 emit_move_insn (target, const0_rtx);
36373 target = gen_rtx_SUBREG (QImode, target, 0);
36375 if ((optimize && !register_operand (op0, mode0))
36376 || !insn_data[d->icode].operand[0].predicate (op0, mode0))
36377 op0 = copy_to_mode_reg (mode0, op0);
36378 if ((optimize && !register_operand (op1, mode1))
36379 || !insn_data[d->icode].operand[1].predicate (op1, mode1))
36380 op1 = copy_to_mode_reg (mode1, op1);
36382 pat = GEN_FCN (d->icode) (op0, op1);
36386 emit_insn (gen_rtx_SET (VOIDmode,
36387 gen_rtx_STRICT_LOW_PART (VOIDmode, target),
36388 gen_rtx_fmt_ee (comparison, QImode,
36392 return SUBREG_REG (target);
36395 /* Subroutines of ix86_expand_args_builtin to take care of round insns. */
36398 ix86_expand_sse_round (const struct builtin_description *d, tree exp,
36402 tree arg0 = CALL_EXPR_ARG (exp, 0);
36403 rtx op1, op0 = expand_normal (arg0);
36404 machine_mode tmode = insn_data[d->icode].operand[0].mode;
36405 machine_mode mode0 = insn_data[d->icode].operand[1].mode;
36407 if (optimize || target == 0
36408 || GET_MODE (target) != tmode
36409 || !insn_data[d->icode].operand[0].predicate (target, tmode))
36410 target = gen_reg_rtx (tmode);
36412 if (VECTOR_MODE_P (mode0))
36413 op0 = safe_vector_operand (op0, mode0);
36415 if ((optimize && !register_operand (op0, mode0))
36416 || !insn_data[d->icode].operand[0].predicate (op0, mode0))
36417 op0 = copy_to_mode_reg (mode0, op0);
36419 op1 = GEN_INT (d->comparison);
36421 pat = GEN_FCN (d->icode) (target, op0, op1);
36429 ix86_expand_sse_round_vec_pack_sfix (const struct builtin_description *d,
36430 tree exp, rtx target)
36433 tree arg0 = CALL_EXPR_ARG (exp, 0);
36434 tree arg1 = CALL_EXPR_ARG (exp, 1);
36435 rtx op0 = expand_normal (arg0);
36436 rtx op1 = expand_normal (arg1);
36438 machine_mode tmode = insn_data[d->icode].operand[0].mode;
36439 machine_mode mode0 = insn_data[d->icode].operand[1].mode;
36440 machine_mode mode1 = insn_data[d->icode].operand[2].mode;
36442 if (optimize || target == 0
36443 || GET_MODE (target) != tmode
36444 || !insn_data[d->icode].operand[0].predicate (target, tmode))
36445 target = gen_reg_rtx (tmode);
36447 op0 = safe_vector_operand (op0, mode0);
36448 op1 = safe_vector_operand (op1, mode1);
36450 if ((optimize && !register_operand (op0, mode0))
36451 || !insn_data[d->icode].operand[0].predicate (op0, mode0))
36452 op0 = copy_to_mode_reg (mode0, op0);
36453 if ((optimize && !register_operand (op1, mode1))
36454 || !insn_data[d->icode].operand[1].predicate (op1, mode1))
36455 op1 = copy_to_mode_reg (mode1, op1);
36457 op2 = GEN_INT (d->comparison);
36459 pat = GEN_FCN (d->icode) (target, op0, op1, op2);
36466 /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
36469 ix86_expand_sse_ptest (const struct builtin_description *d, tree exp,
36473 tree arg0 = CALL_EXPR_ARG (exp, 0);
36474 tree arg1 = CALL_EXPR_ARG (exp, 1);
36475 rtx op0 = expand_normal (arg0);
36476 rtx op1 = expand_normal (arg1);
36477 machine_mode mode0 = insn_data[d->icode].operand[0].mode;
36478 machine_mode mode1 = insn_data[d->icode].operand[1].mode;
36479 enum rtx_code comparison = d->comparison;
36481 if (VECTOR_MODE_P (mode0))
36482 op0 = safe_vector_operand (op0, mode0);
36483 if (VECTOR_MODE_P (mode1))
36484 op1 = safe_vector_operand (op1, mode1);
36486 target = gen_reg_rtx (SImode);
36487 emit_move_insn (target, const0_rtx);
36488 target = gen_rtx_SUBREG (QImode, target, 0);
36490 if ((optimize && !register_operand (op0, mode0))
36491 || !insn_data[d->icode].operand[0].predicate (op0, mode0))
36492 op0 = copy_to_mode_reg (mode0, op0);
36493 if ((optimize && !register_operand (op1, mode1))
36494 || !insn_data[d->icode].operand[1].predicate (op1, mode1))
36495 op1 = copy_to_mode_reg (mode1, op1);
36497 pat = GEN_FCN (d->icode) (op0, op1);
36501 emit_insn (gen_rtx_SET (VOIDmode,
36502 gen_rtx_STRICT_LOW_PART (VOIDmode, target),
36503 gen_rtx_fmt_ee (comparison, QImode,
36507 return SUBREG_REG (target);
36510 /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
36513 ix86_expand_sse_pcmpestr (const struct builtin_description *d,
36514 tree exp, rtx target)
36517 tree arg0 = CALL_EXPR_ARG (exp, 0);
36518 tree arg1 = CALL_EXPR_ARG (exp, 1);
36519 tree arg2 = CALL_EXPR_ARG (exp, 2);
36520 tree arg3 = CALL_EXPR_ARG (exp, 3);
36521 tree arg4 = CALL_EXPR_ARG (exp, 4);
36522 rtx scratch0, scratch1;
36523 rtx op0 = expand_normal (arg0);
36524 rtx op1 = expand_normal (arg1);
36525 rtx op2 = expand_normal (arg2);
36526 rtx op3 = expand_normal (arg3);
36527 rtx op4 = expand_normal (arg4);
36528 machine_mode tmode0, tmode1, modev2, modei3, modev4, modei5, modeimm;
36530 tmode0 = insn_data[d->icode].operand[0].mode;
36531 tmode1 = insn_data[d->icode].operand[1].mode;
36532 modev2 = insn_data[d->icode].operand[2].mode;
36533 modei3 = insn_data[d->icode].operand[3].mode;
36534 modev4 = insn_data[d->icode].operand[4].mode;
36535 modei5 = insn_data[d->icode].operand[5].mode;
36536 modeimm = insn_data[d->icode].operand[6].mode;
36538 if (VECTOR_MODE_P (modev2))
36539 op0 = safe_vector_operand (op0, modev2);
36540 if (VECTOR_MODE_P (modev4))
36541 op2 = safe_vector_operand (op2, modev4);
36543 if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
36544 op0 = copy_to_mode_reg (modev2, op0);
36545 if (!insn_data[d->icode].operand[3].predicate (op1, modei3))
36546 op1 = copy_to_mode_reg (modei3, op1);
36547 if ((optimize && !register_operand (op2, modev4))
36548 || !insn_data[d->icode].operand[4].predicate (op2, modev4))
36549 op2 = copy_to_mode_reg (modev4, op2);
36550 if (!insn_data[d->icode].operand[5].predicate (op3, modei5))
36551 op3 = copy_to_mode_reg (modei5, op3);
36553 if (!insn_data[d->icode].operand[6].predicate (op4, modeimm))
36555 error ("the fifth argument must be an 8-bit immediate");
36559 if (d->code == IX86_BUILTIN_PCMPESTRI128)
36561 if (optimize || !target
36562 || GET_MODE (target) != tmode0
36563 || !insn_data[d->icode].operand[0].predicate (target, tmode0))
36564 target = gen_reg_rtx (tmode0);
36566 scratch1 = gen_reg_rtx (tmode1);
36568 pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2, op3, op4);
36570 else if (d->code == IX86_BUILTIN_PCMPESTRM128)
36572 if (optimize || !target
36573 || GET_MODE (target) != tmode1
36574 || !insn_data[d->icode].operand[1].predicate (target, tmode1))
36575 target = gen_reg_rtx (tmode1);
36577 scratch0 = gen_reg_rtx (tmode0);
36579 pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2, op3, op4);
36583 gcc_assert (d->flag);
36585 scratch0 = gen_reg_rtx (tmode0);
36586 scratch1 = gen_reg_rtx (tmode1);
36588 pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2, op3, op4);
36598 target = gen_reg_rtx (SImode);
36599 emit_move_insn (target, const0_rtx);
36600 target = gen_rtx_SUBREG (QImode, target, 0);
36603 (gen_rtx_SET (VOIDmode, gen_rtx_STRICT_LOW_PART (VOIDmode, target),
36604 gen_rtx_fmt_ee (EQ, QImode,
36605 gen_rtx_REG ((machine_mode) d->flag,
36608 return SUBREG_REG (target);
36615 /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
36618 ix86_expand_sse_pcmpistr (const struct builtin_description *d,
36619 tree exp, rtx target)
36622 tree arg0 = CALL_EXPR_ARG (exp, 0);
36623 tree arg1 = CALL_EXPR_ARG (exp, 1);
36624 tree arg2 = CALL_EXPR_ARG (exp, 2);
36625 rtx scratch0, scratch1;
36626 rtx op0 = expand_normal (arg0);
36627 rtx op1 = expand_normal (arg1);
36628 rtx op2 = expand_normal (arg2);
36629 machine_mode tmode0, tmode1, modev2, modev3, modeimm;
36631 tmode0 = insn_data[d->icode].operand[0].mode;
36632 tmode1 = insn_data[d->icode].operand[1].mode;
36633 modev2 = insn_data[d->icode].operand[2].mode;
36634 modev3 = insn_data[d->icode].operand[3].mode;
36635 modeimm = insn_data[d->icode].operand[4].mode;
36637 if (VECTOR_MODE_P (modev2))
36638 op0 = safe_vector_operand (op0, modev2);
36639 if (VECTOR_MODE_P (modev3))
36640 op1 = safe_vector_operand (op1, modev3);
36642 if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
36643 op0 = copy_to_mode_reg (modev2, op0);
36644 if ((optimize && !register_operand (op1, modev3))
36645 || !insn_data[d->icode].operand[3].predicate (op1, modev3))
36646 op1 = copy_to_mode_reg (modev3, op1);
36648 if (!insn_data[d->icode].operand[4].predicate (op2, modeimm))
36650 error ("the third argument must be an 8-bit immediate");
36654 if (d->code == IX86_BUILTIN_PCMPISTRI128)
36656 if (optimize || !target
36657 || GET_MODE (target) != tmode0
36658 || !insn_data[d->icode].operand[0].predicate (target, tmode0))
36659 target = gen_reg_rtx (tmode0);
36661 scratch1 = gen_reg_rtx (tmode1);
36663 pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2);
36665 else if (d->code == IX86_BUILTIN_PCMPISTRM128)
36667 if (optimize || !target
36668 || GET_MODE (target) != tmode1
36669 || !insn_data[d->icode].operand[1].predicate (target, tmode1))
36670 target = gen_reg_rtx (tmode1);
36672 scratch0 = gen_reg_rtx (tmode0);
36674 pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2);
36678 gcc_assert (d->flag);
36680 scratch0 = gen_reg_rtx (tmode0);
36681 scratch1 = gen_reg_rtx (tmode1);
36683 pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2);
36693 target = gen_reg_rtx (SImode);
36694 emit_move_insn (target, const0_rtx);
36695 target = gen_rtx_SUBREG (QImode, target, 0);
36698 (gen_rtx_SET (VOIDmode, gen_rtx_STRICT_LOW_PART (VOIDmode, target),
36699 gen_rtx_fmt_ee (EQ, QImode,
36700 gen_rtx_REG ((machine_mode) d->flag,
36703 return SUBREG_REG (target);
36709 /* Subroutine of ix86_expand_builtin to take care of insns with
36710 variable number of operands. */
36713 ix86_expand_args_builtin (const struct builtin_description *d,
36714 tree exp, rtx target)
36716 rtx pat, real_target;
36717 unsigned int i, nargs;
36718 unsigned int nargs_constant = 0;
36719 unsigned int mask_pos = 0;
36720 int num_memory = 0;
36726 bool last_arg_count = false;
36727 enum insn_code icode = d->icode;
36728 const struct insn_data_d *insn_p = &insn_data[icode];
36729 machine_mode tmode = insn_p->operand[0].mode;
36730 machine_mode rmode = VOIDmode;
36732 enum rtx_code comparison = d->comparison;
36734 switch ((enum ix86_builtin_func_type) d->flag)
36736 case V2DF_FTYPE_V2DF_ROUND:
36737 case V4DF_FTYPE_V4DF_ROUND:
36738 case V4SF_FTYPE_V4SF_ROUND:
36739 case V8SF_FTYPE_V8SF_ROUND:
36740 case V4SI_FTYPE_V4SF_ROUND:
36741 case V8SI_FTYPE_V8SF_ROUND:
36742 return ix86_expand_sse_round (d, exp, target);
36743 case V4SI_FTYPE_V2DF_V2DF_ROUND:
36744 case V8SI_FTYPE_V4DF_V4DF_ROUND:
36745 case V16SI_FTYPE_V8DF_V8DF_ROUND:
36746 return ix86_expand_sse_round_vec_pack_sfix (d, exp, target);
36747 case INT_FTYPE_V8SF_V8SF_PTEST:
36748 case INT_FTYPE_V4DI_V4DI_PTEST:
36749 case INT_FTYPE_V4DF_V4DF_PTEST:
36750 case INT_FTYPE_V4SF_V4SF_PTEST:
36751 case INT_FTYPE_V2DI_V2DI_PTEST:
36752 case INT_FTYPE_V2DF_V2DF_PTEST:
36753 return ix86_expand_sse_ptest (d, exp, target);
36754 case FLOAT128_FTYPE_FLOAT128:
36755 case FLOAT_FTYPE_FLOAT:
36756 case INT_FTYPE_INT:
36757 case UINT64_FTYPE_INT:
36758 case UINT16_FTYPE_UINT16:
36759 case INT64_FTYPE_INT64:
36760 case INT64_FTYPE_V4SF:
36761 case INT64_FTYPE_V2DF:
36762 case INT_FTYPE_V16QI:
36763 case INT_FTYPE_V8QI:
36764 case INT_FTYPE_V8SF:
36765 case INT_FTYPE_V4DF:
36766 case INT_FTYPE_V4SF:
36767 case INT_FTYPE_V2DF:
36768 case INT_FTYPE_V32QI:
36769 case V16QI_FTYPE_V16QI:
36770 case V8SI_FTYPE_V8SF:
36771 case V8SI_FTYPE_V4SI:
36772 case V8HI_FTYPE_V8HI:
36773 case V8HI_FTYPE_V16QI:
36774 case V8QI_FTYPE_V8QI:
36775 case V8SF_FTYPE_V8SF:
36776 case V8SF_FTYPE_V8SI:
36777 case V8SF_FTYPE_V4SF:
36778 case V8SF_FTYPE_V8HI:
36779 case V4SI_FTYPE_V4SI:
36780 case V4SI_FTYPE_V16QI:
36781 case V4SI_FTYPE_V4SF:
36782 case V4SI_FTYPE_V8SI:
36783 case V4SI_FTYPE_V8HI:
36784 case V4SI_FTYPE_V4DF:
36785 case V4SI_FTYPE_V2DF:
36786 case V4HI_FTYPE_V4HI:
36787 case V4DF_FTYPE_V4DF:
36788 case V4DF_FTYPE_V4SI:
36789 case V4DF_FTYPE_V4SF:
36790 case V4DF_FTYPE_V2DF:
36791 case V4SF_FTYPE_V4SF:
36792 case V4SF_FTYPE_V4SI:
36793 case V4SF_FTYPE_V8SF:
36794 case V4SF_FTYPE_V4DF:
36795 case V4SF_FTYPE_V8HI:
36796 case V4SF_FTYPE_V2DF:
36797 case V2DI_FTYPE_V2DI:
36798 case V2DI_FTYPE_V16QI:
36799 case V2DI_FTYPE_V8HI:
36800 case V2DI_FTYPE_V4SI:
36801 case V2DF_FTYPE_V2DF:
36802 case V2DF_FTYPE_V4SI:
36803 case V2DF_FTYPE_V4DF:
36804 case V2DF_FTYPE_V4SF:
36805 case V2DF_FTYPE_V2SI:
36806 case V2SI_FTYPE_V2SI:
36807 case V2SI_FTYPE_V4SF:
36808 case V2SI_FTYPE_V2SF:
36809 case V2SI_FTYPE_V2DF:
36810 case V2SF_FTYPE_V2SF:
36811 case V2SF_FTYPE_V2SI:
36812 case V32QI_FTYPE_V32QI:
36813 case V32QI_FTYPE_V16QI:
36814 case V16HI_FTYPE_V16HI:
36815 case V16HI_FTYPE_V8HI:
36816 case V8SI_FTYPE_V8SI:
36817 case V16HI_FTYPE_V16QI:
36818 case V8SI_FTYPE_V16QI:
36819 case V4DI_FTYPE_V16QI:
36820 case V8SI_FTYPE_V8HI:
36821 case V4DI_FTYPE_V8HI:
36822 case V4DI_FTYPE_V4SI:
36823 case V4DI_FTYPE_V2DI:
36825 case HI_FTYPE_V16QI:
36826 case SI_FTYPE_V32QI:
36827 case DI_FTYPE_V64QI:
36828 case V16QI_FTYPE_HI:
36829 case V32QI_FTYPE_SI:
36830 case V64QI_FTYPE_DI:
36831 case V8HI_FTYPE_QI:
36832 case V16HI_FTYPE_HI:
36833 case V32HI_FTYPE_SI:
36834 case V4SI_FTYPE_QI:
36835 case V8SI_FTYPE_QI:
36836 case V4SI_FTYPE_HI:
36837 case V8SI_FTYPE_HI:
36838 case QI_FTYPE_V8HI:
36839 case HI_FTYPE_V16HI:
36840 case SI_FTYPE_V32HI:
36841 case QI_FTYPE_V4SI:
36842 case QI_FTYPE_V8SI:
36843 case HI_FTYPE_V16SI:
36844 case QI_FTYPE_V2DI:
36845 case QI_FTYPE_V4DI:
36846 case QI_FTYPE_V8DI:
36847 case UINT_FTYPE_V2DF:
36848 case UINT_FTYPE_V4SF:
36849 case UINT64_FTYPE_V2DF:
36850 case UINT64_FTYPE_V4SF:
36851 case V16QI_FTYPE_V8DI:
36852 case V16HI_FTYPE_V16SI:
36853 case V16SI_FTYPE_HI:
36854 case V2DI_FTYPE_QI:
36855 case V4DI_FTYPE_QI:
36856 case V16SI_FTYPE_V16SI:
36857 case V16SI_FTYPE_INT:
36858 case V16SF_FTYPE_FLOAT:
36859 case V16SF_FTYPE_V8SF:
36860 case V16SI_FTYPE_V8SI:
36861 case V16SF_FTYPE_V4SF:
36862 case V16SI_FTYPE_V4SI:
36863 case V16SF_FTYPE_V16SF:
36864 case V8HI_FTYPE_V8DI:
36865 case V8UHI_FTYPE_V8UHI:
36866 case V8SI_FTYPE_V8DI:
36867 case V8SF_FTYPE_V8DF:
36868 case V8DI_FTYPE_QI:
36869 case V8DI_FTYPE_INT64:
36870 case V8DI_FTYPE_V4DI:
36871 case V8DI_FTYPE_V8DI:
36872 case V8DF_FTYPE_DOUBLE:
36873 case V8DF_FTYPE_V4DF:
36874 case V8DF_FTYPE_V2DF:
36875 case V8DF_FTYPE_V8DF:
36876 case V8DF_FTYPE_V8SI:
36879 case V4SF_FTYPE_V4SF_VEC_MERGE:
36880 case V2DF_FTYPE_V2DF_VEC_MERGE:
36881 return ix86_expand_unop_vec_merge_builtin (icode, exp, target);
36882 case FLOAT128_FTYPE_FLOAT128_FLOAT128:
36883 case V16QI_FTYPE_V16QI_V16QI:
36884 case V16QI_FTYPE_V8HI_V8HI:
36885 case V16SI_FTYPE_V16SI_V16SI:
36886 case V16SF_FTYPE_V16SF_V16SF:
36887 case V16SF_FTYPE_V16SF_V16SI:
36888 case V8QI_FTYPE_V8QI_V8QI:
36889 case V8QI_FTYPE_V4HI_V4HI:
36890 case V8HI_FTYPE_V8HI_V8HI:
36891 case V8HI_FTYPE_V16QI_V16QI:
36892 case V8HI_FTYPE_V4SI_V4SI:
36893 case V8SF_FTYPE_V8SF_V8SF:
36894 case V8SF_FTYPE_V8SF_V8SI:
36895 case V8DI_FTYPE_V8DI_V8DI:
36896 case V8DF_FTYPE_V8DF_V8DF:
36897 case V8DF_FTYPE_V8DF_V8DI:
36898 case V4SI_FTYPE_V4SI_V4SI:
36899 case V4SI_FTYPE_V8HI_V8HI:
36900 case V4SI_FTYPE_V4SF_V4SF:
36901 case V4SI_FTYPE_V2DF_V2DF:
36902 case V4HI_FTYPE_V4HI_V4HI:
36903 case V4HI_FTYPE_V8QI_V8QI:
36904 case V4HI_FTYPE_V2SI_V2SI:
36905 case V4DF_FTYPE_V4DF_V4DF:
36906 case V4DF_FTYPE_V4DF_V4DI:
36907 case V4SF_FTYPE_V4SF_V4SF:
36908 case V4SF_FTYPE_V4SF_V4SI:
36909 case V4SF_FTYPE_V4SF_V2SI:
36910 case V4SF_FTYPE_V4SF_V2DF:
36911 case V4SF_FTYPE_V4SF_UINT:
36912 case V4SF_FTYPE_V4SF_UINT64:
36913 case V4SF_FTYPE_V4SF_DI:
36914 case V4SF_FTYPE_V4SF_SI:
36915 case V2DI_FTYPE_V2DI_V2DI:
36916 case V2DI_FTYPE_V16QI_V16QI:
36917 case V2DI_FTYPE_V4SI_V4SI:
36918 case V2UDI_FTYPE_V4USI_V4USI:
36919 case V2DI_FTYPE_V2DI_V16QI:
36920 case V2DI_FTYPE_V2DF_V2DF:
36921 case V2SI_FTYPE_V2SI_V2SI:
36922 case V2SI_FTYPE_V4HI_V4HI:
36923 case V2SI_FTYPE_V2SF_V2SF:
36924 case V2DF_FTYPE_V2DF_V2DF:
36925 case V2DF_FTYPE_V2DF_V4SF:
36926 case V2DF_FTYPE_V2DF_V2DI:
36927 case V2DF_FTYPE_V2DF_DI:
36928 case V2DF_FTYPE_V2DF_SI:
36929 case V2DF_FTYPE_V2DF_UINT:
36930 case V2DF_FTYPE_V2DF_UINT64:
36931 case V2SF_FTYPE_V2SF_V2SF:
36932 case V1DI_FTYPE_V1DI_V1DI:
36933 case V1DI_FTYPE_V8QI_V8QI:
36934 case V1DI_FTYPE_V2SI_V2SI:
36935 case V32QI_FTYPE_V16HI_V16HI:
36936 case V16HI_FTYPE_V8SI_V8SI:
36937 case V32QI_FTYPE_V32QI_V32QI:
36938 case V16HI_FTYPE_V32QI_V32QI:
36939 case V16HI_FTYPE_V16HI_V16HI:
36940 case V8SI_FTYPE_V4DF_V4DF:
36941 case V8SI_FTYPE_V8SI_V8SI:
36942 case V8SI_FTYPE_V16HI_V16HI:
36943 case V4DI_FTYPE_V4DI_V4DI:
36944 case V4DI_FTYPE_V8SI_V8SI:
36945 case V4UDI_FTYPE_V8USI_V8USI:
36946 case QI_FTYPE_V8DI_V8DI:
36947 case V8DI_FTYPE_V64QI_V64QI:
36948 case HI_FTYPE_V16SI_V16SI:
36949 if (comparison == UNKNOWN)
36950 return ix86_expand_binop_builtin (icode, exp, target);
36953 case V4SF_FTYPE_V4SF_V4SF_SWAP:
36954 case V2DF_FTYPE_V2DF_V2DF_SWAP:
36955 gcc_assert (comparison != UNKNOWN);
36959 case V16HI_FTYPE_V16HI_V8HI_COUNT:
36960 case V16HI_FTYPE_V16HI_SI_COUNT:
36961 case V8SI_FTYPE_V8SI_V4SI_COUNT:
36962 case V8SI_FTYPE_V8SI_SI_COUNT:
36963 case V4DI_FTYPE_V4DI_V2DI_COUNT:
36964 case V4DI_FTYPE_V4DI_INT_COUNT:
36965 case V8HI_FTYPE_V8HI_V8HI_COUNT:
36966 case V8HI_FTYPE_V8HI_SI_COUNT:
36967 case V4SI_FTYPE_V4SI_V4SI_COUNT:
36968 case V4SI_FTYPE_V4SI_SI_COUNT:
36969 case V4HI_FTYPE_V4HI_V4HI_COUNT:
36970 case V4HI_FTYPE_V4HI_SI_COUNT:
36971 case V2DI_FTYPE_V2DI_V2DI_COUNT:
36972 case V2DI_FTYPE_V2DI_SI_COUNT:
36973 case V2SI_FTYPE_V2SI_V2SI_COUNT:
36974 case V2SI_FTYPE_V2SI_SI_COUNT:
36975 case V1DI_FTYPE_V1DI_V1DI_COUNT:
36976 case V1DI_FTYPE_V1DI_SI_COUNT:
36978 last_arg_count = true;
36980 case UINT64_FTYPE_UINT64_UINT64:
36981 case UINT_FTYPE_UINT_UINT:
36982 case UINT_FTYPE_UINT_USHORT:
36983 case UINT_FTYPE_UINT_UCHAR:
36984 case UINT16_FTYPE_UINT16_INT:
36985 case UINT8_FTYPE_UINT8_INT:
36986 case HI_FTYPE_HI_HI:
36987 case SI_FTYPE_SI_SI:
36988 case DI_FTYPE_DI_DI:
36989 case V16SI_FTYPE_V8DF_V8DF:
36992 case V2DI_FTYPE_V2DI_INT_CONVERT:
36995 nargs_constant = 1;
36997 case V4DI_FTYPE_V4DI_INT_CONVERT:
37000 nargs_constant = 1;
37002 case V8DI_FTYPE_V8DI_INT_CONVERT:
37005 nargs_constant = 1;
37007 case V8HI_FTYPE_V8HI_INT:
37008 case V8HI_FTYPE_V8SF_INT:
37009 case V16HI_FTYPE_V16SF_INT:
37010 case V8HI_FTYPE_V4SF_INT:
37011 case V8SF_FTYPE_V8SF_INT:
37012 case V4SF_FTYPE_V16SF_INT:
37013 case V16SF_FTYPE_V16SF_INT:
37014 case V4SI_FTYPE_V4SI_INT:
37015 case V4SI_FTYPE_V8SI_INT:
37016 case V4HI_FTYPE_V4HI_INT:
37017 case V4DF_FTYPE_V4DF_INT:
37018 case V4DF_FTYPE_V8DF_INT:
37019 case V4SF_FTYPE_V4SF_INT:
37020 case V4SF_FTYPE_V8SF_INT:
37021 case V2DI_FTYPE_V2DI_INT:
37022 case V2DF_FTYPE_V2DF_INT:
37023 case V2DF_FTYPE_V4DF_INT:
37024 case V16HI_FTYPE_V16HI_INT:
37025 case V8SI_FTYPE_V8SI_INT:
37026 case V16SI_FTYPE_V16SI_INT:
37027 case V4SI_FTYPE_V16SI_INT:
37028 case V4DI_FTYPE_V4DI_INT:
37029 case V2DI_FTYPE_V4DI_INT:
37030 case V4DI_FTYPE_V8DI_INT:
37031 case HI_FTYPE_HI_INT:
37032 case QI_FTYPE_V4SF_INT:
37033 case QI_FTYPE_V2DF_INT:
37035 nargs_constant = 1;
37037 case V16QI_FTYPE_V16QI_V16QI_V16QI:
37038 case V8SF_FTYPE_V8SF_V8SF_V8SF:
37039 case V4DF_FTYPE_V4DF_V4DF_V4DF:
37040 case V4SF_FTYPE_V4SF_V4SF_V4SF:
37041 case V2DF_FTYPE_V2DF_V2DF_V2DF:
37042 case V32QI_FTYPE_V32QI_V32QI_V32QI:
37043 case HI_FTYPE_V16SI_V16SI_HI:
37044 case QI_FTYPE_V8DI_V8DI_QI:
37045 case V16HI_FTYPE_V16SI_V16HI_HI:
37046 case V16QI_FTYPE_V16SI_V16QI_HI:
37047 case V16QI_FTYPE_V8DI_V16QI_QI:
37048 case V16SF_FTYPE_V16SF_V16SF_HI:
37049 case V16SF_FTYPE_V16SF_V16SF_V16SF:
37050 case V16SF_FTYPE_V16SF_V16SI_V16SF:
37051 case V16SF_FTYPE_V16SI_V16SF_HI:
37052 case V16SF_FTYPE_V16SI_V16SF_V16SF:
37053 case V16SF_FTYPE_V4SF_V16SF_HI:
37054 case V16SI_FTYPE_SI_V16SI_HI:
37055 case V16SI_FTYPE_V16HI_V16SI_HI:
37056 case V16SI_FTYPE_V16QI_V16SI_HI:
37057 case V16SI_FTYPE_V16SF_V16SI_HI:
37058 case V8SF_FTYPE_V4SF_V8SF_QI:
37059 case V4DF_FTYPE_V2DF_V4DF_QI:
37060 case V8SI_FTYPE_V4SI_V8SI_QI:
37061 case V8SI_FTYPE_SI_V8SI_QI:
37062 case V4SI_FTYPE_V4SI_V4SI_QI:
37063 case V4SI_FTYPE_SI_V4SI_QI:
37064 case V4DI_FTYPE_V2DI_V4DI_QI:
37065 case V4DI_FTYPE_DI_V4DI_QI:
37066 case V2DI_FTYPE_V2DI_V2DI_QI:
37067 case V2DI_FTYPE_DI_V2DI_QI:
37068 case V64QI_FTYPE_V64QI_V64QI_DI:
37069 case V64QI_FTYPE_V16QI_V64QI_DI:
37070 case V64QI_FTYPE_QI_V64QI_DI:
37071 case V32QI_FTYPE_V32QI_V32QI_SI:
37072 case V32QI_FTYPE_V16QI_V32QI_SI:
37073 case V32QI_FTYPE_QI_V32QI_SI:
37074 case V16QI_FTYPE_V16QI_V16QI_HI:
37075 case V16QI_FTYPE_QI_V16QI_HI:
37076 case V32HI_FTYPE_V8HI_V32HI_SI:
37077 case V32HI_FTYPE_HI_V32HI_SI:
37078 case V16HI_FTYPE_V8HI_V16HI_HI:
37079 case V16HI_FTYPE_HI_V16HI_HI:
37080 case V8HI_FTYPE_V8HI_V8HI_QI:
37081 case V8HI_FTYPE_HI_V8HI_QI:
37082 case V8SF_FTYPE_V8HI_V8SF_QI:
37083 case V4SF_FTYPE_V8HI_V4SF_QI:
37084 case V8SI_FTYPE_V8SF_V8SI_QI:
37085 case V4SI_FTYPE_V4SF_V4SI_QI:
37086 case V8DI_FTYPE_V8SF_V8DI_QI:
37087 case V4DI_FTYPE_V4SF_V4DI_QI:
37088 case V2DI_FTYPE_V4SF_V2DI_QI:
37089 case V8SF_FTYPE_V8DI_V8SF_QI:
37090 case V4SF_FTYPE_V4DI_V4SF_QI:
37091 case V4SF_FTYPE_V2DI_V4SF_QI:
37092 case V8DF_FTYPE_V8DI_V8DF_QI:
37093 case V4DF_FTYPE_V4DI_V4DF_QI:
37094 case V2DF_FTYPE_V2DI_V2DF_QI:
37095 case V16QI_FTYPE_V8HI_V16QI_QI:
37096 case V16QI_FTYPE_V16HI_V16QI_HI:
37097 case V16QI_FTYPE_V4SI_V16QI_QI:
37098 case V16QI_FTYPE_V8SI_V16QI_QI:
37099 case V8HI_FTYPE_V4SI_V8HI_QI:
37100 case V8HI_FTYPE_V8SI_V8HI_QI:
37101 case V16QI_FTYPE_V2DI_V16QI_QI:
37102 case V16QI_FTYPE_V4DI_V16QI_QI:
37103 case V8HI_FTYPE_V2DI_V8HI_QI:
37104 case V8HI_FTYPE_V4DI_V8HI_QI:
37105 case V4SI_FTYPE_V2DI_V4SI_QI:
37106 case V4SI_FTYPE_V4DI_V4SI_QI:
37107 case V32QI_FTYPE_V32HI_V32QI_SI:
37108 case HI_FTYPE_V16QI_V16QI_HI:
37109 case SI_FTYPE_V32QI_V32QI_SI:
37110 case DI_FTYPE_V64QI_V64QI_DI:
37111 case QI_FTYPE_V8HI_V8HI_QI:
37112 case HI_FTYPE_V16HI_V16HI_HI:
37113 case SI_FTYPE_V32HI_V32HI_SI:
37114 case QI_FTYPE_V4SI_V4SI_QI:
37115 case QI_FTYPE_V8SI_V8SI_QI:
37116 case QI_FTYPE_V2DI_V2DI_QI:
37117 case QI_FTYPE_V4DI_V4DI_QI:
37118 case V4SF_FTYPE_V2DF_V4SF_QI:
37119 case V4SF_FTYPE_V4DF_V4SF_QI:
37120 case V16SI_FTYPE_V16SI_V16SI_HI:
37121 case V16SI_FTYPE_V16SI_V16SI_V16SI:
37122 case V16SI_FTYPE_V4SI_V16SI_HI:
37123 case V2DI_FTYPE_V2DI_V2DI_V2DI:
37124 case V2DI_FTYPE_V4SI_V2DI_QI:
37125 case V2DI_FTYPE_V8HI_V2DI_QI:
37126 case V2DI_FTYPE_V16QI_V2DI_QI:
37127 case V4DI_FTYPE_V4DI_V4DI_QI:
37128 case V4DI_FTYPE_V4SI_V4DI_QI:
37129 case V4DI_FTYPE_V8HI_V4DI_QI:
37130 case V4DI_FTYPE_V16QI_V4DI_QI:
37131 case V8DI_FTYPE_V8DF_V8DI_QI:
37132 case V4DI_FTYPE_V4DF_V4DI_QI:
37133 case V2DI_FTYPE_V2DF_V2DI_QI:
37134 case V4SI_FTYPE_V4DF_V4SI_QI:
37135 case V4SI_FTYPE_V2DF_V4SI_QI:
37136 case V4SI_FTYPE_V8HI_V4SI_QI:
37137 case V4SI_FTYPE_V16QI_V4SI_QI:
37138 case V8SI_FTYPE_V8SI_V8SI_V8SI:
37139 case V4DI_FTYPE_V4DI_V4DI_V4DI:
37140 case V8DF_FTYPE_V2DF_V8DF_QI:
37141 case V8DF_FTYPE_V4DF_V8DF_QI:
37142 case V8DF_FTYPE_V8DF_V8DF_QI:
37143 case V8DF_FTYPE_V8DF_V8DF_V8DF:
37144 case V8SF_FTYPE_V8SF_V8SF_QI:
37145 case V8SF_FTYPE_V8SI_V8SF_QI:
37146 case V4DF_FTYPE_V4DF_V4DF_QI:
37147 case V4SF_FTYPE_V4SF_V4SF_QI:
37148 case V2DF_FTYPE_V2DF_V2DF_QI:
37149 case V2DF_FTYPE_V4SF_V2DF_QI:
37150 case V2DF_FTYPE_V4SI_V2DF_QI:
37151 case V4SF_FTYPE_V4SI_V4SF_QI:
37152 case V4DF_FTYPE_V4SF_V4DF_QI:
37153 case V4DF_FTYPE_V4SI_V4DF_QI:
37154 case V8SI_FTYPE_V8SI_V8SI_QI:
37155 case V8SI_FTYPE_V8HI_V8SI_QI:
37156 case V8SI_FTYPE_V16QI_V8SI_QI:
37157 case V8DF_FTYPE_V8DF_V8DI_V8DF:
37158 case V8DF_FTYPE_V8DI_V8DF_V8DF:
37159 case V8DF_FTYPE_V8SF_V8DF_QI:
37160 case V8DF_FTYPE_V8SI_V8DF_QI:
37161 case V8DI_FTYPE_DI_V8DI_QI:
37162 case V16SF_FTYPE_V8SF_V16SF_HI:
37163 case V16SI_FTYPE_V8SI_V16SI_HI:
37164 case V16HI_FTYPE_V16HI_V16HI_HI:
37165 case V8HI_FTYPE_V16QI_V8HI_QI:
37166 case V16HI_FTYPE_V16QI_V16HI_HI:
37167 case V32HI_FTYPE_V32HI_V32HI_SI:
37168 case V32HI_FTYPE_V32QI_V32HI_SI:
37169 case V8DI_FTYPE_V16QI_V8DI_QI:
37170 case V8DI_FTYPE_V2DI_V8DI_QI:
37171 case V8DI_FTYPE_V4DI_V8DI_QI:
37172 case V8DI_FTYPE_V8DI_V8DI_QI:
37173 case V8DI_FTYPE_V8DI_V8DI_V8DI:
37174 case V8DI_FTYPE_V8HI_V8DI_QI:
37175 case V8DI_FTYPE_V8SI_V8DI_QI:
37176 case V8HI_FTYPE_V8DI_V8HI_QI:
37177 case V8SF_FTYPE_V8DF_V8SF_QI:
37178 case V8SI_FTYPE_V8DF_V8SI_QI:
37179 case V8SI_FTYPE_V8DI_V8SI_QI:
37180 case V4SI_FTYPE_V4SI_V4SI_V4SI:
37183 case V32QI_FTYPE_V32QI_V32QI_INT:
37184 case V16HI_FTYPE_V16HI_V16HI_INT:
37185 case V16QI_FTYPE_V16QI_V16QI_INT:
37186 case V4DI_FTYPE_V4DI_V4DI_INT:
37187 case V8HI_FTYPE_V8HI_V8HI_INT:
37188 case V8SI_FTYPE_V8SI_V8SI_INT:
37189 case V8SI_FTYPE_V8SI_V4SI_INT:
37190 case V8SF_FTYPE_V8SF_V8SF_INT:
37191 case V8SF_FTYPE_V8SF_V4SF_INT:
37192 case V4SI_FTYPE_V4SI_V4SI_INT:
37193 case V4DF_FTYPE_V4DF_V4DF_INT:
37194 case V16SF_FTYPE_V16SF_V16SF_INT:
37195 case V16SF_FTYPE_V16SF_V4SF_INT:
37196 case V16SI_FTYPE_V16SI_V4SI_INT:
37197 case V4DF_FTYPE_V4DF_V2DF_INT:
37198 case V4SF_FTYPE_V4SF_V4SF_INT:
37199 case V2DI_FTYPE_V2DI_V2DI_INT:
37200 case V4DI_FTYPE_V4DI_V2DI_INT:
37201 case V2DF_FTYPE_V2DF_V2DF_INT:
37202 case QI_FTYPE_V8DI_V8DI_INT:
37203 case QI_FTYPE_V8DF_V8DF_INT:
37204 case QI_FTYPE_V2DF_V2DF_INT:
37205 case QI_FTYPE_V4SF_V4SF_INT:
37206 case HI_FTYPE_V16SI_V16SI_INT:
37207 case HI_FTYPE_V16SF_V16SF_INT:
37209 nargs_constant = 1;
37211 case V4DI_FTYPE_V4DI_V4DI_INT_CONVERT:
37214 nargs_constant = 1;
37216 case V2DI_FTYPE_V2DI_V2DI_INT_CONVERT:
37219 nargs_constant = 1;
37221 case V1DI_FTYPE_V1DI_V1DI_INT_CONVERT:
37224 nargs_constant = 1;
37226 case V2DI_FTYPE_V2DI_UINT_UINT:
37228 nargs_constant = 2;
37230 case V8DI_FTYPE_V8DI_V8DI_INT_CONVERT:
37233 nargs_constant = 1;
37235 case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_DI_CONVERT:
37239 nargs_constant = 1;
37241 case QI_FTYPE_V8DF_INT_QI:
37242 case QI_FTYPE_V4DF_INT_QI:
37243 case QI_FTYPE_V2DF_INT_QI:
37244 case HI_FTYPE_V16SF_INT_HI:
37245 case QI_FTYPE_V8SF_INT_QI:
37246 case QI_FTYPE_V4SF_INT_QI:
37249 nargs_constant = 1;
37251 case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_SI_CONVERT:
37255 nargs_constant = 1;
37257 case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_HI_CONVERT:
37261 nargs_constant = 1;
37263 case V32QI_FTYPE_V32QI_V32QI_V32QI_SI:
37264 case V32HI_FTYPE_V32HI_V32HI_V32HI_SI:
37265 case V32HI_FTYPE_V64QI_V64QI_V32HI_SI:
37266 case V16SI_FTYPE_V32HI_V32HI_V16SI_HI:
37267 case V64QI_FTYPE_V64QI_V64QI_V64QI_DI:
37268 case V32HI_FTYPE_V32HI_V8HI_V32HI_SI:
37269 case V16HI_FTYPE_V16HI_V8HI_V16HI_HI:
37270 case V8SI_FTYPE_V8SI_V4SI_V8SI_QI:
37271 case V4DI_FTYPE_V4DI_V2DI_V4DI_QI:
37272 case V64QI_FTYPE_V32HI_V32HI_V64QI_DI:
37273 case V32QI_FTYPE_V16HI_V16HI_V32QI_SI:
37274 case V16QI_FTYPE_V8HI_V8HI_V16QI_HI:
37275 case V32HI_FTYPE_V16SI_V16SI_V32HI_SI:
37276 case V16HI_FTYPE_V8SI_V8SI_V16HI_HI:
37277 case V8HI_FTYPE_V4SI_V4SI_V8HI_QI:
37278 case V4DF_FTYPE_V4DF_V4DI_V4DF_QI:
37279 case V8SF_FTYPE_V8SF_V8SI_V8SF_QI:
37280 case V4SF_FTYPE_V4SF_V4SI_V4SF_QI:
37281 case V2DF_FTYPE_V2DF_V2DI_V2DF_QI:
37282 case V2DI_FTYPE_V4SI_V4SI_V2DI_QI:
37283 case V4DI_FTYPE_V8SI_V8SI_V4DI_QI:
37284 case V4DF_FTYPE_V4DI_V4DF_V4DF_QI:
37285 case V8SF_FTYPE_V8SI_V8SF_V8SF_QI:
37286 case V2DF_FTYPE_V2DI_V2DF_V2DF_QI:
37287 case V4SF_FTYPE_V4SI_V4SF_V4SF_QI:
37288 case V16SF_FTYPE_V16SF_V16SF_V16SF_HI:
37289 case V16SF_FTYPE_V16SF_V16SI_V16SF_HI:
37290 case V16SF_FTYPE_V16SI_V16SF_V16SF_HI:
37291 case V16SI_FTYPE_V16SI_V16SI_V16SI_HI:
37292 case V16SI_FTYPE_V16SI_V4SI_V16SI_HI:
37293 case V8HI_FTYPE_V8HI_V8HI_V8HI_QI:
37294 case V8SI_FTYPE_V8SI_V8SI_V8SI_QI:
37295 case V4SI_FTYPE_V4SI_V4SI_V4SI_QI:
37296 case V8SF_FTYPE_V8SF_V8SF_V8SF_QI:
37297 case V16QI_FTYPE_V16QI_V16QI_V16QI_HI:
37298 case V16HI_FTYPE_V16HI_V16HI_V16HI_HI:
37299 case V2DI_FTYPE_V2DI_V2DI_V2DI_QI:
37300 case V2DF_FTYPE_V2DF_V2DF_V2DF_QI:
37301 case V2DF_FTYPE_V2DF_V4SF_V2DF_QI:
37302 case V4DI_FTYPE_V4DI_V4DI_V4DI_QI:
37303 case V4DF_FTYPE_V4DF_V4DF_V4DF_QI:
37304 case V4SF_FTYPE_V4SF_V2DF_V4SF_QI:
37305 case V4SF_FTYPE_V4SF_V4SF_V4SF_QI:
37306 case V8DF_FTYPE_V8DF_V8DF_V8DF_QI:
37307 case V8DF_FTYPE_V8DF_V8DI_V8DF_QI:
37308 case V8DF_FTYPE_V8DI_V8DF_V8DF_QI:
37309 case V8DI_FTYPE_V16SI_V16SI_V8DI_QI:
37310 case V8DI_FTYPE_V8DI_SI_V8DI_V8DI:
37311 case V8DI_FTYPE_V8DI_V2DI_V8DI_QI:
37312 case V8DI_FTYPE_V8DI_V8DI_V8DI_QI:
37313 case V8HI_FTYPE_V16QI_V16QI_V8HI_QI:
37314 case V16HI_FTYPE_V32QI_V32QI_V16HI_HI:
37315 case V8SI_FTYPE_V16HI_V16HI_V8SI_QI:
37316 case V4SI_FTYPE_V8HI_V8HI_V4SI_QI:
37319 case V2DF_FTYPE_V2DF_V2DF_V2DI_INT:
37320 case V4DF_FTYPE_V4DF_V4DF_V4DI_INT:
37321 case V4SF_FTYPE_V4SF_V4SF_V4SI_INT:
37322 case V8SF_FTYPE_V8SF_V8SF_V8SI_INT:
37323 case V16SF_FTYPE_V16SF_V16SF_V16SI_INT:
37325 nargs_constant = 1;
37327 case QI_FTYPE_V4DI_V4DI_INT_QI:
37328 case QI_FTYPE_V8SI_V8SI_INT_QI:
37329 case QI_FTYPE_V4DF_V4DF_INT_QI:
37330 case QI_FTYPE_V8SF_V8SF_INT_QI:
37331 case QI_FTYPE_V2DI_V2DI_INT_QI:
37332 case QI_FTYPE_V4SI_V4SI_INT_QI:
37333 case QI_FTYPE_V2DF_V2DF_INT_QI:
37334 case QI_FTYPE_V4SF_V4SF_INT_QI:
37335 case DI_FTYPE_V64QI_V64QI_INT_DI:
37336 case SI_FTYPE_V32QI_V32QI_INT_SI:
37337 case HI_FTYPE_V16QI_V16QI_INT_HI:
37338 case SI_FTYPE_V32HI_V32HI_INT_SI:
37339 case HI_FTYPE_V16HI_V16HI_INT_HI:
37340 case QI_FTYPE_V8HI_V8HI_INT_QI:
37343 nargs_constant = 1;
37345 case V2DI_FTYPE_V2DI_V2DI_UINT_UINT:
37347 nargs_constant = 2;
37349 case UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED:
37350 case UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG:
37353 case QI_FTYPE_V8DI_V8DI_INT_QI:
37354 case HI_FTYPE_V16SI_V16SI_INT_HI:
37355 case QI_FTYPE_V8DF_V8DF_INT_QI:
37356 case HI_FTYPE_V16SF_V16SF_INT_HI:
37359 nargs_constant = 1;
37361 case V8SF_FTYPE_V8SF_INT_V8SF_QI:
37362 case V4SF_FTYPE_V4SF_INT_V4SF_QI:
37363 case V2DF_FTYPE_V4DF_INT_V2DF_QI:
37364 case V2DI_FTYPE_V4DI_INT_V2DI_QI:
37365 case V8SF_FTYPE_V16SF_INT_V8SF_QI:
37366 case V8SI_FTYPE_V16SI_INT_V8SI_QI:
37367 case V2DF_FTYPE_V8DF_INT_V2DF_QI:
37368 case V2DI_FTYPE_V8DI_INT_V2DI_QI:
37369 case V4SF_FTYPE_V8SF_INT_V4SF_QI:
37370 case V4SI_FTYPE_V8SI_INT_V4SI_QI:
37371 case V8HI_FTYPE_V8SF_INT_V8HI_QI:
37372 case V8HI_FTYPE_V4SF_INT_V8HI_QI:
37373 case V32HI_FTYPE_V32HI_INT_V32HI_SI:
37374 case V16HI_FTYPE_V16HI_INT_V16HI_HI:
37375 case V8HI_FTYPE_V8HI_INT_V8HI_QI:
37376 case V4DI_FTYPE_V4DI_INT_V4DI_QI:
37377 case V2DI_FTYPE_V2DI_INT_V2DI_QI:
37378 case V8SI_FTYPE_V8SI_INT_V8SI_QI:
37379 case V4SI_FTYPE_V4SI_INT_V4SI_QI:
37380 case V4DF_FTYPE_V4DF_INT_V4DF_QI:
37381 case V2DF_FTYPE_V2DF_INT_V2DF_QI:
37382 case V8DF_FTYPE_V8DF_INT_V8DF_QI:
37383 case V16SF_FTYPE_V16SF_INT_V16SF_HI:
37384 case V16HI_FTYPE_V16SF_INT_V16HI_HI:
37385 case V16SI_FTYPE_V16SI_INT_V16SI_HI:
37386 case V4SI_FTYPE_V16SI_INT_V4SI_QI:
37387 case V4DI_FTYPE_V8DI_INT_V4DI_QI:
37388 case V4DF_FTYPE_V8DF_INT_V4DF_QI:
37389 case V4SF_FTYPE_V16SF_INT_V4SF_QI:
37390 case V8DI_FTYPE_V8DI_INT_V8DI_QI:
37393 nargs_constant = 1;
37395 case V16SF_FTYPE_V16SF_V4SF_INT_V16SF_HI:
37396 case V16SI_FTYPE_V16SI_V4SI_INT_V16SI_HI:
37397 case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI:
37398 case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_QI:
37399 case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI:
37400 case V16SI_FTYPE_V16SI_V16SI_INT_V16SI_HI:
37401 case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI:
37402 case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI:
37403 case V8DF_FTYPE_V8DF_V4DF_INT_V8DF_QI:
37404 case V8DI_FTYPE_V8DI_V4DI_INT_V8DI_QI:
37405 case V4DF_FTYPE_V4DF_V4DF_INT_V4DF_QI:
37406 case V8SF_FTYPE_V8SF_V8SF_INT_V8SF_QI:
37407 case V8DF_FTYPE_V8DF_V2DF_INT_V8DF_QI:
37408 case V8DI_FTYPE_V8DI_V2DI_INT_V8DI_QI:
37409 case V8SI_FTYPE_V8SI_V8SI_INT_V8SI_QI:
37410 case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_QI:
37411 case V4SI_FTYPE_V4SI_V4SI_INT_V4SI_QI:
37412 case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_QI:
37413 case V32HI_FTYPE_V64QI_V64QI_INT_V32HI_SI:
37414 case V16HI_FTYPE_V32QI_V32QI_INT_V16HI_HI:
37415 case V8HI_FTYPE_V16QI_V16QI_INT_V8HI_QI:
37416 case V16SF_FTYPE_V16SF_V8SF_INT_V16SF_HI:
37417 case V16SI_FTYPE_V16SI_V8SI_INT_V16SI_HI:
37418 case V8SF_FTYPE_V8SF_V4SF_INT_V8SF_QI:
37419 case V8SI_FTYPE_V8SI_V4SI_INT_V8SI_QI:
37420 case V4DI_FTYPE_V4DI_V2DI_INT_V4DI_QI:
37421 case V4DF_FTYPE_V4DF_V2DF_INT_V4DF_QI:
37424 nargs_constant = 1;
37426 case V8DI_FTYPE_V8DI_V8DI_V8DI_INT_QI:
37427 case V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI:
37428 case V16SI_FTYPE_V16SI_V16SI_V16SI_INT_HI:
37429 case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI:
37430 case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI:
37431 case V8SF_FTYPE_V8SF_V8SF_V8SI_INT_QI:
37432 case V8SI_FTYPE_V8SI_V8SI_V8SI_INT_QI:
37433 case V4DF_FTYPE_V4DF_V4DF_V4DI_INT_QI:
37434 case V4DI_FTYPE_V4DI_V4DI_V4DI_INT_QI:
37435 case V4SI_FTYPE_V4SI_V4SI_V4SI_INT_QI:
37436 case V2DI_FTYPE_V2DI_V2DI_V2DI_INT_QI:
37440 nargs_constant = 1;
37444 gcc_unreachable ();
37447 gcc_assert (nargs <= ARRAY_SIZE (args));
37449 if (comparison != UNKNOWN)
37451 gcc_assert (nargs == 2);
37452 return ix86_expand_sse_compare (d, exp, target, swap);
37455 if (rmode == VOIDmode || rmode == tmode)
37459 || GET_MODE (target) != tmode
37460 || !insn_p->operand[0].predicate (target, tmode))
37461 target = gen_reg_rtx (tmode);
37462 real_target = target;
37466 real_target = gen_reg_rtx (tmode);
37467 target = simplify_gen_subreg (rmode, real_target, tmode, 0);
37470 for (i = 0; i < nargs; i++)
37472 tree arg = CALL_EXPR_ARG (exp, i);
37473 rtx op = expand_normal (arg);
37474 machine_mode mode = insn_p->operand[i + 1].mode;
37475 bool match = insn_p->operand[i + 1].predicate (op, mode);
37477 if (last_arg_count && (i + 1) == nargs)
37479 /* SIMD shift insns take either an 8-bit immediate or
37480 register as count. But builtin functions take int as
37481 count. If count doesn't match, we put it in register. */
37484 op = simplify_gen_subreg (SImode, op, GET_MODE (op), 0);
37485 if (!insn_p->operand[i + 1].predicate (op, mode))
37486 op = copy_to_reg (op);
37489 else if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
37490 (!mask_pos && (nargs - i) <= nargs_constant))
37495 case CODE_FOR_avx_vinsertf128v4di:
37496 case CODE_FOR_avx_vextractf128v4di:
37497 error ("the last argument must be an 1-bit immediate");
37500 case CODE_FOR_avx512f_cmpv8di3_mask:
37501 case CODE_FOR_avx512f_cmpv16si3_mask:
37502 case CODE_FOR_avx512f_ucmpv8di3_mask:
37503 case CODE_FOR_avx512f_ucmpv16si3_mask:
37504 case CODE_FOR_avx512vl_cmpv4di3_mask:
37505 case CODE_FOR_avx512vl_cmpv8si3_mask:
37506 case CODE_FOR_avx512vl_ucmpv4di3_mask:
37507 case CODE_FOR_avx512vl_ucmpv8si3_mask:
37508 case CODE_FOR_avx512vl_cmpv2di3_mask:
37509 case CODE_FOR_avx512vl_cmpv4si3_mask:
37510 case CODE_FOR_avx512vl_ucmpv2di3_mask:
37511 case CODE_FOR_avx512vl_ucmpv4si3_mask:
37512 error ("the last argument must be a 3-bit immediate");
37515 case CODE_FOR_sse4_1_roundsd:
37516 case CODE_FOR_sse4_1_roundss:
37518 case CODE_FOR_sse4_1_roundpd:
37519 case CODE_FOR_sse4_1_roundps:
37520 case CODE_FOR_avx_roundpd256:
37521 case CODE_FOR_avx_roundps256:
37523 case CODE_FOR_sse4_1_roundpd_vec_pack_sfix:
37524 case CODE_FOR_sse4_1_roundps_sfix:
37525 case CODE_FOR_avx_roundpd_vec_pack_sfix256:
37526 case CODE_FOR_avx_roundps_sfix256:
37528 case CODE_FOR_sse4_1_blendps:
37529 case CODE_FOR_avx_blendpd256:
37530 case CODE_FOR_avx_vpermilv4df:
37531 case CODE_FOR_avx_vpermilv4df_mask:
37532 case CODE_FOR_avx512f_getmantv8df_mask:
37533 case CODE_FOR_avx512f_getmantv16sf_mask:
37534 case CODE_FOR_avx512vl_getmantv8sf_mask:
37535 case CODE_FOR_avx512vl_getmantv4df_mask:
37536 case CODE_FOR_avx512vl_getmantv4sf_mask:
37537 case CODE_FOR_avx512vl_getmantv2df_mask:
37538 case CODE_FOR_avx512dq_rangepv8df_mask_round:
37539 case CODE_FOR_avx512dq_rangepv16sf_mask_round:
37540 case CODE_FOR_avx512dq_rangepv4df_mask:
37541 case CODE_FOR_avx512dq_rangepv8sf_mask:
37542 case CODE_FOR_avx512dq_rangepv2df_mask:
37543 case CODE_FOR_avx512dq_rangepv4sf_mask:
37544 case CODE_FOR_avx_shufpd256_mask:
37545 error ("the last argument must be a 4-bit immediate");
37548 case CODE_FOR_sha1rnds4:
37549 case CODE_FOR_sse4_1_blendpd:
37550 case CODE_FOR_avx_vpermilv2df:
37551 case CODE_FOR_avx_vpermilv2df_mask:
37552 case CODE_FOR_xop_vpermil2v2df3:
37553 case CODE_FOR_xop_vpermil2v4sf3:
37554 case CODE_FOR_xop_vpermil2v4df3:
37555 case CODE_FOR_xop_vpermil2v8sf3:
37556 case CODE_FOR_avx512f_vinsertf32x4_mask:
37557 case CODE_FOR_avx512f_vinserti32x4_mask:
37558 case CODE_FOR_avx512f_vextractf32x4_mask:
37559 case CODE_FOR_avx512f_vextracti32x4_mask:
37560 case CODE_FOR_sse2_shufpd:
37561 case CODE_FOR_sse2_shufpd_mask:
37562 case CODE_FOR_avx512dq_shuf_f64x2_mask:
37563 case CODE_FOR_avx512dq_shuf_i64x2_mask:
37564 case CODE_FOR_avx512vl_shuf_i32x4_mask:
37565 case CODE_FOR_avx512vl_shuf_f32x4_mask:
37566 error ("the last argument must be a 2-bit immediate");
37569 case CODE_FOR_avx_vextractf128v4df:
37570 case CODE_FOR_avx_vextractf128v8sf:
37571 case CODE_FOR_avx_vextractf128v8si:
37572 case CODE_FOR_avx_vinsertf128v4df:
37573 case CODE_FOR_avx_vinsertf128v8sf:
37574 case CODE_FOR_avx_vinsertf128v8si:
37575 case CODE_FOR_avx512f_vinsertf64x4_mask:
37576 case CODE_FOR_avx512f_vinserti64x4_mask:
37577 case CODE_FOR_avx512f_vextractf64x4_mask:
37578 case CODE_FOR_avx512f_vextracti64x4_mask:
37579 case CODE_FOR_avx512dq_vinsertf32x8_mask:
37580 case CODE_FOR_avx512dq_vinserti32x8_mask:
37581 case CODE_FOR_avx512vl_vinsertv4df:
37582 case CODE_FOR_avx512vl_vinsertv4di:
37583 case CODE_FOR_avx512vl_vinsertv8sf:
37584 case CODE_FOR_avx512vl_vinsertv8si:
37585 error ("the last argument must be a 1-bit immediate");
37588 case CODE_FOR_avx_vmcmpv2df3:
37589 case CODE_FOR_avx_vmcmpv4sf3:
37590 case CODE_FOR_avx_cmpv2df3:
37591 case CODE_FOR_avx_cmpv4sf3:
37592 case CODE_FOR_avx_cmpv4df3:
37593 case CODE_FOR_avx_cmpv8sf3:
37594 case CODE_FOR_avx512f_cmpv8df3_mask:
37595 case CODE_FOR_avx512f_cmpv16sf3_mask:
37596 case CODE_FOR_avx512f_vmcmpv2df3_mask:
37597 case CODE_FOR_avx512f_vmcmpv4sf3_mask:
37598 error ("the last argument must be a 5-bit immediate");
37602 switch (nargs_constant)
37605 if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
37606 (!mask_pos && (nargs - i) == nargs_constant))
37608 error ("the next to last argument must be an 8-bit immediate");
37612 error ("the last argument must be an 8-bit immediate");
37615 gcc_unreachable ();
37622 if (VECTOR_MODE_P (mode))
37623 op = safe_vector_operand (op, mode);
37625 /* If we aren't optimizing, only allow one memory operand to
37627 if (memory_operand (op, mode))
37630 op = fixup_modeless_constant (op, mode);
37632 if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
37634 if (optimize || !match || num_memory > 1)
37635 op = copy_to_mode_reg (mode, op);
37639 op = copy_to_reg (op);
37640 op = simplify_gen_subreg (mode, op, GET_MODE (op), 0);
37645 args[i].mode = mode;
37651 pat = GEN_FCN (icode) (real_target, args[0].op);
37654 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op);
37657 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
37661 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
37662 args[2].op, args[3].op);
37665 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
37666 args[2].op, args[3].op, args[4].op);
37668 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
37669 args[2].op, args[3].op, args[4].op,
37673 gcc_unreachable ();
37683 /* Transform pattern of following layout:
37686 (unspec [C] UNSPEC_EMBEDDED_ROUNDING)])
37694 (unspec [C] UNSPEC_EMBEDDED_ROUNDING)
37698 (parallel [ A B ... ]) */
37701 ix86_erase_embedded_rounding (rtx pat)
37703 if (GET_CODE (pat) == INSN)
37704 pat = PATTERN (pat);
37706 gcc_assert (GET_CODE (pat) == PARALLEL);
37708 if (XVECLEN (pat, 0) == 2)
37710 rtx p0 = XVECEXP (pat, 0, 0);
37711 rtx p1 = XVECEXP (pat, 0, 1);
37713 gcc_assert (GET_CODE (p0) == SET
37714 && GET_CODE (p1) == UNSPEC
37715 && XINT (p1, 1) == UNSPEC_EMBEDDED_ROUNDING);
37721 rtx *res = XALLOCAVEC (rtx, XVECLEN (pat, 0));
37725 for (; i < XVECLEN (pat, 0); ++i)
37727 rtx elem = XVECEXP (pat, 0, i);
37728 if (GET_CODE (elem) != UNSPEC
37729 || XINT (elem, 1) != UNSPEC_EMBEDDED_ROUNDING)
37733 /* No more than 1 occurence was removed. */
37734 gcc_assert (j >= XVECLEN (pat, 0) - 1);
37736 return gen_rtx_PARALLEL (GET_MODE (pat), gen_rtvec_v (j, res));
37740 /* Subroutine of ix86_expand_round_builtin to take care of comi insns
37743 ix86_expand_sse_comi_round (const struct builtin_description *d,
37744 tree exp, rtx target)
37747 tree arg0 = CALL_EXPR_ARG (exp, 0);
37748 tree arg1 = CALL_EXPR_ARG (exp, 1);
37749 tree arg2 = CALL_EXPR_ARG (exp, 2);
37750 tree arg3 = CALL_EXPR_ARG (exp, 3);
37751 rtx op0 = expand_normal (arg0);
37752 rtx op1 = expand_normal (arg1);
37753 rtx op2 = expand_normal (arg2);
37754 rtx op3 = expand_normal (arg3);
37755 enum insn_code icode = d->icode;
37756 const struct insn_data_d *insn_p = &insn_data[icode];
37757 machine_mode mode0 = insn_p->operand[0].mode;
37758 machine_mode mode1 = insn_p->operand[1].mode;
37759 enum rtx_code comparison = UNEQ;
37760 bool need_ucomi = false;
37762 /* See avxintrin.h for values. */
37763 enum rtx_code comi_comparisons[32] =
37765 UNEQ, GT, GE, UNORDERED, LTGT, UNLE, UNLT, ORDERED, UNEQ, UNLT,
37766 UNLE, LT, LTGT, GE, GT, LT, UNEQ, GT, GE, UNORDERED, LTGT, UNLE,
37767 UNLT, ORDERED, UNEQ, UNLT, UNLE, LT, LTGT, GE, GT, LT
37769 bool need_ucomi_values[32] =
37771 true, false, false, true, true, false, false, true,
37772 true, false, false, true, true, false, false, true,
37773 false, true, true, false, false, true, true, false,
37774 false, true, true, false, false, true, true, false
37777 if (!CONST_INT_P (op2))
37779 error ("the third argument must be comparison constant");
37782 if (INTVAL (op2) < 0 || INTVAL (op2) >= 32)
37784 error ("incorrect comparison mode");
37788 if (!insn_p->operand[2].predicate (op3, SImode))
37790 error ("incorrect rounding operand");
37794 comparison = comi_comparisons[INTVAL (op2)];
37795 need_ucomi = need_ucomi_values[INTVAL (op2)];
37797 if (VECTOR_MODE_P (mode0))
37798 op0 = safe_vector_operand (op0, mode0);
37799 if (VECTOR_MODE_P (mode1))
37800 op1 = safe_vector_operand (op1, mode1);
37802 target = gen_reg_rtx (SImode);
37803 emit_move_insn (target, const0_rtx);
37804 target = gen_rtx_SUBREG (QImode, target, 0);
37806 if ((optimize && !register_operand (op0, mode0))
37807 || !insn_p->operand[0].predicate (op0, mode0))
37808 op0 = copy_to_mode_reg (mode0, op0);
37809 if ((optimize && !register_operand (op1, mode1))
37810 || !insn_p->operand[1].predicate (op1, mode1))
37811 op1 = copy_to_mode_reg (mode1, op1);
37814 icode = icode == CODE_FOR_sse_comi_round
37815 ? CODE_FOR_sse_ucomi_round
37816 : CODE_FOR_sse2_ucomi_round;
37818 pat = GEN_FCN (icode) (op0, op1, op3);
37822 /* Rounding operand can be either NO_ROUND or ROUND_SAE at this point. */
37823 if (INTVAL (op3) == NO_ROUND)
37825 pat = ix86_erase_embedded_rounding (pat);
37829 set_dst = SET_DEST (pat);
37833 gcc_assert (GET_CODE (XVECEXP (pat, 0, 0)) == SET);
37834 set_dst = SET_DEST (XVECEXP (pat, 0, 0));
37838 emit_insn (gen_rtx_SET (VOIDmode,
37839 gen_rtx_STRICT_LOW_PART (VOIDmode, target),
37840 gen_rtx_fmt_ee (comparison, QImode,
37844 return SUBREG_REG (target);
37848 ix86_expand_round_builtin (const struct builtin_description *d,
37849 tree exp, rtx target)
37852 unsigned int i, nargs;
37858 enum insn_code icode = d->icode;
37859 const struct insn_data_d *insn_p = &insn_data[icode];
37860 machine_mode tmode = insn_p->operand[0].mode;
37861 unsigned int nargs_constant = 0;
37862 unsigned int redundant_embed_rnd = 0;
37864 switch ((enum ix86_builtin_func_type) d->flag)
37866 case UINT64_FTYPE_V2DF_INT:
37867 case UINT64_FTYPE_V4SF_INT:
37868 case UINT_FTYPE_V2DF_INT:
37869 case UINT_FTYPE_V4SF_INT:
37870 case INT64_FTYPE_V2DF_INT:
37871 case INT64_FTYPE_V4SF_INT:
37872 case INT_FTYPE_V2DF_INT:
37873 case INT_FTYPE_V4SF_INT:
37876 case V4SF_FTYPE_V4SF_UINT_INT:
37877 case V4SF_FTYPE_V4SF_UINT64_INT:
37878 case V2DF_FTYPE_V2DF_UINT64_INT:
37879 case V4SF_FTYPE_V4SF_INT_INT:
37880 case V4SF_FTYPE_V4SF_INT64_INT:
37881 case V2DF_FTYPE_V2DF_INT64_INT:
37882 case V4SF_FTYPE_V4SF_V4SF_INT:
37883 case V2DF_FTYPE_V2DF_V2DF_INT:
37884 case V4SF_FTYPE_V4SF_V2DF_INT:
37885 case V2DF_FTYPE_V2DF_V4SF_INT:
37888 case V8SF_FTYPE_V8DF_V8SF_QI_INT:
37889 case V8DF_FTYPE_V8DF_V8DF_QI_INT:
37890 case V8SI_FTYPE_V8DF_V8SI_QI_INT:
37891 case V8DI_FTYPE_V8DF_V8DI_QI_INT:
37892 case V8SF_FTYPE_V8DI_V8SF_QI_INT:
37893 case V8DF_FTYPE_V8DI_V8DF_QI_INT:
37894 case V16SF_FTYPE_V16SF_V16SF_HI_INT:
37895 case V8DI_FTYPE_V8SF_V8DI_QI_INT:
37896 case V16SF_FTYPE_V16SI_V16SF_HI_INT:
37897 case V16SI_FTYPE_V16SF_V16SI_HI_INT:
37898 case V8DF_FTYPE_V8SF_V8DF_QI_INT:
37899 case V16SF_FTYPE_V16HI_V16SF_HI_INT:
37900 case V2DF_FTYPE_V2DF_V2DF_V2DF_INT:
37901 case V4SF_FTYPE_V4SF_V4SF_V4SF_INT:
37904 case V4SF_FTYPE_V4SF_V4SF_INT_INT:
37905 case V2DF_FTYPE_V2DF_V2DF_INT_INT:
37906 nargs_constant = 2;
37909 case INT_FTYPE_V4SF_V4SF_INT_INT:
37910 case INT_FTYPE_V2DF_V2DF_INT_INT:
37911 return ix86_expand_sse_comi_round (d, exp, target);
37912 case V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT:
37913 case V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT:
37914 case V2DF_FTYPE_V2DF_V2DF_V2DF_QI_INT:
37915 case V2DF_FTYPE_V2DF_V4SF_V2DF_QI_INT:
37916 case V4SF_FTYPE_V4SF_V4SF_V4SF_QI_INT:
37917 case V4SF_FTYPE_V4SF_V2DF_V4SF_QI_INT:
37920 case V16SF_FTYPE_V16SF_INT_V16SF_HI_INT:
37921 case V8DF_FTYPE_V8DF_INT_V8DF_QI_INT:
37922 nargs_constant = 4;
37925 case QI_FTYPE_V8DF_V8DF_INT_QI_INT:
37926 case QI_FTYPE_V2DF_V2DF_INT_QI_INT:
37927 case HI_FTYPE_V16SF_V16SF_INT_HI_INT:
37928 case QI_FTYPE_V4SF_V4SF_INT_QI_INT:
37929 nargs_constant = 3;
37932 case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI_INT:
37933 case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI_INT:
37934 case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI_INT:
37935 case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI_INT:
37937 nargs_constant = 4;
37939 case V8DF_FTYPE_V8DF_V8DF_V8DI_INT_QI_INT:
37940 case V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI_INT:
37941 case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI_INT:
37942 case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI_INT:
37944 nargs_constant = 3;
37947 gcc_unreachable ();
37949 gcc_assert (nargs <= ARRAY_SIZE (args));
37953 || GET_MODE (target) != tmode
37954 || !insn_p->operand[0].predicate (target, tmode))
37955 target = gen_reg_rtx (tmode);
37957 for (i = 0; i < nargs; i++)
37959 tree arg = CALL_EXPR_ARG (exp, i);
37960 rtx op = expand_normal (arg);
37961 machine_mode mode = insn_p->operand[i + 1].mode;
37962 bool match = insn_p->operand[i + 1].predicate (op, mode);
37964 if (i == nargs - nargs_constant)
37970 case CODE_FOR_avx512f_getmantv8df_mask_round:
37971 case CODE_FOR_avx512f_getmantv16sf_mask_round:
37972 case CODE_FOR_avx512f_vgetmantv2df_round:
37973 case CODE_FOR_avx512f_vgetmantv4sf_round:
37974 error ("the immediate argument must be a 4-bit immediate");
37976 case CODE_FOR_avx512f_cmpv8df3_mask_round:
37977 case CODE_FOR_avx512f_cmpv16sf3_mask_round:
37978 case CODE_FOR_avx512f_vmcmpv2df3_mask_round:
37979 case CODE_FOR_avx512f_vmcmpv4sf3_mask_round:
37980 error ("the immediate argument must be a 5-bit immediate");
37983 error ("the immediate argument must be an 8-bit immediate");
37988 else if (i == nargs-1)
37990 if (!insn_p->operand[nargs].predicate (op, SImode))
37992 error ("incorrect rounding operand");
37996 /* If there is no rounding use normal version of the pattern. */
37997 if (INTVAL (op) == NO_ROUND)
37998 redundant_embed_rnd = 1;
38002 if (VECTOR_MODE_P (mode))
38003 op = safe_vector_operand (op, mode);
38005 op = fixup_modeless_constant (op, mode);
38007 if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
38009 if (optimize || !match)
38010 op = copy_to_mode_reg (mode, op);
38014 op = copy_to_reg (op);
38015 op = simplify_gen_subreg (mode, op, GET_MODE (op), 0);
38020 args[i].mode = mode;
38026 pat = GEN_FCN (icode) (target, args[0].op);
38029 pat = GEN_FCN (icode) (target, args[0].op, args[1].op);
38032 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
38036 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
38037 args[2].op, args[3].op);
38040 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
38041 args[2].op, args[3].op, args[4].op);
38043 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
38044 args[2].op, args[3].op, args[4].op,
38048 gcc_unreachable ();
38054 if (redundant_embed_rnd)
38055 pat = ix86_erase_embedded_rounding (pat);
38061 /* Subroutine of ix86_expand_builtin to take care of special insns
38062 with variable number of operands. */
38065 ix86_expand_special_args_builtin (const struct builtin_description *d,
38066 tree exp, rtx target)
38070 unsigned int i, nargs, arg_adjust, memory;
38071 bool aligned_mem = false;
38077 enum insn_code icode = d->icode;
38078 bool last_arg_constant = false;
38079 const struct insn_data_d *insn_p = &insn_data[icode];
38080 machine_mode tmode = insn_p->operand[0].mode;
38081 enum { load, store } klass;
38083 switch ((enum ix86_builtin_func_type) d->flag)
38085 case VOID_FTYPE_VOID:
38086 emit_insn (GEN_FCN (icode) (target));
38088 case VOID_FTYPE_UINT64:
38089 case VOID_FTYPE_UNSIGNED:
38095 case INT_FTYPE_VOID:
38096 case USHORT_FTYPE_VOID:
38097 case UINT64_FTYPE_VOID:
38098 case UNSIGNED_FTYPE_VOID:
38103 case UINT64_FTYPE_PUNSIGNED:
38104 case V2DI_FTYPE_PV2DI:
38105 case V4DI_FTYPE_PV4DI:
38106 case V32QI_FTYPE_PCCHAR:
38107 case V16QI_FTYPE_PCCHAR:
38108 case V8SF_FTYPE_PCV4SF:
38109 case V8SF_FTYPE_PCFLOAT:
38110 case V4SF_FTYPE_PCFLOAT:
38111 case V4DF_FTYPE_PCV2DF:
38112 case V4DF_FTYPE_PCDOUBLE:
38113 case V2DF_FTYPE_PCDOUBLE:
38114 case VOID_FTYPE_PVOID:
38115 case V16SI_FTYPE_PV4SI:
38116 case V16SF_FTYPE_PV4SF:
38117 case V8DI_FTYPE_PV4DI:
38118 case V8DI_FTYPE_PV8DI:
38119 case V8DF_FTYPE_PV4DF:
38125 case CODE_FOR_sse4_1_movntdqa:
38126 case CODE_FOR_avx2_movntdqa:
38127 case CODE_FOR_avx512f_movntdqa:
38128 aligned_mem = true;
38134 case VOID_FTYPE_PV2SF_V4SF:
38135 case VOID_FTYPE_PV8DI_V8DI:
38136 case VOID_FTYPE_PV4DI_V4DI:
38137 case VOID_FTYPE_PV2DI_V2DI:
38138 case VOID_FTYPE_PCHAR_V32QI:
38139 case VOID_FTYPE_PCHAR_V16QI:
38140 case VOID_FTYPE_PFLOAT_V16SF:
38141 case VOID_FTYPE_PFLOAT_V8SF:
38142 case VOID_FTYPE_PFLOAT_V4SF:
38143 case VOID_FTYPE_PDOUBLE_V8DF:
38144 case VOID_FTYPE_PDOUBLE_V4DF:
38145 case VOID_FTYPE_PDOUBLE_V2DF:
38146 case VOID_FTYPE_PLONGLONG_LONGLONG:
38147 case VOID_FTYPE_PULONGLONG_ULONGLONG:
38148 case VOID_FTYPE_PINT_INT:
38151 /* Reserve memory operand for target. */
38152 memory = ARRAY_SIZE (args);
38155 /* These builtins and instructions require the memory
38156 to be properly aligned. */
38157 case CODE_FOR_avx_movntv4di:
38158 case CODE_FOR_sse2_movntv2di:
38159 case CODE_FOR_avx_movntv8sf:
38160 case CODE_FOR_sse_movntv4sf:
38161 case CODE_FOR_sse4a_vmmovntv4sf:
38162 case CODE_FOR_avx_movntv4df:
38163 case CODE_FOR_sse2_movntv2df:
38164 case CODE_FOR_sse4a_vmmovntv2df:
38165 case CODE_FOR_sse2_movntidi:
38166 case CODE_FOR_sse_movntq:
38167 case CODE_FOR_sse2_movntisi:
38168 case CODE_FOR_avx512f_movntv16sf:
38169 case CODE_FOR_avx512f_movntv8df:
38170 case CODE_FOR_avx512f_movntv8di:
38171 aligned_mem = true;
38177 case V4SF_FTYPE_V4SF_PCV2SF:
38178 case V2DF_FTYPE_V2DF_PCDOUBLE:
38183 case V8SF_FTYPE_PCV8SF_V8SI:
38184 case V4DF_FTYPE_PCV4DF_V4DI:
38185 case V4SF_FTYPE_PCV4SF_V4SI:
38186 case V2DF_FTYPE_PCV2DF_V2DI:
38187 case V8SI_FTYPE_PCV8SI_V8SI:
38188 case V4DI_FTYPE_PCV4DI_V4DI:
38189 case V4SI_FTYPE_PCV4SI_V4SI:
38190 case V2DI_FTYPE_PCV2DI_V2DI:
38195 case VOID_FTYPE_PV8DF_V8DF_QI:
38196 case VOID_FTYPE_PV16SF_V16SF_HI:
38197 case VOID_FTYPE_PV8DI_V8DI_QI:
38198 case VOID_FTYPE_PV4DI_V4DI_QI:
38199 case VOID_FTYPE_PV2DI_V2DI_QI:
38200 case VOID_FTYPE_PV16SI_V16SI_HI:
38201 case VOID_FTYPE_PV8SI_V8SI_QI:
38202 case VOID_FTYPE_PV4SI_V4SI_QI:
38205 /* These builtins and instructions require the memory
38206 to be properly aligned. */
38207 case CODE_FOR_avx512f_storev16sf_mask:
38208 case CODE_FOR_avx512f_storev16si_mask:
38209 case CODE_FOR_avx512f_storev8df_mask:
38210 case CODE_FOR_avx512f_storev8di_mask:
38211 case CODE_FOR_avx512vl_storev8sf_mask:
38212 case CODE_FOR_avx512vl_storev8si_mask:
38213 case CODE_FOR_avx512vl_storev4df_mask:
38214 case CODE_FOR_avx512vl_storev4di_mask:
38215 case CODE_FOR_avx512vl_storev4sf_mask:
38216 case CODE_FOR_avx512vl_storev4si_mask:
38217 case CODE_FOR_avx512vl_storev2df_mask:
38218 case CODE_FOR_avx512vl_storev2di_mask:
38219 aligned_mem = true;
38225 case VOID_FTYPE_PV8SF_V8SI_V8SF:
38226 case VOID_FTYPE_PV4DF_V4DI_V4DF:
38227 case VOID_FTYPE_PV4SF_V4SI_V4SF:
38228 case VOID_FTYPE_PV2DF_V2DI_V2DF:
38229 case VOID_FTYPE_PV8SI_V8SI_V8SI:
38230 case VOID_FTYPE_PV4DI_V4DI_V4DI:
38231 case VOID_FTYPE_PV4SI_V4SI_V4SI:
38232 case VOID_FTYPE_PV2DI_V2DI_V2DI:
38233 case VOID_FTYPE_PDOUBLE_V2DF_QI:
38234 case VOID_FTYPE_PFLOAT_V4SF_QI:
38235 case VOID_FTYPE_PV8SI_V8DI_QI:
38236 case VOID_FTYPE_PV8HI_V8DI_QI:
38237 case VOID_FTYPE_PV16HI_V16SI_HI:
38238 case VOID_FTYPE_PV16QI_V8DI_QI:
38239 case VOID_FTYPE_PV16QI_V16SI_HI:
38240 case VOID_FTYPE_PV4SI_V4DI_QI:
38241 case VOID_FTYPE_PV4SI_V2DI_QI:
38242 case VOID_FTYPE_PV8HI_V4DI_QI:
38243 case VOID_FTYPE_PV8HI_V2DI_QI:
38244 case VOID_FTYPE_PV8HI_V8SI_QI:
38245 case VOID_FTYPE_PV8HI_V4SI_QI:
38246 case VOID_FTYPE_PV16QI_V4DI_QI:
38247 case VOID_FTYPE_PV16QI_V2DI_QI:
38248 case VOID_FTYPE_PV16QI_V8SI_QI:
38249 case VOID_FTYPE_PV16QI_V4SI_QI:
38250 case VOID_FTYPE_PV8HI_V8HI_QI:
38251 case VOID_FTYPE_PV16HI_V16HI_HI:
38252 case VOID_FTYPE_PV32HI_V32HI_SI:
38253 case VOID_FTYPE_PV16QI_V16QI_HI:
38254 case VOID_FTYPE_PV32QI_V32QI_SI:
38255 case VOID_FTYPE_PV64QI_V64QI_DI:
38256 case VOID_FTYPE_PV4DF_V4DF_QI:
38257 case VOID_FTYPE_PV2DF_V2DF_QI:
38258 case VOID_FTYPE_PV8SF_V8SF_QI:
38259 case VOID_FTYPE_PV4SF_V4SF_QI:
38262 /* Reserve memory operand for target. */
38263 memory = ARRAY_SIZE (args);
38265 case V4SF_FTYPE_PCV4SF_V4SF_QI:
38266 case V8SF_FTYPE_PCV8SF_V8SF_QI:
38267 case V16SF_FTYPE_PCV16SF_V16SF_HI:
38268 case V4SI_FTYPE_PCV4SI_V4SI_QI:
38269 case V8SI_FTYPE_PCV8SI_V8SI_QI:
38270 case V16SI_FTYPE_PCV16SI_V16SI_HI:
38271 case V2DF_FTYPE_PCV2DF_V2DF_QI:
38272 case V4DF_FTYPE_PCV4DF_V4DF_QI:
38273 case V8DF_FTYPE_PCV8DF_V8DF_QI:
38274 case V2DI_FTYPE_PCV2DI_V2DI_QI:
38275 case V4DI_FTYPE_PCV4DI_V4DI_QI:
38276 case V8DI_FTYPE_PCV8DI_V8DI_QI:
38277 case V2DF_FTYPE_PCDOUBLE_V2DF_QI:
38278 case V4SF_FTYPE_PCFLOAT_V4SF_QI:
38279 case V8HI_FTYPE_PCV8HI_V8HI_QI:
38280 case V16HI_FTYPE_PCV16HI_V16HI_HI:
38281 case V32HI_FTYPE_PCV32HI_V32HI_SI:
38282 case V16QI_FTYPE_PCV16QI_V16QI_HI:
38283 case V32QI_FTYPE_PCV32QI_V32QI_SI:
38284 case V64QI_FTYPE_PCV64QI_V64QI_DI:
38290 /* These builtins and instructions require the memory
38291 to be properly aligned. */
38292 case CODE_FOR_avx512f_loadv16sf_mask:
38293 case CODE_FOR_avx512f_loadv16si_mask:
38294 case CODE_FOR_avx512f_loadv8df_mask:
38295 case CODE_FOR_avx512f_loadv8di_mask:
38296 case CODE_FOR_avx512vl_loadv8sf_mask:
38297 case CODE_FOR_avx512vl_loadv8si_mask:
38298 case CODE_FOR_avx512vl_loadv4df_mask:
38299 case CODE_FOR_avx512vl_loadv4di_mask:
38300 case CODE_FOR_avx512vl_loadv4sf_mask:
38301 case CODE_FOR_avx512vl_loadv4si_mask:
38302 case CODE_FOR_avx512vl_loadv2df_mask:
38303 case CODE_FOR_avx512vl_loadv2di_mask:
38304 case CODE_FOR_avx512bw_loadv64qi_mask:
38305 case CODE_FOR_avx512vl_loadv32qi_mask:
38306 case CODE_FOR_avx512vl_loadv16qi_mask:
38307 case CODE_FOR_avx512bw_loadv32hi_mask:
38308 case CODE_FOR_avx512vl_loadv16hi_mask:
38309 case CODE_FOR_avx512vl_loadv8hi_mask:
38310 aligned_mem = true;
38316 case VOID_FTYPE_UINT_UINT_UINT:
38317 case VOID_FTYPE_UINT64_UINT_UINT:
38318 case UCHAR_FTYPE_UINT_UINT_UINT:
38319 case UCHAR_FTYPE_UINT64_UINT_UINT:
38322 memory = ARRAY_SIZE (args);
38323 last_arg_constant = true;
38326 gcc_unreachable ();
38329 gcc_assert (nargs <= ARRAY_SIZE (args));
38331 if (klass == store)
38333 arg = CALL_EXPR_ARG (exp, 0);
38334 op = expand_normal (arg);
38335 gcc_assert (target == 0);
38338 op = ix86_zero_extend_to_Pmode (op);
38339 target = gen_rtx_MEM (tmode, op);
38340 /* target at this point has just BITS_PER_UNIT MEM_ALIGN
38341 on it. Try to improve it using get_pointer_alignment,
38342 and if the special builtin is one that requires strict
38343 mode alignment, also from it's GET_MODE_ALIGNMENT.
38344 Failure to do so could lead to ix86_legitimate_combined_insn
38345 rejecting all changes to such insns. */
38346 unsigned int align = get_pointer_alignment (arg);
38347 if (aligned_mem && align < GET_MODE_ALIGNMENT (tmode))
38348 align = GET_MODE_ALIGNMENT (tmode);
38349 if (MEM_ALIGN (target) < align)
38350 set_mem_align (target, align);
38353 target = force_reg (tmode, op);
38361 || !register_operand (target, tmode)
38362 || GET_MODE (target) != tmode)
38363 target = gen_reg_rtx (tmode);
38366 for (i = 0; i < nargs; i++)
38368 machine_mode mode = insn_p->operand[i + 1].mode;
38371 arg = CALL_EXPR_ARG (exp, i + arg_adjust);
38372 op = expand_normal (arg);
38373 match = insn_p->operand[i + 1].predicate (op, mode);
38375 if (last_arg_constant && (i + 1) == nargs)
38379 if (icode == CODE_FOR_lwp_lwpvalsi3
38380 || icode == CODE_FOR_lwp_lwpinssi3
38381 || icode == CODE_FOR_lwp_lwpvaldi3
38382 || icode == CODE_FOR_lwp_lwpinsdi3)
38383 error ("the last argument must be a 32-bit immediate");
38385 error ("the last argument must be an 8-bit immediate");
38393 /* This must be the memory operand. */
38394 op = ix86_zero_extend_to_Pmode (op);
38395 op = gen_rtx_MEM (mode, op);
38396 /* op at this point has just BITS_PER_UNIT MEM_ALIGN
38397 on it. Try to improve it using get_pointer_alignment,
38398 and if the special builtin is one that requires strict
38399 mode alignment, also from it's GET_MODE_ALIGNMENT.
38400 Failure to do so could lead to ix86_legitimate_combined_insn
38401 rejecting all changes to such insns. */
38402 unsigned int align = get_pointer_alignment (arg);
38403 if (aligned_mem && align < GET_MODE_ALIGNMENT (mode))
38404 align = GET_MODE_ALIGNMENT (mode);
38405 if (MEM_ALIGN (op) < align)
38406 set_mem_align (op, align);
38410 /* This must be register. */
38411 if (VECTOR_MODE_P (mode))
38412 op = safe_vector_operand (op, mode);
38414 op = fixup_modeless_constant (op, mode);
38416 if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
38417 op = copy_to_mode_reg (mode, op);
38420 op = copy_to_reg (op);
38421 op = simplify_gen_subreg (mode, op, GET_MODE (op), 0);
38427 args[i].mode = mode;
38433 pat = GEN_FCN (icode) (target);
38436 pat = GEN_FCN (icode) (target, args[0].op);
38439 pat = GEN_FCN (icode) (target, args[0].op, args[1].op);
38442 pat = GEN_FCN (icode) (target, args[0].op, args[1].op, args[2].op);
38445 gcc_unreachable ();
38451 return klass == store ? 0 : target;
38454 /* Return the integer constant in ARG. Constrain it to be in the range
38455 of the subparts of VEC_TYPE; issue an error if not. */
38458 get_element_number (tree vec_type, tree arg)
38460 unsigned HOST_WIDE_INT elt, max = TYPE_VECTOR_SUBPARTS (vec_type) - 1;
38462 if (!tree_fits_uhwi_p (arg)
38463 || (elt = tree_to_uhwi (arg), elt > max))
38465 error ("selector must be an integer constant in the range 0..%wi", max);
38472 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
38473 ix86_expand_vector_init. We DO have language-level syntax for this, in
38474 the form of (type){ init-list }. Except that since we can't place emms
38475 instructions from inside the compiler, we can't allow the use of MMX
38476 registers unless the user explicitly asks for it. So we do *not* define
38477 vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
38478 we have builtins invoked by mmintrin.h that gives us license to emit
38479 these sorts of instructions. */
38482 ix86_expand_vec_init_builtin (tree type, tree exp, rtx target)
38484 machine_mode tmode = TYPE_MODE (type);
38485 machine_mode inner_mode = GET_MODE_INNER (tmode);
38486 int i, n_elt = GET_MODE_NUNITS (tmode);
38487 rtvec v = rtvec_alloc (n_elt);
38489 gcc_assert (VECTOR_MODE_P (tmode));
38490 gcc_assert (call_expr_nargs (exp) == n_elt);
38492 for (i = 0; i < n_elt; ++i)
38494 rtx x = expand_normal (CALL_EXPR_ARG (exp, i));
38495 RTVEC_ELT (v, i) = gen_lowpart (inner_mode, x);
38498 if (!target || !register_operand (target, tmode))
38499 target = gen_reg_rtx (tmode);
38501 ix86_expand_vector_init (true, target, gen_rtx_PARALLEL (tmode, v));
38505 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
38506 ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
38507 had a language-level syntax for referencing vector elements. */
38510 ix86_expand_vec_ext_builtin (tree exp, rtx target)
38512 machine_mode tmode, mode0;
38517 arg0 = CALL_EXPR_ARG (exp, 0);
38518 arg1 = CALL_EXPR_ARG (exp, 1);
38520 op0 = expand_normal (arg0);
38521 elt = get_element_number (TREE_TYPE (arg0), arg1);
38523 tmode = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
38524 mode0 = TYPE_MODE (TREE_TYPE (arg0));
38525 gcc_assert (VECTOR_MODE_P (mode0));
38527 op0 = force_reg (mode0, op0);
38529 if (optimize || !target || !register_operand (target, tmode))
38530 target = gen_reg_rtx (tmode);
38532 ix86_expand_vector_extract (true, target, op0, elt);
38537 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
38538 ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
38539 a language-level syntax for referencing vector elements. */
38542 ix86_expand_vec_set_builtin (tree exp)
38544 machine_mode tmode, mode1;
38545 tree arg0, arg1, arg2;
38547 rtx op0, op1, target;
38549 arg0 = CALL_EXPR_ARG (exp, 0);
38550 arg1 = CALL_EXPR_ARG (exp, 1);
38551 arg2 = CALL_EXPR_ARG (exp, 2);
38553 tmode = TYPE_MODE (TREE_TYPE (arg0));
38554 mode1 = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
38555 gcc_assert (VECTOR_MODE_P (tmode));
38557 op0 = expand_expr (arg0, NULL_RTX, tmode, EXPAND_NORMAL);
38558 op1 = expand_expr (arg1, NULL_RTX, mode1, EXPAND_NORMAL);
38559 elt = get_element_number (TREE_TYPE (arg0), arg2);
38561 if (GET_MODE (op1) != mode1 && GET_MODE (op1) != VOIDmode)
38562 op1 = convert_modes (mode1, GET_MODE (op1), op1, true);
38564 op0 = force_reg (tmode, op0);
38565 op1 = force_reg (mode1, op1);
38567 /* OP0 is the source of these builtin functions and shouldn't be
38568 modified. Create a copy, use it and return it as target. */
38569 target = gen_reg_rtx (tmode);
38570 emit_move_insn (target, op0);
38571 ix86_expand_vector_set (true, target, op1, elt);
38576 /* Emit conditional move of SRC to DST with condition
38579 ix86_emit_cmove (rtx dst, rtx src, enum rtx_code code, rtx op1, rtx op2)
38585 t = ix86_expand_compare (code, op1, op2);
38586 emit_insn (gen_rtx_SET (VOIDmode, dst,
38587 gen_rtx_IF_THEN_ELSE (GET_MODE (dst), t,
38592 rtx nomove = gen_label_rtx ();
38593 emit_cmp_and_jump_insns (op1, op2, reverse_condition (code),
38594 const0_rtx, GET_MODE (op1), 1, nomove);
38595 emit_move_insn (dst, src);
38596 emit_label (nomove);
38600 /* Choose max of DST and SRC and put it to DST. */
38602 ix86_emit_move_max (rtx dst, rtx src)
38604 ix86_emit_cmove (dst, src, LTU, dst, src);
38607 /* Expand an expression EXP that calls a built-in function,
38608 with result going to TARGET if that's convenient
38609 (and in mode MODE if that's convenient).
38610 SUBTARGET may be used as the target for computing one of EXP's operands.
38611 IGNORE is nonzero if the value is to be ignored. */
38614 ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
38615 machine_mode mode, int ignore)
38617 const struct builtin_description *d;
38619 enum insn_code icode;
38620 tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
38621 tree arg0, arg1, arg2, arg3, arg4;
38622 rtx op0, op1, op2, op3, op4, pat, insn;
38623 machine_mode mode0, mode1, mode2, mode3, mode4;
38624 unsigned int fcode = DECL_FUNCTION_CODE (fndecl);
38626 /* For CPU builtins that can be folded, fold first and expand the fold. */
38629 case IX86_BUILTIN_CPU_INIT:
38631 /* Make it call __cpu_indicator_init in libgcc. */
38632 tree call_expr, fndecl, type;
38633 type = build_function_type_list (integer_type_node, NULL_TREE);
38634 fndecl = build_fn_decl ("__cpu_indicator_init", type);
38635 call_expr = build_call_expr (fndecl, 0);
38636 return expand_expr (call_expr, target, mode, EXPAND_NORMAL);
38638 case IX86_BUILTIN_CPU_IS:
38639 case IX86_BUILTIN_CPU_SUPPORTS:
38641 tree arg0 = CALL_EXPR_ARG (exp, 0);
38642 tree fold_expr = fold_builtin_cpu (fndecl, &arg0);
38643 gcc_assert (fold_expr != NULL_TREE);
38644 return expand_expr (fold_expr, target, mode, EXPAND_NORMAL);
38648 /* Determine whether the builtin function is available under the current ISA.
38649 Originally the builtin was not created if it wasn't applicable to the
38650 current ISA based on the command line switches. With function specific
38651 options, we need to check in the context of the function making the call
38652 whether it is supported. */
38653 if (ix86_builtins_isa[fcode].isa
38654 && !(ix86_builtins_isa[fcode].isa & ix86_isa_flags))
38656 char *opts = ix86_target_string (ix86_builtins_isa[fcode].isa, 0, NULL,
38657 NULL, (enum fpmath_unit) 0, false);
38660 error ("%qE needs unknown isa option", fndecl);
38663 gcc_assert (opts != NULL);
38664 error ("%qE needs isa option %s", fndecl, opts);
38672 case IX86_BUILTIN_BNDMK:
38674 || GET_MODE (target) != BNDmode
38675 || !register_operand (target, BNDmode))
38676 target = gen_reg_rtx (BNDmode);
38678 arg0 = CALL_EXPR_ARG (exp, 0);
38679 arg1 = CALL_EXPR_ARG (exp, 1);
38681 op0 = expand_normal (arg0);
38682 op1 = expand_normal (arg1);
38684 if (!register_operand (op0, Pmode))
38685 op0 = ix86_zero_extend_to_Pmode (op0);
38686 if (!register_operand (op1, Pmode))
38687 op1 = ix86_zero_extend_to_Pmode (op1);
38689 /* Builtin arg1 is size of block but instruction op1 should
38691 op1 = expand_simple_binop (Pmode, PLUS, op1, constm1_rtx,
38692 NULL_RTX, 1, OPTAB_DIRECT);
38694 emit_insn (BNDmode == BND64mode
38695 ? gen_bnd64_mk (target, op0, op1)
38696 : gen_bnd32_mk (target, op0, op1));
38699 case IX86_BUILTIN_BNDSTX:
38700 arg0 = CALL_EXPR_ARG (exp, 0);
38701 arg1 = CALL_EXPR_ARG (exp, 1);
38702 arg2 = CALL_EXPR_ARG (exp, 2);
38704 op0 = expand_normal (arg0);
38705 op1 = expand_normal (arg1);
38706 op2 = expand_normal (arg2);
38708 if (!register_operand (op0, Pmode))
38709 op0 = ix86_zero_extend_to_Pmode (op0);
38710 if (!register_operand (op1, BNDmode))
38711 op1 = copy_to_mode_reg (BNDmode, op1);
38712 if (!register_operand (op2, Pmode))
38713 op2 = ix86_zero_extend_to_Pmode (op2);
38715 emit_insn (BNDmode == BND64mode
38716 ? gen_bnd64_stx (op2, op0, op1)
38717 : gen_bnd32_stx (op2, op0, op1));
38720 case IX86_BUILTIN_BNDLDX:
38722 || GET_MODE (target) != BNDmode
38723 || !register_operand (target, BNDmode))
38724 target = gen_reg_rtx (BNDmode);
38726 arg0 = CALL_EXPR_ARG (exp, 0);
38727 arg1 = CALL_EXPR_ARG (exp, 1);
38729 op0 = expand_normal (arg0);
38730 op1 = expand_normal (arg1);
38732 if (!register_operand (op0, Pmode))
38733 op0 = ix86_zero_extend_to_Pmode (op0);
38734 if (!register_operand (op1, Pmode))
38735 op1 = ix86_zero_extend_to_Pmode (op1);
38737 emit_insn (BNDmode == BND64mode
38738 ? gen_bnd64_ldx (target, op0, op1)
38739 : gen_bnd32_ldx (target, op0, op1));
38742 case IX86_BUILTIN_BNDCL:
38743 arg0 = CALL_EXPR_ARG (exp, 0);
38744 arg1 = CALL_EXPR_ARG (exp, 1);
38746 op0 = expand_normal (arg0);
38747 op1 = expand_normal (arg1);
38749 if (!register_operand (op0, Pmode))
38750 op0 = ix86_zero_extend_to_Pmode (op0);
38751 if (!register_operand (op1, BNDmode))
38752 op1 = copy_to_mode_reg (BNDmode, op1);
38754 emit_insn (BNDmode == BND64mode
38755 ? gen_bnd64_cl (op1, op0)
38756 : gen_bnd32_cl (op1, op0));
38759 case IX86_BUILTIN_BNDCU:
38760 arg0 = CALL_EXPR_ARG (exp, 0);
38761 arg1 = CALL_EXPR_ARG (exp, 1);
38763 op0 = expand_normal (arg0);
38764 op1 = expand_normal (arg1);
38766 if (!register_operand (op0, Pmode))
38767 op0 = ix86_zero_extend_to_Pmode (op0);
38768 if (!register_operand (op1, BNDmode))
38769 op1 = copy_to_mode_reg (BNDmode, op1);
38771 emit_insn (BNDmode == BND64mode
38772 ? gen_bnd64_cu (op1, op0)
38773 : gen_bnd32_cu (op1, op0));
38776 case IX86_BUILTIN_BNDRET:
38777 arg0 = CALL_EXPR_ARG (exp, 0);
38778 gcc_assert (TREE_CODE (arg0) == SSA_NAME);
38779 target = chkp_get_rtl_bounds (arg0);
38781 /* If no bounds were specified for returned value,
38782 then use INIT bounds. It usually happens when
38783 some built-in function is expanded. */
38786 rtx t1 = gen_reg_rtx (Pmode);
38787 rtx t2 = gen_reg_rtx (Pmode);
38788 target = gen_reg_rtx (BNDmode);
38789 emit_move_insn (t1, const0_rtx);
38790 emit_move_insn (t2, constm1_rtx);
38791 emit_insn (BNDmode == BND64mode
38792 ? gen_bnd64_mk (target, t1, t2)
38793 : gen_bnd32_mk (target, t1, t2));
38796 gcc_assert (target && REG_P (target));
38799 case IX86_BUILTIN_BNDNARROW:
38801 rtx m1, m1h1, m1h2, lb, ub, t1;
38803 /* Return value and lb. */
38804 arg0 = CALL_EXPR_ARG (exp, 0);
38806 arg1 = CALL_EXPR_ARG (exp, 1);
38808 arg2 = CALL_EXPR_ARG (exp, 2);
38810 lb = expand_normal (arg0);
38811 op1 = expand_normal (arg1);
38812 op2 = expand_normal (arg2);
38814 /* Size was passed but we need to use (size - 1) as for bndmk. */
38815 op2 = expand_simple_binop (Pmode, PLUS, op2, constm1_rtx,
38816 NULL_RTX, 1, OPTAB_DIRECT);
38818 /* Add LB to size and inverse to get UB. */
38819 op2 = expand_simple_binop (Pmode, PLUS, op2, lb,
38820 op2, 1, OPTAB_DIRECT);
38821 ub = expand_simple_unop (Pmode, NOT, op2, op2, 1);
38823 if (!register_operand (lb, Pmode))
38824 lb = ix86_zero_extend_to_Pmode (lb);
38825 if (!register_operand (ub, Pmode))
38826 ub = ix86_zero_extend_to_Pmode (ub);
38828 /* We need to move bounds to memory before any computations. */
38833 m1 = assign_386_stack_local (BNDmode, SLOT_TEMP);
38834 emit_move_insn (m1, op1);
38837 /* Generate mem expression to be used for access to LB and UB. */
38838 m1h1 = adjust_address (m1, Pmode, 0);
38839 m1h2 = adjust_address (m1, Pmode, GET_MODE_SIZE (Pmode));
38841 t1 = gen_reg_rtx (Pmode);
38844 emit_move_insn (t1, m1h1);
38845 ix86_emit_move_max (t1, lb);
38846 emit_move_insn (m1h1, t1);
38848 /* Compute UB. UB is stored in 1's complement form. Therefore
38849 we also use max here. */
38850 emit_move_insn (t1, m1h2);
38851 ix86_emit_move_max (t1, ub);
38852 emit_move_insn (m1h2, t1);
38854 op2 = gen_reg_rtx (BNDmode);
38855 emit_move_insn (op2, m1);
38857 return chkp_join_splitted_slot (lb, op2);
38860 case IX86_BUILTIN_BNDINT:
38862 rtx res, rh1, rh2, lb1, lb2, ub1, ub2;
38865 || GET_MODE (target) != BNDmode
38866 || !register_operand (target, BNDmode))
38867 target = gen_reg_rtx (BNDmode);
38869 arg0 = CALL_EXPR_ARG (exp, 0);
38870 arg1 = CALL_EXPR_ARG (exp, 1);
38872 op0 = expand_normal (arg0);
38873 op1 = expand_normal (arg1);
38875 res = assign_386_stack_local (BNDmode, SLOT_TEMP);
38876 rh1 = adjust_address (res, Pmode, 0);
38877 rh2 = adjust_address (res, Pmode, GET_MODE_SIZE (Pmode));
38879 /* Put first bounds to temporaries. */
38880 lb1 = gen_reg_rtx (Pmode);
38881 ub1 = gen_reg_rtx (Pmode);
38884 emit_move_insn (lb1, adjust_address (op0, Pmode, 0));
38885 emit_move_insn (ub1, adjust_address (op0, Pmode,
38886 GET_MODE_SIZE (Pmode)));
38890 emit_move_insn (res, op0);
38891 emit_move_insn (lb1, rh1);
38892 emit_move_insn (ub1, rh2);
38895 /* Put second bounds to temporaries. */
38896 lb2 = gen_reg_rtx (Pmode);
38897 ub2 = gen_reg_rtx (Pmode);
38900 emit_move_insn (lb2, adjust_address (op1, Pmode, 0));
38901 emit_move_insn (ub2, adjust_address (op1, Pmode,
38902 GET_MODE_SIZE (Pmode)));
38906 emit_move_insn (res, op1);
38907 emit_move_insn (lb2, rh1);
38908 emit_move_insn (ub2, rh2);
38912 ix86_emit_move_max (lb1, lb2);
38913 emit_move_insn (rh1, lb1);
38915 /* Compute UB. UB is stored in 1's complement form. Therefore
38916 we also use max here. */
38917 ix86_emit_move_max (ub1, ub2);
38918 emit_move_insn (rh2, ub1);
38920 emit_move_insn (target, res);
38925 case IX86_BUILTIN_SIZEOF:
38931 || GET_MODE (target) != Pmode
38932 || !register_operand (target, Pmode))
38933 target = gen_reg_rtx (Pmode);
38935 arg0 = CALL_EXPR_ARG (exp, 0);
38936 gcc_assert (TREE_CODE (arg0) == VAR_DECL);
38938 name = DECL_ASSEMBLER_NAME (arg0);
38939 symbol = gen_rtx_SYMBOL_REF (Pmode, IDENTIFIER_POINTER (name));
38941 emit_insn (Pmode == SImode
38942 ? gen_move_size_reloc_si (target, symbol)
38943 : gen_move_size_reloc_di (target, symbol));
38948 case IX86_BUILTIN_BNDLOWER:
38953 || GET_MODE (target) != Pmode
38954 || !register_operand (target, Pmode))
38955 target = gen_reg_rtx (Pmode);
38957 arg0 = CALL_EXPR_ARG (exp, 0);
38958 op0 = expand_normal (arg0);
38960 /* We need to move bounds to memory first. */
38965 mem = assign_386_stack_local (BNDmode, SLOT_TEMP);
38966 emit_move_insn (mem, op0);
38969 /* Generate mem expression to access LB and load it. */
38970 hmem = adjust_address (mem, Pmode, 0);
38971 emit_move_insn (target, hmem);
38976 case IX86_BUILTIN_BNDUPPER:
38978 rtx mem, hmem, res;
38981 || GET_MODE (target) != Pmode
38982 || !register_operand (target, Pmode))
38983 target = gen_reg_rtx (Pmode);
38985 arg0 = CALL_EXPR_ARG (exp, 0);
38986 op0 = expand_normal (arg0);
38988 /* We need to move bounds to memory first. */
38993 mem = assign_386_stack_local (BNDmode, SLOT_TEMP);
38994 emit_move_insn (mem, op0);
38997 /* Generate mem expression to access UB. */
38998 hmem = adjust_address (mem, Pmode, GET_MODE_SIZE (Pmode));
39000 /* We need to inverse all bits of UB. */
39001 res = expand_simple_unop (Pmode, NOT, hmem, target, 1);
39004 emit_move_insn (target, res);
39009 case IX86_BUILTIN_MASKMOVQ:
39010 case IX86_BUILTIN_MASKMOVDQU:
39011 icode = (fcode == IX86_BUILTIN_MASKMOVQ
39012 ? CODE_FOR_mmx_maskmovq
39013 : CODE_FOR_sse2_maskmovdqu);
39014 /* Note the arg order is different from the operand order. */
39015 arg1 = CALL_EXPR_ARG (exp, 0);
39016 arg2 = CALL_EXPR_ARG (exp, 1);
39017 arg0 = CALL_EXPR_ARG (exp, 2);
39018 op0 = expand_normal (arg0);
39019 op1 = expand_normal (arg1);
39020 op2 = expand_normal (arg2);
39021 mode0 = insn_data[icode].operand[0].mode;
39022 mode1 = insn_data[icode].operand[1].mode;
39023 mode2 = insn_data[icode].operand[2].mode;
39025 op0 = ix86_zero_extend_to_Pmode (op0);
39026 op0 = gen_rtx_MEM (mode1, op0);
39028 if (!insn_data[icode].operand[0].predicate (op0, mode0))
39029 op0 = copy_to_mode_reg (mode0, op0);
39030 if (!insn_data[icode].operand[1].predicate (op1, mode1))
39031 op1 = copy_to_mode_reg (mode1, op1);
39032 if (!insn_data[icode].operand[2].predicate (op2, mode2))
39033 op2 = copy_to_mode_reg (mode2, op2);
39034 pat = GEN_FCN (icode) (op0, op1, op2);
39040 case IX86_BUILTIN_LDMXCSR:
39041 op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
39042 target = assign_386_stack_local (SImode, SLOT_TEMP);
39043 emit_move_insn (target, op0);
39044 emit_insn (gen_sse_ldmxcsr (target));
39047 case IX86_BUILTIN_STMXCSR:
39048 target = assign_386_stack_local (SImode, SLOT_TEMP);
39049 emit_insn (gen_sse_stmxcsr (target));
39050 return copy_to_mode_reg (SImode, target);
39052 case IX86_BUILTIN_CLFLUSH:
39053 arg0 = CALL_EXPR_ARG (exp, 0);
39054 op0 = expand_normal (arg0);
39055 icode = CODE_FOR_sse2_clflush;
39056 if (!insn_data[icode].operand[0].predicate (op0, Pmode))
39057 op0 = ix86_zero_extend_to_Pmode (op0);
39059 emit_insn (gen_sse2_clflush (op0));
39062 case IX86_BUILTIN_CLWB:
39063 arg0 = CALL_EXPR_ARG (exp, 0);
39064 op0 = expand_normal (arg0);
39065 icode = CODE_FOR_clwb;
39066 if (!insn_data[icode].operand[0].predicate (op0, Pmode))
39067 op0 = ix86_zero_extend_to_Pmode (op0);
39069 emit_insn (gen_clwb (op0));
39072 case IX86_BUILTIN_CLFLUSHOPT:
39073 arg0 = CALL_EXPR_ARG (exp, 0);
39074 op0 = expand_normal (arg0);
39075 icode = CODE_FOR_clflushopt;
39076 if (!insn_data[icode].operand[0].predicate (op0, Pmode))
39077 op0 = ix86_zero_extend_to_Pmode (op0);
39079 emit_insn (gen_clflushopt (op0));
39082 case IX86_BUILTIN_MONITOR:
39083 case IX86_BUILTIN_MONITORX:
39084 arg0 = CALL_EXPR_ARG (exp, 0);
39085 arg1 = CALL_EXPR_ARG (exp, 1);
39086 arg2 = CALL_EXPR_ARG (exp, 2);
39087 op0 = expand_normal (arg0);
39088 op1 = expand_normal (arg1);
39089 op2 = expand_normal (arg2);
39091 op0 = ix86_zero_extend_to_Pmode (op0);
39093 op1 = copy_to_mode_reg (SImode, op1);
39095 op2 = copy_to_mode_reg (SImode, op2);
39097 emit_insn (fcode == IX86_BUILTIN_MONITOR
39098 ? ix86_gen_monitor (op0, op1, op2)
39099 : ix86_gen_monitorx (op0, op1, op2));
39102 case IX86_BUILTIN_MWAIT:
39103 arg0 = CALL_EXPR_ARG (exp, 0);
39104 arg1 = CALL_EXPR_ARG (exp, 1);
39105 op0 = expand_normal (arg0);
39106 op1 = expand_normal (arg1);
39108 op0 = copy_to_mode_reg (SImode, op0);
39110 op1 = copy_to_mode_reg (SImode, op1);
39111 emit_insn (gen_sse3_mwait (op0, op1));
39114 case IX86_BUILTIN_MWAITX:
39115 arg0 = CALL_EXPR_ARG (exp, 0);
39116 arg1 = CALL_EXPR_ARG (exp, 1);
39117 arg2 = CALL_EXPR_ARG (exp, 2);
39118 op0 = expand_normal (arg0);
39119 op1 = expand_normal (arg1);
39120 op2 = expand_normal (arg2);
39122 op0 = copy_to_mode_reg (SImode, op0);
39124 op1 = copy_to_mode_reg (SImode, op1);
39126 op2 = copy_to_mode_reg (SImode, op2);
39127 emit_insn (gen_mwaitx (op0, op1, op2));
39130 case IX86_BUILTIN_VEC_INIT_V2SI:
39131 case IX86_BUILTIN_VEC_INIT_V4HI:
39132 case IX86_BUILTIN_VEC_INIT_V8QI:
39133 return ix86_expand_vec_init_builtin (TREE_TYPE (exp), exp, target);
39135 case IX86_BUILTIN_VEC_EXT_V2DF:
39136 case IX86_BUILTIN_VEC_EXT_V2DI:
39137 case IX86_BUILTIN_VEC_EXT_V4SF:
39138 case IX86_BUILTIN_VEC_EXT_V4SI:
39139 case IX86_BUILTIN_VEC_EXT_V8HI:
39140 case IX86_BUILTIN_VEC_EXT_V2SI:
39141 case IX86_BUILTIN_VEC_EXT_V4HI:
39142 case IX86_BUILTIN_VEC_EXT_V16QI:
39143 return ix86_expand_vec_ext_builtin (exp, target);
39145 case IX86_BUILTIN_VEC_SET_V2DI:
39146 case IX86_BUILTIN_VEC_SET_V4SF:
39147 case IX86_BUILTIN_VEC_SET_V4SI:
39148 case IX86_BUILTIN_VEC_SET_V8HI:
39149 case IX86_BUILTIN_VEC_SET_V4HI:
39150 case IX86_BUILTIN_VEC_SET_V16QI:
39151 return ix86_expand_vec_set_builtin (exp);
39153 case IX86_BUILTIN_INFQ:
39154 case IX86_BUILTIN_HUGE_VALQ:
39156 REAL_VALUE_TYPE inf;
39160 tmp = CONST_DOUBLE_FROM_REAL_VALUE (inf, mode);
39162 tmp = validize_mem (force_const_mem (mode, tmp));
39165 target = gen_reg_rtx (mode);
39167 emit_move_insn (target, tmp);
39171 case IX86_BUILTIN_RDPMC:
39172 case IX86_BUILTIN_RDTSC:
39173 case IX86_BUILTIN_RDTSCP:
39175 op0 = gen_reg_rtx (DImode);
39176 op1 = gen_reg_rtx (DImode);
39178 if (fcode == IX86_BUILTIN_RDPMC)
39180 arg0 = CALL_EXPR_ARG (exp, 0);
39181 op2 = expand_normal (arg0);
39182 if (!register_operand (op2, SImode))
39183 op2 = copy_to_mode_reg (SImode, op2);
39185 insn = (TARGET_64BIT
39186 ? gen_rdpmc_rex64 (op0, op1, op2)
39187 : gen_rdpmc (op0, op2));
39190 else if (fcode == IX86_BUILTIN_RDTSC)
39192 insn = (TARGET_64BIT
39193 ? gen_rdtsc_rex64 (op0, op1)
39194 : gen_rdtsc (op0));
39199 op2 = gen_reg_rtx (SImode);
39201 insn = (TARGET_64BIT
39202 ? gen_rdtscp_rex64 (op0, op1, op2)
39203 : gen_rdtscp (op0, op2));
39206 arg0 = CALL_EXPR_ARG (exp, 0);
39207 op4 = expand_normal (arg0);
39208 if (!address_operand (op4, VOIDmode))
39210 op4 = convert_memory_address (Pmode, op4);
39211 op4 = copy_addr_to_reg (op4);
39213 emit_move_insn (gen_rtx_MEM (SImode, op4), op2);
39218 /* mode is VOIDmode if __builtin_rd* has been called
39220 if (mode == VOIDmode)
39222 target = gen_reg_rtx (mode);
39227 op1 = expand_simple_binop (DImode, ASHIFT, op1, GEN_INT (32),
39228 op1, 1, OPTAB_DIRECT);
39229 op0 = expand_simple_binop (DImode, IOR, op0, op1,
39230 op0, 1, OPTAB_DIRECT);
39233 emit_move_insn (target, op0);
39236 case IX86_BUILTIN_FXSAVE:
39237 case IX86_BUILTIN_FXRSTOR:
39238 case IX86_BUILTIN_FXSAVE64:
39239 case IX86_BUILTIN_FXRSTOR64:
39240 case IX86_BUILTIN_FNSTENV:
39241 case IX86_BUILTIN_FLDENV:
39245 case IX86_BUILTIN_FXSAVE:
39246 icode = CODE_FOR_fxsave;
39248 case IX86_BUILTIN_FXRSTOR:
39249 icode = CODE_FOR_fxrstor;
39251 case IX86_BUILTIN_FXSAVE64:
39252 icode = CODE_FOR_fxsave64;
39254 case IX86_BUILTIN_FXRSTOR64:
39255 icode = CODE_FOR_fxrstor64;
39257 case IX86_BUILTIN_FNSTENV:
39258 icode = CODE_FOR_fnstenv;
39260 case IX86_BUILTIN_FLDENV:
39261 icode = CODE_FOR_fldenv;
39264 gcc_unreachable ();
39267 arg0 = CALL_EXPR_ARG (exp, 0);
39268 op0 = expand_normal (arg0);
39270 if (!address_operand (op0, VOIDmode))
39272 op0 = convert_memory_address (Pmode, op0);
39273 op0 = copy_addr_to_reg (op0);
39275 op0 = gen_rtx_MEM (mode0, op0);
39277 pat = GEN_FCN (icode) (op0);
39282 case IX86_BUILTIN_XSAVE:
39283 case IX86_BUILTIN_XRSTOR:
39284 case IX86_BUILTIN_XSAVE64:
39285 case IX86_BUILTIN_XRSTOR64:
39286 case IX86_BUILTIN_XSAVEOPT:
39287 case IX86_BUILTIN_XSAVEOPT64:
39288 case IX86_BUILTIN_XSAVES:
39289 case IX86_BUILTIN_XRSTORS:
39290 case IX86_BUILTIN_XSAVES64:
39291 case IX86_BUILTIN_XRSTORS64:
39292 case IX86_BUILTIN_XSAVEC:
39293 case IX86_BUILTIN_XSAVEC64:
39294 arg0 = CALL_EXPR_ARG (exp, 0);
39295 arg1 = CALL_EXPR_ARG (exp, 1);
39296 op0 = expand_normal (arg0);
39297 op1 = expand_normal (arg1);
39299 if (!address_operand (op0, VOIDmode))
39301 op0 = convert_memory_address (Pmode, op0);
39302 op0 = copy_addr_to_reg (op0);
39304 op0 = gen_rtx_MEM (BLKmode, op0);
39306 op1 = force_reg (DImode, op1);
39310 op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
39311 NULL, 1, OPTAB_DIRECT);
39314 case IX86_BUILTIN_XSAVE:
39315 icode = CODE_FOR_xsave_rex64;
39317 case IX86_BUILTIN_XRSTOR:
39318 icode = CODE_FOR_xrstor_rex64;
39320 case IX86_BUILTIN_XSAVE64:
39321 icode = CODE_FOR_xsave64;
39323 case IX86_BUILTIN_XRSTOR64:
39324 icode = CODE_FOR_xrstor64;
39326 case IX86_BUILTIN_XSAVEOPT:
39327 icode = CODE_FOR_xsaveopt_rex64;
39329 case IX86_BUILTIN_XSAVEOPT64:
39330 icode = CODE_FOR_xsaveopt64;
39332 case IX86_BUILTIN_XSAVES:
39333 icode = CODE_FOR_xsaves_rex64;
39335 case IX86_BUILTIN_XRSTORS:
39336 icode = CODE_FOR_xrstors_rex64;
39338 case IX86_BUILTIN_XSAVES64:
39339 icode = CODE_FOR_xsaves64;
39341 case IX86_BUILTIN_XRSTORS64:
39342 icode = CODE_FOR_xrstors64;
39344 case IX86_BUILTIN_XSAVEC:
39345 icode = CODE_FOR_xsavec_rex64;
39347 case IX86_BUILTIN_XSAVEC64:
39348 icode = CODE_FOR_xsavec64;
39351 gcc_unreachable ();
39354 op2 = gen_lowpart (SImode, op2);
39355 op1 = gen_lowpart (SImode, op1);
39356 pat = GEN_FCN (icode) (op0, op1, op2);
39362 case IX86_BUILTIN_XSAVE:
39363 icode = CODE_FOR_xsave;
39365 case IX86_BUILTIN_XRSTOR:
39366 icode = CODE_FOR_xrstor;
39368 case IX86_BUILTIN_XSAVEOPT:
39369 icode = CODE_FOR_xsaveopt;
39371 case IX86_BUILTIN_XSAVES:
39372 icode = CODE_FOR_xsaves;
39374 case IX86_BUILTIN_XRSTORS:
39375 icode = CODE_FOR_xrstors;
39377 case IX86_BUILTIN_XSAVEC:
39378 icode = CODE_FOR_xsavec;
39381 gcc_unreachable ();
39383 pat = GEN_FCN (icode) (op0, op1);
39390 case IX86_BUILTIN_LLWPCB:
39391 arg0 = CALL_EXPR_ARG (exp, 0);
39392 op0 = expand_normal (arg0);
39393 icode = CODE_FOR_lwp_llwpcb;
39394 if (!insn_data[icode].operand[0].predicate (op0, Pmode))
39395 op0 = ix86_zero_extend_to_Pmode (op0);
39396 emit_insn (gen_lwp_llwpcb (op0));
39399 case IX86_BUILTIN_SLWPCB:
39400 icode = CODE_FOR_lwp_slwpcb;
39402 || !insn_data[icode].operand[0].predicate (target, Pmode))
39403 target = gen_reg_rtx (Pmode);
39404 emit_insn (gen_lwp_slwpcb (target));
39407 case IX86_BUILTIN_BEXTRI32:
39408 case IX86_BUILTIN_BEXTRI64:
39409 arg0 = CALL_EXPR_ARG (exp, 0);
39410 arg1 = CALL_EXPR_ARG (exp, 1);
39411 op0 = expand_normal (arg0);
39412 op1 = expand_normal (arg1);
39413 icode = (fcode == IX86_BUILTIN_BEXTRI32
39414 ? CODE_FOR_tbm_bextri_si
39415 : CODE_FOR_tbm_bextri_di);
39416 if (!CONST_INT_P (op1))
39418 error ("last argument must be an immediate");
39423 unsigned char length = (INTVAL (op1) >> 8) & 0xFF;
39424 unsigned char lsb_index = INTVAL (op1) & 0xFF;
39425 op1 = GEN_INT (length);
39426 op2 = GEN_INT (lsb_index);
39427 pat = GEN_FCN (icode) (target, op0, op1, op2);
39433 case IX86_BUILTIN_RDRAND16_STEP:
39434 icode = CODE_FOR_rdrandhi_1;
39438 case IX86_BUILTIN_RDRAND32_STEP:
39439 icode = CODE_FOR_rdrandsi_1;
39443 case IX86_BUILTIN_RDRAND64_STEP:
39444 icode = CODE_FOR_rdranddi_1;
39448 op0 = gen_reg_rtx (mode0);
39449 emit_insn (GEN_FCN (icode) (op0));
39451 arg0 = CALL_EXPR_ARG (exp, 0);
39452 op1 = expand_normal (arg0);
39453 if (!address_operand (op1, VOIDmode))
39455 op1 = convert_memory_address (Pmode, op1);
39456 op1 = copy_addr_to_reg (op1);
39458 emit_move_insn (gen_rtx_MEM (mode0, op1), op0);
39460 op1 = gen_reg_rtx (SImode);
39461 emit_move_insn (op1, CONST1_RTX (SImode));
39463 /* Emit SImode conditional move. */
39464 if (mode0 == HImode)
39466 op2 = gen_reg_rtx (SImode);
39467 emit_insn (gen_zero_extendhisi2 (op2, op0));
39469 else if (mode0 == SImode)
39472 op2 = gen_rtx_SUBREG (SImode, op0, 0);
39475 || !register_operand (target, SImode))
39476 target = gen_reg_rtx (SImode);
39478 pat = gen_rtx_GEU (VOIDmode, gen_rtx_REG (CCCmode, FLAGS_REG),
39480 emit_insn (gen_rtx_SET (VOIDmode, target,
39481 gen_rtx_IF_THEN_ELSE (SImode, pat, op2, op1)));
39484 case IX86_BUILTIN_RDSEED16_STEP:
39485 icode = CODE_FOR_rdseedhi_1;
39489 case IX86_BUILTIN_RDSEED32_STEP:
39490 icode = CODE_FOR_rdseedsi_1;
39494 case IX86_BUILTIN_RDSEED64_STEP:
39495 icode = CODE_FOR_rdseeddi_1;
39499 op0 = gen_reg_rtx (mode0);
39500 emit_insn (GEN_FCN (icode) (op0));
39502 arg0 = CALL_EXPR_ARG (exp, 0);
39503 op1 = expand_normal (arg0);
39504 if (!address_operand (op1, VOIDmode))
39506 op1 = convert_memory_address (Pmode, op1);
39507 op1 = copy_addr_to_reg (op1);
39509 emit_move_insn (gen_rtx_MEM (mode0, op1), op0);
39511 op2 = gen_reg_rtx (QImode);
39513 pat = gen_rtx_LTU (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
39515 emit_insn (gen_rtx_SET (VOIDmode, op2, pat));
39518 || !register_operand (target, SImode))
39519 target = gen_reg_rtx (SImode);
39521 emit_insn (gen_zero_extendqisi2 (target, op2));
39524 case IX86_BUILTIN_SBB32:
39525 icode = CODE_FOR_subsi3_carry;
39529 case IX86_BUILTIN_SBB64:
39530 icode = CODE_FOR_subdi3_carry;
39534 case IX86_BUILTIN_ADDCARRYX32:
39535 icode = TARGET_ADX ? CODE_FOR_adcxsi3 : CODE_FOR_addsi3_carry;
39539 case IX86_BUILTIN_ADDCARRYX64:
39540 icode = TARGET_ADX ? CODE_FOR_adcxdi3 : CODE_FOR_adddi3_carry;
39544 arg0 = CALL_EXPR_ARG (exp, 0); /* unsigned char c_in. */
39545 arg1 = CALL_EXPR_ARG (exp, 1); /* unsigned int src1. */
39546 arg2 = CALL_EXPR_ARG (exp, 2); /* unsigned int src2. */
39547 arg3 = CALL_EXPR_ARG (exp, 3); /* unsigned int *sum_out. */
39549 op0 = gen_reg_rtx (QImode);
39551 /* Generate CF from input operand. */
39552 op1 = expand_normal (arg0);
39553 op1 = copy_to_mode_reg (QImode, convert_to_mode (QImode, op1, 1));
39554 emit_insn (gen_addqi3_cc (op0, op1, constm1_rtx));
39556 /* Gen ADCX instruction to compute X+Y+CF. */
39557 op2 = expand_normal (arg1);
39558 op3 = expand_normal (arg2);
39561 op2 = copy_to_mode_reg (mode0, op2);
39563 op3 = copy_to_mode_reg (mode0, op3);
39565 op0 = gen_reg_rtx (mode0);
39567 op4 = gen_rtx_REG (CCCmode, FLAGS_REG);
39568 pat = gen_rtx_LTU (VOIDmode, op4, const0_rtx);
39569 emit_insn (GEN_FCN (icode) (op0, op2, op3, op4, pat));
39571 /* Store the result. */
39572 op4 = expand_normal (arg3);
39573 if (!address_operand (op4, VOIDmode))
39575 op4 = convert_memory_address (Pmode, op4);
39576 op4 = copy_addr_to_reg (op4);
39578 emit_move_insn (gen_rtx_MEM (mode0, op4), op0);
39580 /* Return current CF value. */
39582 target = gen_reg_rtx (QImode);
39584 PUT_MODE (pat, QImode);
39585 emit_insn (gen_rtx_SET (VOIDmode, target, pat));
39588 case IX86_BUILTIN_READ_FLAGS:
39589 emit_insn (gen_push (gen_rtx_REG (word_mode, FLAGS_REG)));
39592 || target == NULL_RTX
39593 || !nonimmediate_operand (target, word_mode)
39594 || GET_MODE (target) != word_mode)
39595 target = gen_reg_rtx (word_mode);
39597 emit_insn (gen_pop (target));
39600 case IX86_BUILTIN_WRITE_FLAGS:
39602 arg0 = CALL_EXPR_ARG (exp, 0);
39603 op0 = expand_normal (arg0);
39604 if (!general_no_elim_operand (op0, word_mode))
39605 op0 = copy_to_mode_reg (word_mode, op0);
39607 emit_insn (gen_push (op0));
39608 emit_insn (gen_pop (gen_rtx_REG (word_mode, FLAGS_REG)));
39611 case IX86_BUILTIN_KORTESTC16:
39612 icode = CODE_FOR_kortestchi;
39617 case IX86_BUILTIN_KORTESTZ16:
39618 icode = CODE_FOR_kortestzhi;
39623 arg0 = CALL_EXPR_ARG (exp, 0); /* Mask reg src1. */
39624 arg1 = CALL_EXPR_ARG (exp, 1); /* Mask reg src2. */
39625 op0 = expand_normal (arg0);
39626 op1 = expand_normal (arg1);
39628 op0 = copy_to_reg (op0);
39629 op0 = simplify_gen_subreg (mode0, op0, GET_MODE (op0), 0);
39630 op1 = copy_to_reg (op1);
39631 op1 = simplify_gen_subreg (mode0, op1, GET_MODE (op1), 0);
39633 target = gen_reg_rtx (QImode);
39634 emit_insn (gen_rtx_SET (mode0, target, const0_rtx));
39636 /* Emit kortest. */
39637 emit_insn (GEN_FCN (icode) (op0, op1));
39638 /* And use setcc to return result from flags. */
39639 ix86_expand_setcc (target, EQ,
39640 gen_rtx_REG (mode1, FLAGS_REG), const0_rtx);
39643 case IX86_BUILTIN_GATHERSIV2DF:
39644 icode = CODE_FOR_avx2_gathersiv2df;
39646 case IX86_BUILTIN_GATHERSIV4DF:
39647 icode = CODE_FOR_avx2_gathersiv4df;
39649 case IX86_BUILTIN_GATHERDIV2DF:
39650 icode = CODE_FOR_avx2_gatherdiv2df;
39652 case IX86_BUILTIN_GATHERDIV4DF:
39653 icode = CODE_FOR_avx2_gatherdiv4df;
39655 case IX86_BUILTIN_GATHERSIV4SF:
39656 icode = CODE_FOR_avx2_gathersiv4sf;
39658 case IX86_BUILTIN_GATHERSIV8SF:
39659 icode = CODE_FOR_avx2_gathersiv8sf;
39661 case IX86_BUILTIN_GATHERDIV4SF:
39662 icode = CODE_FOR_avx2_gatherdiv4sf;
39664 case IX86_BUILTIN_GATHERDIV8SF:
39665 icode = CODE_FOR_avx2_gatherdiv8sf;
39667 case IX86_BUILTIN_GATHERSIV2DI:
39668 icode = CODE_FOR_avx2_gathersiv2di;
39670 case IX86_BUILTIN_GATHERSIV4DI:
39671 icode = CODE_FOR_avx2_gathersiv4di;
39673 case IX86_BUILTIN_GATHERDIV2DI:
39674 icode = CODE_FOR_avx2_gatherdiv2di;
39676 case IX86_BUILTIN_GATHERDIV4DI:
39677 icode = CODE_FOR_avx2_gatherdiv4di;
39679 case IX86_BUILTIN_GATHERSIV4SI:
39680 icode = CODE_FOR_avx2_gathersiv4si;
39682 case IX86_BUILTIN_GATHERSIV8SI:
39683 icode = CODE_FOR_avx2_gathersiv8si;
39685 case IX86_BUILTIN_GATHERDIV4SI:
39686 icode = CODE_FOR_avx2_gatherdiv4si;
39688 case IX86_BUILTIN_GATHERDIV8SI:
39689 icode = CODE_FOR_avx2_gatherdiv8si;
39691 case IX86_BUILTIN_GATHERALTSIV4DF:
39692 icode = CODE_FOR_avx2_gathersiv4df;
39694 case IX86_BUILTIN_GATHERALTDIV8SF:
39695 icode = CODE_FOR_avx2_gatherdiv8sf;
39697 case IX86_BUILTIN_GATHERALTSIV4DI:
39698 icode = CODE_FOR_avx2_gathersiv4di;
39700 case IX86_BUILTIN_GATHERALTDIV8SI:
39701 icode = CODE_FOR_avx2_gatherdiv8si;
39703 case IX86_BUILTIN_GATHER3SIV16SF:
39704 icode = CODE_FOR_avx512f_gathersiv16sf;
39706 case IX86_BUILTIN_GATHER3SIV8DF:
39707 icode = CODE_FOR_avx512f_gathersiv8df;
39709 case IX86_BUILTIN_GATHER3DIV16SF:
39710 icode = CODE_FOR_avx512f_gatherdiv16sf;
39712 case IX86_BUILTIN_GATHER3DIV8DF:
39713 icode = CODE_FOR_avx512f_gatherdiv8df;
39715 case IX86_BUILTIN_GATHER3SIV16SI:
39716 icode = CODE_FOR_avx512f_gathersiv16si;
39718 case IX86_BUILTIN_GATHER3SIV8DI:
39719 icode = CODE_FOR_avx512f_gathersiv8di;
39721 case IX86_BUILTIN_GATHER3DIV16SI:
39722 icode = CODE_FOR_avx512f_gatherdiv16si;
39724 case IX86_BUILTIN_GATHER3DIV8DI:
39725 icode = CODE_FOR_avx512f_gatherdiv8di;
39727 case IX86_BUILTIN_GATHER3ALTSIV8DF:
39728 icode = CODE_FOR_avx512f_gathersiv8df;
39730 case IX86_BUILTIN_GATHER3ALTDIV16SF:
39731 icode = CODE_FOR_avx512f_gatherdiv16sf;
39733 case IX86_BUILTIN_GATHER3ALTSIV8DI:
39734 icode = CODE_FOR_avx512f_gathersiv8di;
39736 case IX86_BUILTIN_GATHER3ALTDIV16SI:
39737 icode = CODE_FOR_avx512f_gatherdiv16si;
39739 case IX86_BUILTIN_GATHER3SIV2DF:
39740 icode = CODE_FOR_avx512vl_gathersiv2df;
39742 case IX86_BUILTIN_GATHER3SIV4DF:
39743 icode = CODE_FOR_avx512vl_gathersiv4df;
39745 case IX86_BUILTIN_GATHER3DIV2DF:
39746 icode = CODE_FOR_avx512vl_gatherdiv2df;
39748 case IX86_BUILTIN_GATHER3DIV4DF:
39749 icode = CODE_FOR_avx512vl_gatherdiv4df;
39751 case IX86_BUILTIN_GATHER3SIV4SF:
39752 icode = CODE_FOR_avx512vl_gathersiv4sf;
39754 case IX86_BUILTIN_GATHER3SIV8SF:
39755 icode = CODE_FOR_avx512vl_gathersiv8sf;
39757 case IX86_BUILTIN_GATHER3DIV4SF:
39758 icode = CODE_FOR_avx512vl_gatherdiv4sf;
39760 case IX86_BUILTIN_GATHER3DIV8SF:
39761 icode = CODE_FOR_avx512vl_gatherdiv8sf;
39763 case IX86_BUILTIN_GATHER3SIV2DI:
39764 icode = CODE_FOR_avx512vl_gathersiv2di;
39766 case IX86_BUILTIN_GATHER3SIV4DI:
39767 icode = CODE_FOR_avx512vl_gathersiv4di;
39769 case IX86_BUILTIN_GATHER3DIV2DI:
39770 icode = CODE_FOR_avx512vl_gatherdiv2di;
39772 case IX86_BUILTIN_GATHER3DIV4DI:
39773 icode = CODE_FOR_avx512vl_gatherdiv4di;
39775 case IX86_BUILTIN_GATHER3SIV4SI:
39776 icode = CODE_FOR_avx512vl_gathersiv4si;
39778 case IX86_BUILTIN_GATHER3SIV8SI:
39779 icode = CODE_FOR_avx512vl_gathersiv8si;
39781 case IX86_BUILTIN_GATHER3DIV4SI:
39782 icode = CODE_FOR_avx512vl_gatherdiv4si;
39784 case IX86_BUILTIN_GATHER3DIV8SI:
39785 icode = CODE_FOR_avx512vl_gatherdiv8si;
39787 case IX86_BUILTIN_GATHER3ALTSIV4DF:
39788 icode = CODE_FOR_avx512vl_gathersiv4df;
39790 case IX86_BUILTIN_GATHER3ALTDIV8SF:
39791 icode = CODE_FOR_avx512vl_gatherdiv8sf;
39793 case IX86_BUILTIN_GATHER3ALTSIV4DI:
39794 icode = CODE_FOR_avx512vl_gathersiv4di;
39796 case IX86_BUILTIN_GATHER3ALTDIV8SI:
39797 icode = CODE_FOR_avx512vl_gatherdiv8si;
39799 case IX86_BUILTIN_SCATTERSIV16SF:
39800 icode = CODE_FOR_avx512f_scattersiv16sf;
39802 case IX86_BUILTIN_SCATTERSIV8DF:
39803 icode = CODE_FOR_avx512f_scattersiv8df;
39805 case IX86_BUILTIN_SCATTERDIV16SF:
39806 icode = CODE_FOR_avx512f_scatterdiv16sf;
39808 case IX86_BUILTIN_SCATTERDIV8DF:
39809 icode = CODE_FOR_avx512f_scatterdiv8df;
39811 case IX86_BUILTIN_SCATTERSIV16SI:
39812 icode = CODE_FOR_avx512f_scattersiv16si;
39814 case IX86_BUILTIN_SCATTERSIV8DI:
39815 icode = CODE_FOR_avx512f_scattersiv8di;
39817 case IX86_BUILTIN_SCATTERDIV16SI:
39818 icode = CODE_FOR_avx512f_scatterdiv16si;
39820 case IX86_BUILTIN_SCATTERDIV8DI:
39821 icode = CODE_FOR_avx512f_scatterdiv8di;
39823 case IX86_BUILTIN_SCATTERSIV8SF:
39824 icode = CODE_FOR_avx512vl_scattersiv8sf;
39826 case IX86_BUILTIN_SCATTERSIV4SF:
39827 icode = CODE_FOR_avx512vl_scattersiv4sf;
39829 case IX86_BUILTIN_SCATTERSIV4DF:
39830 icode = CODE_FOR_avx512vl_scattersiv4df;
39832 case IX86_BUILTIN_SCATTERSIV2DF:
39833 icode = CODE_FOR_avx512vl_scattersiv2df;
39835 case IX86_BUILTIN_SCATTERDIV8SF:
39836 icode = CODE_FOR_avx512vl_scatterdiv8sf;
39838 case IX86_BUILTIN_SCATTERDIV4SF:
39839 icode = CODE_FOR_avx512vl_scatterdiv4sf;
39841 case IX86_BUILTIN_SCATTERDIV4DF:
39842 icode = CODE_FOR_avx512vl_scatterdiv4df;
39844 case IX86_BUILTIN_SCATTERDIV2DF:
39845 icode = CODE_FOR_avx512vl_scatterdiv2df;
39847 case IX86_BUILTIN_SCATTERSIV8SI:
39848 icode = CODE_FOR_avx512vl_scattersiv8si;
39850 case IX86_BUILTIN_SCATTERSIV4SI:
39851 icode = CODE_FOR_avx512vl_scattersiv4si;
39853 case IX86_BUILTIN_SCATTERSIV4DI:
39854 icode = CODE_FOR_avx512vl_scattersiv4di;
39856 case IX86_BUILTIN_SCATTERSIV2DI:
39857 icode = CODE_FOR_avx512vl_scattersiv2di;
39859 case IX86_BUILTIN_SCATTERDIV8SI:
39860 icode = CODE_FOR_avx512vl_scatterdiv8si;
39862 case IX86_BUILTIN_SCATTERDIV4SI:
39863 icode = CODE_FOR_avx512vl_scatterdiv4si;
39865 case IX86_BUILTIN_SCATTERDIV4DI:
39866 icode = CODE_FOR_avx512vl_scatterdiv4di;
39868 case IX86_BUILTIN_SCATTERDIV2DI:
39869 icode = CODE_FOR_avx512vl_scatterdiv2di;
39871 case IX86_BUILTIN_GATHERPFDPD:
39872 icode = CODE_FOR_avx512pf_gatherpfv8sidf;
39873 goto vec_prefetch_gen;
39874 case IX86_BUILTIN_GATHERPFDPS:
39875 icode = CODE_FOR_avx512pf_gatherpfv16sisf;
39876 goto vec_prefetch_gen;
39877 case IX86_BUILTIN_GATHERPFQPD:
39878 icode = CODE_FOR_avx512pf_gatherpfv8didf;
39879 goto vec_prefetch_gen;
39880 case IX86_BUILTIN_GATHERPFQPS:
39881 icode = CODE_FOR_avx512pf_gatherpfv8disf;
39882 goto vec_prefetch_gen;
39883 case IX86_BUILTIN_SCATTERPFDPD:
39884 icode = CODE_FOR_avx512pf_scatterpfv8sidf;
39885 goto vec_prefetch_gen;
39886 case IX86_BUILTIN_SCATTERPFDPS:
39887 icode = CODE_FOR_avx512pf_scatterpfv16sisf;
39888 goto vec_prefetch_gen;
39889 case IX86_BUILTIN_SCATTERPFQPD:
39890 icode = CODE_FOR_avx512pf_scatterpfv8didf;
39891 goto vec_prefetch_gen;
39892 case IX86_BUILTIN_SCATTERPFQPS:
39893 icode = CODE_FOR_avx512pf_scatterpfv8disf;
39894 goto vec_prefetch_gen;
39898 rtx (*gen) (rtx, rtx);
39900 arg0 = CALL_EXPR_ARG (exp, 0);
39901 arg1 = CALL_EXPR_ARG (exp, 1);
39902 arg2 = CALL_EXPR_ARG (exp, 2);
39903 arg3 = CALL_EXPR_ARG (exp, 3);
39904 arg4 = CALL_EXPR_ARG (exp, 4);
39905 op0 = expand_normal (arg0);
39906 op1 = expand_normal (arg1);
39907 op2 = expand_normal (arg2);
39908 op3 = expand_normal (arg3);
39909 op4 = expand_normal (arg4);
39910 /* Note the arg order is different from the operand order. */
39911 mode0 = insn_data[icode].operand[1].mode;
39912 mode2 = insn_data[icode].operand[3].mode;
39913 mode3 = insn_data[icode].operand[4].mode;
39914 mode4 = insn_data[icode].operand[5].mode;
39916 if (target == NULL_RTX
39917 || GET_MODE (target) != insn_data[icode].operand[0].mode
39918 || !insn_data[icode].operand[0].predicate (target,
39919 GET_MODE (target)))
39920 subtarget = gen_reg_rtx (insn_data[icode].operand[0].mode);
39922 subtarget = target;
39926 case IX86_BUILTIN_GATHER3ALTSIV8DF:
39927 case IX86_BUILTIN_GATHER3ALTSIV8DI:
39928 half = gen_reg_rtx (V8SImode);
39929 if (!nonimmediate_operand (op2, V16SImode))
39930 op2 = copy_to_mode_reg (V16SImode, op2);
39931 emit_insn (gen_vec_extract_lo_v16si (half, op2));
39934 case IX86_BUILTIN_GATHER3ALTSIV4DF:
39935 case IX86_BUILTIN_GATHER3ALTSIV4DI:
39936 case IX86_BUILTIN_GATHERALTSIV4DF:
39937 case IX86_BUILTIN_GATHERALTSIV4DI:
39938 half = gen_reg_rtx (V4SImode);
39939 if (!nonimmediate_operand (op2, V8SImode))
39940 op2 = copy_to_mode_reg (V8SImode, op2);
39941 emit_insn (gen_vec_extract_lo_v8si (half, op2));
39944 case IX86_BUILTIN_GATHER3ALTDIV16SF:
39945 case IX86_BUILTIN_GATHER3ALTDIV16SI:
39946 half = gen_reg_rtx (mode0);
39947 if (mode0 == V8SFmode)
39948 gen = gen_vec_extract_lo_v16sf;
39950 gen = gen_vec_extract_lo_v16si;
39951 if (!nonimmediate_operand (op0, GET_MODE (op0)))
39952 op0 = copy_to_mode_reg (GET_MODE (op0), op0);
39953 emit_insn (gen (half, op0));
39955 if (GET_MODE (op3) != VOIDmode)
39957 if (!nonimmediate_operand (op3, GET_MODE (op3)))
39958 op3 = copy_to_mode_reg (GET_MODE (op3), op3);
39959 emit_insn (gen (half, op3));
39963 case IX86_BUILTIN_GATHER3ALTDIV8SF:
39964 case IX86_BUILTIN_GATHER3ALTDIV8SI:
39965 case IX86_BUILTIN_GATHERALTDIV8SF:
39966 case IX86_BUILTIN_GATHERALTDIV8SI:
39967 half = gen_reg_rtx (mode0);
39968 if (mode0 == V4SFmode)
39969 gen = gen_vec_extract_lo_v8sf;
39971 gen = gen_vec_extract_lo_v8si;
39972 if (!nonimmediate_operand (op0, GET_MODE (op0)))
39973 op0 = copy_to_mode_reg (GET_MODE (op0), op0);
39974 emit_insn (gen (half, op0));
39976 if (GET_MODE (op3) != VOIDmode)
39978 if (!nonimmediate_operand (op3, GET_MODE (op3)))
39979 op3 = copy_to_mode_reg (GET_MODE (op3), op3);
39980 emit_insn (gen (half, op3));
39988 /* Force memory operand only with base register here. But we
39989 don't want to do it on memory operand for other builtin
39991 op1 = ix86_zero_extend_to_Pmode (op1);
39993 if (!insn_data[icode].operand[1].predicate (op0, mode0))
39994 op0 = copy_to_mode_reg (mode0, op0);
39995 if (!insn_data[icode].operand[2].predicate (op1, Pmode))
39996 op1 = copy_to_mode_reg (Pmode, op1);
39997 if (!insn_data[icode].operand[3].predicate (op2, mode2))
39998 op2 = copy_to_mode_reg (mode2, op2);
40000 op3 = fixup_modeless_constant (op3, mode3);
40002 if (GET_MODE (op3) == mode3 || GET_MODE (op3) == VOIDmode)
40004 if (!insn_data[icode].operand[4].predicate (op3, mode3))
40005 op3 = copy_to_mode_reg (mode3, op3);
40009 op3 = copy_to_reg (op3);
40010 op3 = simplify_gen_subreg (mode3, op3, GET_MODE (op3), 0);
40012 if (!insn_data[icode].operand[5].predicate (op4, mode4))
40014 error ("the last argument must be scale 1, 2, 4, 8");
40018 /* Optimize. If mask is known to have all high bits set,
40019 replace op0 with pc_rtx to signal that the instruction
40020 overwrites the whole destination and doesn't use its
40021 previous contents. */
40024 if (TREE_CODE (arg3) == INTEGER_CST)
40026 if (integer_all_onesp (arg3))
40029 else if (TREE_CODE (arg3) == VECTOR_CST)
40031 unsigned int negative = 0;
40032 for (i = 0; i < VECTOR_CST_NELTS (arg3); ++i)
40034 tree cst = VECTOR_CST_ELT (arg3, i);
40035 if (TREE_CODE (cst) == INTEGER_CST
40036 && tree_int_cst_sign_bit (cst))
40038 else if (TREE_CODE (cst) == REAL_CST
40039 && REAL_VALUE_NEGATIVE (TREE_REAL_CST (cst)))
40042 if (negative == TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg3)))
40045 else if (TREE_CODE (arg3) == SSA_NAME
40046 && TREE_CODE (TREE_TYPE (arg3)) == VECTOR_TYPE)
40048 /* Recognize also when mask is like:
40049 __v2df src = _mm_setzero_pd ();
40050 __v2df mask = _mm_cmpeq_pd (src, src);
40052 __v8sf src = _mm256_setzero_ps ();
40053 __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
40054 as that is a cheaper way to load all ones into
40055 a register than having to load a constant from
40057 gimple def_stmt = SSA_NAME_DEF_STMT (arg3);
40058 if (is_gimple_call (def_stmt))
40060 tree fndecl = gimple_call_fndecl (def_stmt);
40062 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
40063 switch ((unsigned int) DECL_FUNCTION_CODE (fndecl))
40065 case IX86_BUILTIN_CMPPD:
40066 case IX86_BUILTIN_CMPPS:
40067 case IX86_BUILTIN_CMPPD256:
40068 case IX86_BUILTIN_CMPPS256:
40069 if (!integer_zerop (gimple_call_arg (def_stmt, 2)))
40072 case IX86_BUILTIN_CMPEQPD:
40073 case IX86_BUILTIN_CMPEQPS:
40074 if (initializer_zerop (gimple_call_arg (def_stmt, 0))
40075 && initializer_zerop (gimple_call_arg (def_stmt,
40086 pat = GEN_FCN (icode) (subtarget, op0, op1, op2, op3, op4);
40093 case IX86_BUILTIN_GATHER3DIV16SF:
40094 if (target == NULL_RTX)
40095 target = gen_reg_rtx (V8SFmode);
40096 emit_insn (gen_vec_extract_lo_v16sf (target, subtarget));
40098 case IX86_BUILTIN_GATHER3DIV16SI:
40099 if (target == NULL_RTX)
40100 target = gen_reg_rtx (V8SImode);
40101 emit_insn (gen_vec_extract_lo_v16si (target, subtarget));
40103 case IX86_BUILTIN_GATHER3DIV8SF:
40104 case IX86_BUILTIN_GATHERDIV8SF:
40105 if (target == NULL_RTX)
40106 target = gen_reg_rtx (V4SFmode);
40107 emit_insn (gen_vec_extract_lo_v8sf (target, subtarget));
40109 case IX86_BUILTIN_GATHER3DIV8SI:
40110 case IX86_BUILTIN_GATHERDIV8SI:
40111 if (target == NULL_RTX)
40112 target = gen_reg_rtx (V4SImode);
40113 emit_insn (gen_vec_extract_lo_v8si (target, subtarget));
40116 target = subtarget;
40122 arg0 = CALL_EXPR_ARG (exp, 0);
40123 arg1 = CALL_EXPR_ARG (exp, 1);
40124 arg2 = CALL_EXPR_ARG (exp, 2);
40125 arg3 = CALL_EXPR_ARG (exp, 3);
40126 arg4 = CALL_EXPR_ARG (exp, 4);
40127 op0 = expand_normal (arg0);
40128 op1 = expand_normal (arg1);
40129 op2 = expand_normal (arg2);
40130 op3 = expand_normal (arg3);
40131 op4 = expand_normal (arg4);
40132 mode1 = insn_data[icode].operand[1].mode;
40133 mode2 = insn_data[icode].operand[2].mode;
40134 mode3 = insn_data[icode].operand[3].mode;
40135 mode4 = insn_data[icode].operand[4].mode;
40137 /* Force memory operand only with base register here. But we
40138 don't want to do it on memory operand for other builtin
40140 op0 = force_reg (Pmode, convert_to_mode (Pmode, op0, 1));
40142 if (!insn_data[icode].operand[0].predicate (op0, Pmode))
40143 op0 = copy_to_mode_reg (Pmode, op0);
40145 op1 = fixup_modeless_constant (op1, mode1);
40147 if (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode)
40149 if (!insn_data[icode].operand[1].predicate (op1, mode1))
40150 op1 = copy_to_mode_reg (mode1, op1);
40154 op1 = copy_to_reg (op1);
40155 op1 = simplify_gen_subreg (mode1, op1, GET_MODE (op1), 0);
40158 if (!insn_data[icode].operand[2].predicate (op2, mode2))
40159 op2 = copy_to_mode_reg (mode2, op2);
40161 if (!insn_data[icode].operand[3].predicate (op3, mode3))
40162 op3 = copy_to_mode_reg (mode3, op3);
40164 if (!insn_data[icode].operand[4].predicate (op4, mode4))
40166 error ("the last argument must be scale 1, 2, 4, 8");
40170 pat = GEN_FCN (icode) (op0, op1, op2, op3, op4);
40178 arg0 = CALL_EXPR_ARG (exp, 0);
40179 arg1 = CALL_EXPR_ARG (exp, 1);
40180 arg2 = CALL_EXPR_ARG (exp, 2);
40181 arg3 = CALL_EXPR_ARG (exp, 3);
40182 arg4 = CALL_EXPR_ARG (exp, 4);
40183 op0 = expand_normal (arg0);
40184 op1 = expand_normal (arg1);
40185 op2 = expand_normal (arg2);
40186 op3 = expand_normal (arg3);
40187 op4 = expand_normal (arg4);
40188 mode0 = insn_data[icode].operand[0].mode;
40189 mode1 = insn_data[icode].operand[1].mode;
40190 mode3 = insn_data[icode].operand[3].mode;
40191 mode4 = insn_data[icode].operand[4].mode;
40193 op0 = fixup_modeless_constant (op0, mode0);
40195 if (GET_MODE (op0) == mode0
40196 || (GET_MODE (op0) == VOIDmode && op0 != constm1_rtx))
40198 if (!insn_data[icode].operand[0].predicate (op0, mode0))
40199 op0 = copy_to_mode_reg (mode0, op0);
40201 else if (op0 != constm1_rtx)
40203 op0 = copy_to_reg (op0);
40204 op0 = simplify_gen_subreg (mode0, op0, GET_MODE (op0), 0);
40207 if (!insn_data[icode].operand[1].predicate (op1, mode1))
40208 op1 = copy_to_mode_reg (mode1, op1);
40210 /* Force memory operand only with base register here. But we
40211 don't want to do it on memory operand for other builtin
40213 op2 = force_reg (Pmode, convert_to_mode (Pmode, op2, 1));
40215 if (!insn_data[icode].operand[2].predicate (op2, Pmode))
40216 op2 = copy_to_mode_reg (Pmode, op2);
40218 if (!insn_data[icode].operand[3].predicate (op3, mode3))
40220 error ("the forth argument must be scale 1, 2, 4, 8");
40224 if (!insn_data[icode].operand[4].predicate (op4, mode4))
40226 error ("incorrect hint operand");
40230 pat = GEN_FCN (icode) (op0, op1, op2, op3, op4);
40238 case IX86_BUILTIN_XABORT:
40239 icode = CODE_FOR_xabort;
40240 arg0 = CALL_EXPR_ARG (exp, 0);
40241 op0 = expand_normal (arg0);
40242 mode0 = insn_data[icode].operand[0].mode;
40243 if (!insn_data[icode].operand[0].predicate (op0, mode0))
40245 error ("the xabort's argument must be an 8-bit immediate");
40248 emit_insn (gen_xabort (op0));
40255 for (i = 0, d = bdesc_special_args;
40256 i < ARRAY_SIZE (bdesc_special_args);
40258 if (d->code == fcode)
40259 return ix86_expand_special_args_builtin (d, exp, target);
40261 for (i = 0, d = bdesc_args;
40262 i < ARRAY_SIZE (bdesc_args);
40264 if (d->code == fcode)
40267 case IX86_BUILTIN_FABSQ:
40268 case IX86_BUILTIN_COPYSIGNQ:
40270 /* Emit a normal call if SSE isn't available. */
40271 return expand_call (exp, target, ignore);
40273 return ix86_expand_args_builtin (d, exp, target);
40276 for (i = 0, d = bdesc_comi; i < ARRAY_SIZE (bdesc_comi); i++, d++)
40277 if (d->code == fcode)
40278 return ix86_expand_sse_comi (d, exp, target);
40280 for (i = 0, d = bdesc_round_args; i < ARRAY_SIZE (bdesc_round_args); i++, d++)
40281 if (d->code == fcode)
40282 return ix86_expand_round_builtin (d, exp, target);
40284 for (i = 0, d = bdesc_pcmpestr;
40285 i < ARRAY_SIZE (bdesc_pcmpestr);
40287 if (d->code == fcode)
40288 return ix86_expand_sse_pcmpestr (d, exp, target);
40290 for (i = 0, d = bdesc_pcmpistr;
40291 i < ARRAY_SIZE (bdesc_pcmpistr);
40293 if (d->code == fcode)
40294 return ix86_expand_sse_pcmpistr (d, exp, target);
40296 for (i = 0, d = bdesc_multi_arg; i < ARRAY_SIZE (bdesc_multi_arg); i++, d++)
40297 if (d->code == fcode)
40298 return ix86_expand_multi_arg_builtin (d->icode, exp, target,
40299 (enum ix86_builtin_func_type)
40300 d->flag, d->comparison);
40302 gcc_unreachable ();
40305 /* This returns the target-specific builtin with code CODE if
40306 current_function_decl has visibility on this builtin, which is checked
40307 using isa flags. Returns NULL_TREE otherwise. */
40309 static tree ix86_get_builtin (enum ix86_builtins code)
40311 struct cl_target_option *opts;
40312 tree target_tree = NULL_TREE;
40314 /* Determine the isa flags of current_function_decl. */
40316 if (current_function_decl)
40317 target_tree = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl);
40319 if (target_tree == NULL)
40320 target_tree = target_option_default_node;
40322 opts = TREE_TARGET_OPTION (target_tree);
40324 if (ix86_builtins_isa[(int) code].isa & opts->x_ix86_isa_flags)
40325 return ix86_builtin_decl (code, true);
40330 /* Return function decl for target specific builtin
40331 for given MPX builtin passed i FCODE. */
40333 ix86_builtin_mpx_function (unsigned fcode)
40337 case BUILT_IN_CHKP_BNDMK:
40338 return ix86_builtins[IX86_BUILTIN_BNDMK];
40340 case BUILT_IN_CHKP_BNDSTX:
40341 return ix86_builtins[IX86_BUILTIN_BNDSTX];
40343 case BUILT_IN_CHKP_BNDLDX:
40344 return ix86_builtins[IX86_BUILTIN_BNDLDX];
40346 case BUILT_IN_CHKP_BNDCL:
40347 return ix86_builtins[IX86_BUILTIN_BNDCL];
40349 case BUILT_IN_CHKP_BNDCU:
40350 return ix86_builtins[IX86_BUILTIN_BNDCU];
40352 case BUILT_IN_CHKP_BNDRET:
40353 return ix86_builtins[IX86_BUILTIN_BNDRET];
40355 case BUILT_IN_CHKP_INTERSECT:
40356 return ix86_builtins[IX86_BUILTIN_BNDINT];
40358 case BUILT_IN_CHKP_NARROW:
40359 return ix86_builtins[IX86_BUILTIN_BNDNARROW];
40361 case BUILT_IN_CHKP_SIZEOF:
40362 return ix86_builtins[IX86_BUILTIN_SIZEOF];
40364 case BUILT_IN_CHKP_EXTRACT_LOWER:
40365 return ix86_builtins[IX86_BUILTIN_BNDLOWER];
40367 case BUILT_IN_CHKP_EXTRACT_UPPER:
40368 return ix86_builtins[IX86_BUILTIN_BNDUPPER];
40374 gcc_unreachable ();
40377 /* Helper function for ix86_load_bounds and ix86_store_bounds.
40379 Return an address to be used to load/store bounds for pointer
40382 SLOT_NO is an integer constant holding number of a target
40383 dependent special slot to be used in case SLOT is not a memory.
40385 SPECIAL_BASE is a pointer to be used as a base of fake address
40386 to access special slots in Bounds Table. SPECIAL_BASE[-1],
40387 SPECIAL_BASE[-2] etc. will be used as fake pointer locations. */
40390 ix86_get_arg_address_for_bt (rtx slot, rtx slot_no, rtx special_base)
40394 /* NULL slot means we pass bounds for pointer not passed to the
40395 function at all. Register slot means we pass pointer in a
40396 register. In both these cases bounds are passed via Bounds
40397 Table. Since we do not have actual pointer stored in memory,
40398 we have to use fake addresses to access Bounds Table. We
40399 start with (special_base - sizeof (void*)) and decrease this
40400 address by pointer size to get addresses for other slots. */
40401 if (!slot || REG_P (slot))
40403 gcc_assert (CONST_INT_P (slot_no));
40404 addr = plus_constant (Pmode, special_base,
40405 -(INTVAL (slot_no) + 1) * GET_MODE_SIZE (Pmode));
40407 /* If pointer is passed in a memory then its address is used to
40408 access Bounds Table. */
40409 else if (MEM_P (slot))
40411 addr = XEXP (slot, 0);
40412 if (!register_operand (addr, Pmode))
40413 addr = copy_addr_to_reg (addr);
40416 gcc_unreachable ();
40421 /* Expand pass uses this hook to load bounds for function parameter
40422 PTR passed in SLOT in case its bounds are not passed in a register.
40424 If SLOT is a memory, then bounds are loaded as for regular pointer
40425 loaded from memory. PTR may be NULL in case SLOT is a memory.
40426 In such case value of PTR (if required) may be loaded from SLOT.
40428 If SLOT is NULL or a register then SLOT_NO is an integer constant
40429 holding number of the target dependent special slot which should be
40430 used to obtain bounds.
40432 Return loaded bounds. */
40435 ix86_load_bounds (rtx slot, rtx ptr, rtx slot_no)
40437 rtx reg = gen_reg_rtx (BNDmode);
40440 /* Get address to be used to access Bounds Table. Special slots start
40441 at the location of return address of the current function. */
40442 addr = ix86_get_arg_address_for_bt (slot, slot_no, arg_pointer_rtx);
40444 /* Load pointer value from a memory if we don't have it. */
40447 gcc_assert (MEM_P (slot));
40448 ptr = copy_addr_to_reg (slot);
40451 emit_insn (BNDmode == BND64mode
40452 ? gen_bnd64_ldx (reg, addr, ptr)
40453 : gen_bnd32_ldx (reg, addr, ptr));
40458 /* Expand pass uses this hook to store BOUNDS for call argument PTR
40459 passed in SLOT in case BOUNDS are not passed in a register.
40461 If SLOT is a memory, then BOUNDS are stored as for regular pointer
40462 stored in memory. PTR may be NULL in case SLOT is a memory.
40463 In such case value of PTR (if required) may be loaded from SLOT.
40465 If SLOT is NULL or a register then SLOT_NO is an integer constant
40466 holding number of the target dependent special slot which should be
40467 used to store BOUNDS. */
40470 ix86_store_bounds (rtx ptr, rtx slot, rtx bounds, rtx slot_no)
40474 /* Get address to be used to access Bounds Table. Special slots start
40475 at the location of return address of a called function. */
40476 addr = ix86_get_arg_address_for_bt (slot, slot_no, stack_pointer_rtx);
40478 /* Load pointer value from a memory if we don't have it. */
40481 gcc_assert (MEM_P (slot));
40482 ptr = copy_addr_to_reg (slot);
40485 gcc_assert (POINTER_BOUNDS_MODE_P (GET_MODE (bounds)));
40486 if (!register_operand (bounds, BNDmode))
40487 bounds = copy_to_mode_reg (BNDmode, bounds);
40489 emit_insn (BNDmode == BND64mode
40490 ? gen_bnd64_stx (addr, ptr, bounds)
40491 : gen_bnd32_stx (addr, ptr, bounds));
40494 /* Load and return bounds returned by function in SLOT. */
40497 ix86_load_returned_bounds (rtx slot)
40501 gcc_assert (REG_P (slot));
40502 res = gen_reg_rtx (BNDmode);
40503 emit_move_insn (res, slot);
40508 /* Store BOUNDS returned by function into SLOT. */
40511 ix86_store_returned_bounds (rtx slot, rtx bounds)
40513 gcc_assert (REG_P (slot));
40514 emit_move_insn (slot, bounds);
40517 /* Returns a function decl for a vectorized version of the builtin function
40518 with builtin function code FN and the result vector type TYPE, or NULL_TREE
40519 if it is not available. */
40522 ix86_builtin_vectorized_function (tree fndecl, tree type_out,
40525 machine_mode in_mode, out_mode;
40527 enum built_in_function fn = DECL_FUNCTION_CODE (fndecl);
40529 if (TREE_CODE (type_out) != VECTOR_TYPE
40530 || TREE_CODE (type_in) != VECTOR_TYPE
40531 || DECL_BUILT_IN_CLASS (fndecl) != BUILT_IN_NORMAL)
40534 out_mode = TYPE_MODE (TREE_TYPE (type_out));
40535 out_n = TYPE_VECTOR_SUBPARTS (type_out);
40536 in_mode = TYPE_MODE (TREE_TYPE (type_in));
40537 in_n = TYPE_VECTOR_SUBPARTS (type_in);
40541 case BUILT_IN_SQRT:
40542 if (out_mode == DFmode && in_mode == DFmode)
40544 if (out_n == 2 && in_n == 2)
40545 return ix86_get_builtin (IX86_BUILTIN_SQRTPD);
40546 else if (out_n == 4 && in_n == 4)
40547 return ix86_get_builtin (IX86_BUILTIN_SQRTPD256);
40548 else if (out_n == 8 && in_n == 8)
40549 return ix86_get_builtin (IX86_BUILTIN_SQRTPD512);
40553 case BUILT_IN_EXP2F:
40554 if (out_mode == SFmode && in_mode == SFmode)
40556 if (out_n == 16 && in_n == 16)
40557 return ix86_get_builtin (IX86_BUILTIN_EXP2PS);
40561 case BUILT_IN_SQRTF:
40562 if (out_mode == SFmode && in_mode == SFmode)
40564 if (out_n == 4 && in_n == 4)
40565 return ix86_get_builtin (IX86_BUILTIN_SQRTPS_NR);
40566 else if (out_n == 8 && in_n == 8)
40567 return ix86_get_builtin (IX86_BUILTIN_SQRTPS_NR256);
40568 else if (out_n == 16 && in_n == 16)
40569 return ix86_get_builtin (IX86_BUILTIN_SQRTPS_NR512);
40573 case BUILT_IN_IFLOOR:
40574 case BUILT_IN_LFLOOR:
40575 case BUILT_IN_LLFLOOR:
40576 /* The round insn does not trap on denormals. */
40577 if (flag_trapping_math || !TARGET_ROUND)
40580 if (out_mode == SImode && in_mode == DFmode)
40582 if (out_n == 4 && in_n == 2)
40583 return ix86_get_builtin (IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX);
40584 else if (out_n == 8 && in_n == 4)
40585 return ix86_get_builtin (IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256);
40586 else if (out_n == 16 && in_n == 8)
40587 return ix86_get_builtin (IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX512);
40591 case BUILT_IN_IFLOORF:
40592 case BUILT_IN_LFLOORF:
40593 case BUILT_IN_LLFLOORF:
40594 /* The round insn does not trap on denormals. */
40595 if (flag_trapping_math || !TARGET_ROUND)
40598 if (out_mode == SImode && in_mode == SFmode)
40600 if (out_n == 4 && in_n == 4)
40601 return ix86_get_builtin (IX86_BUILTIN_FLOORPS_SFIX);
40602 else if (out_n == 8 && in_n == 8)
40603 return ix86_get_builtin (IX86_BUILTIN_FLOORPS_SFIX256);
40607 case BUILT_IN_ICEIL:
40608 case BUILT_IN_LCEIL:
40609 case BUILT_IN_LLCEIL:
40610 /* The round insn does not trap on denormals. */
40611 if (flag_trapping_math || !TARGET_ROUND)
40614 if (out_mode == SImode && in_mode == DFmode)
40616 if (out_n == 4 && in_n == 2)
40617 return ix86_get_builtin (IX86_BUILTIN_CEILPD_VEC_PACK_SFIX);
40618 else if (out_n == 8 && in_n == 4)
40619 return ix86_get_builtin (IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256);
40620 else if (out_n == 16 && in_n == 8)
40621 return ix86_get_builtin (IX86_BUILTIN_CEILPD_VEC_PACK_SFIX512);
40625 case BUILT_IN_ICEILF:
40626 case BUILT_IN_LCEILF:
40627 case BUILT_IN_LLCEILF:
40628 /* The round insn does not trap on denormals. */
40629 if (flag_trapping_math || !TARGET_ROUND)
40632 if (out_mode == SImode && in_mode == SFmode)
40634 if (out_n == 4 && in_n == 4)
40635 return ix86_get_builtin (IX86_BUILTIN_CEILPS_SFIX);
40636 else if (out_n == 8 && in_n == 8)
40637 return ix86_get_builtin (IX86_BUILTIN_CEILPS_SFIX256);
40641 case BUILT_IN_IRINT:
40642 case BUILT_IN_LRINT:
40643 case BUILT_IN_LLRINT:
40644 if (out_mode == SImode && in_mode == DFmode)
40646 if (out_n == 4 && in_n == 2)
40647 return ix86_get_builtin (IX86_BUILTIN_VEC_PACK_SFIX);
40648 else if (out_n == 8 && in_n == 4)
40649 return ix86_get_builtin (IX86_BUILTIN_VEC_PACK_SFIX256);
40653 case BUILT_IN_IRINTF:
40654 case BUILT_IN_LRINTF:
40655 case BUILT_IN_LLRINTF:
40656 if (out_mode == SImode && in_mode == SFmode)
40658 if (out_n == 4 && in_n == 4)
40659 return ix86_get_builtin (IX86_BUILTIN_CVTPS2DQ);
40660 else if (out_n == 8 && in_n == 8)
40661 return ix86_get_builtin (IX86_BUILTIN_CVTPS2DQ256);
40665 case BUILT_IN_IROUND:
40666 case BUILT_IN_LROUND:
40667 case BUILT_IN_LLROUND:
40668 /* The round insn does not trap on denormals. */
40669 if (flag_trapping_math || !TARGET_ROUND)
40672 if (out_mode == SImode && in_mode == DFmode)
40674 if (out_n == 4 && in_n == 2)
40675 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX);
40676 else if (out_n == 8 && in_n == 4)
40677 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256);
40678 else if (out_n == 16 && in_n == 8)
40679 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX512);
40683 case BUILT_IN_IROUNDF:
40684 case BUILT_IN_LROUNDF:
40685 case BUILT_IN_LLROUNDF:
40686 /* The round insn does not trap on denormals. */
40687 if (flag_trapping_math || !TARGET_ROUND)
40690 if (out_mode == SImode && in_mode == SFmode)
40692 if (out_n == 4 && in_n == 4)
40693 return ix86_get_builtin (IX86_BUILTIN_ROUNDPS_AZ_SFIX);
40694 else if (out_n == 8 && in_n == 8)
40695 return ix86_get_builtin (IX86_BUILTIN_ROUNDPS_AZ_SFIX256);
40699 case BUILT_IN_COPYSIGN:
40700 if (out_mode == DFmode && in_mode == DFmode)
40702 if (out_n == 2 && in_n == 2)
40703 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPD);
40704 else if (out_n == 4 && in_n == 4)
40705 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPD256);
40706 else if (out_n == 8 && in_n == 8)
40707 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPD512);
40711 case BUILT_IN_COPYSIGNF:
40712 if (out_mode == SFmode && in_mode == SFmode)
40714 if (out_n == 4 && in_n == 4)
40715 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPS);
40716 else if (out_n == 8 && in_n == 8)
40717 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPS256);
40718 else if (out_n == 16 && in_n == 16)
40719 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPS512);
40723 case BUILT_IN_FLOOR:
40724 /* The round insn does not trap on denormals. */
40725 if (flag_trapping_math || !TARGET_ROUND)
40728 if (out_mode == DFmode && in_mode == DFmode)
40730 if (out_n == 2 && in_n == 2)
40731 return ix86_get_builtin (IX86_BUILTIN_FLOORPD);
40732 else if (out_n == 4 && in_n == 4)
40733 return ix86_get_builtin (IX86_BUILTIN_FLOORPD256);
40737 case BUILT_IN_FLOORF:
40738 /* The round insn does not trap on denormals. */
40739 if (flag_trapping_math || !TARGET_ROUND)
40742 if (out_mode == SFmode && in_mode == SFmode)
40744 if (out_n == 4 && in_n == 4)
40745 return ix86_get_builtin (IX86_BUILTIN_FLOORPS);
40746 else if (out_n == 8 && in_n == 8)
40747 return ix86_get_builtin (IX86_BUILTIN_FLOORPS256);
40751 case BUILT_IN_CEIL:
40752 /* The round insn does not trap on denormals. */
40753 if (flag_trapping_math || !TARGET_ROUND)
40756 if (out_mode == DFmode && in_mode == DFmode)
40758 if (out_n == 2 && in_n == 2)
40759 return ix86_get_builtin (IX86_BUILTIN_CEILPD);
40760 else if (out_n == 4 && in_n == 4)
40761 return ix86_get_builtin (IX86_BUILTIN_CEILPD256);
40765 case BUILT_IN_CEILF:
40766 /* The round insn does not trap on denormals. */
40767 if (flag_trapping_math || !TARGET_ROUND)
40770 if (out_mode == SFmode && in_mode == SFmode)
40772 if (out_n == 4 && in_n == 4)
40773 return ix86_get_builtin (IX86_BUILTIN_CEILPS);
40774 else if (out_n == 8 && in_n == 8)
40775 return ix86_get_builtin (IX86_BUILTIN_CEILPS256);
40779 case BUILT_IN_TRUNC:
40780 /* The round insn does not trap on denormals. */
40781 if (flag_trapping_math || !TARGET_ROUND)
40784 if (out_mode == DFmode && in_mode == DFmode)
40786 if (out_n == 2 && in_n == 2)
40787 return ix86_get_builtin (IX86_BUILTIN_TRUNCPD);
40788 else if (out_n == 4 && in_n == 4)
40789 return ix86_get_builtin (IX86_BUILTIN_TRUNCPD256);
40793 case BUILT_IN_TRUNCF:
40794 /* The round insn does not trap on denormals. */
40795 if (flag_trapping_math || !TARGET_ROUND)
40798 if (out_mode == SFmode && in_mode == SFmode)
40800 if (out_n == 4 && in_n == 4)
40801 return ix86_get_builtin (IX86_BUILTIN_TRUNCPS);
40802 else if (out_n == 8 && in_n == 8)
40803 return ix86_get_builtin (IX86_BUILTIN_TRUNCPS256);
40807 case BUILT_IN_RINT:
40808 /* The round insn does not trap on denormals. */
40809 if (flag_trapping_math || !TARGET_ROUND)
40812 if (out_mode == DFmode && in_mode == DFmode)
40814 if (out_n == 2 && in_n == 2)
40815 return ix86_get_builtin (IX86_BUILTIN_RINTPD);
40816 else if (out_n == 4 && in_n == 4)
40817 return ix86_get_builtin (IX86_BUILTIN_RINTPD256);
40821 case BUILT_IN_RINTF:
40822 /* The round insn does not trap on denormals. */
40823 if (flag_trapping_math || !TARGET_ROUND)
40826 if (out_mode == SFmode && in_mode == SFmode)
40828 if (out_n == 4 && in_n == 4)
40829 return ix86_get_builtin (IX86_BUILTIN_RINTPS);
40830 else if (out_n == 8 && in_n == 8)
40831 return ix86_get_builtin (IX86_BUILTIN_RINTPS256);
40835 case BUILT_IN_ROUND:
40836 /* The round insn does not trap on denormals. */
40837 if (flag_trapping_math || !TARGET_ROUND)
40840 if (out_mode == DFmode && in_mode == DFmode)
40842 if (out_n == 2 && in_n == 2)
40843 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ);
40844 else if (out_n == 4 && in_n == 4)
40845 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ256);
40849 case BUILT_IN_ROUNDF:
40850 /* The round insn does not trap on denormals. */
40851 if (flag_trapping_math || !TARGET_ROUND)
40854 if (out_mode == SFmode && in_mode == SFmode)
40856 if (out_n == 4 && in_n == 4)
40857 return ix86_get_builtin (IX86_BUILTIN_ROUNDPS_AZ);
40858 else if (out_n == 8 && in_n == 8)
40859 return ix86_get_builtin (IX86_BUILTIN_ROUNDPS_AZ256);
40864 if (out_mode == DFmode && in_mode == DFmode)
40866 if (out_n == 2 && in_n == 2)
40867 return ix86_get_builtin (IX86_BUILTIN_VFMADDPD);
40868 if (out_n == 4 && in_n == 4)
40869 return ix86_get_builtin (IX86_BUILTIN_VFMADDPD256);
40873 case BUILT_IN_FMAF:
40874 if (out_mode == SFmode && in_mode == SFmode)
40876 if (out_n == 4 && in_n == 4)
40877 return ix86_get_builtin (IX86_BUILTIN_VFMADDPS);
40878 if (out_n == 8 && in_n == 8)
40879 return ix86_get_builtin (IX86_BUILTIN_VFMADDPS256);
40887 /* Dispatch to a handler for a vectorization library. */
40888 if (ix86_veclib_handler)
40889 return ix86_veclib_handler ((enum built_in_function) fn, type_out,
40895 /* Handler for an SVML-style interface to
40896 a library with vectorized intrinsics. */
40899 ix86_veclibabi_svml (enum built_in_function fn, tree type_out, tree type_in)
40902 tree fntype, new_fndecl, args;
40905 machine_mode el_mode, in_mode;
40908 /* The SVML is suitable for unsafe math only. */
40909 if (!flag_unsafe_math_optimizations)
40912 el_mode = TYPE_MODE (TREE_TYPE (type_out));
40913 n = TYPE_VECTOR_SUBPARTS (type_out);
40914 in_mode = TYPE_MODE (TREE_TYPE (type_in));
40915 in_n = TYPE_VECTOR_SUBPARTS (type_in);
40916 if (el_mode != in_mode
40924 case BUILT_IN_LOG10:
40926 case BUILT_IN_TANH:
40928 case BUILT_IN_ATAN:
40929 case BUILT_IN_ATAN2:
40930 case BUILT_IN_ATANH:
40931 case BUILT_IN_CBRT:
40932 case BUILT_IN_SINH:
40934 case BUILT_IN_ASINH:
40935 case BUILT_IN_ASIN:
40936 case BUILT_IN_COSH:
40938 case BUILT_IN_ACOSH:
40939 case BUILT_IN_ACOS:
40940 if (el_mode != DFmode || n != 2)
40944 case BUILT_IN_EXPF:
40945 case BUILT_IN_LOGF:
40946 case BUILT_IN_LOG10F:
40947 case BUILT_IN_POWF:
40948 case BUILT_IN_TANHF:
40949 case BUILT_IN_TANF:
40950 case BUILT_IN_ATANF:
40951 case BUILT_IN_ATAN2F:
40952 case BUILT_IN_ATANHF:
40953 case BUILT_IN_CBRTF:
40954 case BUILT_IN_SINHF:
40955 case BUILT_IN_SINF:
40956 case BUILT_IN_ASINHF:
40957 case BUILT_IN_ASINF:
40958 case BUILT_IN_COSHF:
40959 case BUILT_IN_COSF:
40960 case BUILT_IN_ACOSHF:
40961 case BUILT_IN_ACOSF:
40962 if (el_mode != SFmode || n != 4)
40970 bname = IDENTIFIER_POINTER (DECL_NAME (builtin_decl_implicit (fn)));
40972 if (fn == BUILT_IN_LOGF)
40973 strcpy (name, "vmlsLn4");
40974 else if (fn == BUILT_IN_LOG)
40975 strcpy (name, "vmldLn2");
40978 sprintf (name, "vmls%s", bname+10);
40979 name[strlen (name)-1] = '4';
40982 sprintf (name, "vmld%s2", bname+10);
40984 /* Convert to uppercase. */
40988 for (args = DECL_ARGUMENTS (builtin_decl_implicit (fn));
40990 args = TREE_CHAIN (args))
40994 fntype = build_function_type_list (type_out, type_in, NULL);
40996 fntype = build_function_type_list (type_out, type_in, type_in, NULL);
40998 /* Build a function declaration for the vectorized function. */
40999 new_fndecl = build_decl (BUILTINS_LOCATION,
41000 FUNCTION_DECL, get_identifier (name), fntype);
41001 TREE_PUBLIC (new_fndecl) = 1;
41002 DECL_EXTERNAL (new_fndecl) = 1;
41003 DECL_IS_NOVOPS (new_fndecl) = 1;
41004 TREE_READONLY (new_fndecl) = 1;
41009 /* Handler for an ACML-style interface to
41010 a library with vectorized intrinsics. */
41013 ix86_veclibabi_acml (enum built_in_function fn, tree type_out, tree type_in)
41015 char name[20] = "__vr.._";
41016 tree fntype, new_fndecl, args;
41019 machine_mode el_mode, in_mode;
41022 /* The ACML is 64bits only and suitable for unsafe math only as
41023 it does not correctly support parts of IEEE with the required
41024 precision such as denormals. */
41026 || !flag_unsafe_math_optimizations)
41029 el_mode = TYPE_MODE (TREE_TYPE (type_out));
41030 n = TYPE_VECTOR_SUBPARTS (type_out);
41031 in_mode = TYPE_MODE (TREE_TYPE (type_in));
41032 in_n = TYPE_VECTOR_SUBPARTS (type_in);
41033 if (el_mode != in_mode
41043 case BUILT_IN_LOG2:
41044 case BUILT_IN_LOG10:
41047 if (el_mode != DFmode
41052 case BUILT_IN_SINF:
41053 case BUILT_IN_COSF:
41054 case BUILT_IN_EXPF:
41055 case BUILT_IN_POWF:
41056 case BUILT_IN_LOGF:
41057 case BUILT_IN_LOG2F:
41058 case BUILT_IN_LOG10F:
41061 if (el_mode != SFmode
41070 bname = IDENTIFIER_POINTER (DECL_NAME (builtin_decl_implicit (fn)));
41071 sprintf (name + 7, "%s", bname+10);
41074 for (args = DECL_ARGUMENTS (builtin_decl_implicit (fn));
41076 args = TREE_CHAIN (args))
41080 fntype = build_function_type_list (type_out, type_in, NULL);
41082 fntype = build_function_type_list (type_out, type_in, type_in, NULL);
41084 /* Build a function declaration for the vectorized function. */
41085 new_fndecl = build_decl (BUILTINS_LOCATION,
41086 FUNCTION_DECL, get_identifier (name), fntype);
41087 TREE_PUBLIC (new_fndecl) = 1;
41088 DECL_EXTERNAL (new_fndecl) = 1;
41089 DECL_IS_NOVOPS (new_fndecl) = 1;
41090 TREE_READONLY (new_fndecl) = 1;
41095 /* Returns a decl of a function that implements gather load with
41096 memory type MEM_VECTYPE and index type INDEX_VECTYPE and SCALE.
41097 Return NULL_TREE if it is not available. */
41100 ix86_vectorize_builtin_gather (const_tree mem_vectype,
41101 const_tree index_type, int scale)
41104 enum ix86_builtins code;
41109 if ((TREE_CODE (index_type) != INTEGER_TYPE
41110 && !POINTER_TYPE_P (index_type))
41111 || (TYPE_MODE (index_type) != SImode
41112 && TYPE_MODE (index_type) != DImode))
41115 if (TYPE_PRECISION (index_type) > POINTER_SIZE)
41118 /* v*gather* insn sign extends index to pointer mode. */
41119 if (TYPE_PRECISION (index_type) < POINTER_SIZE
41120 && TYPE_UNSIGNED (index_type))
41125 || (scale & (scale - 1)) != 0)
41128 si = TYPE_MODE (index_type) == SImode;
41129 switch (TYPE_MODE (mem_vectype))
41132 if (TARGET_AVX512VL)
41133 code = si ? IX86_BUILTIN_GATHER3SIV2DF : IX86_BUILTIN_GATHER3DIV2DF;
41135 code = si ? IX86_BUILTIN_GATHERSIV2DF : IX86_BUILTIN_GATHERDIV2DF;
41138 if (TARGET_AVX512VL)
41139 code = si ? IX86_BUILTIN_GATHER3ALTSIV4DF : IX86_BUILTIN_GATHER3DIV4DF;
41141 code = si ? IX86_BUILTIN_GATHERALTSIV4DF : IX86_BUILTIN_GATHERDIV4DF;
41144 if (TARGET_AVX512VL)
41145 code = si ? IX86_BUILTIN_GATHER3SIV2DI : IX86_BUILTIN_GATHER3DIV2DI;
41147 code = si ? IX86_BUILTIN_GATHERSIV2DI : IX86_BUILTIN_GATHERDIV2DI;
41150 if (TARGET_AVX512VL)
41151 code = si ? IX86_BUILTIN_GATHER3ALTSIV4DI : IX86_BUILTIN_GATHER3DIV4DI;
41153 code = si ? IX86_BUILTIN_GATHERALTSIV4DI : IX86_BUILTIN_GATHERDIV4DI;
41156 if (TARGET_AVX512VL)
41157 code = si ? IX86_BUILTIN_GATHER3SIV4SF : IX86_BUILTIN_GATHER3DIV4SF;
41159 code = si ? IX86_BUILTIN_GATHERSIV4SF : IX86_BUILTIN_GATHERDIV4SF;
41162 if (TARGET_AVX512VL)
41163 code = si ? IX86_BUILTIN_GATHER3SIV8SF : IX86_BUILTIN_GATHER3ALTDIV8SF;
41165 code = si ? IX86_BUILTIN_GATHERSIV8SF : IX86_BUILTIN_GATHERALTDIV8SF;
41168 if (TARGET_AVX512VL)
41169 code = si ? IX86_BUILTIN_GATHER3SIV4SI : IX86_BUILTIN_GATHER3DIV4SI;
41171 code = si ? IX86_BUILTIN_GATHERSIV4SI : IX86_BUILTIN_GATHERDIV4SI;
41174 if (TARGET_AVX512VL)
41175 code = si ? IX86_BUILTIN_GATHER3SIV8SI : IX86_BUILTIN_GATHER3ALTDIV8SI;
41177 code = si ? IX86_BUILTIN_GATHERSIV8SI : IX86_BUILTIN_GATHERALTDIV8SI;
41180 if (TARGET_AVX512F)
41181 code = si ? IX86_BUILTIN_GATHER3ALTSIV8DF : IX86_BUILTIN_GATHER3DIV8DF;
41186 if (TARGET_AVX512F)
41187 code = si ? IX86_BUILTIN_GATHER3ALTSIV8DI : IX86_BUILTIN_GATHER3DIV8DI;
41192 if (TARGET_AVX512F)
41193 code = si ? IX86_BUILTIN_GATHER3SIV16SF : IX86_BUILTIN_GATHER3ALTDIV16SF;
41198 if (TARGET_AVX512F)
41199 code = si ? IX86_BUILTIN_GATHER3SIV16SI : IX86_BUILTIN_GATHER3ALTDIV16SI;
41207 return ix86_get_builtin (code);
41210 /* Returns a code for a target-specific builtin that implements
41211 reciprocal of the function, or NULL_TREE if not available. */
41214 ix86_builtin_reciprocal (unsigned int fn, bool md_fn, bool)
41216 if (! (TARGET_SSE_MATH && !optimize_insn_for_size_p ()
41217 && flag_finite_math_only && !flag_trapping_math
41218 && flag_unsafe_math_optimizations))
41222 /* Machine dependent builtins. */
41225 /* Vectorized version of sqrt to rsqrt conversion. */
41226 case IX86_BUILTIN_SQRTPS_NR:
41227 return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR);
41229 case IX86_BUILTIN_SQRTPS_NR256:
41230 return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR256);
41236 /* Normal builtins. */
41239 /* Sqrt to rsqrt conversion. */
41240 case BUILT_IN_SQRTF:
41241 return ix86_get_builtin (IX86_BUILTIN_RSQRTF);
41248 /* Helper for avx_vpermilps256_operand et al. This is also used by
41249 the expansion functions to turn the parallel back into a mask.
41250 The return value is 0 for no match and the imm8+1 for a match. */
41253 avx_vpermilp_parallel (rtx par, machine_mode mode)
41255 unsigned i, nelt = GET_MODE_NUNITS (mode);
41257 unsigned char ipar[16] = {}; /* Silence -Wuninitialized warning. */
41259 if (XVECLEN (par, 0) != (int) nelt)
41262 /* Validate that all of the elements are constants, and not totally
41263 out of range. Copy the data into an integral array to make the
41264 subsequent checks easier. */
41265 for (i = 0; i < nelt; ++i)
41267 rtx er = XVECEXP (par, 0, i);
41268 unsigned HOST_WIDE_INT ei;
41270 if (!CONST_INT_P (er))
41281 /* In the 512-bit DFmode case, we can only move elements within
41282 a 128-bit lane. First fill the second part of the mask,
41284 for (i = 4; i < 6; ++i)
41286 if (ipar[i] < 4 || ipar[i] >= 6)
41288 mask |= (ipar[i] - 4) << i;
41290 for (i = 6; i < 8; ++i)
41294 mask |= (ipar[i] - 6) << i;
41299 /* In the 256-bit DFmode case, we can only move elements within
41301 for (i = 0; i < 2; ++i)
41305 mask |= ipar[i] << i;
41307 for (i = 2; i < 4; ++i)
41311 mask |= (ipar[i] - 2) << i;
41316 /* In 512 bit SFmode case, permutation in the upper 256 bits
41317 must mirror the permutation in the lower 256-bits. */
41318 for (i = 0; i < 8; ++i)
41319 if (ipar[i] + 8 != ipar[i + 8])
41324 /* In 256 bit SFmode case, we have full freedom of
41325 movement within the low 128-bit lane, but the high 128-bit
41326 lane must mirror the exact same pattern. */
41327 for (i = 0; i < 4; ++i)
41328 if (ipar[i] + 4 != ipar[i + 4])
41335 /* In the 128-bit case, we've full freedom in the placement of
41336 the elements from the source operand. */
41337 for (i = 0; i < nelt; ++i)
41338 mask |= ipar[i] << (i * (nelt / 2));
41342 gcc_unreachable ();
41345 /* Make sure success has a non-zero value by adding one. */
41349 /* Helper for avx_vperm2f128_v4df_operand et al. This is also used by
41350 the expansion functions to turn the parallel back into a mask.
41351 The return value is 0 for no match and the imm8+1 for a match. */
41354 avx_vperm2f128_parallel (rtx par, machine_mode mode)
41356 unsigned i, nelt = GET_MODE_NUNITS (mode), nelt2 = nelt / 2;
41358 unsigned char ipar[8] = {}; /* Silence -Wuninitialized warning. */
41360 if (XVECLEN (par, 0) != (int) nelt)
41363 /* Validate that all of the elements are constants, and not totally
41364 out of range. Copy the data into an integral array to make the
41365 subsequent checks easier. */
41366 for (i = 0; i < nelt; ++i)
41368 rtx er = XVECEXP (par, 0, i);
41369 unsigned HOST_WIDE_INT ei;
41371 if (!CONST_INT_P (er))
41374 if (ei >= 2 * nelt)
41379 /* Validate that the halves of the permute are halves. */
41380 for (i = 0; i < nelt2 - 1; ++i)
41381 if (ipar[i] + 1 != ipar[i + 1])
41383 for (i = nelt2; i < nelt - 1; ++i)
41384 if (ipar[i] + 1 != ipar[i + 1])
41387 /* Reconstruct the mask. */
41388 for (i = 0; i < 2; ++i)
41390 unsigned e = ipar[i * nelt2];
41394 mask |= e << (i * 4);
41397 /* Make sure success has a non-zero value by adding one. */
41401 /* Return a register priority for hard reg REGNO. */
41403 ix86_register_priority (int hard_regno)
41405 /* ebp and r13 as the base always wants a displacement, r12 as the
41406 base always wants an index. So discourage their usage in an
41408 if (hard_regno == R12_REG || hard_regno == R13_REG)
41410 if (hard_regno == BP_REG)
41412 /* New x86-64 int registers result in bigger code size. Discourage
41414 if (FIRST_REX_INT_REG <= hard_regno && hard_regno <= LAST_REX_INT_REG)
41416 /* New x86-64 SSE registers result in bigger code size. Discourage
41418 if (FIRST_REX_SSE_REG <= hard_regno && hard_regno <= LAST_REX_SSE_REG)
41420 /* Usage of AX register results in smaller code. Prefer it. */
41421 if (hard_regno == AX_REG)
41426 /* Implement TARGET_PREFERRED_RELOAD_CLASS.
41428 Put float CONST_DOUBLE in the constant pool instead of fp regs.
41429 QImode must go into class Q_REGS.
41430 Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
41431 movdf to do mem-to-mem moves through integer regs. */
41434 ix86_preferred_reload_class (rtx x, reg_class_t regclass)
41436 machine_mode mode = GET_MODE (x);
41438 /* We're only allowed to return a subclass of CLASS. Many of the
41439 following checks fail for NO_REGS, so eliminate that early. */
41440 if (regclass == NO_REGS)
41443 /* All classes can load zeros. */
41444 if (x == CONST0_RTX (mode))
41447 /* Force constants into memory if we are loading a (nonzero) constant into
41448 an MMX, SSE or MASK register. This is because there are no MMX/SSE/MASK
41449 instructions to load from a constant. */
41451 && (MAYBE_MMX_CLASS_P (regclass)
41452 || MAYBE_SSE_CLASS_P (regclass)
41453 || MAYBE_MASK_CLASS_P (regclass)))
41456 /* Prefer SSE regs only, if we can use them for math. */
41457 if (TARGET_SSE_MATH && !TARGET_MIX_SSE_I387 && SSE_FLOAT_MODE_P (mode))
41458 return SSE_CLASS_P (regclass) ? regclass : NO_REGS;
41460 /* Floating-point constants need more complex checks. */
41461 if (GET_CODE (x) == CONST_DOUBLE && GET_MODE (x) != VOIDmode)
41463 /* General regs can load everything. */
41464 if (reg_class_subset_p (regclass, GENERAL_REGS))
41467 /* Floats can load 0 and 1 plus some others. Note that we eliminated
41468 zero above. We only want to wind up preferring 80387 registers if
41469 we plan on doing computation with them. */
41471 && standard_80387_constant_p (x) > 0)
41473 /* Limit class to non-sse. */
41474 if (regclass == FLOAT_SSE_REGS)
41476 if (regclass == FP_TOP_SSE_REGS)
41478 if (regclass == FP_SECOND_SSE_REGS)
41479 return FP_SECOND_REG;
41480 if (regclass == FLOAT_INT_REGS || regclass == FLOAT_REGS)
41487 /* Generally when we see PLUS here, it's the function invariant
41488 (plus soft-fp const_int). Which can only be computed into general
41490 if (GET_CODE (x) == PLUS)
41491 return reg_class_subset_p (regclass, GENERAL_REGS) ? regclass : NO_REGS;
41493 /* QImode constants are easy to load, but non-constant QImode data
41494 must go into Q_REGS. */
41495 if (GET_MODE (x) == QImode && !CONSTANT_P (x))
41497 if (reg_class_subset_p (regclass, Q_REGS))
41499 if (reg_class_subset_p (Q_REGS, regclass))
41507 /* Discourage putting floating-point values in SSE registers unless
41508 SSE math is being used, and likewise for the 387 registers. */
41510 ix86_preferred_output_reload_class (rtx x, reg_class_t regclass)
41512 machine_mode mode = GET_MODE (x);
41514 /* Restrict the output reload class to the register bank that we are doing
41515 math on. If we would like not to return a subset of CLASS, reject this
41516 alternative: if reload cannot do this, it will still use its choice. */
41517 mode = GET_MODE (x);
41518 if (TARGET_SSE_MATH && SSE_FLOAT_MODE_P (mode))
41519 return MAYBE_SSE_CLASS_P (regclass) ? ALL_SSE_REGS : NO_REGS;
41521 if (X87_FLOAT_MODE_P (mode))
41523 if (regclass == FP_TOP_SSE_REGS)
41525 else if (regclass == FP_SECOND_SSE_REGS)
41526 return FP_SECOND_REG;
41528 return FLOAT_CLASS_P (regclass) ? regclass : NO_REGS;
41535 ix86_secondary_reload (bool in_p, rtx x, reg_class_t rclass,
41536 machine_mode mode, secondary_reload_info *sri)
41538 /* Double-word spills from general registers to non-offsettable memory
41539 references (zero-extended addresses) require special handling. */
41542 && GET_MODE_SIZE (mode) > UNITS_PER_WORD
41543 && INTEGER_CLASS_P (rclass)
41544 && !offsettable_memref_p (x))
41547 ? CODE_FOR_reload_noff_load
41548 : CODE_FOR_reload_noff_store);
41549 /* Add the cost of moving address to a temporary. */
41550 sri->extra_cost = 1;
41555 /* QImode spills from non-QI registers require
41556 intermediate register on 32bit targets. */
41558 && (MAYBE_MASK_CLASS_P (rclass)
41559 || (!TARGET_64BIT && !in_p
41560 && INTEGER_CLASS_P (rclass)
41561 && MAYBE_NON_Q_CLASS_P (rclass))))
41570 if (regno >= FIRST_PSEUDO_REGISTER || GET_CODE (x) == SUBREG)
41571 regno = true_regnum (x);
41573 /* Return Q_REGS if the operand is in memory. */
41578 /* This condition handles corner case where an expression involving
41579 pointers gets vectorized. We're trying to use the address of a
41580 stack slot as a vector initializer.
41582 (set (reg:V2DI 74 [ vect_cst_.2 ])
41583 (vec_duplicate:V2DI (reg/f:DI 20 frame)))
41585 Eventually frame gets turned into sp+offset like this:
41587 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
41588 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
41589 (const_int 392 [0x188]))))
41591 That later gets turned into:
41593 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
41594 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
41595 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))))
41597 We'll have the following reload recorded:
41599 Reload 0: reload_in (DI) =
41600 (plus:DI (reg/f:DI 7 sp)
41601 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))
41602 reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
41603 SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine
41604 reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188]))
41605 reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
41606 reload_reg_rtx: (reg:V2DI 22 xmm1)
41608 Which isn't going to work since SSE instructions can't handle scalar
41609 additions. Returning GENERAL_REGS forces the addition into integer
41610 register and reload can handle subsequent reloads without problems. */
41612 if (in_p && GET_CODE (x) == PLUS
41613 && SSE_CLASS_P (rclass)
41614 && SCALAR_INT_MODE_P (mode))
41615 return GENERAL_REGS;
41620 /* Implement TARGET_CLASS_LIKELY_SPILLED_P. */
41623 ix86_class_likely_spilled_p (reg_class_t rclass)
41634 case SSE_FIRST_REG:
41636 case FP_SECOND_REG:
41647 /* If we are copying between general and FP registers, we need a memory
41648 location. The same is true for SSE and MMX registers.
41650 To optimize register_move_cost performance, allow inline variant.
41652 The macro can't work reliably when one of the CLASSES is class containing
41653 registers from multiple units (SSE, MMX, integer). We avoid this by never
41654 combining those units in single alternative in the machine description.
41655 Ensure that this constraint holds to avoid unexpected surprises.
41657 When STRICT is false, we are being called from REGISTER_MOVE_COST, so do not
41658 enforce these sanity checks. */
41661 inline_secondary_memory_needed (enum reg_class class1, enum reg_class class2,
41662 machine_mode mode, int strict)
41664 if (lra_in_progress && (class1 == NO_REGS || class2 == NO_REGS))
41666 if (MAYBE_FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class1)
41667 || MAYBE_FLOAT_CLASS_P (class2) != FLOAT_CLASS_P (class2)
41668 || MAYBE_SSE_CLASS_P (class1) != SSE_CLASS_P (class1)
41669 || MAYBE_SSE_CLASS_P (class2) != SSE_CLASS_P (class2)
41670 || MAYBE_MMX_CLASS_P (class1) != MMX_CLASS_P (class1)
41671 || MAYBE_MMX_CLASS_P (class2) != MMX_CLASS_P (class2))
41673 gcc_assert (!strict || lra_in_progress);
41677 if (FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class2))
41680 /* Between mask and general, we have moves no larger than word size. */
41681 if ((MAYBE_MASK_CLASS_P (class1) != MAYBE_MASK_CLASS_P (class2))
41682 && (GET_MODE_SIZE (mode) > UNITS_PER_WORD))
41685 /* ??? This is a lie. We do have moves between mmx/general, and for
41686 mmx/sse2. But by saying we need secondary memory we discourage the
41687 register allocator from using the mmx registers unless needed. */
41688 if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2))
41691 if (SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
41693 /* SSE1 doesn't have any direct moves from other classes. */
41697 /* If the target says that inter-unit moves are more expensive
41698 than moving through memory, then don't generate them. */
41699 if ((SSE_CLASS_P (class1) && !TARGET_INTER_UNIT_MOVES_FROM_VEC)
41700 || (SSE_CLASS_P (class2) && !TARGET_INTER_UNIT_MOVES_TO_VEC))
41703 /* Between SSE and general, we have moves no larger than word size. */
41704 if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
41712 ix86_secondary_memory_needed (enum reg_class class1, enum reg_class class2,
41713 machine_mode mode, int strict)
41715 return inline_secondary_memory_needed (class1, class2, mode, strict);
41718 /* Implement the TARGET_CLASS_MAX_NREGS hook.
41720 On the 80386, this is the size of MODE in words,
41721 except in the FP regs, where a single reg is always enough. */
41723 static unsigned char
41724 ix86_class_max_nregs (reg_class_t rclass, machine_mode mode)
41726 if (MAYBE_INTEGER_CLASS_P (rclass))
41728 if (mode == XFmode)
41729 return (TARGET_64BIT ? 2 : 3);
41730 else if (mode == XCmode)
41731 return (TARGET_64BIT ? 4 : 6);
41733 return ((GET_MODE_SIZE (mode) + UNITS_PER_WORD - 1) / UNITS_PER_WORD);
41737 if (COMPLEX_MODE_P (mode))
41744 /* Return true if the registers in CLASS cannot represent the change from
41745 modes FROM to TO. */
41748 ix86_cannot_change_mode_class (machine_mode from, machine_mode to,
41749 enum reg_class regclass)
41754 /* x87 registers can't do subreg at all, as all values are reformatted
41755 to extended precision. */
41756 if (MAYBE_FLOAT_CLASS_P (regclass))
41759 if (MAYBE_SSE_CLASS_P (regclass) || MAYBE_MMX_CLASS_P (regclass))
41761 /* Vector registers do not support QI or HImode loads. If we don't
41762 disallow a change to these modes, reload will assume it's ok to
41763 drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
41764 the vec_dupv4hi pattern. */
41765 if (GET_MODE_SIZE (from) < 4)
41772 /* Return the cost of moving data of mode M between a
41773 register and memory. A value of 2 is the default; this cost is
41774 relative to those in `REGISTER_MOVE_COST'.
41776 This function is used extensively by register_move_cost that is used to
41777 build tables at startup. Make it inline in this case.
41778 When IN is 2, return maximum of in and out move cost.
41780 If moving between registers and memory is more expensive than
41781 between two registers, you should define this macro to express the
41784 Model also increased moving costs of QImode registers in non
41788 inline_memory_move_cost (machine_mode mode, enum reg_class regclass,
41792 if (FLOAT_CLASS_P (regclass))
41810 return MAX (ix86_cost->fp_load [index], ix86_cost->fp_store [index]);
41811 return in ? ix86_cost->fp_load [index] : ix86_cost->fp_store [index];
41813 if (SSE_CLASS_P (regclass))
41816 switch (GET_MODE_SIZE (mode))
41831 return MAX (ix86_cost->sse_load [index], ix86_cost->sse_store [index]);
41832 return in ? ix86_cost->sse_load [index] : ix86_cost->sse_store [index];
41834 if (MMX_CLASS_P (regclass))
41837 switch (GET_MODE_SIZE (mode))
41849 return MAX (ix86_cost->mmx_load [index], ix86_cost->mmx_store [index]);
41850 return in ? ix86_cost->mmx_load [index] : ix86_cost->mmx_store [index];
41852 switch (GET_MODE_SIZE (mode))
41855 if (Q_CLASS_P (regclass) || TARGET_64BIT)
41858 return ix86_cost->int_store[0];
41859 if (TARGET_PARTIAL_REG_DEPENDENCY
41860 && optimize_function_for_speed_p (cfun))
41861 cost = ix86_cost->movzbl_load;
41863 cost = ix86_cost->int_load[0];
41865 return MAX (cost, ix86_cost->int_store[0]);
41871 return MAX (ix86_cost->movzbl_load, ix86_cost->int_store[0] + 4);
41873 return ix86_cost->movzbl_load;
41875 return ix86_cost->int_store[0] + 4;
41880 return MAX (ix86_cost->int_load[1], ix86_cost->int_store[1]);
41881 return in ? ix86_cost->int_load[1] : ix86_cost->int_store[1];
41883 /* Compute number of 32bit moves needed. TFmode is moved as XFmode. */
41884 if (mode == TFmode)
41887 cost = MAX (ix86_cost->int_load[2] , ix86_cost->int_store[2]);
41889 cost = ix86_cost->int_load[2];
41891 cost = ix86_cost->int_store[2];
41892 return (cost * (((int) GET_MODE_SIZE (mode)
41893 + UNITS_PER_WORD - 1) / UNITS_PER_WORD));
41898 ix86_memory_move_cost (machine_mode mode, reg_class_t regclass,
41901 return inline_memory_move_cost (mode, (enum reg_class) regclass, in ? 1 : 0);
41905 /* Return the cost of moving data from a register in class CLASS1 to
41906 one in class CLASS2.
41908 It is not required that the cost always equal 2 when FROM is the same as TO;
41909 on some machines it is expensive to move between registers if they are not
41910 general registers. */
41913 ix86_register_move_cost (machine_mode mode, reg_class_t class1_i,
41914 reg_class_t class2_i)
41916 enum reg_class class1 = (enum reg_class) class1_i;
41917 enum reg_class class2 = (enum reg_class) class2_i;
41919 /* In case we require secondary memory, compute cost of the store followed
41920 by load. In order to avoid bad register allocation choices, we need
41921 for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
41923 if (inline_secondary_memory_needed (class1, class2, mode, 0))
41927 cost += inline_memory_move_cost (mode, class1, 2);
41928 cost += inline_memory_move_cost (mode, class2, 2);
41930 /* In case of copying from general_purpose_register we may emit multiple
41931 stores followed by single load causing memory size mismatch stall.
41932 Count this as arbitrarily high cost of 20. */
41933 if (targetm.class_max_nregs (class1, mode)
41934 > targetm.class_max_nregs (class2, mode))
41937 /* In the case of FP/MMX moves, the registers actually overlap, and we
41938 have to switch modes in order to treat them differently. */
41939 if ((MMX_CLASS_P (class1) && MAYBE_FLOAT_CLASS_P (class2))
41940 || (MMX_CLASS_P (class2) && MAYBE_FLOAT_CLASS_P (class1)))
41946 /* Moves between SSE/MMX and integer unit are expensive. */
41947 if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2)
41948 || SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
41950 /* ??? By keeping returned value relatively high, we limit the number
41951 of moves between integer and MMX/SSE registers for all targets.
41952 Additionally, high value prevents problem with x86_modes_tieable_p(),
41953 where integer modes in MMX/SSE registers are not tieable
41954 because of missing QImode and HImode moves to, from or between
41955 MMX/SSE registers. */
41956 return MAX (8, ix86_cost->mmxsse_to_integer);
41958 if (MAYBE_FLOAT_CLASS_P (class1))
41959 return ix86_cost->fp_move;
41960 if (MAYBE_SSE_CLASS_P (class1))
41961 return ix86_cost->sse_move;
41962 if (MAYBE_MMX_CLASS_P (class1))
41963 return ix86_cost->mmx_move;
41967 /* Return TRUE if hard register REGNO can hold a value of machine-mode
41971 ix86_hard_regno_mode_ok (int regno, machine_mode mode)
41973 /* Flags and only flags can only hold CCmode values. */
41974 if (CC_REGNO_P (regno))
41975 return GET_MODE_CLASS (mode) == MODE_CC;
41976 if (GET_MODE_CLASS (mode) == MODE_CC
41977 || GET_MODE_CLASS (mode) == MODE_RANDOM
41978 || GET_MODE_CLASS (mode) == MODE_PARTIAL_INT)
41980 if (STACK_REGNO_P (regno))
41981 return VALID_FP_MODE_P (mode);
41982 if (MASK_REGNO_P (regno))
41983 return (VALID_MASK_REG_MODE (mode)
41984 || (TARGET_AVX512BW
41985 && VALID_MASK_AVX512BW_MODE (mode)));
41986 if (BND_REGNO_P (regno))
41987 return VALID_BND_REG_MODE (mode);
41988 if (SSE_REGNO_P (regno))
41990 /* We implement the move patterns for all vector modes into and
41991 out of SSE registers, even when no operation instructions
41994 /* For AVX-512 we allow, regardless of regno:
41996 - any of 512-bit wide vector mode
41997 - any scalar mode. */
42000 || VALID_AVX512F_REG_MODE (mode)
42001 || VALID_AVX512F_SCALAR_MODE (mode)))
42004 /* TODO check for QI/HI scalars. */
42005 /* AVX512VL allows sse regs16+ for 128/256 bit modes. */
42006 if (TARGET_AVX512VL
42009 || VALID_AVX256_REG_MODE (mode)
42010 || VALID_AVX512VL_128_REG_MODE (mode)))
42013 /* xmm16-xmm31 are only available for AVX-512. */
42014 if (EXT_REX_SSE_REGNO_P (regno))
42017 /* OImode and AVX modes are available only when AVX is enabled. */
42018 return ((TARGET_AVX
42019 && VALID_AVX256_REG_OR_OI_MODE (mode))
42020 || VALID_SSE_REG_MODE (mode)
42021 || VALID_SSE2_REG_MODE (mode)
42022 || VALID_MMX_REG_MODE (mode)
42023 || VALID_MMX_REG_MODE_3DNOW (mode));
42025 if (MMX_REGNO_P (regno))
42027 /* We implement the move patterns for 3DNOW modes even in MMX mode,
42028 so if the register is available at all, then we can move data of
42029 the given mode into or out of it. */
42030 return (VALID_MMX_REG_MODE (mode)
42031 || VALID_MMX_REG_MODE_3DNOW (mode));
42034 if (mode == QImode)
42036 /* Take care for QImode values - they can be in non-QI regs,
42037 but then they do cause partial register stalls. */
42038 if (ANY_QI_REGNO_P (regno))
42040 if (!TARGET_PARTIAL_REG_STALL)
42042 /* LRA checks if the hard register is OK for the given mode.
42043 QImode values can live in non-QI regs, so we allow all
42045 if (lra_in_progress)
42047 return !can_create_pseudo_p ();
42049 /* We handle both integer and floats in the general purpose registers. */
42050 else if (VALID_INT_MODE_P (mode))
42052 else if (VALID_FP_MODE_P (mode))
42054 else if (VALID_DFP_MODE_P (mode))
42056 /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
42057 on to use that value in smaller contexts, this can easily force a
42058 pseudo to be allocated to GENERAL_REGS. Since this is no worse than
42059 supporting DImode, allow it. */
42060 else if (VALID_MMX_REG_MODE_3DNOW (mode) || VALID_MMX_REG_MODE (mode))
42066 /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
42067 tieable integer mode. */
42070 ix86_tieable_integer_mode_p (machine_mode mode)
42079 return TARGET_64BIT || !TARGET_PARTIAL_REG_STALL;
42082 return TARGET_64BIT;
42089 /* Return true if MODE1 is accessible in a register that can hold MODE2
42090 without copying. That is, all register classes that can hold MODE2
42091 can also hold MODE1. */
42094 ix86_modes_tieable_p (machine_mode mode1, machine_mode mode2)
42096 if (mode1 == mode2)
42099 if (ix86_tieable_integer_mode_p (mode1)
42100 && ix86_tieable_integer_mode_p (mode2))
42103 /* MODE2 being XFmode implies fp stack or general regs, which means we
42104 can tie any smaller floating point modes to it. Note that we do not
42105 tie this with TFmode. */
42106 if (mode2 == XFmode)
42107 return mode1 == SFmode || mode1 == DFmode;
42109 /* MODE2 being DFmode implies fp stack, general or sse regs, which means
42110 that we can tie it with SFmode. */
42111 if (mode2 == DFmode)
42112 return mode1 == SFmode;
42114 /* If MODE2 is only appropriate for an SSE register, then tie with
42115 any other mode acceptable to SSE registers. */
42116 if (GET_MODE_SIZE (mode2) == 32
42117 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
42118 return (GET_MODE_SIZE (mode1) == 32
42119 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1));
42120 if (GET_MODE_SIZE (mode2) == 16
42121 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
42122 return (GET_MODE_SIZE (mode1) == 16
42123 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1));
42125 /* If MODE2 is appropriate for an MMX register, then tie
42126 with any other mode acceptable to MMX registers. */
42127 if (GET_MODE_SIZE (mode2) == 8
42128 && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode2))
42129 return (GET_MODE_SIZE (mode1) == 8
42130 && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode1));
42135 /* Return the cost of moving between two registers of mode MODE. */
42138 ix86_set_reg_reg_cost (machine_mode mode)
42140 unsigned int units = UNITS_PER_WORD;
42142 switch (GET_MODE_CLASS (mode))
42148 units = GET_MODE_SIZE (CCmode);
42152 if ((TARGET_SSE && mode == TFmode)
42153 || (TARGET_80387 && mode == XFmode)
42154 || ((TARGET_80387 || TARGET_SSE2) && mode == DFmode)
42155 || ((TARGET_80387 || TARGET_SSE) && mode == SFmode))
42156 units = GET_MODE_SIZE (mode);
42159 case MODE_COMPLEX_FLOAT:
42160 if ((TARGET_SSE && mode == TCmode)
42161 || (TARGET_80387 && mode == XCmode)
42162 || ((TARGET_80387 || TARGET_SSE2) && mode == DCmode)
42163 || ((TARGET_80387 || TARGET_SSE) && mode == SCmode))
42164 units = GET_MODE_SIZE (mode);
42167 case MODE_VECTOR_INT:
42168 case MODE_VECTOR_FLOAT:
42169 if ((TARGET_AVX512F && VALID_AVX512F_REG_MODE (mode))
42170 || (TARGET_AVX && VALID_AVX256_REG_MODE (mode))
42171 || (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
42172 || (TARGET_SSE && VALID_SSE_REG_MODE (mode))
42173 || (TARGET_MMX && VALID_MMX_REG_MODE (mode)))
42174 units = GET_MODE_SIZE (mode);
42177 /* Return the cost of moving between two registers of mode MODE,
42178 assuming that the move will be in pieces of at most UNITS bytes. */
42179 return COSTS_N_INSNS ((GET_MODE_SIZE (mode) + units - 1) / units);
42182 /* Compute a (partial) cost for rtx X. Return true if the complete
42183 cost has been computed, and false if subexpressions should be
42184 scanned. In either case, *TOTAL contains the cost result. */
42187 ix86_rtx_costs (rtx x, int code_i, int outer_code_i, int opno, int *total,
42191 enum rtx_code code = (enum rtx_code) code_i;
42192 enum rtx_code outer_code = (enum rtx_code) outer_code_i;
42193 machine_mode mode = GET_MODE (x);
42194 const struct processor_costs *cost = speed ? ix86_cost : &ix86_size_cost;
42199 if (register_operand (SET_DEST (x), VOIDmode)
42200 && reg_or_0_operand (SET_SRC (x), VOIDmode))
42202 *total = ix86_set_reg_reg_cost (GET_MODE (SET_DEST (x)));
42211 if (TARGET_64BIT && !x86_64_immediate_operand (x, VOIDmode))
42213 else if (TARGET_64BIT && !x86_64_zext_immediate_operand (x, VOIDmode))
42215 else if (flag_pic && SYMBOLIC_CONST (x)
42217 && (GET_CODE (x) == LABEL_REF
42218 || (GET_CODE (x) == SYMBOL_REF
42219 && SYMBOL_REF_LOCAL_P (x)))))
42226 if (mode == VOIDmode)
42231 switch (standard_80387_constant_p (x))
42236 default: /* Other constants */
42243 if (SSE_FLOAT_MODE_P (mode))
42246 switch (standard_sse_constant_p (x))
42250 case 1: /* 0: xor eliminates false dependency */
42253 default: /* -1: cmp contains false dependency */
42258 /* Fall back to (MEM (SYMBOL_REF)), since that's where
42259 it'll probably end up. Add a penalty for size. */
42260 *total = (COSTS_N_INSNS (1)
42261 + (flag_pic != 0 && !TARGET_64BIT)
42262 + (mode == SFmode ? 0 : mode == DFmode ? 1 : 2));
42266 /* The zero extensions is often completely free on x86_64, so make
42267 it as cheap as possible. */
42268 if (TARGET_64BIT && mode == DImode
42269 && GET_MODE (XEXP (x, 0)) == SImode)
42271 else if (TARGET_ZERO_EXTEND_WITH_AND)
42272 *total = cost->add;
42274 *total = cost->movzx;
42278 *total = cost->movsx;
42282 if (SCALAR_INT_MODE_P (mode)
42283 && GET_MODE_SIZE (mode) < UNITS_PER_WORD
42284 && CONST_INT_P (XEXP (x, 1)))
42286 HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
42289 *total = cost->add;
42292 if ((value == 2 || value == 3)
42293 && cost->lea <= cost->shift_const)
42295 *total = cost->lea;
42305 if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
42307 /* ??? Should be SSE vector operation cost. */
42308 /* At least for published AMD latencies, this really is the same
42309 as the latency for a simple fpu operation like fabs. */
42310 /* V*QImode is emulated with 1-11 insns. */
42311 if (mode == V16QImode || mode == V32QImode)
42314 if (TARGET_XOP && mode == V16QImode)
42316 /* For XOP we use vpshab, which requires a broadcast of the
42317 value to the variable shift insn. For constants this
42318 means a V16Q const in mem; even when we can perform the
42319 shift with one insn set the cost to prefer paddb. */
42320 if (CONSTANT_P (XEXP (x, 1)))
42322 *total = (cost->fabs
42323 + rtx_cost (XEXP (x, 0), code, 0, speed)
42324 + (speed ? 2 : COSTS_N_BYTES (16)));
42329 else if (TARGET_SSSE3)
42331 *total = cost->fabs * count;
42334 *total = cost->fabs;
42336 else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
42338 if (CONST_INT_P (XEXP (x, 1)))
42340 if (INTVAL (XEXP (x, 1)) > 32)
42341 *total = cost->shift_const + COSTS_N_INSNS (2);
42343 *total = cost->shift_const * 2;
42347 if (GET_CODE (XEXP (x, 1)) == AND)
42348 *total = cost->shift_var * 2;
42350 *total = cost->shift_var * 6 + COSTS_N_INSNS (2);
42355 if (CONST_INT_P (XEXP (x, 1)))
42356 *total = cost->shift_const;
42357 else if (GET_CODE (XEXP (x, 1)) == SUBREG
42358 && GET_CODE (XEXP (XEXP (x, 1), 0)) == AND)
42360 /* Return the cost after shift-and truncation. */
42361 *total = cost->shift_var;
42365 *total = cost->shift_var;
42373 gcc_assert (FLOAT_MODE_P (mode));
42374 gcc_assert (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F);
42376 /* ??? SSE scalar/vector cost should be used here. */
42377 /* ??? Bald assumption that fma has the same cost as fmul. */
42378 *total = cost->fmul;
42379 *total += rtx_cost (XEXP (x, 1), FMA, 1, speed);
42381 /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */
42383 if (GET_CODE (sub) == NEG)
42384 sub = XEXP (sub, 0);
42385 *total += rtx_cost (sub, FMA, 0, speed);
42388 if (GET_CODE (sub) == NEG)
42389 sub = XEXP (sub, 0);
42390 *total += rtx_cost (sub, FMA, 2, speed);
42395 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42397 /* ??? SSE scalar cost should be used here. */
42398 *total = cost->fmul;
42401 else if (X87_FLOAT_MODE_P (mode))
42403 *total = cost->fmul;
42406 else if (FLOAT_MODE_P (mode))
42408 /* ??? SSE vector cost should be used here. */
42409 *total = cost->fmul;
42412 else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
42414 /* V*QImode is emulated with 7-13 insns. */
42415 if (mode == V16QImode || mode == V32QImode)
42418 if (TARGET_XOP && mode == V16QImode)
42420 else if (TARGET_SSSE3)
42422 *total = cost->fmul * 2 + cost->fabs * extra;
42424 /* V*DImode is emulated with 5-8 insns. */
42425 else if (mode == V2DImode || mode == V4DImode)
42427 if (TARGET_XOP && mode == V2DImode)
42428 *total = cost->fmul * 2 + cost->fabs * 3;
42430 *total = cost->fmul * 3 + cost->fabs * 5;
42432 /* Without sse4.1, we don't have PMULLD; it's emulated with 7
42433 insns, including two PMULUDQ. */
42434 else if (mode == V4SImode && !(TARGET_SSE4_1 || TARGET_AVX))
42435 *total = cost->fmul * 2 + cost->fabs * 5;
42437 *total = cost->fmul;
42442 rtx op0 = XEXP (x, 0);
42443 rtx op1 = XEXP (x, 1);
42445 if (CONST_INT_P (XEXP (x, 1)))
42447 unsigned HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
42448 for (nbits = 0; value != 0; value &= value - 1)
42452 /* This is arbitrary. */
42455 /* Compute costs correctly for widening multiplication. */
42456 if ((GET_CODE (op0) == SIGN_EXTEND || GET_CODE (op0) == ZERO_EXTEND)
42457 && GET_MODE_SIZE (GET_MODE (XEXP (op0, 0))) * 2
42458 == GET_MODE_SIZE (mode))
42460 int is_mulwiden = 0;
42461 machine_mode inner_mode = GET_MODE (op0);
42463 if (GET_CODE (op0) == GET_CODE (op1))
42464 is_mulwiden = 1, op1 = XEXP (op1, 0);
42465 else if (CONST_INT_P (op1))
42467 if (GET_CODE (op0) == SIGN_EXTEND)
42468 is_mulwiden = trunc_int_for_mode (INTVAL (op1), inner_mode)
42471 is_mulwiden = !(INTVAL (op1) & ~GET_MODE_MASK (inner_mode));
42475 op0 = XEXP (op0, 0), mode = GET_MODE (op0);
42478 *total = (cost->mult_init[MODE_INDEX (mode)]
42479 + nbits * cost->mult_bit
42480 + rtx_cost (op0, outer_code, opno, speed)
42481 + rtx_cost (op1, outer_code, opno, speed));
42490 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42491 /* ??? SSE cost should be used here. */
42492 *total = cost->fdiv;
42493 else if (X87_FLOAT_MODE_P (mode))
42494 *total = cost->fdiv;
42495 else if (FLOAT_MODE_P (mode))
42496 /* ??? SSE vector cost should be used here. */
42497 *total = cost->fdiv;
42499 *total = cost->divide[MODE_INDEX (mode)];
42503 if (GET_MODE_CLASS (mode) == MODE_INT
42504 && GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
42506 if (GET_CODE (XEXP (x, 0)) == PLUS
42507 && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
42508 && CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 0), 1))
42509 && CONSTANT_P (XEXP (x, 1)))
42511 HOST_WIDE_INT val = INTVAL (XEXP (XEXP (XEXP (x, 0), 0), 1));
42512 if (val == 2 || val == 4 || val == 8)
42514 *total = cost->lea;
42515 *total += rtx_cost (XEXP (XEXP (x, 0), 1),
42516 outer_code, opno, speed);
42517 *total += rtx_cost (XEXP (XEXP (XEXP (x, 0), 0), 0),
42518 outer_code, opno, speed);
42519 *total += rtx_cost (XEXP (x, 1), outer_code, opno, speed);
42523 else if (GET_CODE (XEXP (x, 0)) == MULT
42524 && CONST_INT_P (XEXP (XEXP (x, 0), 1)))
42526 HOST_WIDE_INT val = INTVAL (XEXP (XEXP (x, 0), 1));
42527 if (val == 2 || val == 4 || val == 8)
42529 *total = cost->lea;
42530 *total += rtx_cost (XEXP (XEXP (x, 0), 0),
42531 outer_code, opno, speed);
42532 *total += rtx_cost (XEXP (x, 1), outer_code, opno, speed);
42536 else if (GET_CODE (XEXP (x, 0)) == PLUS)
42538 *total = cost->lea;
42539 *total += rtx_cost (XEXP (XEXP (x, 0), 0),
42540 outer_code, opno, speed);
42541 *total += rtx_cost (XEXP (XEXP (x, 0), 1),
42542 outer_code, opno, speed);
42543 *total += rtx_cost (XEXP (x, 1), outer_code, opno, speed);
42550 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42552 /* ??? SSE cost should be used here. */
42553 *total = cost->fadd;
42556 else if (X87_FLOAT_MODE_P (mode))
42558 *total = cost->fadd;
42561 else if (FLOAT_MODE_P (mode))
42563 /* ??? SSE vector cost should be used here. */
42564 *total = cost->fadd;
42572 if (GET_MODE_CLASS (mode) == MODE_INT
42573 && GET_MODE_SIZE (mode) > UNITS_PER_WORD)
42575 *total = (cost->add * 2
42576 + (rtx_cost (XEXP (x, 0), outer_code, opno, speed)
42577 << (GET_MODE (XEXP (x, 0)) != DImode))
42578 + (rtx_cost (XEXP (x, 1), outer_code, opno, speed)
42579 << (GET_MODE (XEXP (x, 1)) != DImode)));
42585 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42587 /* ??? SSE cost should be used here. */
42588 *total = cost->fchs;
42591 else if (X87_FLOAT_MODE_P (mode))
42593 *total = cost->fchs;
42596 else if (FLOAT_MODE_P (mode))
42598 /* ??? SSE vector cost should be used here. */
42599 *total = cost->fchs;
42605 if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
42607 /* ??? Should be SSE vector operation cost. */
42608 /* At least for published AMD latencies, this really is the same
42609 as the latency for a simple fpu operation like fabs. */
42610 *total = cost->fabs;
42612 else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
42613 *total = cost->add * 2;
42615 *total = cost->add;
42619 if (GET_CODE (XEXP (x, 0)) == ZERO_EXTRACT
42620 && XEXP (XEXP (x, 0), 1) == const1_rtx
42621 && CONST_INT_P (XEXP (XEXP (x, 0), 2))
42622 && XEXP (x, 1) == const0_rtx)
42624 /* This kind of construct is implemented using test[bwl].
42625 Treat it as if we had an AND. */
42626 *total = (cost->add
42627 + rtx_cost (XEXP (XEXP (x, 0), 0), outer_code, opno, speed)
42628 + rtx_cost (const1_rtx, outer_code, opno, speed));
42634 if (!(SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH))
42639 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42640 /* ??? SSE cost should be used here. */
42641 *total = cost->fabs;
42642 else if (X87_FLOAT_MODE_P (mode))
42643 *total = cost->fabs;
42644 else if (FLOAT_MODE_P (mode))
42645 /* ??? SSE vector cost should be used here. */
42646 *total = cost->fabs;
42650 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42651 /* ??? SSE cost should be used here. */
42652 *total = cost->fsqrt;
42653 else if (X87_FLOAT_MODE_P (mode))
42654 *total = cost->fsqrt;
42655 else if (FLOAT_MODE_P (mode))
42656 /* ??? SSE vector cost should be used here. */
42657 *total = cost->fsqrt;
42661 if (XINT (x, 1) == UNSPEC_TP)
42667 case VEC_DUPLICATE:
42668 /* ??? Assume all of these vector manipulation patterns are
42669 recognizable. In which case they all pretty much have the
42671 *total = cost->fabs;
42674 mask = XEXP (x, 2);
42675 /* This is masked instruction, assume the same cost,
42676 as nonmasked variant. */
42677 if (TARGET_AVX512F && register_operand (mask, GET_MODE (mask)))
42678 *total = rtx_cost (XEXP (x, 0), outer_code, opno, speed);
42680 *total = cost->fabs;
42690 static int current_machopic_label_num;
42692 /* Given a symbol name and its associated stub, write out the
42693 definition of the stub. */
42696 machopic_output_stub (FILE *file, const char *symb, const char *stub)
42698 unsigned int length;
42699 char *binder_name, *symbol_name, lazy_ptr_name[32];
42700 int label = ++current_machopic_label_num;
42702 /* For 64-bit we shouldn't get here. */
42703 gcc_assert (!TARGET_64BIT);
42705 /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
42706 symb = targetm.strip_name_encoding (symb);
42708 length = strlen (stub);
42709 binder_name = XALLOCAVEC (char, length + 32);
42710 GEN_BINDER_NAME_FOR_STUB (binder_name, stub, length);
42712 length = strlen (symb);
42713 symbol_name = XALLOCAVEC (char, length + 32);
42714 GEN_SYMBOL_NAME_FOR_SYMBOL (symbol_name, symb, length);
42716 sprintf (lazy_ptr_name, "L%d$lz", label);
42718 if (MACHOPIC_ATT_STUB)
42719 switch_to_section (darwin_sections[machopic_picsymbol_stub3_section]);
42720 else if (MACHOPIC_PURE)
42721 switch_to_section (darwin_sections[machopic_picsymbol_stub2_section]);
42723 switch_to_section (darwin_sections[machopic_symbol_stub_section]);
42725 fprintf (file, "%s:\n", stub);
42726 fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
42728 if (MACHOPIC_ATT_STUB)
42730 fprintf (file, "\thlt ; hlt ; hlt ; hlt ; hlt\n");
42732 else if (MACHOPIC_PURE)
42735 /* 25-byte PIC stub using "CALL get_pc_thunk". */
42736 rtx tmp = gen_rtx_REG (SImode, 2 /* ECX */);
42737 output_set_got (tmp, NULL_RTX); /* "CALL ___<cpu>.get_pc_thunk.cx". */
42738 fprintf (file, "LPC$%d:\tmovl\t%s-LPC$%d(%%ecx),%%ecx\n",
42739 label, lazy_ptr_name, label);
42740 fprintf (file, "\tjmp\t*%%ecx\n");
42743 fprintf (file, "\tjmp\t*%s\n", lazy_ptr_name);
42745 /* The AT&T-style ("self-modifying") stub is not lazily bound, thus
42746 it needs no stub-binding-helper. */
42747 if (MACHOPIC_ATT_STUB)
42750 fprintf (file, "%s:\n", binder_name);
42754 fprintf (file, "\tlea\t%s-%s(%%ecx),%%ecx\n", lazy_ptr_name, binder_name);
42755 fprintf (file, "\tpushl\t%%ecx\n");
42758 fprintf (file, "\tpushl\t$%s\n", lazy_ptr_name);
42760 fputs ("\tjmp\tdyld_stub_binding_helper\n", file);
42762 /* N.B. Keep the correspondence of these
42763 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the
42764 old-pic/new-pic/non-pic stubs; altering this will break
42765 compatibility with existing dylibs. */
42768 /* 25-byte PIC stub using "CALL get_pc_thunk". */
42769 switch_to_section (darwin_sections[machopic_lazy_symbol_ptr2_section]);
42772 /* 16-byte -mdynamic-no-pic stub. */
42773 switch_to_section(darwin_sections[machopic_lazy_symbol_ptr3_section]);
42775 fprintf (file, "%s:\n", lazy_ptr_name);
42776 fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
42777 fprintf (file, ASM_LONG "%s\n", binder_name);
42779 #endif /* TARGET_MACHO */
42781 /* Order the registers for register allocator. */
42784 x86_order_regs_for_local_alloc (void)
42789 /* First allocate the local general purpose registers. */
42790 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
42791 if (GENERAL_REGNO_P (i) && call_used_regs[i])
42792 reg_alloc_order [pos++] = i;
42794 /* Global general purpose registers. */
42795 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
42796 if (GENERAL_REGNO_P (i) && !call_used_regs[i])
42797 reg_alloc_order [pos++] = i;
42799 /* x87 registers come first in case we are doing FP math
42801 if (!TARGET_SSE_MATH)
42802 for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
42803 reg_alloc_order [pos++] = i;
42805 /* SSE registers. */
42806 for (i = FIRST_SSE_REG; i <= LAST_SSE_REG; i++)
42807 reg_alloc_order [pos++] = i;
42808 for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
42809 reg_alloc_order [pos++] = i;
42811 /* Extended REX SSE registers. */
42812 for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
42813 reg_alloc_order [pos++] = i;
42815 /* Mask register. */
42816 for (i = FIRST_MASK_REG; i <= LAST_MASK_REG; i++)
42817 reg_alloc_order [pos++] = i;
42819 /* MPX bound registers. */
42820 for (i = FIRST_BND_REG; i <= LAST_BND_REG; i++)
42821 reg_alloc_order [pos++] = i;
42823 /* x87 registers. */
42824 if (TARGET_SSE_MATH)
42825 for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
42826 reg_alloc_order [pos++] = i;
42828 for (i = FIRST_MMX_REG; i <= LAST_MMX_REG; i++)
42829 reg_alloc_order [pos++] = i;
42831 /* Initialize the rest of array as we do not allocate some registers
42833 while (pos < FIRST_PSEUDO_REGISTER)
42834 reg_alloc_order [pos++] = 0;
42837 /* Handle a "callee_pop_aggregate_return" attribute; arguments as
42838 in struct attribute_spec handler. */
42840 ix86_handle_callee_pop_aggregate_return (tree *node, tree name,
42843 bool *no_add_attrs)
42845 if (TREE_CODE (*node) != FUNCTION_TYPE
42846 && TREE_CODE (*node) != METHOD_TYPE
42847 && TREE_CODE (*node) != FIELD_DECL
42848 && TREE_CODE (*node) != TYPE_DECL)
42850 warning (OPT_Wattributes, "%qE attribute only applies to functions",
42852 *no_add_attrs = true;
42857 warning (OPT_Wattributes, "%qE attribute only available for 32-bit",
42859 *no_add_attrs = true;
42862 if (is_attribute_p ("callee_pop_aggregate_return", name))
42866 cst = TREE_VALUE (args);
42867 if (TREE_CODE (cst) != INTEGER_CST)
42869 warning (OPT_Wattributes,
42870 "%qE attribute requires an integer constant argument",
42872 *no_add_attrs = true;
42874 else if (compare_tree_int (cst, 0) != 0
42875 && compare_tree_int (cst, 1) != 0)
42877 warning (OPT_Wattributes,
42878 "argument to %qE attribute is neither zero, nor one",
42880 *no_add_attrs = true;
42889 /* Handle a "ms_abi" or "sysv" attribute; arguments as in
42890 struct attribute_spec.handler. */
42892 ix86_handle_abi_attribute (tree *node, tree name, tree, int,
42893 bool *no_add_attrs)
42895 if (TREE_CODE (*node) != FUNCTION_TYPE
42896 && TREE_CODE (*node) != METHOD_TYPE
42897 && TREE_CODE (*node) != FIELD_DECL
42898 && TREE_CODE (*node) != TYPE_DECL)
42900 warning (OPT_Wattributes, "%qE attribute only applies to functions",
42902 *no_add_attrs = true;
42906 /* Can combine regparm with all attributes but fastcall. */
42907 if (is_attribute_p ("ms_abi", name))
42909 if (lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (*node)))
42911 error ("ms_abi and sysv_abi attributes are not compatible");
42916 else if (is_attribute_p ("sysv_abi", name))
42918 if (lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (*node)))
42920 error ("ms_abi and sysv_abi attributes are not compatible");
42929 /* Handle a "ms_struct" or "gcc_struct" attribute; arguments as in
42930 struct attribute_spec.handler. */
42932 ix86_handle_struct_attribute (tree *node, tree name, tree, int,
42933 bool *no_add_attrs)
42936 if (DECL_P (*node))
42938 if (TREE_CODE (*node) == TYPE_DECL)
42939 type = &TREE_TYPE (*node);
42944 if (!(type && RECORD_OR_UNION_TYPE_P (*type)))
42946 warning (OPT_Wattributes, "%qE attribute ignored",
42948 *no_add_attrs = true;
42951 else if ((is_attribute_p ("ms_struct", name)
42952 && lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (*type)))
42953 || ((is_attribute_p ("gcc_struct", name)
42954 && lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (*type)))))
42956 warning (OPT_Wattributes, "%qE incompatible attribute ignored",
42958 *no_add_attrs = true;
42965 ix86_handle_fndecl_attribute (tree *node, tree name, tree, int,
42966 bool *no_add_attrs)
42968 if (TREE_CODE (*node) != FUNCTION_DECL)
42970 warning (OPT_Wattributes, "%qE attribute only applies to functions",
42972 *no_add_attrs = true;
42978 ix86_ms_bitfield_layout_p (const_tree record_type)
42980 return ((TARGET_MS_BITFIELD_LAYOUT
42981 && !lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (record_type)))
42982 || lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (record_type)));
42985 /* Returns an expression indicating where the this parameter is
42986 located on entry to the FUNCTION. */
42989 x86_this_parameter (tree function)
42991 tree type = TREE_TYPE (function);
42992 bool aggr = aggregate_value_p (TREE_TYPE (type), type) != 0;
42997 const int *parm_regs;
42999 if (ix86_function_type_abi (type) == MS_ABI)
43000 parm_regs = x86_64_ms_abi_int_parameter_registers;
43002 parm_regs = x86_64_int_parameter_registers;
43003 return gen_rtx_REG (Pmode, parm_regs[aggr]);
43006 nregs = ix86_function_regparm (type, function);
43008 if (nregs > 0 && !stdarg_p (type))
43011 unsigned int ccvt = ix86_get_callcvt (type);
43013 if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
43014 regno = aggr ? DX_REG : CX_REG;
43015 else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
43019 return gen_rtx_MEM (SImode,
43020 plus_constant (Pmode, stack_pointer_rtx, 4));
43029 return gen_rtx_MEM (SImode,
43030 plus_constant (Pmode,
43031 stack_pointer_rtx, 4));
43034 return gen_rtx_REG (SImode, regno);
43037 return gen_rtx_MEM (SImode, plus_constant (Pmode, stack_pointer_rtx,
43041 /* Determine whether x86_output_mi_thunk can succeed. */
43044 x86_can_output_mi_thunk (const_tree, HOST_WIDE_INT, HOST_WIDE_INT vcall_offset,
43045 const_tree function)
43047 /* 64-bit can handle anything. */
43051 /* For 32-bit, everything's fine if we have one free register. */
43052 if (ix86_function_regparm (TREE_TYPE (function), function) < 3)
43055 /* Need a free register for vcall_offset. */
43059 /* Need a free register for GOT references. */
43060 if (flag_pic && !targetm.binds_local_p (function))
43063 /* Otherwise ok. */
43067 /* Output the assembler code for a thunk function. THUNK_DECL is the
43068 declaration for the thunk function itself, FUNCTION is the decl for
43069 the target function. DELTA is an immediate constant offset to be
43070 added to THIS. If VCALL_OFFSET is nonzero, the word at
43071 *(*this + vcall_offset) should be added to THIS. */
43074 x86_output_mi_thunk (FILE *file, tree, HOST_WIDE_INT delta,
43075 HOST_WIDE_INT vcall_offset, tree function)
43077 rtx this_param = x86_this_parameter (function);
43078 rtx this_reg, tmp, fnaddr;
43079 unsigned int tmp_regno;
43083 tmp_regno = R10_REG;
43086 unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (function));
43087 if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
43088 tmp_regno = AX_REG;
43089 else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
43090 tmp_regno = DX_REG;
43092 tmp_regno = CX_REG;
43095 emit_note (NOTE_INSN_PROLOGUE_END);
43097 /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
43098 pull it in now and let DELTA benefit. */
43099 if (REG_P (this_param))
43100 this_reg = this_param;
43101 else if (vcall_offset)
43103 /* Put the this parameter into %eax. */
43104 this_reg = gen_rtx_REG (Pmode, AX_REG);
43105 emit_move_insn (this_reg, this_param);
43108 this_reg = NULL_RTX;
43110 /* Adjust the this parameter by a fixed constant. */
43113 rtx delta_rtx = GEN_INT (delta);
43114 rtx delta_dst = this_reg ? this_reg : this_param;
43118 if (!x86_64_general_operand (delta_rtx, Pmode))
43120 tmp = gen_rtx_REG (Pmode, tmp_regno);
43121 emit_move_insn (tmp, delta_rtx);
43126 ix86_emit_binop (PLUS, Pmode, delta_dst, delta_rtx);
43129 /* Adjust the this parameter by a value stored in the vtable. */
43132 rtx vcall_addr, vcall_mem, this_mem;
43134 tmp = gen_rtx_REG (Pmode, tmp_regno);
43136 this_mem = gen_rtx_MEM (ptr_mode, this_reg);
43137 if (Pmode != ptr_mode)
43138 this_mem = gen_rtx_ZERO_EXTEND (Pmode, this_mem);
43139 emit_move_insn (tmp, this_mem);
43141 /* Adjust the this parameter. */
43142 vcall_addr = plus_constant (Pmode, tmp, vcall_offset);
43144 && !ix86_legitimate_address_p (ptr_mode, vcall_addr, true))
43146 rtx tmp2 = gen_rtx_REG (Pmode, R11_REG);
43147 emit_move_insn (tmp2, GEN_INT (vcall_offset));
43148 vcall_addr = gen_rtx_PLUS (Pmode, tmp, tmp2);
43151 vcall_mem = gen_rtx_MEM (ptr_mode, vcall_addr);
43152 if (Pmode != ptr_mode)
43153 emit_insn (gen_addsi_1_zext (this_reg,
43154 gen_rtx_REG (ptr_mode,
43158 ix86_emit_binop (PLUS, Pmode, this_reg, vcall_mem);
43161 /* If necessary, drop THIS back to its stack slot. */
43162 if (this_reg && this_reg != this_param)
43163 emit_move_insn (this_param, this_reg);
43165 fnaddr = XEXP (DECL_RTL (function), 0);
43168 if (!flag_pic || targetm.binds_local_p (function)
43173 tmp = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, fnaddr), UNSPEC_GOTPCREL);
43174 tmp = gen_rtx_CONST (Pmode, tmp);
43175 fnaddr = gen_const_mem (Pmode, tmp);
43180 if (!flag_pic || targetm.binds_local_p (function))
43183 else if (TARGET_MACHO)
43185 fnaddr = machopic_indirect_call_target (DECL_RTL (function));
43186 fnaddr = XEXP (fnaddr, 0);
43188 #endif /* TARGET_MACHO */
43191 tmp = gen_rtx_REG (Pmode, CX_REG);
43192 output_set_got (tmp, NULL_RTX);
43194 fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, fnaddr), UNSPEC_GOT);
43195 fnaddr = gen_rtx_CONST (Pmode, fnaddr);
43196 fnaddr = gen_rtx_PLUS (Pmode, tmp, fnaddr);
43197 fnaddr = gen_const_mem (Pmode, fnaddr);
43201 /* Our sibling call patterns do not allow memories, because we have no
43202 predicate that can distinguish between frame and non-frame memory.
43203 For our purposes here, we can get away with (ab)using a jump pattern,
43204 because we're going to do no optimization. */
43205 if (MEM_P (fnaddr))
43207 if (sibcall_insn_operand (fnaddr, word_mode))
43209 fnaddr = XEXP (DECL_RTL (function), 0);
43210 tmp = gen_rtx_MEM (QImode, fnaddr);
43211 tmp = gen_rtx_CALL (VOIDmode, tmp, const0_rtx);
43212 tmp = emit_call_insn (tmp);
43213 SIBLING_CALL_P (tmp) = 1;
43216 emit_jump_insn (gen_indirect_jump (fnaddr));
43220 if (ix86_cmodel == CM_LARGE_PIC && SYMBOLIC_CONST (fnaddr))
43222 // CM_LARGE_PIC always uses pseudo PIC register which is
43223 // uninitialized. Since FUNCTION is local and calling it
43224 // doesn't go through PLT, we use scratch register %r11 as
43225 // PIC register and initialize it here.
43226 pic_offset_table_rtx = gen_rtx_REG (Pmode, R11_REG);
43227 ix86_init_large_pic_reg (tmp_regno);
43228 fnaddr = legitimize_pic_address (fnaddr,
43229 gen_rtx_REG (Pmode, tmp_regno));
43232 if (!sibcall_insn_operand (fnaddr, word_mode))
43234 tmp = gen_rtx_REG (word_mode, tmp_regno);
43235 if (GET_MODE (fnaddr) != word_mode)
43236 fnaddr = gen_rtx_ZERO_EXTEND (word_mode, fnaddr);
43237 emit_move_insn (tmp, fnaddr);
43241 tmp = gen_rtx_MEM (QImode, fnaddr);
43242 tmp = gen_rtx_CALL (VOIDmode, tmp, const0_rtx);
43243 tmp = emit_call_insn (tmp);
43244 SIBLING_CALL_P (tmp) = 1;
43248 /* Emit just enough of rest_of_compilation to get the insns emitted.
43249 Note that use_thunk calls assemble_start_function et al. */
43250 insn = get_insns ();
43251 shorten_branches (insn);
43252 final_start_function (insn, file, 1);
43253 final (insn, file, 1);
43254 final_end_function ();
43258 x86_file_start (void)
43260 default_file_start ();
43262 fputs ("\t.code16gcc\n", asm_out_file);
43264 darwin_file_start ();
43266 if (X86_FILE_START_VERSION_DIRECTIVE)
43267 fputs ("\t.version\t\"01.01\"\n", asm_out_file);
43268 if (X86_FILE_START_FLTUSED)
43269 fputs ("\t.global\t__fltused\n", asm_out_file);
43270 if (ix86_asm_dialect == ASM_INTEL)
43271 fputs ("\t.intel_syntax noprefix\n", asm_out_file);
43275 x86_field_alignment (tree field, int computed)
43278 tree type = TREE_TYPE (field);
43280 if (TARGET_64BIT || TARGET_ALIGN_DOUBLE)
43282 mode = TYPE_MODE (strip_array_types (type));
43283 if (mode == DFmode || mode == DCmode
43284 || GET_MODE_CLASS (mode) == MODE_INT
43285 || GET_MODE_CLASS (mode) == MODE_COMPLEX_INT)
43286 return MIN (32, computed);
43290 /* Print call to TARGET to FILE. */
43293 x86_print_call_or_nop (FILE *file, const char *target)
43295 if (flag_nop_mcount)
43296 fprintf (file, "1:\tnopl 0x00(%%eax,%%eax,1)\n"); /* 5 byte nop. */
43298 fprintf (file, "1:\tcall\t%s\n", target);
43301 /* Output assembler code to FILE to increment profiler label # LABELNO
43302 for profiling a function entry. */
43304 x86_function_profiler (FILE *file, int labelno ATTRIBUTE_UNUSED)
43306 const char *mcount_name = (flag_fentry ? MCOUNT_NAME_BEFORE_PROLOGUE
43310 #ifndef NO_PROFILE_COUNTERS
43311 fprintf (file, "\tleaq\t%sP%d(%%rip),%%r11\n", LPREFIX, labelno);
43314 if (!TARGET_PECOFF && flag_pic)
43315 fprintf (file, "1:\tcall\t*%s@GOTPCREL(%%rip)\n", mcount_name);
43317 x86_print_call_or_nop (file, mcount_name);
43321 #ifndef NO_PROFILE_COUNTERS
43322 fprintf (file, "\tleal\t%sP%d@GOTOFF(%%ebx),%%" PROFILE_COUNT_REGISTER "\n",
43325 fprintf (file, "1:\tcall\t*%s@GOT(%%ebx)\n", mcount_name);
43329 #ifndef NO_PROFILE_COUNTERS
43330 fprintf (file, "\tmovl\t$%sP%d,%%" PROFILE_COUNT_REGISTER "\n",
43333 x86_print_call_or_nop (file, mcount_name);
43336 if (flag_record_mcount)
43338 fprintf (file, "\t.section __mcount_loc, \"a\",@progbits\n");
43339 fprintf (file, "\t.%s 1b\n", TARGET_64BIT ? "quad" : "long");
43340 fprintf (file, "\t.previous\n");
43344 /* We don't have exact information about the insn sizes, but we may assume
43345 quite safely that we are informed about all 1 byte insns and memory
43346 address sizes. This is enough to eliminate unnecessary padding in
43350 min_insn_size (rtx_insn *insn)
43354 if (!INSN_P (insn) || !active_insn_p (insn))
43357 /* Discard alignments we've emit and jump instructions. */
43358 if (GET_CODE (PATTERN (insn)) == UNSPEC_VOLATILE
43359 && XINT (PATTERN (insn), 1) == UNSPECV_ALIGN)
43362 /* Important case - calls are always 5 bytes.
43363 It is common to have many calls in the row. */
43365 && symbolic_reference_mentioned_p (PATTERN (insn))
43366 && !SIBLING_CALL_P (insn))
43368 len = get_attr_length (insn);
43372 /* For normal instructions we rely on get_attr_length being exact,
43373 with a few exceptions. */
43374 if (!JUMP_P (insn))
43376 enum attr_type type = get_attr_type (insn);
43381 if (GET_CODE (PATTERN (insn)) == ASM_INPUT
43382 || asm_noperands (PATTERN (insn)) >= 0)
43389 /* Otherwise trust get_attr_length. */
43393 l = get_attr_length_address (insn);
43394 if (l < 4 && symbolic_reference_mentioned_p (PATTERN (insn)))
43403 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
43405 /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
43409 ix86_avoid_jump_mispredicts (void)
43411 rtx_insn *insn, *start = get_insns ();
43412 int nbytes = 0, njumps = 0;
43413 bool isjump = false;
43415 /* Look for all minimal intervals of instructions containing 4 jumps.
43416 The intervals are bounded by START and INSN. NBYTES is the total
43417 size of instructions in the interval including INSN and not including
43418 START. When the NBYTES is smaller than 16 bytes, it is possible
43419 that the end of START and INSN ends up in the same 16byte page.
43421 The smallest offset in the page INSN can start is the case where START
43422 ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
43423 We add p2align to 16byte window with maxskip 15 - NBYTES + sizeof (INSN).
43425 Don't consider asm goto as jump, while it can contain a jump, it doesn't
43426 have to, control transfer to label(s) can be performed through other
43427 means, and also we estimate minimum length of all asm stmts as 0. */
43428 for (insn = start; insn; insn = NEXT_INSN (insn))
43432 if (LABEL_P (insn))
43434 int align = label_to_alignment (insn);
43435 int max_skip = label_to_max_skip (insn);
43439 /* If align > 3, only up to 16 - max_skip - 1 bytes can be
43440 already in the current 16 byte page, because otherwise
43441 ASM_OUTPUT_MAX_SKIP_ALIGN could skip max_skip or fewer
43442 bytes to reach 16 byte boundary. */
43444 || (align <= 3 && max_skip != (1 << align) - 1))
43447 fprintf (dump_file, "Label %i with max_skip %i\n",
43448 INSN_UID (insn), max_skip);
43451 while (nbytes + max_skip >= 16)
43453 start = NEXT_INSN (start);
43454 if ((JUMP_P (start) && asm_noperands (PATTERN (start)) < 0)
43456 njumps--, isjump = true;
43459 nbytes -= min_insn_size (start);
43465 min_size = min_insn_size (insn);
43466 nbytes += min_size;
43468 fprintf (dump_file, "Insn %i estimated to %i bytes\n",
43469 INSN_UID (insn), min_size);
43470 if ((JUMP_P (insn) && asm_noperands (PATTERN (insn)) < 0)
43478 start = NEXT_INSN (start);
43479 if ((JUMP_P (start) && asm_noperands (PATTERN (start)) < 0)
43481 njumps--, isjump = true;
43484 nbytes -= min_insn_size (start);
43486 gcc_assert (njumps >= 0);
43488 fprintf (dump_file, "Interval %i to %i has %i bytes\n",
43489 INSN_UID (start), INSN_UID (insn), nbytes);
43491 if (njumps == 3 && isjump && nbytes < 16)
43493 int padsize = 15 - nbytes + min_insn_size (insn);
43496 fprintf (dump_file, "Padding insn %i by %i bytes!\n",
43497 INSN_UID (insn), padsize);
43498 emit_insn_before (gen_pad (GEN_INT (padsize)), insn);
43504 /* AMD Athlon works faster
43505 when RET is not destination of conditional jump or directly preceded
43506 by other jump instruction. We avoid the penalty by inserting NOP just
43507 before the RET instructions in such cases. */
43509 ix86_pad_returns (void)
43514 FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
43516 basic_block bb = e->src;
43517 rtx_insn *ret = BB_END (bb);
43519 bool replace = false;
43521 if (!JUMP_P (ret) || !ANY_RETURN_P (PATTERN (ret))
43522 || optimize_bb_for_size_p (bb))
43524 for (prev = PREV_INSN (ret); prev; prev = PREV_INSN (prev))
43525 if (active_insn_p (prev) || LABEL_P (prev))
43527 if (prev && LABEL_P (prev))
43532 FOR_EACH_EDGE (e, ei, bb->preds)
43533 if (EDGE_FREQUENCY (e) && e->src->index >= 0
43534 && !(e->flags & EDGE_FALLTHRU))
43542 prev = prev_active_insn (ret);
43544 && ((JUMP_P (prev) && any_condjump_p (prev))
43547 /* Empty functions get branch mispredict even when
43548 the jump destination is not visible to us. */
43549 if (!prev && !optimize_function_for_size_p (cfun))
43554 emit_jump_insn_before (gen_simple_return_internal_long (), ret);
43560 /* Count the minimum number of instructions in BB. Return 4 if the
43561 number of instructions >= 4. */
43564 ix86_count_insn_bb (basic_block bb)
43567 int insn_count = 0;
43569 /* Count number of instructions in this block. Return 4 if the number
43570 of instructions >= 4. */
43571 FOR_BB_INSNS (bb, insn)
43573 /* Only happen in exit blocks. */
43575 && ANY_RETURN_P (PATTERN (insn)))
43578 if (NONDEBUG_INSN_P (insn)
43579 && GET_CODE (PATTERN (insn)) != USE
43580 && GET_CODE (PATTERN (insn)) != CLOBBER)
43583 if (insn_count >= 4)
43592 /* Count the minimum number of instructions in code path in BB.
43593 Return 4 if the number of instructions >= 4. */
43596 ix86_count_insn (basic_block bb)
43600 int min_prev_count;
43602 /* Only bother counting instructions along paths with no
43603 more than 2 basic blocks between entry and exit. Given
43604 that BB has an edge to exit, determine if a predecessor
43605 of BB has an edge from entry. If so, compute the number
43606 of instructions in the predecessor block. If there
43607 happen to be multiple such blocks, compute the minimum. */
43608 min_prev_count = 4;
43609 FOR_EACH_EDGE (e, ei, bb->preds)
43612 edge_iterator prev_ei;
43614 if (e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
43616 min_prev_count = 0;
43619 FOR_EACH_EDGE (prev_e, prev_ei, e->src->preds)
43621 if (prev_e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
43623 int count = ix86_count_insn_bb (e->src);
43624 if (count < min_prev_count)
43625 min_prev_count = count;
43631 if (min_prev_count < 4)
43632 min_prev_count += ix86_count_insn_bb (bb);
43634 return min_prev_count;
43637 /* Pad short function to 4 instructions. */
43640 ix86_pad_short_function (void)
43645 FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
43647 rtx_insn *ret = BB_END (e->src);
43648 if (JUMP_P (ret) && ANY_RETURN_P (PATTERN (ret)))
43650 int insn_count = ix86_count_insn (e->src);
43652 /* Pad short function. */
43653 if (insn_count < 4)
43655 rtx_insn *insn = ret;
43657 /* Find epilogue. */
43660 || NOTE_KIND (insn) != NOTE_INSN_EPILOGUE_BEG))
43661 insn = PREV_INSN (insn);
43666 /* Two NOPs count as one instruction. */
43667 insn_count = 2 * (4 - insn_count);
43668 emit_insn_before (gen_nops (GEN_INT (insn_count)), insn);
43674 /* Fix up a Windows system unwinder issue. If an EH region falls through into
43675 the epilogue, the Windows system unwinder will apply epilogue logic and
43676 produce incorrect offsets. This can be avoided by adding a nop between
43677 the last insn that can throw and the first insn of the epilogue. */
43680 ix86_seh_fixup_eh_fallthru (void)
43685 FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
43687 rtx_insn *insn, *next;
43689 /* Find the beginning of the epilogue. */
43690 for (insn = BB_END (e->src); insn != NULL; insn = PREV_INSN (insn))
43691 if (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_EPILOGUE_BEG)
43696 /* We only care about preceding insns that can throw. */
43697 insn = prev_active_insn (insn);
43698 if (insn == NULL || !can_throw_internal (insn))
43701 /* Do not separate calls from their debug information. */
43702 for (next = NEXT_INSN (insn); next != NULL; next = NEXT_INSN (next))
43704 && (NOTE_KIND (next) == NOTE_INSN_VAR_LOCATION
43705 || NOTE_KIND (next) == NOTE_INSN_CALL_ARG_LOCATION))
43710 emit_insn_after (gen_nops (const1_rtx), insn);
43714 /* Implement machine specific optimizations. We implement padding of returns
43715 for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
43719 /* We are freeing block_for_insn in the toplev to keep compatibility
43720 with old MDEP_REORGS that are not CFG based. Recompute it now. */
43721 compute_bb_for_insn ();
43723 if (TARGET_SEH && current_function_has_exception_handlers ())
43724 ix86_seh_fixup_eh_fallthru ();
43726 if (optimize && optimize_function_for_speed_p (cfun))
43728 if (TARGET_PAD_SHORT_FUNCTION)
43729 ix86_pad_short_function ();
43730 else if (TARGET_PAD_RETURNS)
43731 ix86_pad_returns ();
43732 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
43733 if (TARGET_FOUR_JUMP_LIMIT)
43734 ix86_avoid_jump_mispredicts ();
43739 /* Return nonzero when QImode register that must be represented via REX prefix
43742 x86_extended_QIreg_mentioned_p (rtx_insn *insn)
43745 extract_insn_cached (insn);
43746 for (i = 0; i < recog_data.n_operands; i++)
43747 if (GENERAL_REG_P (recog_data.operand[i])
43748 && !QI_REGNO_P (REGNO (recog_data.operand[i])))
43753 /* Return true when INSN mentions register that must be encoded using REX
43756 x86_extended_reg_mentioned_p (rtx insn)
43758 subrtx_iterator::array_type array;
43759 FOR_EACH_SUBRTX (iter, array, INSN_P (insn) ? PATTERN (insn) : insn, NONCONST)
43761 const_rtx x = *iter;
43763 && (REX_INT_REGNO_P (REGNO (x)) || REX_SSE_REGNO_P (REGNO (x))))
43769 /* If profitable, negate (without causing overflow) integer constant
43770 of mode MODE at location LOC. Return true in this case. */
43772 x86_maybe_negate_const_int (rtx *loc, machine_mode mode)
43776 if (!CONST_INT_P (*loc))
43782 /* DImode x86_64 constants must fit in 32 bits. */
43783 gcc_assert (x86_64_immediate_operand (*loc, mode));
43794 gcc_unreachable ();
43797 /* Avoid overflows. */
43798 if (mode_signbit_p (mode, *loc))
43801 val = INTVAL (*loc);
43803 /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'.
43804 Exceptions: -128 encodes smaller than 128, so swap sign and op. */
43805 if ((val < 0 && val != -128)
43808 *loc = GEN_INT (-val);
43815 /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
43816 optabs would emit if we didn't have TFmode patterns. */
43819 x86_emit_floatuns (rtx operands[2])
43821 rtx_code_label *neglab, *donelab;
43822 rtx i0, i1, f0, in, out;
43823 machine_mode mode, inmode;
43825 inmode = GET_MODE (operands[1]);
43826 gcc_assert (inmode == SImode || inmode == DImode);
43829 in = force_reg (inmode, operands[1]);
43830 mode = GET_MODE (out);
43831 neglab = gen_label_rtx ();
43832 donelab = gen_label_rtx ();
43833 f0 = gen_reg_rtx (mode);
43835 emit_cmp_and_jump_insns (in, const0_rtx, LT, const0_rtx, inmode, 0, neglab);
43837 expand_float (out, in, 0);
43839 emit_jump_insn (gen_jump (donelab));
43842 emit_label (neglab);
43844 i0 = expand_simple_binop (inmode, LSHIFTRT, in, const1_rtx, NULL,
43846 i1 = expand_simple_binop (inmode, AND, in, const1_rtx, NULL,
43848 i0 = expand_simple_binop (inmode, IOR, i0, i1, i0, 1, OPTAB_DIRECT);
43850 expand_float (f0, i0, 0);
43852 emit_insn (gen_rtx_SET (VOIDmode, out, gen_rtx_PLUS (mode, f0, f0)));
43854 emit_label (donelab);
43857 static bool canonicalize_perm (struct expand_vec_perm_d *d);
43858 static bool expand_vec_perm_1 (struct expand_vec_perm_d *d);
43859 static bool expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d);
43860 static bool expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool);
43862 /* Get a vector mode of the same size as the original but with elements
43863 twice as wide. This is only guaranteed to apply to integral vectors. */
43865 static inline machine_mode
43866 get_mode_wider_vector (machine_mode o)
43868 /* ??? Rely on the ordering that genmodes.c gives to vectors. */
43869 machine_mode n = GET_MODE_WIDER_MODE (o);
43870 gcc_assert (GET_MODE_NUNITS (o) == GET_MODE_NUNITS (n) * 2);
43871 gcc_assert (GET_MODE_SIZE (o) == GET_MODE_SIZE (n));
43875 /* A subroutine of ix86_expand_vector_init_duplicate. Tries to
43876 fill target with val via vec_duplicate. */
43879 ix86_vector_duplicate_value (machine_mode mode, rtx target, rtx val)
43885 /* First attempt to recognize VAL as-is. */
43886 dup = gen_rtx_VEC_DUPLICATE (mode, val);
43887 insn = emit_insn (gen_rtx_SET (VOIDmode, target, dup));
43888 if (recog_memoized (insn) < 0)
43891 /* If that fails, force VAL into a register. */
43894 XEXP (dup, 0) = force_reg (GET_MODE_INNER (mode), val);
43895 seq = get_insns ();
43898 emit_insn_before (seq, insn);
43900 ok = recog_memoized (insn) >= 0;
43906 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
43907 with all elements equal to VAR. Return true if successful. */
43910 ix86_expand_vector_init_duplicate (bool mmx_ok, machine_mode mode,
43911 rtx target, rtx val)
43935 return ix86_vector_duplicate_value (mode, target, val);
43940 if (TARGET_SSE || TARGET_3DNOW_A)
43944 val = gen_lowpart (SImode, val);
43945 x = gen_rtx_TRUNCATE (HImode, val);
43946 x = gen_rtx_VEC_DUPLICATE (mode, x);
43947 emit_insn (gen_rtx_SET (VOIDmode, target, x));
43959 return ix86_vector_duplicate_value (mode, target, val);
43963 struct expand_vec_perm_d dperm;
43967 memset (&dperm, 0, sizeof (dperm));
43968 dperm.target = target;
43969 dperm.vmode = mode;
43970 dperm.nelt = GET_MODE_NUNITS (mode);
43971 dperm.op0 = dperm.op1 = gen_reg_rtx (mode);
43972 dperm.one_operand_p = true;
43974 /* Extend to SImode using a paradoxical SUBREG. */
43975 tmp1 = gen_reg_rtx (SImode);
43976 emit_move_insn (tmp1, gen_lowpart (SImode, val));
43978 /* Insert the SImode value as low element of a V4SImode vector. */
43979 tmp2 = gen_reg_rtx (V4SImode);
43980 emit_insn (gen_vec_setv4si_0 (tmp2, CONST0_RTX (V4SImode), tmp1));
43981 emit_move_insn (dperm.op0, gen_lowpart (mode, tmp2));
43983 ok = (expand_vec_perm_1 (&dperm)
43984 || expand_vec_perm_broadcast_1 (&dperm));
43992 return ix86_vector_duplicate_value (mode, target, val);
43999 /* Replicate the value once into the next wider mode and recurse. */
44001 machine_mode smode, wsmode, wvmode;
44004 smode = GET_MODE_INNER (mode);
44005 wvmode = get_mode_wider_vector (mode);
44006 wsmode = GET_MODE_INNER (wvmode);
44008 val = convert_modes (wsmode, smode, val, true);
44009 x = expand_simple_binop (wsmode, ASHIFT, val,
44010 GEN_INT (GET_MODE_BITSIZE (smode)),
44011 NULL_RTX, 1, OPTAB_LIB_WIDEN);
44012 val = expand_simple_binop (wsmode, IOR, val, x, x, 1, OPTAB_LIB_WIDEN);
44014 x = gen_reg_rtx (wvmode);
44015 ok = ix86_expand_vector_init_duplicate (mmx_ok, wvmode, x, val);
44017 emit_move_insn (target, gen_lowpart (GET_MODE (target), x));
44024 return ix86_vector_duplicate_value (mode, target, val);
44027 machine_mode hvmode = (mode == V16HImode ? V8HImode : V16QImode);
44028 rtx x = gen_reg_rtx (hvmode);
44030 ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
44033 x = gen_rtx_VEC_CONCAT (mode, x, x);
44034 emit_insn (gen_rtx_SET (VOIDmode, target, x));
44040 if (TARGET_AVX512BW)
44041 return ix86_vector_duplicate_value (mode, target, val);
44044 machine_mode hvmode = (mode == V32HImode ? V16HImode : V32QImode);
44045 rtx x = gen_reg_rtx (hvmode);
44047 ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
44050 x = gen_rtx_VEC_CONCAT (mode, x, x);
44051 emit_insn (gen_rtx_SET (VOIDmode, target, x));
44060 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
44061 whose ONE_VAR element is VAR, and other elements are zero. Return true
44065 ix86_expand_vector_init_one_nonzero (bool mmx_ok, machine_mode mode,
44066 rtx target, rtx var, int one_var)
44068 machine_mode vsimode;
44071 bool use_vector_set = false;
44076 /* For SSE4.1, we normally use vector set. But if the second
44077 element is zero and inter-unit moves are OK, we use movq
44079 use_vector_set = (TARGET_64BIT && TARGET_SSE4_1
44080 && !(TARGET_INTER_UNIT_MOVES_TO_VEC
44086 use_vector_set = TARGET_SSE4_1;
44089 use_vector_set = TARGET_SSE2;
44092 use_vector_set = TARGET_SSE || TARGET_3DNOW_A;
44099 use_vector_set = TARGET_AVX;
44102 /* Use ix86_expand_vector_set in 64bit mode only. */
44103 use_vector_set = TARGET_AVX && TARGET_64BIT;
44109 if (use_vector_set)
44111 emit_insn (gen_rtx_SET (VOIDmode, target, CONST0_RTX (mode)));
44112 var = force_reg (GET_MODE_INNER (mode), var);
44113 ix86_expand_vector_set (mmx_ok, target, var, one_var);
44129 var = force_reg (GET_MODE_INNER (mode), var);
44130 x = gen_rtx_VEC_CONCAT (mode, var, CONST0_RTX (GET_MODE_INNER (mode)));
44131 emit_insn (gen_rtx_SET (VOIDmode, target, x));
44136 if (!REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
44137 new_target = gen_reg_rtx (mode);
44139 new_target = target;
44140 var = force_reg (GET_MODE_INNER (mode), var);
44141 x = gen_rtx_VEC_DUPLICATE (mode, var);
44142 x = gen_rtx_VEC_MERGE (mode, x, CONST0_RTX (mode), const1_rtx);
44143 emit_insn (gen_rtx_SET (VOIDmode, new_target, x));
44146 /* We need to shuffle the value to the correct position, so
44147 create a new pseudo to store the intermediate result. */
44149 /* With SSE2, we can use the integer shuffle insns. */
44150 if (mode != V4SFmode && TARGET_SSE2)
44152 emit_insn (gen_sse2_pshufd_1 (new_target, new_target,
44154 GEN_INT (one_var == 1 ? 0 : 1),
44155 GEN_INT (one_var == 2 ? 0 : 1),
44156 GEN_INT (one_var == 3 ? 0 : 1)));
44157 if (target != new_target)
44158 emit_move_insn (target, new_target);
44162 /* Otherwise convert the intermediate result to V4SFmode and
44163 use the SSE1 shuffle instructions. */
44164 if (mode != V4SFmode)
44166 tmp = gen_reg_rtx (V4SFmode);
44167 emit_move_insn (tmp, gen_lowpart (V4SFmode, new_target));
44172 emit_insn (gen_sse_shufps_v4sf (tmp, tmp, tmp,
44174 GEN_INT (one_var == 1 ? 0 : 1),
44175 GEN_INT (one_var == 2 ? 0+4 : 1+4),
44176 GEN_INT (one_var == 3 ? 0+4 : 1+4)));
44178 if (mode != V4SFmode)
44179 emit_move_insn (target, gen_lowpart (V4SImode, tmp));
44180 else if (tmp != target)
44181 emit_move_insn (target, tmp);
44183 else if (target != new_target)
44184 emit_move_insn (target, new_target);
44189 vsimode = V4SImode;
44195 vsimode = V2SImode;
44201 /* Zero extend the variable element to SImode and recurse. */
44202 var = convert_modes (SImode, GET_MODE_INNER (mode), var, true);
44204 x = gen_reg_rtx (vsimode);
44205 if (!ix86_expand_vector_init_one_nonzero (mmx_ok, vsimode, x,
44207 gcc_unreachable ();
44209 emit_move_insn (target, gen_lowpart (mode, x));
44217 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
44218 consisting of the values in VALS. It is known that all elements
44219 except ONE_VAR are constants. Return true if successful. */
44222 ix86_expand_vector_init_one_var (bool mmx_ok, machine_mode mode,
44223 rtx target, rtx vals, int one_var)
44225 rtx var = XVECEXP (vals, 0, one_var);
44226 machine_mode wmode;
44229 const_vec = copy_rtx (vals);
44230 XVECEXP (const_vec, 0, one_var) = CONST0_RTX (GET_MODE_INNER (mode));
44231 const_vec = gen_rtx_CONST_VECTOR (mode, XVEC (const_vec, 0));
44239 /* For the two element vectors, it's just as easy to use
44240 the general case. */
44244 /* Use ix86_expand_vector_set in 64bit mode only. */
44267 /* There's no way to set one QImode entry easily. Combine
44268 the variable value with its adjacent constant value, and
44269 promote to an HImode set. */
44270 x = XVECEXP (vals, 0, one_var ^ 1);
44273 var = convert_modes (HImode, QImode, var, true);
44274 var = expand_simple_binop (HImode, ASHIFT, var, GEN_INT (8),
44275 NULL_RTX, 1, OPTAB_LIB_WIDEN);
44276 x = GEN_INT (INTVAL (x) & 0xff);
44280 var = convert_modes (HImode, QImode, var, true);
44281 x = gen_int_mode (INTVAL (x) << 8, HImode);
44283 if (x != const0_rtx)
44284 var = expand_simple_binop (HImode, IOR, var, x, var,
44285 1, OPTAB_LIB_WIDEN);
44287 x = gen_reg_rtx (wmode);
44288 emit_move_insn (x, gen_lowpart (wmode, const_vec));
44289 ix86_expand_vector_set (mmx_ok, x, var, one_var >> 1);
44291 emit_move_insn (target, gen_lowpart (mode, x));
44298 emit_move_insn (target, const_vec);
44299 ix86_expand_vector_set (mmx_ok, target, var, one_var);
44303 /* A subroutine of ix86_expand_vector_init_general. Use vector
44304 concatenate to handle the most general case: all values variable,
44305 and none identical. */
44308 ix86_expand_vector_init_concat (machine_mode mode,
44309 rtx target, rtx *ops, int n)
44311 machine_mode cmode, hmode = VOIDmode, gmode = VOIDmode;
44312 rtx first[16], second[8], third[4];
44364 gcc_unreachable ();
44367 if (!register_operand (ops[1], cmode))
44368 ops[1] = force_reg (cmode, ops[1]);
44369 if (!register_operand (ops[0], cmode))
44370 ops[0] = force_reg (cmode, ops[0]);
44371 emit_insn (gen_rtx_SET (VOIDmode, target,
44372 gen_rtx_VEC_CONCAT (mode, ops[0],
44392 gcc_unreachable ();
44416 gcc_unreachable ();
44434 gcc_unreachable ();
44439 /* FIXME: We process inputs backward to help RA. PR 36222. */
44442 for (; i > 0; i -= 2, j--)
44444 first[j] = gen_reg_rtx (cmode);
44445 v = gen_rtvec (2, ops[i - 1], ops[i]);
44446 ix86_expand_vector_init (false, first[j],
44447 gen_rtx_PARALLEL (cmode, v));
44453 gcc_assert (hmode != VOIDmode);
44454 gcc_assert (gmode != VOIDmode);
44455 for (i = j = 0; i < n; i += 2, j++)
44457 second[j] = gen_reg_rtx (hmode);
44458 ix86_expand_vector_init_concat (hmode, second [j],
44462 for (i = j = 0; i < n; i += 2, j++)
44464 third[j] = gen_reg_rtx (gmode);
44465 ix86_expand_vector_init_concat (gmode, third[j],
44469 ix86_expand_vector_init_concat (mode, target, third, n);
44473 gcc_assert (hmode != VOIDmode);
44474 for (i = j = 0; i < n; i += 2, j++)
44476 second[j] = gen_reg_rtx (hmode);
44477 ix86_expand_vector_init_concat (hmode, second [j],
44481 ix86_expand_vector_init_concat (mode, target, second, n);
44484 ix86_expand_vector_init_concat (mode, target, first, n);
44488 gcc_unreachable ();
44492 /* A subroutine of ix86_expand_vector_init_general. Use vector
44493 interleave to handle the most general case: all values variable,
44494 and none identical. */
44497 ix86_expand_vector_init_interleave (machine_mode mode,
44498 rtx target, rtx *ops, int n)
44500 machine_mode first_imode, second_imode, third_imode, inner_mode;
44503 rtx (*gen_load_even) (rtx, rtx, rtx);
44504 rtx (*gen_interleave_first_low) (rtx, rtx, rtx);
44505 rtx (*gen_interleave_second_low) (rtx, rtx, rtx);
44510 gen_load_even = gen_vec_setv8hi;
44511 gen_interleave_first_low = gen_vec_interleave_lowv4si;
44512 gen_interleave_second_low = gen_vec_interleave_lowv2di;
44513 inner_mode = HImode;
44514 first_imode = V4SImode;
44515 second_imode = V2DImode;
44516 third_imode = VOIDmode;
44519 gen_load_even = gen_vec_setv16qi;
44520 gen_interleave_first_low = gen_vec_interleave_lowv8hi;
44521 gen_interleave_second_low = gen_vec_interleave_lowv4si;
44522 inner_mode = QImode;
44523 first_imode = V8HImode;
44524 second_imode = V4SImode;
44525 third_imode = V2DImode;
44528 gcc_unreachable ();
44531 for (i = 0; i < n; i++)
44533 /* Extend the odd elment to SImode using a paradoxical SUBREG. */
44534 op0 = gen_reg_rtx (SImode);
44535 emit_move_insn (op0, gen_lowpart (SImode, ops [i + i]));
44537 /* Insert the SImode value as low element of V4SImode vector. */
44538 op1 = gen_reg_rtx (V4SImode);
44539 op0 = gen_rtx_VEC_MERGE (V4SImode,
44540 gen_rtx_VEC_DUPLICATE (V4SImode,
44542 CONST0_RTX (V4SImode),
44544 emit_insn (gen_rtx_SET (VOIDmode, op1, op0));
44546 /* Cast the V4SImode vector back to a vector in orignal mode. */
44547 op0 = gen_reg_rtx (mode);
44548 emit_move_insn (op0, gen_lowpart (mode, op1));
44550 /* Load even elements into the second position. */
44551 emit_insn (gen_load_even (op0,
44552 force_reg (inner_mode,
44556 /* Cast vector to FIRST_IMODE vector. */
44557 ops[i] = gen_reg_rtx (first_imode);
44558 emit_move_insn (ops[i], gen_lowpart (first_imode, op0));
44561 /* Interleave low FIRST_IMODE vectors. */
44562 for (i = j = 0; i < n; i += 2, j++)
44564 op0 = gen_reg_rtx (first_imode);
44565 emit_insn (gen_interleave_first_low (op0, ops[i], ops[i + 1]));
44567 /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
44568 ops[j] = gen_reg_rtx (second_imode);
44569 emit_move_insn (ops[j], gen_lowpart (second_imode, op0));
44572 /* Interleave low SECOND_IMODE vectors. */
44573 switch (second_imode)
44576 for (i = j = 0; i < n / 2; i += 2, j++)
44578 op0 = gen_reg_rtx (second_imode);
44579 emit_insn (gen_interleave_second_low (op0, ops[i],
44582 /* Cast the SECOND_IMODE vector to the THIRD_IMODE
44584 ops[j] = gen_reg_rtx (third_imode);
44585 emit_move_insn (ops[j], gen_lowpart (third_imode, op0));
44587 second_imode = V2DImode;
44588 gen_interleave_second_low = gen_vec_interleave_lowv2di;
44592 op0 = gen_reg_rtx (second_imode);
44593 emit_insn (gen_interleave_second_low (op0, ops[0],
44596 /* Cast the SECOND_IMODE vector back to a vector on original
44598 emit_insn (gen_rtx_SET (VOIDmode, target,
44599 gen_lowpart (mode, op0)));
44603 gcc_unreachable ();
44607 /* A subroutine of ix86_expand_vector_init. Handle the most general case:
44608 all values variable, and none identical. */
44611 ix86_expand_vector_init_general (bool mmx_ok, machine_mode mode,
44612 rtx target, rtx vals)
44614 rtx ops[64], op0, op1, op2, op3, op4, op5;
44615 machine_mode half_mode = VOIDmode;
44616 machine_mode quarter_mode = VOIDmode;
44623 if (!mmx_ok && !TARGET_SSE)
44639 n = GET_MODE_NUNITS (mode);
44640 for (i = 0; i < n; i++)
44641 ops[i] = XVECEXP (vals, 0, i);
44642 ix86_expand_vector_init_concat (mode, target, ops, n);
44646 half_mode = V16QImode;
44650 half_mode = V8HImode;
44654 n = GET_MODE_NUNITS (mode);
44655 for (i = 0; i < n; i++)
44656 ops[i] = XVECEXP (vals, 0, i);
44657 op0 = gen_reg_rtx (half_mode);
44658 op1 = gen_reg_rtx (half_mode);
44659 ix86_expand_vector_init_interleave (half_mode, op0, ops,
44661 ix86_expand_vector_init_interleave (half_mode, op1,
44662 &ops [n >> 1], n >> 2);
44663 emit_insn (gen_rtx_SET (VOIDmode, target,
44664 gen_rtx_VEC_CONCAT (mode, op0, op1)));
44668 quarter_mode = V16QImode;
44669 half_mode = V32QImode;
44673 quarter_mode = V8HImode;
44674 half_mode = V16HImode;
44678 n = GET_MODE_NUNITS (mode);
44679 for (i = 0; i < n; i++)
44680 ops[i] = XVECEXP (vals, 0, i);
44681 op0 = gen_reg_rtx (quarter_mode);
44682 op1 = gen_reg_rtx (quarter_mode);
44683 op2 = gen_reg_rtx (quarter_mode);
44684 op3 = gen_reg_rtx (quarter_mode);
44685 op4 = gen_reg_rtx (half_mode);
44686 op5 = gen_reg_rtx (half_mode);
44687 ix86_expand_vector_init_interleave (quarter_mode, op0, ops,
44689 ix86_expand_vector_init_interleave (quarter_mode, op1,
44690 &ops [n >> 2], n >> 3);
44691 ix86_expand_vector_init_interleave (quarter_mode, op2,
44692 &ops [n >> 1], n >> 3);
44693 ix86_expand_vector_init_interleave (quarter_mode, op3,
44694 &ops [(n >> 1) | (n >> 2)], n >> 3);
44695 emit_insn (gen_rtx_SET (VOIDmode, op4,
44696 gen_rtx_VEC_CONCAT (half_mode, op0, op1)));
44697 emit_insn (gen_rtx_SET (VOIDmode, op5,
44698 gen_rtx_VEC_CONCAT (half_mode, op2, op3)));
44699 emit_insn (gen_rtx_SET (VOIDmode, target,
44700 gen_rtx_VEC_CONCAT (mode, op4, op5)));
44704 if (!TARGET_SSE4_1)
44712 /* Don't use ix86_expand_vector_init_interleave if we can't
44713 move from GPR to SSE register directly. */
44714 if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
44717 n = GET_MODE_NUNITS (mode);
44718 for (i = 0; i < n; i++)
44719 ops[i] = XVECEXP (vals, 0, i);
44720 ix86_expand_vector_init_interleave (mode, target, ops, n >> 1);
44728 gcc_unreachable ();
44732 int i, j, n_elts, n_words, n_elt_per_word;
44733 machine_mode inner_mode;
44734 rtx words[4], shift;
44736 inner_mode = GET_MODE_INNER (mode);
44737 n_elts = GET_MODE_NUNITS (mode);
44738 n_words = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
44739 n_elt_per_word = n_elts / n_words;
44740 shift = GEN_INT (GET_MODE_BITSIZE (inner_mode));
44742 for (i = 0; i < n_words; ++i)
44744 rtx word = NULL_RTX;
44746 for (j = 0; j < n_elt_per_word; ++j)
44748 rtx elt = XVECEXP (vals, 0, (i+1)*n_elt_per_word - j - 1);
44749 elt = convert_modes (word_mode, inner_mode, elt, true);
44755 word = expand_simple_binop (word_mode, ASHIFT, word, shift,
44756 word, 1, OPTAB_LIB_WIDEN);
44757 word = expand_simple_binop (word_mode, IOR, word, elt,
44758 word, 1, OPTAB_LIB_WIDEN);
44766 emit_move_insn (target, gen_lowpart (mode, words[0]));
44767 else if (n_words == 2)
44769 rtx tmp = gen_reg_rtx (mode);
44770 emit_clobber (tmp);
44771 emit_move_insn (gen_lowpart (word_mode, tmp), words[0]);
44772 emit_move_insn (gen_highpart (word_mode, tmp), words[1]);
44773 emit_move_insn (target, tmp);
44775 else if (n_words == 4)
44777 rtx tmp = gen_reg_rtx (V4SImode);
44778 gcc_assert (word_mode == SImode);
44779 vals = gen_rtx_PARALLEL (V4SImode, gen_rtvec_v (4, words));
44780 ix86_expand_vector_init_general (false, V4SImode, tmp, vals);
44781 emit_move_insn (target, gen_lowpart (mode, tmp));
44784 gcc_unreachable ();
44788 /* Initialize vector TARGET via VALS. Suppress the use of MMX
44789 instructions unless MMX_OK is true. */
44792 ix86_expand_vector_init (bool mmx_ok, rtx target, rtx vals)
44794 machine_mode mode = GET_MODE (target);
44795 machine_mode inner_mode = GET_MODE_INNER (mode);
44796 int n_elts = GET_MODE_NUNITS (mode);
44797 int n_var = 0, one_var = -1;
44798 bool all_same = true, all_const_zero = true;
44802 for (i = 0; i < n_elts; ++i)
44804 x = XVECEXP (vals, 0, i);
44805 if (!(CONST_INT_P (x)
44806 || GET_CODE (x) == CONST_DOUBLE
44807 || GET_CODE (x) == CONST_FIXED))
44808 n_var++, one_var = i;
44809 else if (x != CONST0_RTX (inner_mode))
44810 all_const_zero = false;
44811 if (i > 0 && !rtx_equal_p (x, XVECEXP (vals, 0, 0)))
44815 /* Constants are best loaded from the constant pool. */
44818 emit_move_insn (target, gen_rtx_CONST_VECTOR (mode, XVEC (vals, 0)));
44822 /* If all values are identical, broadcast the value. */
44824 && ix86_expand_vector_init_duplicate (mmx_ok, mode, target,
44825 XVECEXP (vals, 0, 0)))
44828 /* Values where only one field is non-constant are best loaded from
44829 the pool and overwritten via move later. */
44833 && ix86_expand_vector_init_one_nonzero (mmx_ok, mode, target,
44834 XVECEXP (vals, 0, one_var),
44838 if (ix86_expand_vector_init_one_var (mmx_ok, mode, target, vals, one_var))
44842 ix86_expand_vector_init_general (mmx_ok, mode, target, vals);
44846 ix86_expand_vector_set (bool mmx_ok, rtx target, rtx val, int elt)
44848 machine_mode mode = GET_MODE (target);
44849 machine_mode inner_mode = GET_MODE_INNER (mode);
44850 machine_mode half_mode;
44851 bool use_vec_merge = false;
44853 static rtx (*gen_extract[6][2]) (rtx, rtx)
44855 { gen_vec_extract_lo_v32qi, gen_vec_extract_hi_v32qi },
44856 { gen_vec_extract_lo_v16hi, gen_vec_extract_hi_v16hi },
44857 { gen_vec_extract_lo_v8si, gen_vec_extract_hi_v8si },
44858 { gen_vec_extract_lo_v4di, gen_vec_extract_hi_v4di },
44859 { gen_vec_extract_lo_v8sf, gen_vec_extract_hi_v8sf },
44860 { gen_vec_extract_lo_v4df, gen_vec_extract_hi_v4df }
44862 static rtx (*gen_insert[6][2]) (rtx, rtx, rtx)
44864 { gen_vec_set_lo_v32qi, gen_vec_set_hi_v32qi },
44865 { gen_vec_set_lo_v16hi, gen_vec_set_hi_v16hi },
44866 { gen_vec_set_lo_v8si, gen_vec_set_hi_v8si },
44867 { gen_vec_set_lo_v4di, gen_vec_set_hi_v4di },
44868 { gen_vec_set_lo_v8sf, gen_vec_set_hi_v8sf },
44869 { gen_vec_set_lo_v4df, gen_vec_set_hi_v4df }
44872 machine_mode mmode = VOIDmode;
44873 rtx (*gen_blendm) (rtx, rtx, rtx, rtx);
44881 tmp = gen_reg_rtx (GET_MODE_INNER (mode));
44882 ix86_expand_vector_extract (true, tmp, target, 1 - elt);
44884 tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
44886 tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
44887 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
44893 use_vec_merge = TARGET_SSE4_1 && TARGET_64BIT;
44897 tmp = gen_reg_rtx (GET_MODE_INNER (mode));
44898 ix86_expand_vector_extract (false, tmp, target, 1 - elt);
44900 tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
44902 tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
44903 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
44910 /* For the two element vectors, we implement a VEC_CONCAT with
44911 the extraction of the other element. */
44913 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (1 - elt)));
44914 tmp = gen_rtx_VEC_SELECT (inner_mode, target, tmp);
44917 op0 = val, op1 = tmp;
44919 op0 = tmp, op1 = val;
44921 tmp = gen_rtx_VEC_CONCAT (mode, op0, op1);
44922 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
44927 use_vec_merge = TARGET_SSE4_1;
44934 use_vec_merge = true;
44938 /* tmp = target = A B C D */
44939 tmp = copy_to_reg (target);
44940 /* target = A A B B */
44941 emit_insn (gen_vec_interleave_lowv4sf (target, target, target));
44942 /* target = X A B B */
44943 ix86_expand_vector_set (false, target, val, 0);
44944 /* target = A X C D */
44945 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
44946 const1_rtx, const0_rtx,
44947 GEN_INT (2+4), GEN_INT (3+4)));
44951 /* tmp = target = A B C D */
44952 tmp = copy_to_reg (target);
44953 /* tmp = X B C D */
44954 ix86_expand_vector_set (false, tmp, val, 0);
44955 /* target = A B X D */
44956 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
44957 const0_rtx, const1_rtx,
44958 GEN_INT (0+4), GEN_INT (3+4)));
44962 /* tmp = target = A B C D */
44963 tmp = copy_to_reg (target);
44964 /* tmp = X B C D */
44965 ix86_expand_vector_set (false, tmp, val, 0);
44966 /* target = A B X D */
44967 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
44968 const0_rtx, const1_rtx,
44969 GEN_INT (2+4), GEN_INT (0+4)));
44973 gcc_unreachable ();
44978 use_vec_merge = TARGET_SSE4_1;
44982 /* Element 0 handled by vec_merge below. */
44985 use_vec_merge = true;
44991 /* With SSE2, use integer shuffles to swap element 0 and ELT,
44992 store into element 0, then shuffle them back. */
44996 order[0] = GEN_INT (elt);
44997 order[1] = const1_rtx;
44998 order[2] = const2_rtx;
44999 order[3] = GEN_INT (3);
45000 order[elt] = const0_rtx;
45002 emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
45003 order[1], order[2], order[3]));
45005 ix86_expand_vector_set (false, target, val, 0);
45007 emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
45008 order[1], order[2], order[3]));
45012 /* For SSE1, we have to reuse the V4SF code. */
45013 rtx t = gen_reg_rtx (V4SFmode);
45014 ix86_expand_vector_set (false, t, gen_lowpart (SFmode, val), elt);
45015 emit_move_insn (target, gen_lowpart (mode, t));
45020 use_vec_merge = TARGET_SSE2;
45023 use_vec_merge = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
45027 use_vec_merge = TARGET_SSE4_1;
45034 half_mode = V16QImode;
45040 half_mode = V8HImode;
45046 half_mode = V4SImode;
45052 half_mode = V2DImode;
45058 half_mode = V4SFmode;
45064 half_mode = V2DFmode;
45070 /* Compute offset. */
45074 gcc_assert (i <= 1);
45076 /* Extract the half. */
45077 tmp = gen_reg_rtx (half_mode);
45078 emit_insn (gen_extract[j][i] (tmp, target));
45080 /* Put val in tmp at elt. */
45081 ix86_expand_vector_set (false, tmp, val, elt);
45084 emit_insn (gen_insert[j][i] (target, target, tmp));
45088 if (TARGET_AVX512F)
45091 gen_blendm = gen_avx512f_blendmv8df;
45096 if (TARGET_AVX512F)
45099 gen_blendm = gen_avx512f_blendmv8di;
45104 if (TARGET_AVX512F)
45107 gen_blendm = gen_avx512f_blendmv16si;
45112 if (TARGET_AVX512F)
45115 gen_blendm = gen_avx512f_blendmv16si;
45120 if (TARGET_AVX512F && TARGET_AVX512BW)
45123 gen_blendm = gen_avx512bw_blendmv32hi;
45128 if (TARGET_AVX512F && TARGET_AVX512BW)
45131 gen_blendm = gen_avx512bw_blendmv64qi;
45139 if (mmode != VOIDmode)
45141 tmp = gen_reg_rtx (mode);
45142 emit_insn (gen_rtx_SET (VOIDmode, tmp,
45143 gen_rtx_VEC_DUPLICATE (mode, val)));
45144 emit_insn (gen_blendm (target, tmp, target,
45146 gen_int_mode (1 << elt, mmode))));
45148 else if (use_vec_merge)
45150 tmp = gen_rtx_VEC_DUPLICATE (mode, val);
45151 tmp = gen_rtx_VEC_MERGE (mode, tmp, target, GEN_INT (1 << elt));
45152 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
45156 rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
45158 emit_move_insn (mem, target);
45160 tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
45161 emit_move_insn (tmp, val);
45163 emit_move_insn (target, mem);
45168 ix86_expand_vector_extract (bool mmx_ok, rtx target, rtx vec, int elt)
45170 machine_mode mode = GET_MODE (vec);
45171 machine_mode inner_mode = GET_MODE_INNER (mode);
45172 bool use_vec_extr = false;
45185 use_vec_extr = true;
45189 use_vec_extr = TARGET_SSE4_1;
45201 tmp = gen_reg_rtx (mode);
45202 emit_insn (gen_sse_shufps_v4sf (tmp, vec, vec,
45203 GEN_INT (elt), GEN_INT (elt),
45204 GEN_INT (elt+4), GEN_INT (elt+4)));
45208 tmp = gen_reg_rtx (mode);
45209 emit_insn (gen_vec_interleave_highv4sf (tmp, vec, vec));
45213 gcc_unreachable ();
45216 use_vec_extr = true;
45221 use_vec_extr = TARGET_SSE4_1;
45235 tmp = gen_reg_rtx (mode);
45236 emit_insn (gen_sse2_pshufd_1 (tmp, vec,
45237 GEN_INT (elt), GEN_INT (elt),
45238 GEN_INT (elt), GEN_INT (elt)));
45242 tmp = gen_reg_rtx (mode);
45243 emit_insn (gen_vec_interleave_highv4si (tmp, vec, vec));
45247 gcc_unreachable ();
45250 use_vec_extr = true;
45255 /* For SSE1, we have to reuse the V4SF code. */
45256 ix86_expand_vector_extract (false, gen_lowpart (SFmode, target),
45257 gen_lowpart (V4SFmode, vec), elt);
45263 use_vec_extr = TARGET_SSE2;
45266 use_vec_extr = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
45270 use_vec_extr = TARGET_SSE4_1;
45276 tmp = gen_reg_rtx (V4SFmode);
45278 emit_insn (gen_vec_extract_lo_v8sf (tmp, vec));
45280 emit_insn (gen_vec_extract_hi_v8sf (tmp, vec));
45281 ix86_expand_vector_extract (false, target, tmp, elt & 3);
45289 tmp = gen_reg_rtx (V2DFmode);
45291 emit_insn (gen_vec_extract_lo_v4df (tmp, vec));
45293 emit_insn (gen_vec_extract_hi_v4df (tmp, vec));
45294 ix86_expand_vector_extract (false, target, tmp, elt & 1);
45302 tmp = gen_reg_rtx (V16QImode);
45304 emit_insn (gen_vec_extract_lo_v32qi (tmp, vec));
45306 emit_insn (gen_vec_extract_hi_v32qi (tmp, vec));
45307 ix86_expand_vector_extract (false, target, tmp, elt & 15);
45315 tmp = gen_reg_rtx (V8HImode);
45317 emit_insn (gen_vec_extract_lo_v16hi (tmp, vec));
45319 emit_insn (gen_vec_extract_hi_v16hi (tmp, vec));
45320 ix86_expand_vector_extract (false, target, tmp, elt & 7);
45328 tmp = gen_reg_rtx (V4SImode);
45330 emit_insn (gen_vec_extract_lo_v8si (tmp, vec));
45332 emit_insn (gen_vec_extract_hi_v8si (tmp, vec));
45333 ix86_expand_vector_extract (false, target, tmp, elt & 3);
45341 tmp = gen_reg_rtx (V2DImode);
45343 emit_insn (gen_vec_extract_lo_v4di (tmp, vec));
45345 emit_insn (gen_vec_extract_hi_v4di (tmp, vec));
45346 ix86_expand_vector_extract (false, target, tmp, elt & 1);
45352 if (TARGET_AVX512BW)
45354 tmp = gen_reg_rtx (V16HImode);
45356 emit_insn (gen_vec_extract_lo_v32hi (tmp, vec));
45358 emit_insn (gen_vec_extract_hi_v32hi (tmp, vec));
45359 ix86_expand_vector_extract (false, target, tmp, elt & 15);
45365 if (TARGET_AVX512BW)
45367 tmp = gen_reg_rtx (V32QImode);
45369 emit_insn (gen_vec_extract_lo_v64qi (tmp, vec));
45371 emit_insn (gen_vec_extract_hi_v64qi (tmp, vec));
45372 ix86_expand_vector_extract (false, target, tmp, elt & 31);
45378 tmp = gen_reg_rtx (V8SFmode);
45380 emit_insn (gen_vec_extract_lo_v16sf (tmp, vec));
45382 emit_insn (gen_vec_extract_hi_v16sf (tmp, vec));
45383 ix86_expand_vector_extract (false, target, tmp, elt & 7);
45387 tmp = gen_reg_rtx (V4DFmode);
45389 emit_insn (gen_vec_extract_lo_v8df (tmp, vec));
45391 emit_insn (gen_vec_extract_hi_v8df (tmp, vec));
45392 ix86_expand_vector_extract (false, target, tmp, elt & 3);
45396 tmp = gen_reg_rtx (V8SImode);
45398 emit_insn (gen_vec_extract_lo_v16si (tmp, vec));
45400 emit_insn (gen_vec_extract_hi_v16si (tmp, vec));
45401 ix86_expand_vector_extract (false, target, tmp, elt & 7);
45405 tmp = gen_reg_rtx (V4DImode);
45407 emit_insn (gen_vec_extract_lo_v8di (tmp, vec));
45409 emit_insn (gen_vec_extract_hi_v8di (tmp, vec));
45410 ix86_expand_vector_extract (false, target, tmp, elt & 3);
45414 /* ??? Could extract the appropriate HImode element and shift. */
45421 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (elt)));
45422 tmp = gen_rtx_VEC_SELECT (inner_mode, vec, tmp);
45424 /* Let the rtl optimizers know about the zero extension performed. */
45425 if (inner_mode == QImode || inner_mode == HImode)
45427 tmp = gen_rtx_ZERO_EXTEND (SImode, tmp);
45428 target = gen_lowpart (SImode, target);
45431 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
45435 rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
45437 emit_move_insn (mem, vec);
45439 tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
45440 emit_move_insn (target, tmp);
45444 /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
45445 to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
45446 The upper bits of DEST are undefined, though they shouldn't cause
45447 exceptions (some bits from src or all zeros are ok). */
45450 emit_reduc_half (rtx dest, rtx src, int i)
45453 switch (GET_MODE (src))
45457 tem = gen_sse_movhlps (dest, src, src);
45459 tem = gen_sse_shufps_v4sf (dest, src, src, const1_rtx, const1_rtx,
45460 GEN_INT (1 + 4), GEN_INT (1 + 4));
45463 tem = gen_vec_interleave_highv2df (dest, src, src);
45469 d = gen_reg_rtx (V1TImode);
45470 tem = gen_sse2_lshrv1ti3 (d, gen_lowpart (V1TImode, src),
45475 tem = gen_avx_vperm2f128v8sf3 (dest, src, src, const1_rtx);
45477 tem = gen_avx_shufps256 (dest, src, src,
45478 GEN_INT (i == 128 ? 2 + (3 << 2) : 1));
45482 tem = gen_avx_vperm2f128v4df3 (dest, src, src, const1_rtx);
45484 tem = gen_avx_shufpd256 (dest, src, src, const1_rtx);
45492 if (GET_MODE (dest) != V4DImode)
45493 d = gen_reg_rtx (V4DImode);
45494 tem = gen_avx2_permv2ti (d, gen_lowpart (V4DImode, src),
45495 gen_lowpart (V4DImode, src),
45500 d = gen_reg_rtx (V2TImode);
45501 tem = gen_avx2_lshrv2ti3 (d, gen_lowpart (V2TImode, src),
45512 tem = gen_avx512f_shuf_i32x4_1 (gen_lowpart (V16SImode, dest),
45513 gen_lowpart (V16SImode, src),
45514 gen_lowpart (V16SImode, src),
45515 GEN_INT (0x4 + (i == 512 ? 4 : 0)),
45516 GEN_INT (0x5 + (i == 512 ? 4 : 0)),
45517 GEN_INT (0x6 + (i == 512 ? 4 : 0)),
45518 GEN_INT (0x7 + (i == 512 ? 4 : 0)),
45519 GEN_INT (0xC), GEN_INT (0xD),
45520 GEN_INT (0xE), GEN_INT (0xF),
45521 GEN_INT (0x10), GEN_INT (0x11),
45522 GEN_INT (0x12), GEN_INT (0x13),
45523 GEN_INT (0x14), GEN_INT (0x15),
45524 GEN_INT (0x16), GEN_INT (0x17));
45526 tem = gen_avx512f_pshufd_1 (gen_lowpart (V16SImode, dest),
45527 gen_lowpart (V16SImode, src),
45528 GEN_INT (i == 128 ? 0x2 : 0x1),
45532 GEN_INT (i == 128 ? 0x6 : 0x5),
45536 GEN_INT (i == 128 ? 0xA : 0x9),
45540 GEN_INT (i == 128 ? 0xE : 0xD),
45546 gcc_unreachable ();
45550 emit_move_insn (dest, gen_lowpart (GET_MODE (dest), d));
45553 /* Expand a vector reduction. FN is the binary pattern to reduce;
45554 DEST is the destination; IN is the input vector. */
45557 ix86_expand_reduc (rtx (*fn) (rtx, rtx, rtx), rtx dest, rtx in)
45559 rtx half, dst, vec = in;
45560 machine_mode mode = GET_MODE (in);
45563 /* SSE4 has a special instruction for V8HImode UMIN reduction. */
45565 && mode == V8HImode
45566 && fn == gen_uminv8hi3)
45568 emit_insn (gen_sse4_1_phminposuw (dest, in));
45572 for (i = GET_MODE_BITSIZE (mode);
45573 i > GET_MODE_BITSIZE (GET_MODE_INNER (mode));
45576 half = gen_reg_rtx (mode);
45577 emit_reduc_half (half, vec, i);
45578 if (i == GET_MODE_BITSIZE (GET_MODE_INNER (mode)) * 2)
45581 dst = gen_reg_rtx (mode);
45582 emit_insn (fn (dst, half, vec));
45587 /* Target hook for scalar_mode_supported_p. */
45589 ix86_scalar_mode_supported_p (machine_mode mode)
45591 if (DECIMAL_FLOAT_MODE_P (mode))
45592 return default_decimal_float_supported_p ();
45593 else if (mode == TFmode)
45596 return default_scalar_mode_supported_p (mode);
45599 /* Implements target hook vector_mode_supported_p. */
45601 ix86_vector_mode_supported_p (machine_mode mode)
45603 if (TARGET_SSE && VALID_SSE_REG_MODE (mode))
45605 if (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
45607 if (TARGET_AVX && VALID_AVX256_REG_MODE (mode))
45609 if (TARGET_AVX512F && VALID_AVX512F_REG_MODE (mode))
45611 if (TARGET_MMX && VALID_MMX_REG_MODE (mode))
45613 if (TARGET_3DNOW && VALID_MMX_REG_MODE_3DNOW (mode))
45618 /* Implement target hook libgcc_floating_mode_supported_p. */
45620 ix86_libgcc_floating_mode_supported_p (machine_mode mode)
45630 #ifdef IX86_NO_LIBGCC_TFMODE
45632 #elif defined IX86_MAYBE_NO_LIBGCC_TFMODE
45633 return TARGET_LONG_DOUBLE_128;
45643 /* Target hook for c_mode_for_suffix. */
45644 static machine_mode
45645 ix86_c_mode_for_suffix (char suffix)
45655 /* Worker function for TARGET_MD_ASM_CLOBBERS.
45657 We do this in the new i386 backend to maintain source compatibility
45658 with the old cc0-based compiler. */
45661 ix86_md_asm_clobbers (tree, tree, tree clobbers)
45663 clobbers = tree_cons (NULL_TREE, build_string (5, "flags"),
45665 clobbers = tree_cons (NULL_TREE, build_string (4, "fpsr"),
45670 /* Implements target vector targetm.asm.encode_section_info. */
45672 static void ATTRIBUTE_UNUSED
45673 ix86_encode_section_info (tree decl, rtx rtl, int first)
45675 default_encode_section_info (decl, rtl, first);
45677 if (ix86_in_large_data_p (decl))
45678 SYMBOL_REF_FLAGS (XEXP (rtl, 0)) |= SYMBOL_FLAG_FAR_ADDR;
45681 /* Worker function for REVERSE_CONDITION. */
45684 ix86_reverse_condition (enum rtx_code code, machine_mode mode)
45686 return (mode != CCFPmode && mode != CCFPUmode
45687 ? reverse_condition (code)
45688 : reverse_condition_maybe_unordered (code));
45691 /* Output code to perform an x87 FP register move, from OPERANDS[1]
45695 output_387_reg_move (rtx insn, rtx *operands)
45697 if (REG_P (operands[0]))
45699 if (REG_P (operands[1])
45700 && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
45702 if (REGNO (operands[0]) == FIRST_STACK_REG)
45703 return output_387_ffreep (operands, 0);
45704 return "fstp\t%y0";
45706 if (STACK_TOP_P (operands[0]))
45707 return "fld%Z1\t%y1";
45710 else if (MEM_P (operands[0]))
45712 gcc_assert (REG_P (operands[1]));
45713 if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
45714 return "fstp%Z0\t%y0";
45717 /* There is no non-popping store to memory for XFmode.
45718 So if we need one, follow the store with a load. */
45719 if (GET_MODE (operands[0]) == XFmode)
45720 return "fstp%Z0\t%y0\n\tfld%Z0\t%y0";
45722 return "fst%Z0\t%y0";
45729 /* Output code to perform a conditional jump to LABEL, if C2 flag in
45730 FP status register is set. */
45733 ix86_emit_fp_unordered_jump (rtx label)
45735 rtx reg = gen_reg_rtx (HImode);
45738 emit_insn (gen_x86_fnstsw_1 (reg));
45740 if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
45742 emit_insn (gen_x86_sahf_1 (reg));
45744 temp = gen_rtx_REG (CCmode, FLAGS_REG);
45745 temp = gen_rtx_UNORDERED (VOIDmode, temp, const0_rtx);
45749 emit_insn (gen_testqi_ext_ccno_0 (reg, GEN_INT (0x04)));
45751 temp = gen_rtx_REG (CCNOmode, FLAGS_REG);
45752 temp = gen_rtx_NE (VOIDmode, temp, const0_rtx);
45755 temp = gen_rtx_IF_THEN_ELSE (VOIDmode, temp,
45756 gen_rtx_LABEL_REF (VOIDmode, label),
45758 temp = gen_rtx_SET (VOIDmode, pc_rtx, temp);
45760 emit_jump_insn (temp);
45761 predict_jump (REG_BR_PROB_BASE * 10 / 100);
45764 /* Output code to perform a log1p XFmode calculation. */
45766 void ix86_emit_i387_log1p (rtx op0, rtx op1)
45768 rtx_code_label *label1 = gen_label_rtx ();
45769 rtx_code_label *label2 = gen_label_rtx ();
45771 rtx tmp = gen_reg_rtx (XFmode);
45772 rtx tmp2 = gen_reg_rtx (XFmode);
45775 emit_insn (gen_absxf2 (tmp, op1));
45776 test = gen_rtx_GE (VOIDmode, tmp,
45777 CONST_DOUBLE_FROM_REAL_VALUE (
45778 REAL_VALUE_ATOF ("0.29289321881345247561810596348408353", XFmode),
45780 emit_jump_insn (gen_cbranchxf4 (test, XEXP (test, 0), XEXP (test, 1), label1));
45782 emit_move_insn (tmp2, standard_80387_constant_rtx (4)); /* fldln2 */
45783 emit_insn (gen_fyl2xp1xf3_i387 (op0, op1, tmp2));
45784 emit_jump (label2);
45786 emit_label (label1);
45787 emit_move_insn (tmp, CONST1_RTX (XFmode));
45788 emit_insn (gen_addxf3 (tmp, op1, tmp));
45789 emit_move_insn (tmp2, standard_80387_constant_rtx (4)); /* fldln2 */
45790 emit_insn (gen_fyl2xxf3_i387 (op0, tmp, tmp2));
45792 emit_label (label2);
45795 /* Emit code for round calculation. */
45796 void ix86_emit_i387_round (rtx op0, rtx op1)
45798 machine_mode inmode = GET_MODE (op1);
45799 machine_mode outmode = GET_MODE (op0);
45800 rtx e1, e2, res, tmp, tmp1, half;
45801 rtx scratch = gen_reg_rtx (HImode);
45802 rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
45803 rtx_code_label *jump_label = gen_label_rtx ();
45805 rtx (*gen_abs) (rtx, rtx);
45806 rtx (*gen_neg) (rtx, rtx);
45811 gen_abs = gen_abssf2;
45814 gen_abs = gen_absdf2;
45817 gen_abs = gen_absxf2;
45820 gcc_unreachable ();
45826 gen_neg = gen_negsf2;
45829 gen_neg = gen_negdf2;
45832 gen_neg = gen_negxf2;
45835 gen_neg = gen_neghi2;
45838 gen_neg = gen_negsi2;
45841 gen_neg = gen_negdi2;
45844 gcc_unreachable ();
45847 e1 = gen_reg_rtx (inmode);
45848 e2 = gen_reg_rtx (inmode);
45849 res = gen_reg_rtx (outmode);
45851 half = CONST_DOUBLE_FROM_REAL_VALUE (dconsthalf, inmode);
45853 /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
45855 /* scratch = fxam(op1) */
45856 emit_insn (gen_rtx_SET (VOIDmode, scratch,
45857 gen_rtx_UNSPEC (HImode, gen_rtvec (1, op1),
45859 /* e1 = fabs(op1) */
45860 emit_insn (gen_abs (e1, op1));
45862 /* e2 = e1 + 0.5 */
45863 half = force_reg (inmode, half);
45864 emit_insn (gen_rtx_SET (VOIDmode, e2,
45865 gen_rtx_PLUS (inmode, e1, half)));
45867 /* res = floor(e2) */
45868 if (inmode != XFmode)
45870 tmp1 = gen_reg_rtx (XFmode);
45872 emit_insn (gen_rtx_SET (VOIDmode, tmp1,
45873 gen_rtx_FLOAT_EXTEND (XFmode, e2)));
45883 rtx tmp0 = gen_reg_rtx (XFmode);
45885 emit_insn (gen_frndintxf2_floor (tmp0, tmp1));
45887 emit_insn (gen_rtx_SET (VOIDmode, res,
45888 gen_rtx_UNSPEC (outmode, gen_rtvec (1, tmp0),
45889 UNSPEC_TRUNC_NOOP)));
45893 emit_insn (gen_frndintxf2_floor (res, tmp1));
45896 emit_insn (gen_lfloorxfhi2 (res, tmp1));
45899 emit_insn (gen_lfloorxfsi2 (res, tmp1));
45902 emit_insn (gen_lfloorxfdi2 (res, tmp1));
45905 gcc_unreachable ();
45908 /* flags = signbit(a) */
45909 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x02)));
45911 /* if (flags) then res = -res */
45912 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
45913 gen_rtx_EQ (VOIDmode, flags, const0_rtx),
45914 gen_rtx_LABEL_REF (VOIDmode, jump_label),
45916 insn = emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp));
45917 predict_jump (REG_BR_PROB_BASE * 50 / 100);
45918 JUMP_LABEL (insn) = jump_label;
45920 emit_insn (gen_neg (res, res));
45922 emit_label (jump_label);
45923 LABEL_NUSES (jump_label) = 1;
45925 emit_move_insn (op0, res);
45928 /* Output code to perform a Newton-Rhapson approximation of a single precision
45929 floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
45931 void ix86_emit_swdivsf (rtx res, rtx a, rtx b, machine_mode mode)
45933 rtx x0, x1, e0, e1;
45935 x0 = gen_reg_rtx (mode);
45936 e0 = gen_reg_rtx (mode);
45937 e1 = gen_reg_rtx (mode);
45938 x1 = gen_reg_rtx (mode);
45940 /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
45942 b = force_reg (mode, b);
45944 /* x0 = rcp(b) estimate */
45945 if (mode == V16SFmode || mode == V8DFmode)
45946 emit_insn (gen_rtx_SET (VOIDmode, x0,
45947 gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
45950 emit_insn (gen_rtx_SET (VOIDmode, x0,
45951 gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
45955 emit_insn (gen_rtx_SET (VOIDmode, e0,
45956 gen_rtx_MULT (mode, x0, b)));
45959 emit_insn (gen_rtx_SET (VOIDmode, e0,
45960 gen_rtx_MULT (mode, x0, e0)));
45963 emit_insn (gen_rtx_SET (VOIDmode, e1,
45964 gen_rtx_PLUS (mode, x0, x0)));
45967 emit_insn (gen_rtx_SET (VOIDmode, x1,
45968 gen_rtx_MINUS (mode, e1, e0)));
45971 emit_insn (gen_rtx_SET (VOIDmode, res,
45972 gen_rtx_MULT (mode, a, x1)));
45975 /* Output code to perform a Newton-Rhapson approximation of a
45976 single precision floating point [reciprocal] square root. */
45978 void ix86_emit_swsqrtsf (rtx res, rtx a, machine_mode mode,
45981 rtx x0, e0, e1, e2, e3, mthree, mhalf;
45985 x0 = gen_reg_rtx (mode);
45986 e0 = gen_reg_rtx (mode);
45987 e1 = gen_reg_rtx (mode);
45988 e2 = gen_reg_rtx (mode);
45989 e3 = gen_reg_rtx (mode);
45991 real_from_integer (&r, VOIDmode, -3, SIGNED);
45992 mthree = CONST_DOUBLE_FROM_REAL_VALUE (r, SFmode);
45994 real_arithmetic (&r, NEGATE_EXPR, &dconsthalf, NULL);
45995 mhalf = CONST_DOUBLE_FROM_REAL_VALUE (r, SFmode);
45996 unspec = UNSPEC_RSQRT;
45998 if (VECTOR_MODE_P (mode))
46000 mthree = ix86_build_const_vector (mode, true, mthree);
46001 mhalf = ix86_build_const_vector (mode, true, mhalf);
46002 /* There is no 512-bit rsqrt. There is however rsqrt14. */
46003 if (GET_MODE_SIZE (mode) == 64)
46004 unspec = UNSPEC_RSQRT14;
46007 /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
46008 rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
46010 a = force_reg (mode, a);
46012 /* x0 = rsqrt(a) estimate */
46013 emit_insn (gen_rtx_SET (VOIDmode, x0,
46014 gen_rtx_UNSPEC (mode, gen_rtvec (1, a),
46017 /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
46022 zero = gen_reg_rtx (mode);
46023 mask = gen_reg_rtx (mode);
46025 zero = force_reg (mode, CONST0_RTX(mode));
46027 /* Handle masked compare. */
46028 if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
46030 mask = gen_reg_rtx (HImode);
46031 /* Imm value 0x4 corresponds to not-equal comparison. */
46032 emit_insn (gen_avx512f_cmpv16sf3 (mask, zero, a, GEN_INT (0x4)));
46033 emit_insn (gen_avx512f_blendmv16sf (x0, zero, x0, mask));
46037 emit_insn (gen_rtx_SET (VOIDmode, mask,
46038 gen_rtx_NE (mode, zero, a)));
46040 emit_insn (gen_rtx_SET (VOIDmode, x0,
46041 gen_rtx_AND (mode, x0, mask)));
46046 emit_insn (gen_rtx_SET (VOIDmode, e0,
46047 gen_rtx_MULT (mode, x0, a)));
46049 emit_insn (gen_rtx_SET (VOIDmode, e1,
46050 gen_rtx_MULT (mode, e0, x0)));
46053 mthree = force_reg (mode, mthree);
46054 emit_insn (gen_rtx_SET (VOIDmode, e2,
46055 gen_rtx_PLUS (mode, e1, mthree)));
46057 mhalf = force_reg (mode, mhalf);
46059 /* e3 = -.5 * x0 */
46060 emit_insn (gen_rtx_SET (VOIDmode, e3,
46061 gen_rtx_MULT (mode, x0, mhalf)));
46063 /* e3 = -.5 * e0 */
46064 emit_insn (gen_rtx_SET (VOIDmode, e3,
46065 gen_rtx_MULT (mode, e0, mhalf)));
46066 /* ret = e2 * e3 */
46067 emit_insn (gen_rtx_SET (VOIDmode, res,
46068 gen_rtx_MULT (mode, e2, e3)));
46071 #ifdef TARGET_SOLARIS
46072 /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
46075 i386_solaris_elf_named_section (const char *name, unsigned int flags,
46078 /* With Binutils 2.15, the "@unwind" marker must be specified on
46079 every occurrence of the ".eh_frame" section, not just the first
46082 && strcmp (name, ".eh_frame") == 0)
46084 fprintf (asm_out_file, "\t.section\t%s,\"%s\",@unwind\n", name,
46085 flags & SECTION_WRITE ? "aw" : "a");
46090 if (HAVE_COMDAT_GROUP && flags & SECTION_LINKONCE)
46092 solaris_elf_asm_comdat_section (name, flags, decl);
46097 default_elf_asm_named_section (name, flags, decl);
46099 #endif /* TARGET_SOLARIS */
46101 /* Return the mangling of TYPE if it is an extended fundamental type. */
46103 static const char *
46104 ix86_mangle_type (const_tree type)
46106 type = TYPE_MAIN_VARIANT (type);
46108 if (TREE_CODE (type) != VOID_TYPE && TREE_CODE (type) != BOOLEAN_TYPE
46109 && TREE_CODE (type) != INTEGER_TYPE && TREE_CODE (type) != REAL_TYPE)
46112 switch (TYPE_MODE (type))
46115 /* __float128 is "g". */
46118 /* "long double" or __float80 is "e". */
46125 /* For 32-bit code we can save PIC register setup by using
46126 __stack_chk_fail_local hidden function instead of calling
46127 __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
46128 register, so it is better to call __stack_chk_fail directly. */
46130 static tree ATTRIBUTE_UNUSED
46131 ix86_stack_protect_fail (void)
46133 return TARGET_64BIT
46134 ? default_external_stack_protect_fail ()
46135 : default_hidden_stack_protect_fail ();
46138 /* Select a format to encode pointers in exception handling data. CODE
46139 is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
46140 true if the symbol may be affected by dynamic relocations.
46142 ??? All x86 object file formats are capable of representing this.
46143 After all, the relocation needed is the same as for the call insn.
46144 Whether or not a particular assembler allows us to enter such, I
46145 guess we'll have to see. */
46147 asm_preferred_eh_data_format (int code, int global)
46151 int type = DW_EH_PE_sdata8;
46153 || ix86_cmodel == CM_SMALL_PIC
46154 || (ix86_cmodel == CM_MEDIUM_PIC && (global || code)))
46155 type = DW_EH_PE_sdata4;
46156 return (global ? DW_EH_PE_indirect : 0) | DW_EH_PE_pcrel | type;
46158 if (ix86_cmodel == CM_SMALL
46159 || (ix86_cmodel == CM_MEDIUM && code))
46160 return DW_EH_PE_udata4;
46161 return DW_EH_PE_absptr;
46164 /* Expand copysign from SIGN to the positive value ABS_VALUE
46165 storing in RESULT. If MASK is non-null, it shall be a mask to mask out
46168 ix86_sse_copysign_to_positive (rtx result, rtx abs_value, rtx sign, rtx mask)
46170 machine_mode mode = GET_MODE (sign);
46171 rtx sgn = gen_reg_rtx (mode);
46172 if (mask == NULL_RTX)
46174 machine_mode vmode;
46176 if (mode == SFmode)
46178 else if (mode == DFmode)
46183 mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), false);
46184 if (!VECTOR_MODE_P (mode))
46186 /* We need to generate a scalar mode mask in this case. */
46187 rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
46188 tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
46189 mask = gen_reg_rtx (mode);
46190 emit_insn (gen_rtx_SET (VOIDmode, mask, tmp));
46194 mask = gen_rtx_NOT (mode, mask);
46195 emit_insn (gen_rtx_SET (VOIDmode, sgn,
46196 gen_rtx_AND (mode, mask, sign)));
46197 emit_insn (gen_rtx_SET (VOIDmode, result,
46198 gen_rtx_IOR (mode, abs_value, sgn)));
46201 /* Expand fabs (OP0) and return a new rtx that holds the result. The
46202 mask for masking out the sign-bit is stored in *SMASK, if that is
46205 ix86_expand_sse_fabs (rtx op0, rtx *smask)
46207 machine_mode vmode, mode = GET_MODE (op0);
46210 xa = gen_reg_rtx (mode);
46211 if (mode == SFmode)
46213 else if (mode == DFmode)
46217 mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), true);
46218 if (!VECTOR_MODE_P (mode))
46220 /* We need to generate a scalar mode mask in this case. */
46221 rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
46222 tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
46223 mask = gen_reg_rtx (mode);
46224 emit_insn (gen_rtx_SET (VOIDmode, mask, tmp));
46226 emit_insn (gen_rtx_SET (VOIDmode, xa,
46227 gen_rtx_AND (mode, op0, mask)));
46235 /* Expands a comparison of OP0 with OP1 using comparison code CODE,
46236 swapping the operands if SWAP_OPERANDS is true. The expanded
46237 code is a forward jump to a newly created label in case the
46238 comparison is true. The generated label rtx is returned. */
46239 static rtx_code_label *
46240 ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1,
46241 bool swap_operands)
46243 machine_mode fpcmp_mode = ix86_fp_compare_mode (code);
46244 rtx_code_label *label;
46248 std::swap (op0, op1);
46250 label = gen_label_rtx ();
46251 tmp = gen_rtx_REG (fpcmp_mode, FLAGS_REG);
46252 emit_insn (gen_rtx_SET (VOIDmode, tmp,
46253 gen_rtx_COMPARE (fpcmp_mode, op0, op1)));
46254 tmp = gen_rtx_fmt_ee (code, VOIDmode, tmp, const0_rtx);
46255 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
46256 gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx);
46257 tmp = emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp));
46258 JUMP_LABEL (tmp) = label;
46263 /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
46264 using comparison code CODE. Operands are swapped for the comparison if
46265 SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
46267 ix86_expand_sse_compare_mask (enum rtx_code code, rtx op0, rtx op1,
46268 bool swap_operands)
46270 rtx (*insn)(rtx, rtx, rtx, rtx);
46271 machine_mode mode = GET_MODE (op0);
46272 rtx mask = gen_reg_rtx (mode);
46275 std::swap (op0, op1);
46277 insn = mode == DFmode ? gen_setcc_df_sse : gen_setcc_sf_sse;
46279 emit_insn (insn (mask, op0, op1,
46280 gen_rtx_fmt_ee (code, mode, op0, op1)));
46284 /* Generate and return a rtx of mode MODE for 2**n where n is the number
46285 of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
46287 ix86_gen_TWO52 (machine_mode mode)
46289 REAL_VALUE_TYPE TWO52r;
46292 real_ldexp (&TWO52r, &dconst1, mode == DFmode ? 52 : 23);
46293 TWO52 = const_double_from_real_value (TWO52r, mode);
46294 TWO52 = force_reg (mode, TWO52);
46299 /* Expand SSE sequence for computing lround from OP1 storing
46302 ix86_expand_lround (rtx op0, rtx op1)
46304 /* C code for the stuff we're doing below:
46305 tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
46308 machine_mode mode = GET_MODE (op1);
46309 const struct real_format *fmt;
46310 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
46313 /* load nextafter (0.5, 0.0) */
46314 fmt = REAL_MODE_FORMAT (mode);
46315 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
46316 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
46318 /* adj = copysign (0.5, op1) */
46319 adj = force_reg (mode, const_double_from_real_value (pred_half, mode));
46320 ix86_sse_copysign_to_positive (adj, adj, force_reg (mode, op1), NULL_RTX);
46322 /* adj = op1 + adj */
46323 adj = expand_simple_binop (mode, PLUS, adj, op1, NULL_RTX, 0, OPTAB_DIRECT);
46325 /* op0 = (imode)adj */
46326 expand_fix (op0, adj, 0);
46329 /* Expand SSE2 sequence for computing lround from OPERAND1 storing
46332 ix86_expand_lfloorceil (rtx op0, rtx op1, bool do_floor)
46334 /* C code for the stuff we're doing below (for do_floor):
46336 xi -= (double)xi > op1 ? 1 : 0;
46339 machine_mode fmode = GET_MODE (op1);
46340 machine_mode imode = GET_MODE (op0);
46341 rtx ireg, freg, tmp;
46342 rtx_code_label *label;
46344 /* reg = (long)op1 */
46345 ireg = gen_reg_rtx (imode);
46346 expand_fix (ireg, op1, 0);
46348 /* freg = (double)reg */
46349 freg = gen_reg_rtx (fmode);
46350 expand_float (freg, ireg, 0);
46352 /* ireg = (freg > op1) ? ireg - 1 : ireg */
46353 label = ix86_expand_sse_compare_and_jump (UNLE,
46354 freg, op1, !do_floor);
46355 tmp = expand_simple_binop (imode, do_floor ? MINUS : PLUS,
46356 ireg, const1_rtx, NULL_RTX, 0, OPTAB_DIRECT);
46357 emit_move_insn (ireg, tmp);
46359 emit_label (label);
46360 LABEL_NUSES (label) = 1;
46362 emit_move_insn (op0, ireg);
46365 /* Expand rint (IEEE round to nearest) rounding OPERAND1 and storing the
46366 result in OPERAND0. */
46368 ix86_expand_rint (rtx operand0, rtx operand1)
46370 /* C code for the stuff we're doing below:
46371 xa = fabs (operand1);
46372 if (!isless (xa, 2**52))
46374 xa = xa + 2**52 - 2**52;
46375 return copysign (xa, operand1);
46377 machine_mode mode = GET_MODE (operand0);
46378 rtx res, xa, TWO52, mask;
46379 rtx_code_label *label;
46381 res = gen_reg_rtx (mode);
46382 emit_move_insn (res, operand1);
46384 /* xa = abs (operand1) */
46385 xa = ix86_expand_sse_fabs (res, &mask);
46387 /* if (!isless (xa, TWO52)) goto label; */
46388 TWO52 = ix86_gen_TWO52 (mode);
46389 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46391 xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
46392 xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
46394 ix86_sse_copysign_to_positive (res, xa, res, mask);
46396 emit_label (label);
46397 LABEL_NUSES (label) = 1;
46399 emit_move_insn (operand0, res);
46402 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
46405 ix86_expand_floorceildf_32 (rtx operand0, rtx operand1, bool do_floor)
46407 /* C code for the stuff we expand below.
46408 double xa = fabs (x), x2;
46409 if (!isless (xa, TWO52))
46411 xa = xa + TWO52 - TWO52;
46412 x2 = copysign (xa, x);
46421 machine_mode mode = GET_MODE (operand0);
46422 rtx xa, TWO52, tmp, one, res, mask;
46423 rtx_code_label *label;
46425 TWO52 = ix86_gen_TWO52 (mode);
46427 /* Temporary for holding the result, initialized to the input
46428 operand to ease control flow. */
46429 res = gen_reg_rtx (mode);
46430 emit_move_insn (res, operand1);
46432 /* xa = abs (operand1) */
46433 xa = ix86_expand_sse_fabs (res, &mask);
46435 /* if (!isless (xa, TWO52)) goto label; */
46436 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46438 /* xa = xa + TWO52 - TWO52; */
46439 xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
46440 xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
46442 /* xa = copysign (xa, operand1) */
46443 ix86_sse_copysign_to_positive (xa, xa, res, mask);
46445 /* generate 1.0 or -1.0 */
46446 one = force_reg (mode,
46447 const_double_from_real_value (do_floor
46448 ? dconst1 : dconstm1, mode));
46450 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
46451 tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
46452 emit_insn (gen_rtx_SET (VOIDmode, tmp,
46453 gen_rtx_AND (mode, one, tmp)));
46454 /* We always need to subtract here to preserve signed zero. */
46455 tmp = expand_simple_binop (mode, MINUS,
46456 xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
46457 emit_move_insn (res, tmp);
46459 emit_label (label);
46460 LABEL_NUSES (label) = 1;
46462 emit_move_insn (operand0, res);
46465 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
46468 ix86_expand_floorceil (rtx operand0, rtx operand1, bool do_floor)
46470 /* C code for the stuff we expand below.
46471 double xa = fabs (x), x2;
46472 if (!isless (xa, TWO52))
46474 x2 = (double)(long)x;
46481 if (HONOR_SIGNED_ZEROS (mode))
46482 return copysign (x2, x);
46485 machine_mode mode = GET_MODE (operand0);
46486 rtx xa, xi, TWO52, tmp, one, res, mask;
46487 rtx_code_label *label;
46489 TWO52 = ix86_gen_TWO52 (mode);
46491 /* Temporary for holding the result, initialized to the input
46492 operand to ease control flow. */
46493 res = gen_reg_rtx (mode);
46494 emit_move_insn (res, operand1);
46496 /* xa = abs (operand1) */
46497 xa = ix86_expand_sse_fabs (res, &mask);
46499 /* if (!isless (xa, TWO52)) goto label; */
46500 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46502 /* xa = (double)(long)x */
46503 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
46504 expand_fix (xi, res, 0);
46505 expand_float (xa, xi, 0);
46508 one = force_reg (mode, const_double_from_real_value (dconst1, mode));
46510 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
46511 tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
46512 emit_insn (gen_rtx_SET (VOIDmode, tmp,
46513 gen_rtx_AND (mode, one, tmp)));
46514 tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
46515 xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
46516 emit_move_insn (res, tmp);
46518 if (HONOR_SIGNED_ZEROS (mode))
46519 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), mask);
46521 emit_label (label);
46522 LABEL_NUSES (label) = 1;
46524 emit_move_insn (operand0, res);
46527 /* Expand SSE sequence for computing round from OPERAND1 storing
46528 into OPERAND0. Sequence that works without relying on DImode truncation
46529 via cvttsd2siq that is only available on 64bit targets. */
46531 ix86_expand_rounddf_32 (rtx operand0, rtx operand1)
46533 /* C code for the stuff we expand below.
46534 double xa = fabs (x), xa2, x2;
46535 if (!isless (xa, TWO52))
46537 Using the absolute value and copying back sign makes
46538 -0.0 -> -0.0 correct.
46539 xa2 = xa + TWO52 - TWO52;
46544 else if (dxa > 0.5)
46546 x2 = copysign (xa2, x);
46549 machine_mode mode = GET_MODE (operand0);
46550 rtx xa, xa2, dxa, TWO52, tmp, half, mhalf, one, res, mask;
46551 rtx_code_label *label;
46553 TWO52 = ix86_gen_TWO52 (mode);
46555 /* Temporary for holding the result, initialized to the input
46556 operand to ease control flow. */
46557 res = gen_reg_rtx (mode);
46558 emit_move_insn (res, operand1);
46560 /* xa = abs (operand1) */
46561 xa = ix86_expand_sse_fabs (res, &mask);
46563 /* if (!isless (xa, TWO52)) goto label; */
46564 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46566 /* xa2 = xa + TWO52 - TWO52; */
46567 xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
46568 xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
46570 /* dxa = xa2 - xa; */
46571 dxa = expand_simple_binop (mode, MINUS, xa2, xa, NULL_RTX, 0, OPTAB_DIRECT);
46573 /* generate 0.5, 1.0 and -0.5 */
46574 half = force_reg (mode, const_double_from_real_value (dconsthalf, mode));
46575 one = expand_simple_binop (mode, PLUS, half, half, NULL_RTX, 0, OPTAB_DIRECT);
46576 mhalf = expand_simple_binop (mode, MINUS, half, one, NULL_RTX,
46580 tmp = gen_reg_rtx (mode);
46581 /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
46582 tmp = ix86_expand_sse_compare_mask (UNGT, dxa, half, false);
46583 emit_insn (gen_rtx_SET (VOIDmode, tmp,
46584 gen_rtx_AND (mode, one, tmp)));
46585 xa2 = expand_simple_binop (mode, MINUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
46586 /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
46587 tmp = ix86_expand_sse_compare_mask (UNGE, mhalf, dxa, false);
46588 emit_insn (gen_rtx_SET (VOIDmode, tmp,
46589 gen_rtx_AND (mode, one, tmp)));
46590 xa2 = expand_simple_binop (mode, PLUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
46592 /* res = copysign (xa2, operand1) */
46593 ix86_sse_copysign_to_positive (res, xa2, force_reg (mode, operand1), mask);
46595 emit_label (label);
46596 LABEL_NUSES (label) = 1;
46598 emit_move_insn (operand0, res);
46601 /* Expand SSE sequence for computing trunc from OPERAND1 storing
46604 ix86_expand_trunc (rtx operand0, rtx operand1)
46606 /* C code for SSE variant we expand below.
46607 double xa = fabs (x), x2;
46608 if (!isless (xa, TWO52))
46610 x2 = (double)(long)x;
46611 if (HONOR_SIGNED_ZEROS (mode))
46612 return copysign (x2, x);
46615 machine_mode mode = GET_MODE (operand0);
46616 rtx xa, xi, TWO52, res, mask;
46617 rtx_code_label *label;
46619 TWO52 = ix86_gen_TWO52 (mode);
46621 /* Temporary for holding the result, initialized to the input
46622 operand to ease control flow. */
46623 res = gen_reg_rtx (mode);
46624 emit_move_insn (res, operand1);
46626 /* xa = abs (operand1) */
46627 xa = ix86_expand_sse_fabs (res, &mask);
46629 /* if (!isless (xa, TWO52)) goto label; */
46630 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46632 /* x = (double)(long)x */
46633 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
46634 expand_fix (xi, res, 0);
46635 expand_float (res, xi, 0);
46637 if (HONOR_SIGNED_ZEROS (mode))
46638 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), mask);
46640 emit_label (label);
46641 LABEL_NUSES (label) = 1;
46643 emit_move_insn (operand0, res);
46646 /* Expand SSE sequence for computing trunc from OPERAND1 storing
46649 ix86_expand_truncdf_32 (rtx operand0, rtx operand1)
46651 machine_mode mode = GET_MODE (operand0);
46652 rtx xa, mask, TWO52, one, res, smask, tmp;
46653 rtx_code_label *label;
46655 /* C code for SSE variant we expand below.
46656 double xa = fabs (x), x2;
46657 if (!isless (xa, TWO52))
46659 xa2 = xa + TWO52 - TWO52;
46663 x2 = copysign (xa2, x);
46667 TWO52 = ix86_gen_TWO52 (mode);
46669 /* Temporary for holding the result, initialized to the input
46670 operand to ease control flow. */
46671 res = gen_reg_rtx (mode);
46672 emit_move_insn (res, operand1);
46674 /* xa = abs (operand1) */
46675 xa = ix86_expand_sse_fabs (res, &smask);
46677 /* if (!isless (xa, TWO52)) goto label; */
46678 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46680 /* res = xa + TWO52 - TWO52; */
46681 tmp = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
46682 tmp = expand_simple_binop (mode, MINUS, tmp, TWO52, tmp, 0, OPTAB_DIRECT);
46683 emit_move_insn (res, tmp);
46686 one = force_reg (mode, const_double_from_real_value (dconst1, mode));
46688 /* Compensate: res = xa2 - (res > xa ? 1 : 0) */
46689 mask = ix86_expand_sse_compare_mask (UNGT, res, xa, false);
46690 emit_insn (gen_rtx_SET (VOIDmode, mask,
46691 gen_rtx_AND (mode, mask, one)));
46692 tmp = expand_simple_binop (mode, MINUS,
46693 res, mask, NULL_RTX, 0, OPTAB_DIRECT);
46694 emit_move_insn (res, tmp);
46696 /* res = copysign (res, operand1) */
46697 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), smask);
46699 emit_label (label);
46700 LABEL_NUSES (label) = 1;
46702 emit_move_insn (operand0, res);
46705 /* Expand SSE sequence for computing round from OPERAND1 storing
46708 ix86_expand_round (rtx operand0, rtx operand1)
46710 /* C code for the stuff we're doing below:
46711 double xa = fabs (x);
46712 if (!isless (xa, TWO52))
46714 xa = (double)(long)(xa + nextafter (0.5, 0.0));
46715 return copysign (xa, x);
46717 machine_mode mode = GET_MODE (operand0);
46718 rtx res, TWO52, xa, xi, half, mask;
46719 rtx_code_label *label;
46720 const struct real_format *fmt;
46721 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
46723 /* Temporary for holding the result, initialized to the input
46724 operand to ease control flow. */
46725 res = gen_reg_rtx (mode);
46726 emit_move_insn (res, operand1);
46728 TWO52 = ix86_gen_TWO52 (mode);
46729 xa = ix86_expand_sse_fabs (res, &mask);
46730 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46732 /* load nextafter (0.5, 0.0) */
46733 fmt = REAL_MODE_FORMAT (mode);
46734 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
46735 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
46737 /* xa = xa + 0.5 */
46738 half = force_reg (mode, const_double_from_real_value (pred_half, mode));
46739 xa = expand_simple_binop (mode, PLUS, xa, half, NULL_RTX, 0, OPTAB_DIRECT);
46741 /* xa = (double)(int64_t)xa */
46742 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
46743 expand_fix (xi, xa, 0);
46744 expand_float (xa, xi, 0);
46746 /* res = copysign (xa, operand1) */
46747 ix86_sse_copysign_to_positive (res, xa, force_reg (mode, operand1), mask);
46749 emit_label (label);
46750 LABEL_NUSES (label) = 1;
46752 emit_move_insn (operand0, res);
46755 /* Expand SSE sequence for computing round
46756 from OP1 storing into OP0 using sse4 round insn. */
46758 ix86_expand_round_sse4 (rtx op0, rtx op1)
46760 machine_mode mode = GET_MODE (op0);
46761 rtx e1, e2, res, half;
46762 const struct real_format *fmt;
46763 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
46764 rtx (*gen_copysign) (rtx, rtx, rtx);
46765 rtx (*gen_round) (rtx, rtx, rtx);
46770 gen_copysign = gen_copysignsf3;
46771 gen_round = gen_sse4_1_roundsf2;
46774 gen_copysign = gen_copysigndf3;
46775 gen_round = gen_sse4_1_rounddf2;
46778 gcc_unreachable ();
46781 /* round (a) = trunc (a + copysign (0.5, a)) */
46783 /* load nextafter (0.5, 0.0) */
46784 fmt = REAL_MODE_FORMAT (mode);
46785 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
46786 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
46787 half = const_double_from_real_value (pred_half, mode);
46789 /* e1 = copysign (0.5, op1) */
46790 e1 = gen_reg_rtx (mode);
46791 emit_insn (gen_copysign (e1, half, op1));
46793 /* e2 = op1 + e1 */
46794 e2 = expand_simple_binop (mode, PLUS, op1, e1, NULL_RTX, 0, OPTAB_DIRECT);
46796 /* res = trunc (e2) */
46797 res = gen_reg_rtx (mode);
46798 emit_insn (gen_round (res, e2, GEN_INT (ROUND_TRUNC)));
46800 emit_move_insn (op0, res);
46804 /* Table of valid machine attributes. */
46805 static const struct attribute_spec ix86_attribute_table[] =
46807 /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler,
46808 affects_type_identity } */
46809 /* Stdcall attribute says callee is responsible for popping arguments
46810 if they are not variable. */
46811 { "stdcall", 0, 0, false, true, true, ix86_handle_cconv_attribute,
46813 /* Fastcall attribute says callee is responsible for popping arguments
46814 if they are not variable. */
46815 { "fastcall", 0, 0, false, true, true, ix86_handle_cconv_attribute,
46817 /* Thiscall attribute says callee is responsible for popping arguments
46818 if they are not variable. */
46819 { "thiscall", 0, 0, false, true, true, ix86_handle_cconv_attribute,
46821 /* Cdecl attribute says the callee is a normal C declaration */
46822 { "cdecl", 0, 0, false, true, true, ix86_handle_cconv_attribute,
46824 /* Regparm attribute specifies how many integer arguments are to be
46825 passed in registers. */
46826 { "regparm", 1, 1, false, true, true, ix86_handle_cconv_attribute,
46828 /* Sseregparm attribute says we are using x86_64 calling conventions
46829 for FP arguments. */
46830 { "sseregparm", 0, 0, false, true, true, ix86_handle_cconv_attribute,
46832 /* The transactional memory builtins are implicitly regparm or fastcall
46833 depending on the ABI. Override the generic do-nothing attribute that
46834 these builtins were declared with. */
46835 { "*tm regparm", 0, 0, false, true, true, ix86_handle_tm_regparm_attribute,
46837 /* force_align_arg_pointer says this function realigns the stack at entry. */
46838 { (const char *)&ix86_force_align_arg_pointer_string, 0, 0,
46839 false, true, true, ix86_handle_cconv_attribute, false },
46840 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
46841 { "dllimport", 0, 0, false, false, false, handle_dll_attribute, false },
46842 { "dllexport", 0, 0, false, false, false, handle_dll_attribute, false },
46843 { "shared", 0, 0, true, false, false, ix86_handle_shared_attribute,
46846 { "ms_struct", 0, 0, false, false, false, ix86_handle_struct_attribute,
46848 { "gcc_struct", 0, 0, false, false, false, ix86_handle_struct_attribute,
46850 #ifdef SUBTARGET_ATTRIBUTE_TABLE
46851 SUBTARGET_ATTRIBUTE_TABLE,
46853 /* ms_abi and sysv_abi calling convention function attributes. */
46854 { "ms_abi", 0, 0, false, true, true, ix86_handle_abi_attribute, true },
46855 { "sysv_abi", 0, 0, false, true, true, ix86_handle_abi_attribute, true },
46856 { "ms_hook_prologue", 0, 0, true, false, false, ix86_handle_fndecl_attribute,
46858 { "callee_pop_aggregate_return", 1, 1, false, true, true,
46859 ix86_handle_callee_pop_aggregate_return, true },
46861 { NULL, 0, 0, false, false, false, NULL, false }
46864 /* Implement targetm.vectorize.builtin_vectorization_cost. */
46866 ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
46871 switch (type_of_cost)
46874 return ix86_cost->scalar_stmt_cost;
46877 return ix86_cost->scalar_load_cost;
46880 return ix86_cost->scalar_store_cost;
46883 return ix86_cost->vec_stmt_cost;
46886 return ix86_cost->vec_align_load_cost;
46889 return ix86_cost->vec_store_cost;
46891 case vec_to_scalar:
46892 return ix86_cost->vec_to_scalar_cost;
46894 case scalar_to_vec:
46895 return ix86_cost->scalar_to_vec_cost;
46897 case unaligned_load:
46898 case unaligned_store:
46899 return ix86_cost->vec_unalign_load_cost;
46901 case cond_branch_taken:
46902 return ix86_cost->cond_taken_branch_cost;
46904 case cond_branch_not_taken:
46905 return ix86_cost->cond_not_taken_branch_cost;
46908 case vec_promote_demote:
46909 return ix86_cost->vec_stmt_cost;
46911 case vec_construct:
46912 elements = TYPE_VECTOR_SUBPARTS (vectype);
46913 return elements / 2 + 1;
46916 gcc_unreachable ();
46920 /* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
46921 insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
46922 insn every time. */
46924 static GTY(()) rtx_insn *vselect_insn;
46926 /* Initialize vselect_insn. */
46929 init_vselect_insn (void)
46934 x = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (MAX_VECT_LEN));
46935 for (i = 0; i < MAX_VECT_LEN; ++i)
46936 XVECEXP (x, 0, i) = const0_rtx;
46937 x = gen_rtx_VEC_SELECT (V2DFmode, gen_rtx_VEC_CONCAT (V4DFmode, const0_rtx,
46939 x = gen_rtx_SET (VOIDmode, const0_rtx, x);
46941 vselect_insn = emit_insn (x);
46945 /* Construct (set target (vec_select op0 (parallel perm))) and
46946 return true if that's a valid instruction in the active ISA. */
46949 expand_vselect (rtx target, rtx op0, const unsigned char *perm,
46950 unsigned nelt, bool testing_p)
46953 rtx x, save_vconcat;
46956 if (vselect_insn == NULL_RTX)
46957 init_vselect_insn ();
46959 x = XEXP (SET_SRC (PATTERN (vselect_insn)), 1);
46960 PUT_NUM_ELEM (XVEC (x, 0), nelt);
46961 for (i = 0; i < nelt; ++i)
46962 XVECEXP (x, 0, i) = GEN_INT (perm[i]);
46963 save_vconcat = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
46964 XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = op0;
46965 PUT_MODE (SET_SRC (PATTERN (vselect_insn)), GET_MODE (target));
46966 SET_DEST (PATTERN (vselect_insn)) = target;
46967 icode = recog_memoized (vselect_insn);
46969 if (icode >= 0 && !testing_p)
46970 emit_insn (copy_rtx (PATTERN (vselect_insn)));
46972 SET_DEST (PATTERN (vselect_insn)) = const0_rtx;
46973 XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = save_vconcat;
46974 INSN_CODE (vselect_insn) = -1;
46979 /* Similar, but generate a vec_concat from op0 and op1 as well. */
46982 expand_vselect_vconcat (rtx target, rtx op0, rtx op1,
46983 const unsigned char *perm, unsigned nelt,
46986 machine_mode v2mode;
46990 if (vselect_insn == NULL_RTX)
46991 init_vselect_insn ();
46993 v2mode = GET_MODE_2XWIDER_MODE (GET_MODE (op0));
46994 x = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
46995 PUT_MODE (x, v2mode);
46998 ok = expand_vselect (target, x, perm, nelt, testing_p);
46999 XEXP (x, 0) = const0_rtx;
47000 XEXP (x, 1) = const0_rtx;
47004 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
47005 in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
47008 expand_vec_perm_blend (struct expand_vec_perm_d *d)
47010 machine_mode mmode, vmode = d->vmode;
47011 unsigned i, mask, nelt = d->nelt;
47012 rtx target, op0, op1, maskop, x;
47013 rtx rperm[32], vperm;
47015 if (d->one_operand_p)
47017 if (TARGET_AVX512F && GET_MODE_SIZE (vmode) == 64
47018 && (TARGET_AVX512BW
47019 || GET_MODE_SIZE (GET_MODE_INNER (vmode)) >= 4))
47021 else if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
47023 else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
47025 else if (TARGET_SSE4_1 && GET_MODE_SIZE (vmode) == 16)
47030 /* This is a blend, not a permute. Elements must stay in their
47031 respective lanes. */
47032 for (i = 0; i < nelt; ++i)
47034 unsigned e = d->perm[i];
47035 if (!(e == i || e == i + nelt))
47042 /* ??? Without SSE4.1, we could implement this with and/andn/or. This
47043 decision should be extracted elsewhere, so that we only try that
47044 sequence once all budget==3 options have been tried. */
47045 target = d->target;
47064 for (i = 0; i < nelt; ++i)
47065 mask |= (d->perm[i] >= nelt) << i;
47069 for (i = 0; i < 2; ++i)
47070 mask |= (d->perm[i] >= 2 ? 15 : 0) << (i * 4);
47075 for (i = 0; i < 4; ++i)
47076 mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
47081 /* See if bytes move in pairs so we can use pblendw with
47082 an immediate argument, rather than pblendvb with a vector
47084 for (i = 0; i < 16; i += 2)
47085 if (d->perm[i] + 1 != d->perm[i + 1])
47088 for (i = 0; i < nelt; ++i)
47089 rperm[i] = (d->perm[i] < nelt ? const0_rtx : constm1_rtx);
47092 vperm = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (nelt, rperm));
47093 vperm = force_reg (vmode, vperm);
47095 if (GET_MODE_SIZE (vmode) == 16)
47096 emit_insn (gen_sse4_1_pblendvb (target, op0, op1, vperm));
47098 emit_insn (gen_avx2_pblendvb (target, op0, op1, vperm));
47099 if (target != d->target)
47100 emit_move_insn (d->target, gen_lowpart (d->vmode, target));
47104 for (i = 0; i < 8; ++i)
47105 mask |= (d->perm[i * 2] >= 16) << i;
47110 target = gen_reg_rtx (vmode);
47111 op0 = gen_lowpart (vmode, op0);
47112 op1 = gen_lowpart (vmode, op1);
47116 /* See if bytes move in pairs. If not, vpblendvb must be used. */
47117 for (i = 0; i < 32; i += 2)
47118 if (d->perm[i] + 1 != d->perm[i + 1])
47120 /* See if bytes move in quadruplets. If yes, vpblendd
47121 with immediate can be used. */
47122 for (i = 0; i < 32; i += 4)
47123 if (d->perm[i] + 2 != d->perm[i + 2])
47127 /* See if bytes move the same in both lanes. If yes,
47128 vpblendw with immediate can be used. */
47129 for (i = 0; i < 16; i += 2)
47130 if (d->perm[i] + 16 != d->perm[i + 16])
47133 /* Use vpblendw. */
47134 for (i = 0; i < 16; ++i)
47135 mask |= (d->perm[i * 2] >= 32) << i;
47140 /* Use vpblendd. */
47141 for (i = 0; i < 8; ++i)
47142 mask |= (d->perm[i * 4] >= 32) << i;
47147 /* See if words move in pairs. If yes, vpblendd can be used. */
47148 for (i = 0; i < 16; i += 2)
47149 if (d->perm[i] + 1 != d->perm[i + 1])
47153 /* See if words move the same in both lanes. If not,
47154 vpblendvb must be used. */
47155 for (i = 0; i < 8; i++)
47156 if (d->perm[i] + 8 != d->perm[i + 8])
47158 /* Use vpblendvb. */
47159 for (i = 0; i < 32; ++i)
47160 rperm[i] = (d->perm[i / 2] < 16 ? const0_rtx : constm1_rtx);
47164 target = gen_reg_rtx (vmode);
47165 op0 = gen_lowpart (vmode, op0);
47166 op1 = gen_lowpart (vmode, op1);
47167 goto finish_pblendvb;
47170 /* Use vpblendw. */
47171 for (i = 0; i < 16; ++i)
47172 mask |= (d->perm[i] >= 16) << i;
47176 /* Use vpblendd. */
47177 for (i = 0; i < 8; ++i)
47178 mask |= (d->perm[i * 2] >= 16) << i;
47183 /* Use vpblendd. */
47184 for (i = 0; i < 4; ++i)
47185 mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
47190 gcc_unreachable ();
47213 if (mmode != VOIDmode)
47214 maskop = force_reg (mmode, gen_int_mode (mask, mmode));
47216 maskop = GEN_INT (mask);
47218 /* This matches five different patterns with the different modes. */
47219 x = gen_rtx_VEC_MERGE (vmode, op1, op0, maskop);
47220 x = gen_rtx_SET (VOIDmode, target, x);
47222 if (target != d->target)
47223 emit_move_insn (d->target, gen_lowpart (d->vmode, target));
47228 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
47229 in terms of the variable form of vpermilps.
47231 Note that we will have already failed the immediate input vpermilps,
47232 which requires that the high and low part shuffle be identical; the
47233 variable form doesn't require that. */
47236 expand_vec_perm_vpermil (struct expand_vec_perm_d *d)
47238 rtx rperm[8], vperm;
47241 if (!TARGET_AVX || d->vmode != V8SFmode || !d->one_operand_p)
47244 /* We can only permute within the 128-bit lane. */
47245 for (i = 0; i < 8; ++i)
47247 unsigned e = d->perm[i];
47248 if (i < 4 ? e >= 4 : e < 4)
47255 for (i = 0; i < 8; ++i)
47257 unsigned e = d->perm[i];
47259 /* Within each 128-bit lane, the elements of op0 are numbered
47260 from 0 and the elements of op1 are numbered from 4. */
47266 rperm[i] = GEN_INT (e);
47269 vperm = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, rperm));
47270 vperm = force_reg (V8SImode, vperm);
47271 emit_insn (gen_avx_vpermilvarv8sf3 (d->target, d->op0, vperm));
47276 /* Return true if permutation D can be performed as VMODE permutation
47280 valid_perm_using_mode_p (machine_mode vmode, struct expand_vec_perm_d *d)
47282 unsigned int i, j, chunk;
47284 if (GET_MODE_CLASS (vmode) != MODE_VECTOR_INT
47285 || GET_MODE_CLASS (d->vmode) != MODE_VECTOR_INT
47286 || GET_MODE_SIZE (vmode) != GET_MODE_SIZE (d->vmode))
47289 if (GET_MODE_NUNITS (vmode) >= d->nelt)
47292 chunk = d->nelt / GET_MODE_NUNITS (vmode);
47293 for (i = 0; i < d->nelt; i += chunk)
47294 if (d->perm[i] & (chunk - 1))
47297 for (j = 1; j < chunk; ++j)
47298 if (d->perm[i] + j != d->perm[i + j])
47304 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
47305 in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128. */
47308 expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
47310 unsigned i, nelt, eltsz, mask;
47311 unsigned char perm[64];
47312 machine_mode vmode = V16QImode;
47313 rtx rperm[64], vperm, target, op0, op1;
47317 if (!d->one_operand_p)
47319 if (!TARGET_XOP || GET_MODE_SIZE (d->vmode) != 16)
47322 && valid_perm_using_mode_p (V2TImode, d))
47327 /* Use vperm2i128 insn. The pattern uses
47328 V4DImode instead of V2TImode. */
47329 target = d->target;
47330 if (d->vmode != V4DImode)
47331 target = gen_reg_rtx (V4DImode);
47332 op0 = gen_lowpart (V4DImode, d->op0);
47333 op1 = gen_lowpart (V4DImode, d->op1);
47335 = GEN_INT ((d->perm[0] / (nelt / 2))
47336 | ((d->perm[nelt / 2] / (nelt / 2)) * 16));
47337 emit_insn (gen_avx2_permv2ti (target, op0, op1, rperm[0]));
47338 if (target != d->target)
47339 emit_move_insn (d->target, gen_lowpart (d->vmode, target));
47347 if (GET_MODE_SIZE (d->vmode) == 16)
47352 else if (GET_MODE_SIZE (d->vmode) == 32)
47357 /* V4DImode should be already handled through
47358 expand_vselect by vpermq instruction. */
47359 gcc_assert (d->vmode != V4DImode);
47362 if (d->vmode == V8SImode
47363 || d->vmode == V16HImode
47364 || d->vmode == V32QImode)
47366 /* First see if vpermq can be used for
47367 V8SImode/V16HImode/V32QImode. */
47368 if (valid_perm_using_mode_p (V4DImode, d))
47370 for (i = 0; i < 4; i++)
47371 perm[i] = (d->perm[i * nelt / 4] * 4 / nelt) & 3;
47374 target = gen_reg_rtx (V4DImode);
47375 if (expand_vselect (target, gen_lowpart (V4DImode, d->op0),
47378 emit_move_insn (d->target,
47379 gen_lowpart (d->vmode, target));
47385 /* Next see if vpermd can be used. */
47386 if (valid_perm_using_mode_p (V8SImode, d))
47389 /* Or if vpermps can be used. */
47390 else if (d->vmode == V8SFmode)
47393 if (vmode == V32QImode)
47395 /* vpshufb only works intra lanes, it is not
47396 possible to shuffle bytes in between the lanes. */
47397 for (i = 0; i < nelt; ++i)
47398 if ((d->perm[i] ^ i) & (nelt / 2))
47402 else if (GET_MODE_SIZE (d->vmode) == 64)
47404 if (!TARGET_AVX512BW)
47407 /* If vpermq didn't work, vpshufb won't work either. */
47408 if (d->vmode == V8DFmode || d->vmode == V8DImode)
47412 if (d->vmode == V16SImode
47413 || d->vmode == V32HImode
47414 || d->vmode == V64QImode)
47416 /* First see if vpermq can be used for
47417 V16SImode/V32HImode/V64QImode. */
47418 if (valid_perm_using_mode_p (V8DImode, d))
47420 for (i = 0; i < 8; i++)
47421 perm[i] = (d->perm[i * nelt / 8] * 8 / nelt) & 7;
47424 target = gen_reg_rtx (V8DImode);
47425 if (expand_vselect (target, gen_lowpart (V8DImode, d->op0),
47428 emit_move_insn (d->target,
47429 gen_lowpart (d->vmode, target));
47435 /* Next see if vpermd can be used. */
47436 if (valid_perm_using_mode_p (V16SImode, d))
47439 /* Or if vpermps can be used. */
47440 else if (d->vmode == V16SFmode)
47442 if (vmode == V64QImode)
47444 /* vpshufb only works intra lanes, it is not
47445 possible to shuffle bytes in between the lanes. */
47446 for (i = 0; i < nelt; ++i)
47447 if ((d->perm[i] ^ i) & (nelt / 4))
47458 if (vmode == V8SImode)
47459 for (i = 0; i < 8; ++i)
47460 rperm[i] = GEN_INT ((d->perm[i * nelt / 8] * 8 / nelt) & 7);
47461 else if (vmode == V16SImode)
47462 for (i = 0; i < 16; ++i)
47463 rperm[i] = GEN_INT ((d->perm[i * nelt / 16] * 16 / nelt) & 15);
47466 eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
47467 if (!d->one_operand_p)
47468 mask = 2 * nelt - 1;
47469 else if (vmode == V16QImode)
47471 else if (vmode == V64QImode)
47472 mask = nelt / 4 - 1;
47474 mask = nelt / 2 - 1;
47476 for (i = 0; i < nelt; ++i)
47478 unsigned j, e = d->perm[i] & mask;
47479 for (j = 0; j < eltsz; ++j)
47480 rperm[i * eltsz + j] = GEN_INT (e * eltsz + j);
47484 vperm = gen_rtx_CONST_VECTOR (vmode,
47485 gen_rtvec_v (GET_MODE_NUNITS (vmode), rperm));
47486 vperm = force_reg (vmode, vperm);
47488 target = d->target;
47489 if (d->vmode != vmode)
47490 target = gen_reg_rtx (vmode);
47491 op0 = gen_lowpart (vmode, d->op0);
47492 if (d->one_operand_p)
47494 if (vmode == V16QImode)
47495 emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, vperm));
47496 else if (vmode == V32QImode)
47497 emit_insn (gen_avx2_pshufbv32qi3 (target, op0, vperm));
47498 else if (vmode == V64QImode)
47499 emit_insn (gen_avx512bw_pshufbv64qi3 (target, op0, vperm));
47500 else if (vmode == V8SFmode)
47501 emit_insn (gen_avx2_permvarv8sf (target, op0, vperm));
47502 else if (vmode == V8SImode)
47503 emit_insn (gen_avx2_permvarv8si (target, op0, vperm));
47504 else if (vmode == V16SFmode)
47505 emit_insn (gen_avx512f_permvarv16sf (target, op0, vperm));
47506 else if (vmode == V16SImode)
47507 emit_insn (gen_avx512f_permvarv16si (target, op0, vperm));
47509 gcc_unreachable ();
47513 op1 = gen_lowpart (vmode, d->op1);
47514 emit_insn (gen_xop_pperm (target, op0, op1, vperm));
47516 if (target != d->target)
47517 emit_move_insn (d->target, gen_lowpart (d->vmode, target));
47522 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to instantiate D
47523 in a single instruction. */
47526 expand_vec_perm_1 (struct expand_vec_perm_d *d)
47528 unsigned i, nelt = d->nelt;
47529 unsigned char perm2[MAX_VECT_LEN];
47531 /* Check plain VEC_SELECT first, because AVX has instructions that could
47532 match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
47533 input where SEL+CONCAT may not. */
47534 if (d->one_operand_p)
47536 int mask = nelt - 1;
47537 bool identity_perm = true;
47538 bool broadcast_perm = true;
47540 for (i = 0; i < nelt; i++)
47542 perm2[i] = d->perm[i] & mask;
47544 identity_perm = false;
47546 broadcast_perm = false;
47552 emit_move_insn (d->target, d->op0);
47555 else if (broadcast_perm && TARGET_AVX2)
47557 /* Use vpbroadcast{b,w,d}. */
47558 rtx (*gen) (rtx, rtx) = NULL;
47562 if (TARGET_AVX512BW)
47563 gen = gen_avx512bw_vec_dupv64qi_1;
47566 gen = gen_avx2_pbroadcastv32qi_1;
47569 if (TARGET_AVX512BW)
47570 gen = gen_avx512bw_vec_dupv32hi_1;
47573 gen = gen_avx2_pbroadcastv16hi_1;
47576 if (TARGET_AVX512F)
47577 gen = gen_avx512f_vec_dupv16si_1;
47580 gen = gen_avx2_pbroadcastv8si_1;
47583 gen = gen_avx2_pbroadcastv16qi;
47586 gen = gen_avx2_pbroadcastv8hi;
47589 if (TARGET_AVX512F)
47590 gen = gen_avx512f_vec_dupv16sf_1;
47593 gen = gen_avx2_vec_dupv8sf_1;
47596 if (TARGET_AVX512F)
47597 gen = gen_avx512f_vec_dupv8df_1;
47600 if (TARGET_AVX512F)
47601 gen = gen_avx512f_vec_dupv8di_1;
47603 /* For other modes prefer other shuffles this function creates. */
47609 emit_insn (gen (d->target, d->op0));
47614 if (expand_vselect (d->target, d->op0, perm2, nelt, d->testing_p))
47617 /* There are plenty of patterns in sse.md that are written for
47618 SEL+CONCAT and are not replicated for a single op. Perhaps
47619 that should be changed, to avoid the nastiness here. */
47621 /* Recognize interleave style patterns, which means incrementing
47622 every other permutation operand. */
47623 for (i = 0; i < nelt; i += 2)
47625 perm2[i] = d->perm[i] & mask;
47626 perm2[i + 1] = (d->perm[i + 1] & mask) + nelt;
47628 if (expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, nelt,
47632 /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
47635 for (i = 0; i < nelt; i += 4)
47637 perm2[i + 0] = d->perm[i + 0] & mask;
47638 perm2[i + 1] = d->perm[i + 1] & mask;
47639 perm2[i + 2] = (d->perm[i + 2] & mask) + nelt;
47640 perm2[i + 3] = (d->perm[i + 3] & mask) + nelt;
47643 if (expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, nelt,
47649 /* Finally, try the fully general two operand permute. */
47650 if (expand_vselect_vconcat (d->target, d->op0, d->op1, d->perm, nelt,
47654 /* Recognize interleave style patterns with reversed operands. */
47655 if (!d->one_operand_p)
47657 for (i = 0; i < nelt; ++i)
47659 unsigned e = d->perm[i];
47667 if (expand_vselect_vconcat (d->target, d->op1, d->op0, perm2, nelt,
47672 /* Try the SSE4.1 blend variable merge instructions. */
47673 if (expand_vec_perm_blend (d))
47676 /* Try one of the AVX vpermil variable permutations. */
47677 if (expand_vec_perm_vpermil (d))
47680 /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
47681 vpshufb, vpermd, vpermps or vpermq variable permutation. */
47682 if (expand_vec_perm_pshufb (d))
47685 /* Try the AVX2 vpalignr instruction. */
47686 if (expand_vec_perm_palignr (d, true))
47689 /* Try the AVX512F vpermi2 instructions. */
47690 if (ix86_expand_vec_perm_vpermi2 (NULL_RTX, NULL_RTX, NULL_RTX, NULL_RTX, d))
47696 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
47697 in terms of a pair of pshuflw + pshufhw instructions. */
47700 expand_vec_perm_pshuflw_pshufhw (struct expand_vec_perm_d *d)
47702 unsigned char perm2[MAX_VECT_LEN];
47706 if (d->vmode != V8HImode || !d->one_operand_p)
47709 /* The two permutations only operate in 64-bit lanes. */
47710 for (i = 0; i < 4; ++i)
47711 if (d->perm[i] >= 4)
47713 for (i = 4; i < 8; ++i)
47714 if (d->perm[i] < 4)
47720 /* Emit the pshuflw. */
47721 memcpy (perm2, d->perm, 4);
47722 for (i = 4; i < 8; ++i)
47724 ok = expand_vselect (d->target, d->op0, perm2, 8, d->testing_p);
47727 /* Emit the pshufhw. */
47728 memcpy (perm2 + 4, d->perm + 4, 4);
47729 for (i = 0; i < 4; ++i)
47731 ok = expand_vselect (d->target, d->target, perm2, 8, d->testing_p);
47737 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
47738 the permutation using the SSSE3 palignr instruction. This succeeds
47739 when all of the elements in PERM fit within one vector and we merely
47740 need to shift them down so that a single vector permutation has a
47741 chance to succeed. If SINGLE_INSN_ONLY_P, succeed if only
47742 the vpalignr instruction itself can perform the requested permutation. */
47745 expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool single_insn_only_p)
47747 unsigned i, nelt = d->nelt;
47748 unsigned min, max, minswap, maxswap;
47749 bool in_order, ok, swap = false;
47751 struct expand_vec_perm_d dcopy;
47753 /* Even with AVX, palignr only operates on 128-bit vectors,
47754 in AVX2 palignr operates on both 128-bit lanes. */
47755 if ((!TARGET_SSSE3 || GET_MODE_SIZE (d->vmode) != 16)
47756 && (!TARGET_AVX2 || GET_MODE_SIZE (d->vmode) != 32))
47761 minswap = 2 * nelt;
47763 for (i = 0; i < nelt; ++i)
47765 unsigned e = d->perm[i];
47766 unsigned eswap = d->perm[i] ^ nelt;
47767 if (GET_MODE_SIZE (d->vmode) == 32)
47769 e = (e & ((nelt / 2) - 1)) | ((e & nelt) >> 1);
47770 eswap = e ^ (nelt / 2);
47776 if (eswap < minswap)
47778 if (eswap > maxswap)
47782 || max - min >= (GET_MODE_SIZE (d->vmode) == 32 ? nelt / 2 : nelt))
47784 if (d->one_operand_p
47786 || maxswap - minswap >= (GET_MODE_SIZE (d->vmode) == 32
47787 ? nelt / 2 : nelt))
47794 /* Given that we have SSSE3, we know we'll be able to implement the
47795 single operand permutation after the palignr with pshufb for
47796 128-bit vectors. If SINGLE_INSN_ONLY_P, in_order has to be computed
47798 if (d->testing_p && GET_MODE_SIZE (d->vmode) == 16 && !single_insn_only_p)
47804 dcopy.op0 = d->op1;
47805 dcopy.op1 = d->op0;
47806 for (i = 0; i < nelt; ++i)
47807 dcopy.perm[i] ^= nelt;
47811 for (i = 0; i < nelt; ++i)
47813 unsigned e = dcopy.perm[i];
47814 if (GET_MODE_SIZE (d->vmode) == 32
47816 && (e & (nelt / 2 - 1)) < min)
47817 e = e - min - (nelt / 2);
47824 dcopy.one_operand_p = true;
47826 if (single_insn_only_p && !in_order)
47829 /* For AVX2, test whether we can permute the result in one instruction. */
47834 dcopy.op1 = dcopy.op0;
47835 return expand_vec_perm_1 (&dcopy);
47838 shift = GEN_INT (min * GET_MODE_BITSIZE (GET_MODE_INNER (d->vmode)));
47839 if (GET_MODE_SIZE (d->vmode) == 16)
47841 target = gen_reg_rtx (TImode);
47842 emit_insn (gen_ssse3_palignrti (target, gen_lowpart (TImode, dcopy.op1),
47843 gen_lowpart (TImode, dcopy.op0), shift));
47847 target = gen_reg_rtx (V2TImode);
47848 emit_insn (gen_avx2_palignrv2ti (target,
47849 gen_lowpart (V2TImode, dcopy.op1),
47850 gen_lowpart (V2TImode, dcopy.op0),
47854 dcopy.op0 = dcopy.op1 = gen_lowpart (d->vmode, target);
47856 /* Test for the degenerate case where the alignment by itself
47857 produces the desired permutation. */
47860 emit_move_insn (d->target, dcopy.op0);
47864 ok = expand_vec_perm_1 (&dcopy);
47865 gcc_assert (ok || GET_MODE_SIZE (d->vmode) == 32);
47870 /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
47871 the permutation using the SSE4_1 pblendv instruction. Potentially
47872 reduces permutation from 2 pshufb and or to 1 pshufb and pblendv. */
47875 expand_vec_perm_pblendv (struct expand_vec_perm_d *d)
47877 unsigned i, which, nelt = d->nelt;
47878 struct expand_vec_perm_d dcopy, dcopy1;
47879 machine_mode vmode = d->vmode;
47882 /* Use the same checks as in expand_vec_perm_blend. */
47883 if (d->one_operand_p)
47885 if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
47887 else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
47889 else if (TARGET_SSE4_1 && GET_MODE_SIZE (vmode) == 16)
47894 /* Figure out where permutation elements stay not in their
47895 respective lanes. */
47896 for (i = 0, which = 0; i < nelt; ++i)
47898 unsigned e = d->perm[i];
47900 which |= (e < nelt ? 1 : 2);
47902 /* We can pblend the part where elements stay not in their
47903 respective lanes only when these elements are all in one
47904 half of a permutation.
47905 {0 1 8 3 4 5 9 7} is ok as 8, 9 are at not at their respective
47906 lanes, but both 8 and 9 >= 8
47907 {0 1 8 3 4 5 2 7} is not ok as 2 and 8 are not at their
47908 respective lanes and 8 >= 8, but 2 not. */
47909 if (which != 1 && which != 2)
47911 if (d->testing_p && GET_MODE_SIZE (vmode) == 16)
47914 /* First we apply one operand permutation to the part where
47915 elements stay not in their respective lanes. */
47918 dcopy.op0 = dcopy.op1 = d->op1;
47920 dcopy.op0 = dcopy.op1 = d->op0;
47922 dcopy.target = gen_reg_rtx (vmode);
47923 dcopy.one_operand_p = true;
47925 for (i = 0; i < nelt; ++i)
47926 dcopy.perm[i] = d->perm[i] & (nelt - 1);
47928 ok = expand_vec_perm_1 (&dcopy);
47929 if (GET_MODE_SIZE (vmode) != 16 && !ok)
47936 /* Next we put permuted elements into their positions. */
47939 dcopy1.op1 = dcopy.target;
47941 dcopy1.op0 = dcopy.target;
47943 for (i = 0; i < nelt; ++i)
47944 dcopy1.perm[i] = ((d->perm[i] >= nelt) ? (nelt + i) : i);
47946 ok = expand_vec_perm_blend (&dcopy1);
47952 static bool expand_vec_perm_interleave3 (struct expand_vec_perm_d *d);
47954 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
47955 a two vector permutation into a single vector permutation by using
47956 an interleave operation to merge the vectors. */
47959 expand_vec_perm_interleave2 (struct expand_vec_perm_d *d)
47961 struct expand_vec_perm_d dremap, dfinal;
47962 unsigned i, nelt = d->nelt, nelt2 = nelt / 2;
47963 unsigned HOST_WIDE_INT contents;
47964 unsigned char remap[2 * MAX_VECT_LEN];
47966 bool ok, same_halves = false;
47968 if (GET_MODE_SIZE (d->vmode) == 16)
47970 if (d->one_operand_p)
47973 else if (GET_MODE_SIZE (d->vmode) == 32)
47977 /* For 32-byte modes allow even d->one_operand_p.
47978 The lack of cross-lane shuffling in some instructions
47979 might prevent a single insn shuffle. */
47981 dfinal.testing_p = true;
47982 /* If expand_vec_perm_interleave3 can expand this into
47983 a 3 insn sequence, give up and let it be expanded as
47984 3 insn sequence. While that is one insn longer,
47985 it doesn't need a memory operand and in the common
47986 case that both interleave low and high permutations
47987 with the same operands are adjacent needs 4 insns
47988 for both after CSE. */
47989 if (expand_vec_perm_interleave3 (&dfinal))
47995 /* Examine from whence the elements come. */
47997 for (i = 0; i < nelt; ++i)
47998 contents |= ((unsigned HOST_WIDE_INT) 1) << d->perm[i];
48000 memset (remap, 0xff, sizeof (remap));
48003 if (GET_MODE_SIZE (d->vmode) == 16)
48005 unsigned HOST_WIDE_INT h1, h2, h3, h4;
48007 /* Split the two input vectors into 4 halves. */
48008 h1 = (((unsigned HOST_WIDE_INT) 1) << nelt2) - 1;
48013 /* If the elements from the low halves use interleave low, and similarly
48014 for interleave high. If the elements are from mis-matched halves, we
48015 can use shufps for V4SF/V4SI or do a DImode shuffle. */
48016 if ((contents & (h1 | h3)) == contents)
48019 for (i = 0; i < nelt2; ++i)
48022 remap[i + nelt] = i * 2 + 1;
48023 dremap.perm[i * 2] = i;
48024 dremap.perm[i * 2 + 1] = i + nelt;
48026 if (!TARGET_SSE2 && d->vmode == V4SImode)
48027 dremap.vmode = V4SFmode;
48029 else if ((contents & (h2 | h4)) == contents)
48032 for (i = 0; i < nelt2; ++i)
48034 remap[i + nelt2] = i * 2;
48035 remap[i + nelt + nelt2] = i * 2 + 1;
48036 dremap.perm[i * 2] = i + nelt2;
48037 dremap.perm[i * 2 + 1] = i + nelt + nelt2;
48039 if (!TARGET_SSE2 && d->vmode == V4SImode)
48040 dremap.vmode = V4SFmode;
48042 else if ((contents & (h1 | h4)) == contents)
48045 for (i = 0; i < nelt2; ++i)
48048 remap[i + nelt + nelt2] = i + nelt2;
48049 dremap.perm[i] = i;
48050 dremap.perm[i + nelt2] = i + nelt + nelt2;
48055 dremap.vmode = V2DImode;
48057 dremap.perm[0] = 0;
48058 dremap.perm[1] = 3;
48061 else if ((contents & (h2 | h3)) == contents)
48064 for (i = 0; i < nelt2; ++i)
48066 remap[i + nelt2] = i;
48067 remap[i + nelt] = i + nelt2;
48068 dremap.perm[i] = i + nelt2;
48069 dremap.perm[i + nelt2] = i + nelt;
48074 dremap.vmode = V2DImode;
48076 dremap.perm[0] = 1;
48077 dremap.perm[1] = 2;
48085 unsigned int nelt4 = nelt / 4, nzcnt = 0;
48086 unsigned HOST_WIDE_INT q[8];
48087 unsigned int nonzero_halves[4];
48089 /* Split the two input vectors into 8 quarters. */
48090 q[0] = (((unsigned HOST_WIDE_INT) 1) << nelt4) - 1;
48091 for (i = 1; i < 8; ++i)
48092 q[i] = q[0] << (nelt4 * i);
48093 for (i = 0; i < 4; ++i)
48094 if (((q[2 * i] | q[2 * i + 1]) & contents) != 0)
48096 nonzero_halves[nzcnt] = i;
48102 gcc_assert (d->one_operand_p);
48103 nonzero_halves[1] = nonzero_halves[0];
48104 same_halves = true;
48106 else if (d->one_operand_p)
48108 gcc_assert (nonzero_halves[0] == 0);
48109 gcc_assert (nonzero_halves[1] == 1);
48114 if (d->perm[0] / nelt2 == nonzero_halves[1])
48116 /* Attempt to increase the likelihood that dfinal
48117 shuffle will be intra-lane. */
48118 char tmph = nonzero_halves[0];
48119 nonzero_halves[0] = nonzero_halves[1];
48120 nonzero_halves[1] = tmph;
48123 /* vperm2f128 or vperm2i128. */
48124 for (i = 0; i < nelt2; ++i)
48126 remap[i + nonzero_halves[1] * nelt2] = i + nelt2;
48127 remap[i + nonzero_halves[0] * nelt2] = i;
48128 dremap.perm[i + nelt2] = i + nonzero_halves[1] * nelt2;
48129 dremap.perm[i] = i + nonzero_halves[0] * nelt2;
48132 if (d->vmode != V8SFmode
48133 && d->vmode != V4DFmode
48134 && d->vmode != V8SImode)
48136 dremap.vmode = V8SImode;
48138 for (i = 0; i < 4; ++i)
48140 dremap.perm[i] = i + nonzero_halves[0] * 4;
48141 dremap.perm[i + 4] = i + nonzero_halves[1] * 4;
48145 else if (d->one_operand_p)
48147 else if (TARGET_AVX2
48148 && (contents & (q[0] | q[2] | q[4] | q[6])) == contents)
48151 for (i = 0; i < nelt4; ++i)
48154 remap[i + nelt] = i * 2 + 1;
48155 remap[i + nelt2] = i * 2 + nelt2;
48156 remap[i + nelt + nelt2] = i * 2 + nelt2 + 1;
48157 dremap.perm[i * 2] = i;
48158 dremap.perm[i * 2 + 1] = i + nelt;
48159 dremap.perm[i * 2 + nelt2] = i + nelt2;
48160 dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2;
48163 else if (TARGET_AVX2
48164 && (contents & (q[1] | q[3] | q[5] | q[7])) == contents)
48167 for (i = 0; i < nelt4; ++i)
48169 remap[i + nelt4] = i * 2;
48170 remap[i + nelt + nelt4] = i * 2 + 1;
48171 remap[i + nelt2 + nelt4] = i * 2 + nelt2;
48172 remap[i + nelt + nelt2 + nelt4] = i * 2 + nelt2 + 1;
48173 dremap.perm[i * 2] = i + nelt4;
48174 dremap.perm[i * 2 + 1] = i + nelt + nelt4;
48175 dremap.perm[i * 2 + nelt2] = i + nelt2 + nelt4;
48176 dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2 + nelt4;
48183 /* Use the remapping array set up above to move the elements from their
48184 swizzled locations into their final destinations. */
48186 for (i = 0; i < nelt; ++i)
48188 unsigned e = remap[d->perm[i]];
48189 gcc_assert (e < nelt);
48190 /* If same_halves is true, both halves of the remapped vector are the
48191 same. Avoid cross-lane accesses if possible. */
48192 if (same_halves && i >= nelt2)
48194 gcc_assert (e < nelt2);
48195 dfinal.perm[i] = e + nelt2;
48198 dfinal.perm[i] = e;
48202 dremap.target = gen_reg_rtx (dremap.vmode);
48203 dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
48205 dfinal.op1 = dfinal.op0;
48206 dfinal.one_operand_p = true;
48208 /* Test if the final remap can be done with a single insn. For V4SFmode or
48209 V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
48211 ok = expand_vec_perm_1 (&dfinal);
48212 seq = get_insns ();
48221 if (dremap.vmode != dfinal.vmode)
48223 dremap.op0 = gen_lowpart (dremap.vmode, dremap.op0);
48224 dremap.op1 = gen_lowpart (dremap.vmode, dremap.op1);
48227 ok = expand_vec_perm_1 (&dremap);
48234 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
48235 a single vector cross-lane permutation into vpermq followed
48236 by any of the single insn permutations. */
48239 expand_vec_perm_vpermq_perm_1 (struct expand_vec_perm_d *d)
48241 struct expand_vec_perm_d dremap, dfinal;
48242 unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, nelt4 = nelt / 4;
48243 unsigned contents[2];
48247 && (d->vmode == V32QImode || d->vmode == V16HImode)
48248 && d->one_operand_p))
48253 for (i = 0; i < nelt2; ++i)
48255 contents[0] |= 1u << (d->perm[i] / nelt4);
48256 contents[1] |= 1u << (d->perm[i + nelt2] / nelt4);
48259 for (i = 0; i < 2; ++i)
48261 unsigned int cnt = 0;
48262 for (j = 0; j < 4; ++j)
48263 if ((contents[i] & (1u << j)) != 0 && ++cnt > 2)
48271 dremap.vmode = V4DImode;
48273 dremap.target = gen_reg_rtx (V4DImode);
48274 dremap.op0 = gen_lowpart (V4DImode, d->op0);
48275 dremap.op1 = dremap.op0;
48276 dremap.one_operand_p = true;
48277 for (i = 0; i < 2; ++i)
48279 unsigned int cnt = 0;
48280 for (j = 0; j < 4; ++j)
48281 if ((contents[i] & (1u << j)) != 0)
48282 dremap.perm[2 * i + cnt++] = j;
48283 for (; cnt < 2; ++cnt)
48284 dremap.perm[2 * i + cnt] = 0;
48288 dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
48289 dfinal.op1 = dfinal.op0;
48290 dfinal.one_operand_p = true;
48291 for (i = 0, j = 0; i < nelt; ++i)
48295 dfinal.perm[i] = (d->perm[i] & (nelt4 - 1)) | (j ? nelt2 : 0);
48296 if ((d->perm[i] / nelt4) == dremap.perm[j])
48298 else if ((d->perm[i] / nelt4) == dremap.perm[j + 1])
48299 dfinal.perm[i] |= nelt4;
48301 gcc_unreachable ();
48304 ok = expand_vec_perm_1 (&dremap);
48307 ok = expand_vec_perm_1 (&dfinal);
48313 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to expand
48314 a vector permutation using two instructions, vperm2f128 resp.
48315 vperm2i128 followed by any single in-lane permutation. */
48318 expand_vec_perm_vperm2f128 (struct expand_vec_perm_d *d)
48320 struct expand_vec_perm_d dfirst, dsecond;
48321 unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, perm;
48325 || GET_MODE_SIZE (d->vmode) != 32
48326 || (d->vmode != V8SFmode && d->vmode != V4DFmode && !TARGET_AVX2))
48330 dsecond.one_operand_p = false;
48331 dsecond.testing_p = true;
48333 /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
48334 immediate. For perm < 16 the second permutation uses
48335 d->op0 as first operand, for perm >= 16 it uses d->op1
48336 as first operand. The second operand is the result of
48338 for (perm = 0; perm < 32; perm++)
48340 /* Ignore permutations which do not move anything cross-lane. */
48343 /* The second shuffle for e.g. V4DFmode has
48344 0123 and ABCD operands.
48345 Ignore AB23, as 23 is already in the second lane
48346 of the first operand. */
48347 if ((perm & 0xc) == (1 << 2)) continue;
48348 /* And 01CD, as 01 is in the first lane of the first
48350 if ((perm & 3) == 0) continue;
48351 /* And 4567, as then the vperm2[fi]128 doesn't change
48352 anything on the original 4567 second operand. */
48353 if ((perm & 0xf) == ((3 << 2) | 2)) continue;
48357 /* The second shuffle for e.g. V4DFmode has
48358 4567 and ABCD operands.
48359 Ignore AB67, as 67 is already in the second lane
48360 of the first operand. */
48361 if ((perm & 0xc) == (3 << 2)) continue;
48362 /* And 45CD, as 45 is in the first lane of the first
48364 if ((perm & 3) == 2) continue;
48365 /* And 0123, as then the vperm2[fi]128 doesn't change
48366 anything on the original 0123 first operand. */
48367 if ((perm & 0xf) == (1 << 2)) continue;
48370 for (i = 0; i < nelt; i++)
48372 j = d->perm[i] / nelt2;
48373 if (j == ((perm >> (2 * (i >= nelt2))) & 3))
48374 dsecond.perm[i] = nelt + (i & nelt2) + (d->perm[i] & (nelt2 - 1));
48375 else if (j == (unsigned) (i >= nelt2) + 2 * (perm >= 16))
48376 dsecond.perm[i] = d->perm[i] & (nelt - 1);
48384 ok = expand_vec_perm_1 (&dsecond);
48395 /* Found a usable second shuffle. dfirst will be
48396 vperm2f128 on d->op0 and d->op1. */
48397 dsecond.testing_p = false;
48399 dfirst.target = gen_reg_rtx (d->vmode);
48400 for (i = 0; i < nelt; i++)
48401 dfirst.perm[i] = (i & (nelt2 - 1))
48402 + ((perm >> (2 * (i >= nelt2))) & 3) * nelt2;
48404 canonicalize_perm (&dfirst);
48405 ok = expand_vec_perm_1 (&dfirst);
48408 /* And dsecond is some single insn shuffle, taking
48409 d->op0 and result of vperm2f128 (if perm < 16) or
48410 d->op1 and result of vperm2f128 (otherwise). */
48412 dsecond.op0 = dsecond.op1;
48413 dsecond.op1 = dfirst.target;
48415 ok = expand_vec_perm_1 (&dsecond);
48421 /* For one operand, the only useful vperm2f128 permutation is 0x01
48423 if (d->one_operand_p)
48430 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
48431 a two vector permutation using 2 intra-lane interleave insns
48432 and cross-lane shuffle for 32-byte vectors. */
48435 expand_vec_perm_interleave3 (struct expand_vec_perm_d *d)
48438 rtx (*gen) (rtx, rtx, rtx);
48440 if (d->one_operand_p)
48442 if (TARGET_AVX2 && GET_MODE_SIZE (d->vmode) == 32)
48444 else if (TARGET_AVX && (d->vmode == V8SFmode || d->vmode == V4DFmode))
48450 if (d->perm[0] != 0 && d->perm[0] != nelt / 2)
48452 for (i = 0; i < nelt; i += 2)
48453 if (d->perm[i] != d->perm[0] + i / 2
48454 || d->perm[i + 1] != d->perm[0] + i / 2 + nelt)
48464 gen = gen_vec_interleave_highv32qi;
48466 gen = gen_vec_interleave_lowv32qi;
48470 gen = gen_vec_interleave_highv16hi;
48472 gen = gen_vec_interleave_lowv16hi;
48476 gen = gen_vec_interleave_highv8si;
48478 gen = gen_vec_interleave_lowv8si;
48482 gen = gen_vec_interleave_highv4di;
48484 gen = gen_vec_interleave_lowv4di;
48488 gen = gen_vec_interleave_highv8sf;
48490 gen = gen_vec_interleave_lowv8sf;
48494 gen = gen_vec_interleave_highv4df;
48496 gen = gen_vec_interleave_lowv4df;
48499 gcc_unreachable ();
48502 emit_insn (gen (d->target, d->op0, d->op1));
48506 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement
48507 a single vector permutation using a single intra-lane vector
48508 permutation, vperm2f128 swapping the lanes and vblend* insn blending
48509 the non-swapped and swapped vectors together. */
48512 expand_vec_perm_vperm2f128_vblend (struct expand_vec_perm_d *d)
48514 struct expand_vec_perm_d dfirst, dsecond;
48515 unsigned i, j, msk, nelt = d->nelt, nelt2 = nelt / 2;
48518 rtx (*blend) (rtx, rtx, rtx, rtx) = NULL;
48522 || (d->vmode != V8SFmode && d->vmode != V4DFmode)
48523 || !d->one_operand_p)
48527 for (i = 0; i < nelt; i++)
48528 dfirst.perm[i] = 0xff;
48529 for (i = 0, msk = 0; i < nelt; i++)
48531 j = (d->perm[i] & nelt2) ? i | nelt2 : i & ~nelt2;
48532 if (dfirst.perm[j] != 0xff && dfirst.perm[j] != d->perm[i])
48534 dfirst.perm[j] = d->perm[i];
48538 for (i = 0; i < nelt; i++)
48539 if (dfirst.perm[i] == 0xff)
48540 dfirst.perm[i] = i;
48543 dfirst.target = gen_reg_rtx (dfirst.vmode);
48546 ok = expand_vec_perm_1 (&dfirst);
48547 seq = get_insns ();
48559 dsecond.op0 = dfirst.target;
48560 dsecond.op1 = dfirst.target;
48561 dsecond.one_operand_p = true;
48562 dsecond.target = gen_reg_rtx (dsecond.vmode);
48563 for (i = 0; i < nelt; i++)
48564 dsecond.perm[i] = i ^ nelt2;
48566 ok = expand_vec_perm_1 (&dsecond);
48569 blend = d->vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256;
48570 emit_insn (blend (d->target, dfirst.target, dsecond.target, GEN_INT (msk)));
48574 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement a V4DF
48575 permutation using two vperm2f128, followed by a vshufpd insn blending
48576 the two vectors together. */
48579 expand_vec_perm_2vperm2f128_vshuf (struct expand_vec_perm_d *d)
48581 struct expand_vec_perm_d dfirst, dsecond, dthird;
48584 if (!TARGET_AVX || (d->vmode != V4DFmode))
48594 dfirst.perm[0] = (d->perm[0] & ~1);
48595 dfirst.perm[1] = (d->perm[0] & ~1) + 1;
48596 dfirst.perm[2] = (d->perm[2] & ~1);
48597 dfirst.perm[3] = (d->perm[2] & ~1) + 1;
48598 dsecond.perm[0] = (d->perm[1] & ~1);
48599 dsecond.perm[1] = (d->perm[1] & ~1) + 1;
48600 dsecond.perm[2] = (d->perm[3] & ~1);
48601 dsecond.perm[3] = (d->perm[3] & ~1) + 1;
48602 dthird.perm[0] = (d->perm[0] % 2);
48603 dthird.perm[1] = (d->perm[1] % 2) + 4;
48604 dthird.perm[2] = (d->perm[2] % 2) + 2;
48605 dthird.perm[3] = (d->perm[3] % 2) + 6;
48607 dfirst.target = gen_reg_rtx (dfirst.vmode);
48608 dsecond.target = gen_reg_rtx (dsecond.vmode);
48609 dthird.op0 = dfirst.target;
48610 dthird.op1 = dsecond.target;
48611 dthird.one_operand_p = false;
48613 canonicalize_perm (&dfirst);
48614 canonicalize_perm (&dsecond);
48616 ok = expand_vec_perm_1 (&dfirst)
48617 && expand_vec_perm_1 (&dsecond)
48618 && expand_vec_perm_1 (&dthird);
48625 /* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
48626 permutation with two pshufb insns and an ior. We should have already
48627 failed all two instruction sequences. */
48630 expand_vec_perm_pshufb2 (struct expand_vec_perm_d *d)
48632 rtx rperm[2][16], vperm, l, h, op, m128;
48633 unsigned int i, nelt, eltsz;
48635 if (!TARGET_SSSE3 || GET_MODE_SIZE (d->vmode) != 16)
48637 gcc_assert (!d->one_operand_p);
48643 eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
48645 /* Generate two permutation masks. If the required element is within
48646 the given vector it is shuffled into the proper lane. If the required
48647 element is in the other vector, force a zero into the lane by setting
48648 bit 7 in the permutation mask. */
48649 m128 = GEN_INT (-128);
48650 for (i = 0; i < nelt; ++i)
48652 unsigned j, e = d->perm[i];
48653 unsigned which = (e >= nelt);
48657 for (j = 0; j < eltsz; ++j)
48659 rperm[which][i*eltsz + j] = GEN_INT (e*eltsz + j);
48660 rperm[1-which][i*eltsz + j] = m128;
48664 vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[0]));
48665 vperm = force_reg (V16QImode, vperm);
48667 l = gen_reg_rtx (V16QImode);
48668 op = gen_lowpart (V16QImode, d->op0);
48669 emit_insn (gen_ssse3_pshufbv16qi3 (l, op, vperm));
48671 vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[1]));
48672 vperm = force_reg (V16QImode, vperm);
48674 h = gen_reg_rtx (V16QImode);
48675 op = gen_lowpart (V16QImode, d->op1);
48676 emit_insn (gen_ssse3_pshufbv16qi3 (h, op, vperm));
48679 if (d->vmode != V16QImode)
48680 op = gen_reg_rtx (V16QImode);
48681 emit_insn (gen_iorv16qi3 (op, l, h));
48682 if (op != d->target)
48683 emit_move_insn (d->target, gen_lowpart (d->vmode, op));
48688 /* Implement arbitrary permutation of one V32QImode and V16QImode operand
48689 with two vpshufb insns, vpermq and vpor. We should have already failed
48690 all two or three instruction sequences. */
48693 expand_vec_perm_vpshufb2_vpermq (struct expand_vec_perm_d *d)
48695 rtx rperm[2][32], vperm, l, h, hp, op, m128;
48696 unsigned int i, nelt, eltsz;
48699 || !d->one_operand_p
48700 || (d->vmode != V32QImode && d->vmode != V16HImode))
48707 eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
48709 /* Generate two permutation masks. If the required element is within
48710 the same lane, it is shuffled in. If the required element from the
48711 other lane, force a zero by setting bit 7 in the permutation mask.
48712 In the other mask the mask has non-negative elements if element
48713 is requested from the other lane, but also moved to the other lane,
48714 so that the result of vpshufb can have the two V2TImode halves
48716 m128 = GEN_INT (-128);
48717 for (i = 0; i < nelt; ++i)
48719 unsigned j, e = d->perm[i] & (nelt / 2 - 1);
48720 unsigned which = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
48722 for (j = 0; j < eltsz; ++j)
48724 rperm[!!which][(i * eltsz + j) ^ which] = GEN_INT (e * eltsz + j);
48725 rperm[!which][(i * eltsz + j) ^ (which ^ 16)] = m128;
48729 vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
48730 vperm = force_reg (V32QImode, vperm);
48732 h = gen_reg_rtx (V32QImode);
48733 op = gen_lowpart (V32QImode, d->op0);
48734 emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
48736 /* Swap the 128-byte lanes of h into hp. */
48737 hp = gen_reg_rtx (V4DImode);
48738 op = gen_lowpart (V4DImode, h);
48739 emit_insn (gen_avx2_permv4di_1 (hp, op, const2_rtx, GEN_INT (3), const0_rtx,
48742 vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
48743 vperm = force_reg (V32QImode, vperm);
48745 l = gen_reg_rtx (V32QImode);
48746 op = gen_lowpart (V32QImode, d->op0);
48747 emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
48750 if (d->vmode != V32QImode)
48751 op = gen_reg_rtx (V32QImode);
48752 emit_insn (gen_iorv32qi3 (op, l, gen_lowpart (V32QImode, hp)));
48753 if (op != d->target)
48754 emit_move_insn (d->target, gen_lowpart (d->vmode, op));
48759 /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
48760 and extract-odd permutations of two V32QImode and V16QImode operand
48761 with two vpshufb insns, vpor and vpermq. We should have already
48762 failed all two or three instruction sequences. */
48765 expand_vec_perm_vpshufb2_vpermq_even_odd (struct expand_vec_perm_d *d)
48767 rtx rperm[2][32], vperm, l, h, ior, op, m128;
48768 unsigned int i, nelt, eltsz;
48771 || d->one_operand_p
48772 || (d->vmode != V32QImode && d->vmode != V16HImode))
48775 for (i = 0; i < d->nelt; ++i)
48776 if ((d->perm[i] ^ (i * 2)) & (3 * d->nelt / 2))
48783 eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
48785 /* Generate two permutation masks. In the first permutation mask
48786 the first quarter will contain indexes for the first half
48787 of the op0, the second quarter will contain bit 7 set, third quarter
48788 will contain indexes for the second half of the op0 and the
48789 last quarter bit 7 set. In the second permutation mask
48790 the first quarter will contain bit 7 set, the second quarter
48791 indexes for the first half of the op1, the third quarter bit 7 set
48792 and last quarter indexes for the second half of the op1.
48793 I.e. the first mask e.g. for V32QImode extract even will be:
48794 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
48795 (all values masked with 0xf except for -128) and second mask
48796 for extract even will be
48797 -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
48798 m128 = GEN_INT (-128);
48799 for (i = 0; i < nelt; ++i)
48801 unsigned j, e = d->perm[i] & (nelt / 2 - 1);
48802 unsigned which = d->perm[i] >= nelt;
48803 unsigned xorv = (i >= nelt / 4 && i < 3 * nelt / 4) ? 24 : 0;
48805 for (j = 0; j < eltsz; ++j)
48807 rperm[which][(i * eltsz + j) ^ xorv] = GEN_INT (e * eltsz + j);
48808 rperm[1 - which][(i * eltsz + j) ^ xorv] = m128;
48812 vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
48813 vperm = force_reg (V32QImode, vperm);
48815 l = gen_reg_rtx (V32QImode);
48816 op = gen_lowpart (V32QImode, d->op0);
48817 emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
48819 vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
48820 vperm = force_reg (V32QImode, vperm);
48822 h = gen_reg_rtx (V32QImode);
48823 op = gen_lowpart (V32QImode, d->op1);
48824 emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
48826 ior = gen_reg_rtx (V32QImode);
48827 emit_insn (gen_iorv32qi3 (ior, l, h));
48829 /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
48830 op = gen_reg_rtx (V4DImode);
48831 ior = gen_lowpart (V4DImode, ior);
48832 emit_insn (gen_avx2_permv4di_1 (op, ior, const0_rtx, const2_rtx,
48833 const1_rtx, GEN_INT (3)));
48834 emit_move_insn (d->target, gen_lowpart (d->vmode, op));
48839 /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
48840 and extract-odd permutations of two V16QI, V8HI, V16HI or V32QI operands
48841 with two "and" and "pack" or two "shift" and "pack" insns. We should
48842 have already failed all two instruction sequences. */
48845 expand_vec_perm_even_odd_pack (struct expand_vec_perm_d *d)
48847 rtx op, dop0, dop1, t, rperm[16];
48848 unsigned i, odd, c, s, nelt = d->nelt;
48849 bool end_perm = false;
48850 machine_mode half_mode;
48851 rtx (*gen_and) (rtx, rtx, rtx);
48852 rtx (*gen_pack) (rtx, rtx, rtx);
48853 rtx (*gen_shift) (rtx, rtx, rtx);
48855 if (d->one_operand_p)
48861 /* Required for "pack". */
48862 if (!TARGET_SSE4_1)
48866 half_mode = V4SImode;
48867 gen_and = gen_andv4si3;
48868 gen_pack = gen_sse4_1_packusdw;
48869 gen_shift = gen_lshrv4si3;
48872 /* No check as all instructions are SSE2. */
48875 half_mode = V8HImode;
48876 gen_and = gen_andv8hi3;
48877 gen_pack = gen_sse2_packuswb;
48878 gen_shift = gen_lshrv8hi3;
48885 half_mode = V8SImode;
48886 gen_and = gen_andv8si3;
48887 gen_pack = gen_avx2_packusdw;
48888 gen_shift = gen_lshrv8si3;
48896 half_mode = V16HImode;
48897 gen_and = gen_andv16hi3;
48898 gen_pack = gen_avx2_packuswb;
48899 gen_shift = gen_lshrv16hi3;
48903 /* Only V8HI, V16QI, V16HI and V32QI modes are more profitable than
48904 general shuffles. */
48908 /* Check that permutation is even or odd. */
48913 for (i = 1; i < nelt; ++i)
48914 if (d->perm[i] != 2 * i + odd)
48920 dop0 = gen_reg_rtx (half_mode);
48921 dop1 = gen_reg_rtx (half_mode);
48924 for (i = 0; i < nelt / 2; i++)
48925 rperm[i] = GEN_INT (c);
48926 t = gen_rtx_CONST_VECTOR (half_mode, gen_rtvec_v (nelt / 2, rperm));
48927 t = force_reg (half_mode, t);
48928 emit_insn (gen_and (dop0, t, gen_lowpart (half_mode, d->op0)));
48929 emit_insn (gen_and (dop1, t, gen_lowpart (half_mode, d->op1)));
48933 emit_insn (gen_shift (dop0,
48934 gen_lowpart (half_mode, d->op0),
48936 emit_insn (gen_shift (dop1,
48937 gen_lowpart (half_mode, d->op1),
48940 /* In AVX2 for 256 bit case we need to permute pack result. */
48941 if (TARGET_AVX2 && end_perm)
48943 op = gen_reg_rtx (d->vmode);
48944 t = gen_reg_rtx (V4DImode);
48945 emit_insn (gen_pack (op, dop0, dop1));
48946 emit_insn (gen_avx2_permv4di_1 (t,
48947 gen_lowpart (V4DImode, op),
48952 emit_move_insn (d->target, gen_lowpart (d->vmode, t));
48955 emit_insn (gen_pack (d->target, dop0, dop1));
48960 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement extract-even
48961 and extract-odd permutations. */
48964 expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
48966 rtx t1, t2, t3, t4, t5;
48973 t1 = gen_reg_rtx (V4DFmode);
48974 t2 = gen_reg_rtx (V4DFmode);
48976 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
48977 emit_insn (gen_avx_vperm2f128v4df3 (t1, d->op0, d->op1, GEN_INT (0x20)));
48978 emit_insn (gen_avx_vperm2f128v4df3 (t2, d->op0, d->op1, GEN_INT (0x31)));
48980 /* Now an unpck[lh]pd will produce the result required. */
48982 t3 = gen_avx_unpckhpd256 (d->target, t1, t2);
48984 t3 = gen_avx_unpcklpd256 (d->target, t1, t2);
48990 int mask = odd ? 0xdd : 0x88;
48994 t1 = gen_reg_rtx (V8SFmode);
48995 t2 = gen_reg_rtx (V8SFmode);
48996 t3 = gen_reg_rtx (V8SFmode);
48998 /* Shuffle within the 128-bit lanes to produce:
48999 { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
49000 emit_insn (gen_avx_shufps256 (t1, d->op0, d->op1,
49003 /* Shuffle the lanes around to produce:
49004 { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
49005 emit_insn (gen_avx_vperm2f128v8sf3 (t2, t1, t1,
49008 /* Shuffle within the 128-bit lanes to produce:
49009 { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
49010 emit_insn (gen_avx_shufps256 (t3, t1, t2, GEN_INT (0x44)));
49012 /* Shuffle within the 128-bit lanes to produce:
49013 { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
49014 emit_insn (gen_avx_shufps256 (t2, t1, t2, GEN_INT (0xee)));
49016 /* Shuffle the lanes around to produce:
49017 { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
49018 emit_insn (gen_avx_vperm2f128v8sf3 (d->target, t3, t2,
49027 /* These are always directly implementable by expand_vec_perm_1. */
49028 gcc_unreachable ();
49032 return expand_vec_perm_even_odd_pack (d);
49033 else if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
49034 return expand_vec_perm_pshufb2 (d);
49039 /* We need 2*log2(N)-1 operations to achieve odd/even
49040 with interleave. */
49041 t1 = gen_reg_rtx (V8HImode);
49042 t2 = gen_reg_rtx (V8HImode);
49043 emit_insn (gen_vec_interleave_highv8hi (t1, d->op0, d->op1));
49044 emit_insn (gen_vec_interleave_lowv8hi (d->target, d->op0, d->op1));
49045 emit_insn (gen_vec_interleave_highv8hi (t2, d->target, t1));
49046 emit_insn (gen_vec_interleave_lowv8hi (d->target, d->target, t1));
49048 t3 = gen_vec_interleave_highv8hi (d->target, d->target, t2);
49050 t3 = gen_vec_interleave_lowv8hi (d->target, d->target, t2);
49056 return expand_vec_perm_even_odd_pack (d);
49060 return expand_vec_perm_even_odd_pack (d);
49065 struct expand_vec_perm_d d_copy = *d;
49066 d_copy.vmode = V4DFmode;
49068 d_copy.target = gen_lowpart (V4DFmode, d->target);
49070 d_copy.target = gen_reg_rtx (V4DFmode);
49071 d_copy.op0 = gen_lowpart (V4DFmode, d->op0);
49072 d_copy.op1 = gen_lowpart (V4DFmode, d->op1);
49073 if (expand_vec_perm_even_odd_1 (&d_copy, odd))
49076 emit_move_insn (d->target,
49077 gen_lowpart (V4DImode, d_copy.target));
49086 t1 = gen_reg_rtx (V4DImode);
49087 t2 = gen_reg_rtx (V4DImode);
49089 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
49090 emit_insn (gen_avx2_permv2ti (t1, d->op0, d->op1, GEN_INT (0x20)));
49091 emit_insn (gen_avx2_permv2ti (t2, d->op0, d->op1, GEN_INT (0x31)));
49093 /* Now an vpunpck[lh]qdq will produce the result required. */
49095 t3 = gen_avx2_interleave_highv4di (d->target, t1, t2);
49097 t3 = gen_avx2_interleave_lowv4di (d->target, t1, t2);
49104 struct expand_vec_perm_d d_copy = *d;
49105 d_copy.vmode = V8SFmode;
49107 d_copy.target = gen_lowpart (V8SFmode, d->target);
49109 d_copy.target = gen_reg_rtx (V8SFmode);
49110 d_copy.op0 = gen_lowpart (V8SFmode, d->op0);
49111 d_copy.op1 = gen_lowpart (V8SFmode, d->op1);
49112 if (expand_vec_perm_even_odd_1 (&d_copy, odd))
49115 emit_move_insn (d->target,
49116 gen_lowpart (V8SImode, d_copy.target));
49125 t1 = gen_reg_rtx (V8SImode);
49126 t2 = gen_reg_rtx (V8SImode);
49127 t3 = gen_reg_rtx (V4DImode);
49128 t4 = gen_reg_rtx (V4DImode);
49129 t5 = gen_reg_rtx (V4DImode);
49131 /* Shuffle the lanes around into
49132 { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
49133 emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, d->op0),
49134 gen_lowpart (V4DImode, d->op1),
49136 emit_insn (gen_avx2_permv2ti (t4, gen_lowpart (V4DImode, d->op0),
49137 gen_lowpart (V4DImode, d->op1),
49140 /* Swap the 2nd and 3rd position in each lane into
49141 { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
49142 emit_insn (gen_avx2_pshufdv3 (t1, gen_lowpart (V8SImode, t3),
49143 GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
49144 emit_insn (gen_avx2_pshufdv3 (t2, gen_lowpart (V8SImode, t4),
49145 GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
49147 /* Now an vpunpck[lh]qdq will produce
49148 { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
49150 t3 = gen_avx2_interleave_highv4di (t5, gen_lowpart (V4DImode, t1),
49151 gen_lowpart (V4DImode, t2));
49153 t3 = gen_avx2_interleave_lowv4di (t5, gen_lowpart (V4DImode, t1),
49154 gen_lowpart (V4DImode, t2));
49156 emit_move_insn (d->target, gen_lowpart (V8SImode, t5));
49160 gcc_unreachable ();
49166 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
49167 extract-even and extract-odd permutations. */
49170 expand_vec_perm_even_odd (struct expand_vec_perm_d *d)
49172 unsigned i, odd, nelt = d->nelt;
49175 if (odd != 0 && odd != 1)
49178 for (i = 1; i < nelt; ++i)
49179 if (d->perm[i] != 2 * i + odd)
49182 return expand_vec_perm_even_odd_1 (d, odd);
49185 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement broadcast
49186 permutations. We assume that expand_vec_perm_1 has already failed. */
49189 expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d)
49191 unsigned elt = d->perm[0], nelt2 = d->nelt / 2;
49192 machine_mode vmode = d->vmode;
49193 unsigned char perm2[4];
49194 rtx op0 = d->op0, dest;
49201 /* These are special-cased in sse.md so that we can optionally
49202 use the vbroadcast instruction. They expand to two insns
49203 if the input happens to be in a register. */
49204 gcc_unreachable ();
49210 /* These are always implementable using standard shuffle patterns. */
49211 gcc_unreachable ();
49215 /* These can be implemented via interleave. We save one insn by
49216 stopping once we have promoted to V4SImode and then use pshufd. */
49222 rtx (*gen) (rtx, rtx, rtx)
49223 = vmode == V16QImode ? gen_vec_interleave_lowv16qi
49224 : gen_vec_interleave_lowv8hi;
49228 gen = vmode == V16QImode ? gen_vec_interleave_highv16qi
49229 : gen_vec_interleave_highv8hi;
49234 dest = gen_reg_rtx (vmode);
49235 emit_insn (gen (dest, op0, op0));
49236 vmode = get_mode_wider_vector (vmode);
49237 op0 = gen_lowpart (vmode, dest);
49239 while (vmode != V4SImode);
49241 memset (perm2, elt, 4);
49242 dest = gen_reg_rtx (V4SImode);
49243 ok = expand_vselect (dest, op0, perm2, 4, d->testing_p);
49246 emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
49254 /* For AVX2 broadcasts of the first element vpbroadcast* or
49255 vpermq should be used by expand_vec_perm_1. */
49256 gcc_assert (!TARGET_AVX2 || d->perm[0]);
49260 gcc_unreachable ();
49264 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
49265 broadcast permutations. */
49268 expand_vec_perm_broadcast (struct expand_vec_perm_d *d)
49270 unsigned i, elt, nelt = d->nelt;
49272 if (!d->one_operand_p)
49276 for (i = 1; i < nelt; ++i)
49277 if (d->perm[i] != elt)
49280 return expand_vec_perm_broadcast_1 (d);
49283 /* Implement arbitrary permutations of two V64QImode operands
49284 will 2 vpermi2w, 2 vpshufb and one vpor instruction. */
49286 expand_vec_perm_vpermi2_vpshub2 (struct expand_vec_perm_d *d)
49288 if (!TARGET_AVX512BW || !(d->vmode == V64QImode))
49294 struct expand_vec_perm_d ds[2];
49295 rtx rperm[128], vperm, target0, target1;
49296 unsigned int i, nelt;
49297 machine_mode vmode;
49302 for (i = 0; i < 2; i++)
49305 ds[i].vmode = V32HImode;
49307 ds[i].target = gen_reg_rtx (V32HImode);
49308 ds[i].op0 = gen_lowpart (V32HImode, d->op0);
49309 ds[i].op1 = gen_lowpart (V32HImode, d->op1);
49312 /* Prepare permutations such that the first one takes care of
49313 putting the even bytes into the right positions or one higher
49314 positions (ds[0]) and the second one takes care of
49315 putting the odd bytes into the right positions or one below
49318 for (i = 0; i < nelt; i++)
49320 ds[i & 1].perm[i / 2] = d->perm[i] / 2;
49323 rperm[i] = constm1_rtx;
49324 rperm[i + 64] = GEN_INT ((i & 14) + (d->perm[i] & 1));
49328 rperm[i] = GEN_INT ((i & 14) + (d->perm[i] & 1));
49329 rperm[i + 64] = constm1_rtx;
49333 bool ok = expand_vec_perm_1 (&ds[0]);
49335 ds[0].target = gen_lowpart (V64QImode, ds[0].target);
49337 ok = expand_vec_perm_1 (&ds[1]);
49339 ds[1].target = gen_lowpart (V64QImode, ds[1].target);
49341 vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm));
49342 vperm = force_reg (vmode, vperm);
49343 target0 = gen_reg_rtx (V64QImode);
49344 emit_insn (gen_avx512bw_pshufbv64qi3 (target0, ds[0].target, vperm));
49346 vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm + 64));
49347 vperm = force_reg (vmode, vperm);
49348 target1 = gen_reg_rtx (V64QImode);
49349 emit_insn (gen_avx512bw_pshufbv64qi3 (target1, ds[1].target, vperm));
49351 emit_insn (gen_iorv64qi3 (d->target, target0, target1));
49355 /* Implement arbitrary permutation of two V32QImode and V16QImode operands
49356 with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
49357 all the shorter instruction sequences. */
49360 expand_vec_perm_vpshufb4_vpermq2 (struct expand_vec_perm_d *d)
49362 rtx rperm[4][32], vperm, l[2], h[2], op, m128;
49363 unsigned int i, nelt, eltsz;
49367 || d->one_operand_p
49368 || (d->vmode != V32QImode && d->vmode != V16HImode))
49375 eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
49377 /* Generate 4 permutation masks. If the required element is within
49378 the same lane, it is shuffled in. If the required element from the
49379 other lane, force a zero by setting bit 7 in the permutation mask.
49380 In the other mask the mask has non-negative elements if element
49381 is requested from the other lane, but also moved to the other lane,
49382 so that the result of vpshufb can have the two V2TImode halves
49384 m128 = GEN_INT (-128);
49385 for (i = 0; i < 32; ++i)
49387 rperm[0][i] = m128;
49388 rperm[1][i] = m128;
49389 rperm[2][i] = m128;
49390 rperm[3][i] = m128;
49396 for (i = 0; i < nelt; ++i)
49398 unsigned j, e = d->perm[i] & (nelt / 2 - 1);
49399 unsigned xlane = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
49400 unsigned int which = ((d->perm[i] & nelt) ? 2 : 0) + (xlane ? 1 : 0);
49402 for (j = 0; j < eltsz; ++j)
49403 rperm[which][(i * eltsz + j) ^ xlane] = GEN_INT (e * eltsz + j);
49404 used[which] = true;
49407 for (i = 0; i < 2; ++i)
49409 if (!used[2 * i + 1])
49414 vperm = gen_rtx_CONST_VECTOR (V32QImode,
49415 gen_rtvec_v (32, rperm[2 * i + 1]));
49416 vperm = force_reg (V32QImode, vperm);
49417 h[i] = gen_reg_rtx (V32QImode);
49418 op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
49419 emit_insn (gen_avx2_pshufbv32qi3 (h[i], op, vperm));
49422 /* Swap the 128-byte lanes of h[X]. */
49423 for (i = 0; i < 2; ++i)
49425 if (h[i] == NULL_RTX)
49427 op = gen_reg_rtx (V4DImode);
49428 emit_insn (gen_avx2_permv4di_1 (op, gen_lowpart (V4DImode, h[i]),
49429 const2_rtx, GEN_INT (3), const0_rtx,
49431 h[i] = gen_lowpart (V32QImode, op);
49434 for (i = 0; i < 2; ++i)
49441 vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[2 * i]));
49442 vperm = force_reg (V32QImode, vperm);
49443 l[i] = gen_reg_rtx (V32QImode);
49444 op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
49445 emit_insn (gen_avx2_pshufbv32qi3 (l[i], op, vperm));
49448 for (i = 0; i < 2; ++i)
49452 op = gen_reg_rtx (V32QImode);
49453 emit_insn (gen_iorv32qi3 (op, l[i], h[i]));
49460 gcc_assert (l[0] && l[1]);
49462 if (d->vmode != V32QImode)
49463 op = gen_reg_rtx (V32QImode);
49464 emit_insn (gen_iorv32qi3 (op, l[0], l[1]));
49465 if (op != d->target)
49466 emit_move_insn (d->target, gen_lowpart (d->vmode, op));
49470 /* The guts of ix86_expand_vec_perm_const, also used by the ok hook.
49471 With all of the interface bits taken care of, perform the expansion
49472 in D and return true on success. */
49475 ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
49477 /* Try a single instruction expansion. */
49478 if (expand_vec_perm_1 (d))
49481 /* Try sequences of two instructions. */
49483 if (expand_vec_perm_pshuflw_pshufhw (d))
49486 if (expand_vec_perm_palignr (d, false))
49489 if (expand_vec_perm_interleave2 (d))
49492 if (expand_vec_perm_broadcast (d))
49495 if (expand_vec_perm_vpermq_perm_1 (d))
49498 if (expand_vec_perm_vperm2f128 (d))
49501 if (expand_vec_perm_pblendv (d))
49504 /* Try sequences of three instructions. */
49506 if (expand_vec_perm_even_odd_pack (d))
49509 if (expand_vec_perm_2vperm2f128_vshuf (d))
49512 if (expand_vec_perm_pshufb2 (d))
49515 if (expand_vec_perm_interleave3 (d))
49518 if (expand_vec_perm_vperm2f128_vblend (d))
49521 /* Try sequences of four instructions. */
49523 if (expand_vec_perm_vpshufb2_vpermq (d))
49526 if (expand_vec_perm_vpshufb2_vpermq_even_odd (d))
49529 if (expand_vec_perm_vpermi2_vpshub2 (d))
49532 /* ??? Look for narrow permutations whose element orderings would
49533 allow the promotion to a wider mode. */
49535 /* ??? Look for sequences of interleave or a wider permute that place
49536 the data into the correct lanes for a half-vector shuffle like
49537 pshuf[lh]w or vpermilps. */
49539 /* ??? Look for sequences of interleave that produce the desired results.
49540 The combinatorics of punpck[lh] get pretty ugly... */
49542 if (expand_vec_perm_even_odd (d))
49545 /* Even longer sequences. */
49546 if (expand_vec_perm_vpshufb4_vpermq2 (d))
49552 /* If a permutation only uses one operand, make it clear. Returns true
49553 if the permutation references both operands. */
49556 canonicalize_perm (struct expand_vec_perm_d *d)
49558 int i, which, nelt = d->nelt;
49560 for (i = which = 0; i < nelt; ++i)
49561 which |= (d->perm[i] < nelt ? 1 : 2);
49563 d->one_operand_p = true;
49570 if (!rtx_equal_p (d->op0, d->op1))
49572 d->one_operand_p = false;
49575 /* The elements of PERM do not suggest that only the first operand
49576 is used, but both operands are identical. Allow easier matching
49577 of the permutation by folding the permutation into the single
49582 for (i = 0; i < nelt; ++i)
49583 d->perm[i] &= nelt - 1;
49592 return (which == 3);
49596 ix86_expand_vec_perm_const (rtx operands[4])
49598 struct expand_vec_perm_d d;
49599 unsigned char perm[MAX_VECT_LEN];
49604 d.target = operands[0];
49605 d.op0 = operands[1];
49606 d.op1 = operands[2];
49609 d.vmode = GET_MODE (d.target);
49610 gcc_assert (VECTOR_MODE_P (d.vmode));
49611 d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
49612 d.testing_p = false;
49614 gcc_assert (GET_CODE (sel) == CONST_VECTOR);
49615 gcc_assert (XVECLEN (sel, 0) == nelt);
49616 gcc_checking_assert (sizeof (d.perm) == sizeof (perm));
49618 for (i = 0; i < nelt; ++i)
49620 rtx e = XVECEXP (sel, 0, i);
49621 int ei = INTVAL (e) & (2 * nelt - 1);
49626 two_args = canonicalize_perm (&d);
49628 if (ix86_expand_vec_perm_const_1 (&d))
49631 /* If the selector says both arguments are needed, but the operands are the
49632 same, the above tried to expand with one_operand_p and flattened selector.
49633 If that didn't work, retry without one_operand_p; we succeeded with that
49635 if (two_args && d.one_operand_p)
49637 d.one_operand_p = false;
49638 memcpy (d.perm, perm, sizeof (perm));
49639 return ix86_expand_vec_perm_const_1 (&d);
49645 /* Implement targetm.vectorize.vec_perm_const_ok. */
49648 ix86_vectorize_vec_perm_const_ok (machine_mode vmode,
49649 const unsigned char *sel)
49651 struct expand_vec_perm_d d;
49652 unsigned int i, nelt, which;
49656 d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
49657 d.testing_p = true;
49659 /* Given sufficient ISA support we can just return true here
49660 for selected vector modes. */
49667 if (TARGET_AVX512F)
49668 /* All implementable with a single vpermi2 insn. */
49672 if (TARGET_AVX512BW)
49673 /* All implementable with a single vpermi2 insn. */
49677 if (TARGET_AVX512BW)
49678 /* Implementable with 2 vpermi2, 2 vpshufb and 1 or insn. */
49685 if (TARGET_AVX512VL)
49686 /* All implementable with a single vpermi2 insn. */
49691 /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */
49696 /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */
49703 /* All implementable with a single vpperm insn. */
49706 /* All implementable with 2 pshufb + 1 ior. */
49712 /* All implementable with shufpd or unpck[lh]pd. */
49718 /* Extract the values from the vector CST into the permutation
49720 memcpy (d.perm, sel, nelt);
49721 for (i = which = 0; i < nelt; ++i)
49723 unsigned char e = d.perm[i];
49724 gcc_assert (e < 2 * nelt);
49725 which |= (e < nelt ? 1 : 2);
49728 /* For all elements from second vector, fold the elements to first. */
49730 for (i = 0; i < nelt; ++i)
49733 /* Check whether the mask can be applied to the vector type. */
49734 d.one_operand_p = (which != 3);
49736 /* Implementable with shufps or pshufd. */
49737 if (d.one_operand_p && (d.vmode == V4SFmode || d.vmode == V4SImode))
49740 /* Otherwise we have to go through the motions and see if we can
49741 figure out how to generate the requested permutation. */
49742 d.target = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 1);
49743 d.op1 = d.op0 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 2);
49744 if (!d.one_operand_p)
49745 d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3);
49748 ret = ix86_expand_vec_perm_const_1 (&d);
49755 ix86_expand_vec_extract_even_odd (rtx targ, rtx op0, rtx op1, unsigned odd)
49757 struct expand_vec_perm_d d;
49763 d.vmode = GET_MODE (targ);
49764 d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
49765 d.one_operand_p = false;
49766 d.testing_p = false;
49768 for (i = 0; i < nelt; ++i)
49769 d.perm[i] = i * 2 + odd;
49771 /* We'll either be able to implement the permutation directly... */
49772 if (expand_vec_perm_1 (&d))
49775 /* ... or we use the special-case patterns. */
49776 expand_vec_perm_even_odd_1 (&d, odd);
49780 ix86_expand_vec_interleave (rtx targ, rtx op0, rtx op1, bool high_p)
49782 struct expand_vec_perm_d d;
49783 unsigned i, nelt, base;
49789 d.vmode = GET_MODE (targ);
49790 d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
49791 d.one_operand_p = false;
49792 d.testing_p = false;
49794 base = high_p ? nelt / 2 : 0;
49795 for (i = 0; i < nelt / 2; ++i)
49797 d.perm[i * 2] = i + base;
49798 d.perm[i * 2 + 1] = i + base + nelt;
49801 /* Note that for AVX this isn't one instruction. */
49802 ok = ix86_expand_vec_perm_const_1 (&d);
49807 /* Expand a vector operation CODE for a V*QImode in terms of the
49808 same operation on V*HImode. */
49811 ix86_expand_vecop_qihi (enum rtx_code code, rtx dest, rtx op1, rtx op2)
49813 machine_mode qimode = GET_MODE (dest);
49814 machine_mode himode;
49815 rtx (*gen_il) (rtx, rtx, rtx);
49816 rtx (*gen_ih) (rtx, rtx, rtx);
49817 rtx op1_l, op1_h, op2_l, op2_h, res_l, res_h;
49818 struct expand_vec_perm_d d;
49819 bool ok, full_interleave;
49820 bool uns_p = false;
49827 gen_il = gen_vec_interleave_lowv16qi;
49828 gen_ih = gen_vec_interleave_highv16qi;
49831 himode = V16HImode;
49832 gen_il = gen_avx2_interleave_lowv32qi;
49833 gen_ih = gen_avx2_interleave_highv32qi;
49836 himode = V32HImode;
49837 gen_il = gen_avx512bw_interleave_lowv64qi;
49838 gen_ih = gen_avx512bw_interleave_highv64qi;
49841 gcc_unreachable ();
49844 op2_l = op2_h = op2;
49848 /* Unpack data such that we've got a source byte in each low byte of
49849 each word. We don't care what goes into the high byte of each word.
49850 Rather than trying to get zero in there, most convenient is to let
49851 it be a copy of the low byte. */
49852 op2_l = gen_reg_rtx (qimode);
49853 op2_h = gen_reg_rtx (qimode);
49854 emit_insn (gen_il (op2_l, op2, op2));
49855 emit_insn (gen_ih (op2_h, op2, op2));
49858 op1_l = gen_reg_rtx (qimode);
49859 op1_h = gen_reg_rtx (qimode);
49860 emit_insn (gen_il (op1_l, op1, op1));
49861 emit_insn (gen_ih (op1_h, op1, op1));
49862 full_interleave = qimode == V16QImode;
49870 op1_l = gen_reg_rtx (himode);
49871 op1_h = gen_reg_rtx (himode);
49872 ix86_expand_sse_unpack (op1_l, op1, uns_p, false);
49873 ix86_expand_sse_unpack (op1_h, op1, uns_p, true);
49874 full_interleave = true;
49877 gcc_unreachable ();
49880 /* Perform the operation. */
49881 res_l = expand_simple_binop (himode, code, op1_l, op2_l, NULL_RTX,
49883 res_h = expand_simple_binop (himode, code, op1_h, op2_h, NULL_RTX,
49885 gcc_assert (res_l && res_h);
49887 /* Merge the data back into the right place. */
49889 d.op0 = gen_lowpart (qimode, res_l);
49890 d.op1 = gen_lowpart (qimode, res_h);
49892 d.nelt = GET_MODE_NUNITS (qimode);
49893 d.one_operand_p = false;
49894 d.testing_p = false;
49896 if (full_interleave)
49898 /* For SSE2, we used an full interleave, so the desired
49899 results are in the even elements. */
49900 for (i = 0; i < 64; ++i)
49905 /* For AVX, the interleave used above was not cross-lane. So the
49906 extraction is evens but with the second and third quarter swapped.
49907 Happily, that is even one insn shorter than even extraction. */
49908 for (i = 0; i < 64; ++i)
49909 d.perm[i] = i * 2 + ((i & 24) == 8 ? 16 : (i & 24) == 16 ? -16 : 0);
49912 ok = ix86_expand_vec_perm_const_1 (&d);
49915 set_unique_reg_note (get_last_insn (), REG_EQUAL,
49916 gen_rtx_fmt_ee (code, qimode, op1, op2));
49919 /* Helper function of ix86_expand_mul_widen_evenodd. Return true
49920 if op is CONST_VECTOR with all odd elements equal to their
49921 preceding element. */
49924 const_vector_equal_evenodd_p (rtx op)
49926 machine_mode mode = GET_MODE (op);
49927 int i, nunits = GET_MODE_NUNITS (mode);
49928 if (GET_CODE (op) != CONST_VECTOR
49929 || nunits != CONST_VECTOR_NUNITS (op))
49931 for (i = 0; i < nunits; i += 2)
49932 if (CONST_VECTOR_ELT (op, i) != CONST_VECTOR_ELT (op, i + 1))
49938 ix86_expand_mul_widen_evenodd (rtx dest, rtx op1, rtx op2,
49939 bool uns_p, bool odd_p)
49941 machine_mode mode = GET_MODE (op1);
49942 machine_mode wmode = GET_MODE (dest);
49944 rtx orig_op1 = op1, orig_op2 = op2;
49946 if (!nonimmediate_operand (op1, mode))
49947 op1 = force_reg (mode, op1);
49948 if (!nonimmediate_operand (op2, mode))
49949 op2 = force_reg (mode, op2);
49951 /* We only play even/odd games with vectors of SImode. */
49952 gcc_assert (mode == V4SImode || mode == V8SImode || mode == V16SImode);
49954 /* If we're looking for the odd results, shift those members down to
49955 the even slots. For some cpus this is faster than a PSHUFD. */
49958 /* For XOP use vpmacsdqh, but only for smult, as it is only
49960 if (TARGET_XOP && mode == V4SImode && !uns_p)
49962 x = force_reg (wmode, CONST0_RTX (wmode));
49963 emit_insn (gen_xop_pmacsdqh (dest, op1, op2, x));
49967 x = GEN_INT (GET_MODE_UNIT_BITSIZE (mode));
49968 if (!const_vector_equal_evenodd_p (orig_op1))
49969 op1 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op1),
49970 x, NULL, 1, OPTAB_DIRECT);
49971 if (!const_vector_equal_evenodd_p (orig_op2))
49972 op2 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op2),
49973 x, NULL, 1, OPTAB_DIRECT);
49974 op1 = gen_lowpart (mode, op1);
49975 op2 = gen_lowpart (mode, op2);
49978 if (mode == V16SImode)
49981 x = gen_vec_widen_umult_even_v16si (dest, op1, op2);
49983 x = gen_vec_widen_smult_even_v16si (dest, op1, op2);
49985 else if (mode == V8SImode)
49988 x = gen_vec_widen_umult_even_v8si (dest, op1, op2);
49990 x = gen_vec_widen_smult_even_v8si (dest, op1, op2);
49993 x = gen_vec_widen_umult_even_v4si (dest, op1, op2);
49994 else if (TARGET_SSE4_1)
49995 x = gen_sse4_1_mulv2siv2di3 (dest, op1, op2);
49998 rtx s1, s2, t0, t1, t2;
50000 /* The easiest way to implement this without PMULDQ is to go through
50001 the motions as if we are performing a full 64-bit multiply. With
50002 the exception that we need to do less shuffling of the elements. */
50004 /* Compute the sign-extension, aka highparts, of the two operands. */
50005 s1 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
50006 op1, pc_rtx, pc_rtx);
50007 s2 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
50008 op2, pc_rtx, pc_rtx);
50010 /* Multiply LO(A) * HI(B), and vice-versa. */
50011 t1 = gen_reg_rtx (wmode);
50012 t2 = gen_reg_rtx (wmode);
50013 emit_insn (gen_vec_widen_umult_even_v4si (t1, s1, op2));
50014 emit_insn (gen_vec_widen_umult_even_v4si (t2, s2, op1));
50016 /* Multiply LO(A) * LO(B). */
50017 t0 = gen_reg_rtx (wmode);
50018 emit_insn (gen_vec_widen_umult_even_v4si (t0, op1, op2));
50020 /* Combine and shift the highparts into place. */
50021 t1 = expand_binop (wmode, add_optab, t1, t2, t1, 1, OPTAB_DIRECT);
50022 t1 = expand_binop (wmode, ashl_optab, t1, GEN_INT (32), t1,
50025 /* Combine high and low parts. */
50026 force_expand_binop (wmode, add_optab, t0, t1, dest, 1, OPTAB_DIRECT);
50033 ix86_expand_mul_widen_hilo (rtx dest, rtx op1, rtx op2,
50034 bool uns_p, bool high_p)
50036 machine_mode wmode = GET_MODE (dest);
50037 machine_mode mode = GET_MODE (op1);
50038 rtx t1, t2, t3, t4, mask;
50043 t1 = gen_reg_rtx (mode);
50044 t2 = gen_reg_rtx (mode);
50045 if (TARGET_XOP && !uns_p)
50047 /* With XOP, we have pmacsdqh, aka mul_widen_odd. In this case,
50048 shuffle the elements once so that all elements are in the right
50049 place for immediate use: { A C B D }. */
50050 emit_insn (gen_sse2_pshufd_1 (t1, op1, const0_rtx, const2_rtx,
50051 const1_rtx, GEN_INT (3)));
50052 emit_insn (gen_sse2_pshufd_1 (t2, op2, const0_rtx, const2_rtx,
50053 const1_rtx, GEN_INT (3)));
50057 /* Put the elements into place for the multiply. */
50058 ix86_expand_vec_interleave (t1, op1, op1, high_p);
50059 ix86_expand_vec_interleave (t2, op2, op2, high_p);
50062 ix86_expand_mul_widen_evenodd (dest, t1, t2, uns_p, high_p);
50066 /* Shuffle the elements between the lanes. After this we
50067 have { A B E F | C D G H } for each operand. */
50068 t1 = gen_reg_rtx (V4DImode);
50069 t2 = gen_reg_rtx (V4DImode);
50070 emit_insn (gen_avx2_permv4di_1 (t1, gen_lowpart (V4DImode, op1),
50071 const0_rtx, const2_rtx,
50072 const1_rtx, GEN_INT (3)));
50073 emit_insn (gen_avx2_permv4di_1 (t2, gen_lowpart (V4DImode, op2),
50074 const0_rtx, const2_rtx,
50075 const1_rtx, GEN_INT (3)));
50077 /* Shuffle the elements within the lanes. After this we
50078 have { A A B B | C C D D } or { E E F F | G G H H }. */
50079 t3 = gen_reg_rtx (V8SImode);
50080 t4 = gen_reg_rtx (V8SImode);
50081 mask = GEN_INT (high_p
50082 ? 2 + (2 << 2) + (3 << 4) + (3 << 6)
50083 : 0 + (0 << 2) + (1 << 4) + (1 << 6));
50084 emit_insn (gen_avx2_pshufdv3 (t3, gen_lowpart (V8SImode, t1), mask));
50085 emit_insn (gen_avx2_pshufdv3 (t4, gen_lowpart (V8SImode, t2), mask));
50087 ix86_expand_mul_widen_evenodd (dest, t3, t4, uns_p, false);
50092 t1 = expand_binop (mode, smul_optab, op1, op2, NULL_RTX,
50093 uns_p, OPTAB_DIRECT);
50094 t2 = expand_binop (mode,
50095 uns_p ? umul_highpart_optab : smul_highpart_optab,
50096 op1, op2, NULL_RTX, uns_p, OPTAB_DIRECT);
50097 gcc_assert (t1 && t2);
50099 t3 = gen_reg_rtx (mode);
50100 ix86_expand_vec_interleave (t3, t1, t2, high_p);
50101 emit_move_insn (dest, gen_lowpart (wmode, t3));
50109 t1 = gen_reg_rtx (wmode);
50110 t2 = gen_reg_rtx (wmode);
50111 ix86_expand_sse_unpack (t1, op1, uns_p, high_p);
50112 ix86_expand_sse_unpack (t2, op2, uns_p, high_p);
50114 emit_insn (gen_rtx_SET (VOIDmode, dest, gen_rtx_MULT (wmode, t1, t2)));
50118 gcc_unreachable ();
50123 ix86_expand_sse2_mulv4si3 (rtx op0, rtx op1, rtx op2)
50125 rtx res_1, res_2, res_3, res_4;
50127 res_1 = gen_reg_rtx (V4SImode);
50128 res_2 = gen_reg_rtx (V4SImode);
50129 res_3 = gen_reg_rtx (V2DImode);
50130 res_4 = gen_reg_rtx (V2DImode);
50131 ix86_expand_mul_widen_evenodd (res_3, op1, op2, true, false);
50132 ix86_expand_mul_widen_evenodd (res_4, op1, op2, true, true);
50134 /* Move the results in element 2 down to element 1; we don't care
50135 what goes in elements 2 and 3. Then we can merge the parts
50136 back together with an interleave.
50138 Note that two other sequences were tried:
50139 (1) Use interleaves at the start instead of psrldq, which allows
50140 us to use a single shufps to merge things back at the end.
50141 (2) Use shufps here to combine the two vectors, then pshufd to
50142 put the elements in the correct order.
50143 In both cases the cost of the reformatting stall was too high
50144 and the overall sequence slower. */
50146 emit_insn (gen_sse2_pshufd_1 (res_1, gen_lowpart (V4SImode, res_3),
50147 const0_rtx, const2_rtx,
50148 const0_rtx, const0_rtx));
50149 emit_insn (gen_sse2_pshufd_1 (res_2, gen_lowpart (V4SImode, res_4),
50150 const0_rtx, const2_rtx,
50151 const0_rtx, const0_rtx));
50152 res_1 = emit_insn (gen_vec_interleave_lowv4si (op0, res_1, res_2));
50154 set_unique_reg_note (res_1, REG_EQUAL, gen_rtx_MULT (V4SImode, op1, op2));
50158 ix86_expand_sse2_mulvxdi3 (rtx op0, rtx op1, rtx op2)
50160 machine_mode mode = GET_MODE (op0);
50161 rtx t1, t2, t3, t4, t5, t6;
50163 if (TARGET_AVX512DQ && mode == V8DImode)
50164 emit_insn (gen_avx512dq_mulv8di3 (op0, op1, op2));
50165 else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V4DImode)
50166 emit_insn (gen_avx512dq_mulv4di3 (op0, op1, op2));
50167 else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V2DImode)
50168 emit_insn (gen_avx512dq_mulv2di3 (op0, op1, op2));
50169 else if (TARGET_XOP && mode == V2DImode)
50171 /* op1: A,B,C,D, op2: E,F,G,H */
50172 op1 = gen_lowpart (V4SImode, op1);
50173 op2 = gen_lowpart (V4SImode, op2);
50175 t1 = gen_reg_rtx (V4SImode);
50176 t2 = gen_reg_rtx (V4SImode);
50177 t3 = gen_reg_rtx (V2DImode);
50178 t4 = gen_reg_rtx (V2DImode);
50181 emit_insn (gen_sse2_pshufd_1 (t1, op1,
50187 /* t2: (B*E),(A*F),(D*G),(C*H) */
50188 emit_insn (gen_mulv4si3 (t2, t1, op2));
50190 /* t3: (B*E)+(A*F), (D*G)+(C*H) */
50191 emit_insn (gen_xop_phadddq (t3, t2));
50193 /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
50194 emit_insn (gen_ashlv2di3 (t4, t3, GEN_INT (32)));
50196 /* Multiply lower parts and add all */
50197 t5 = gen_reg_rtx (V2DImode);
50198 emit_insn (gen_vec_widen_umult_even_v4si (t5,
50199 gen_lowpart (V4SImode, op1),
50200 gen_lowpart (V4SImode, op2)));
50201 op0 = expand_binop (mode, add_optab, t5, t4, op0, 1, OPTAB_DIRECT);
50206 machine_mode nmode;
50207 rtx (*umul) (rtx, rtx, rtx);
50209 if (mode == V2DImode)
50211 umul = gen_vec_widen_umult_even_v4si;
50214 else if (mode == V4DImode)
50216 umul = gen_vec_widen_umult_even_v8si;
50219 else if (mode == V8DImode)
50221 umul = gen_vec_widen_umult_even_v16si;
50225 gcc_unreachable ();
50228 /* Multiply low parts. */
50229 t1 = gen_reg_rtx (mode);
50230 emit_insn (umul (t1, gen_lowpart (nmode, op1), gen_lowpart (nmode, op2)));
50232 /* Shift input vectors right 32 bits so we can multiply high parts. */
50234 t2 = expand_binop (mode, lshr_optab, op1, t6, NULL, 1, OPTAB_DIRECT);
50235 t3 = expand_binop (mode, lshr_optab, op2, t6, NULL, 1, OPTAB_DIRECT);
50237 /* Multiply high parts by low parts. */
50238 t4 = gen_reg_rtx (mode);
50239 t5 = gen_reg_rtx (mode);
50240 emit_insn (umul (t4, gen_lowpart (nmode, t2), gen_lowpart (nmode, op2)));
50241 emit_insn (umul (t5, gen_lowpart (nmode, t3), gen_lowpart (nmode, op1)));
50243 /* Combine and shift the highparts back. */
50244 t4 = expand_binop (mode, add_optab, t4, t5, t4, 1, OPTAB_DIRECT);
50245 t4 = expand_binop (mode, ashl_optab, t4, t6, t4, 1, OPTAB_DIRECT);
50247 /* Combine high and low parts. */
50248 force_expand_binop (mode, add_optab, t1, t4, op0, 1, OPTAB_DIRECT);
50251 set_unique_reg_note (get_last_insn (), REG_EQUAL,
50252 gen_rtx_MULT (mode, op1, op2));
50255 /* Return 1 if control tansfer instruction INSN
50256 should be encoded with bnd prefix.
50257 If insn is NULL then return 1 when control
50258 transfer instructions should be prefixed with
50259 bnd by default for current function. */
50262 ix86_bnd_prefixed_insn_p (rtx insn)
50264 /* For call insns check special flag. */
50265 if (insn && CALL_P (insn))
50267 rtx call = get_call_rtx_from (insn);
50269 return CALL_EXPR_WITH_BOUNDS_P (call);
50272 /* All other insns are prefixed only if function is instrumented. */
50273 return chkp_function_instrumented_p (current_function_decl);
50276 /* Calculate integer abs() using only SSE2 instructions. */
50279 ix86_expand_sse2_abs (rtx target, rtx input)
50281 machine_mode mode = GET_MODE (target);
50286 /* For 32-bit signed integer X, the best way to calculate the absolute
50287 value of X is (((signed) X >> (W-1)) ^ X) - ((signed) X >> (W-1)). */
50289 tmp0 = expand_simple_binop (mode, ASHIFTRT, input,
50290 GEN_INT (GET_MODE_BITSIZE
50291 (GET_MODE_INNER (mode)) - 1),
50292 NULL, 0, OPTAB_DIRECT);
50293 tmp1 = expand_simple_binop (mode, XOR, tmp0, input,
50294 NULL, 0, OPTAB_DIRECT);
50295 x = expand_simple_binop (mode, MINUS, tmp1, tmp0,
50296 target, 0, OPTAB_DIRECT);
50299 /* For 16-bit signed integer X, the best way to calculate the absolute
50300 value of X is max (X, -X), as SSE2 provides the PMAXSW insn. */
50302 tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
50304 x = expand_simple_binop (mode, SMAX, tmp0, input,
50305 target, 0, OPTAB_DIRECT);
50308 /* For 8-bit signed integer X, the best way to calculate the absolute
50309 value of X is min ((unsigned char) X, (unsigned char) (-X)),
50310 as SSE2 provides the PMINUB insn. */
50312 tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
50314 x = expand_simple_binop (V16QImode, UMIN, tmp0, input,
50315 target, 0, OPTAB_DIRECT);
50319 gcc_unreachable ();
50323 emit_move_insn (target, x);
50326 /* Expand an insert into a vector register through pinsr insn.
50327 Return true if successful. */
50330 ix86_expand_pinsr (rtx *operands)
50332 rtx dst = operands[0];
50333 rtx src = operands[3];
50335 unsigned int size = INTVAL (operands[1]);
50336 unsigned int pos = INTVAL (operands[2]);
50338 if (GET_CODE (src) == SUBREG)
50340 /* Reject non-lowpart subregs. */
50341 if (SUBREG_BYTE (src) != 0)
50343 src = SUBREG_REG (src);
50346 if (GET_CODE (dst) == SUBREG)
50348 pos += SUBREG_BYTE (dst) * BITS_PER_UNIT;
50349 dst = SUBREG_REG (dst);
50352 switch (GET_MODE (dst))
50359 machine_mode srcmode, dstmode;
50360 rtx (*pinsr)(rtx, rtx, rtx, rtx);
50362 srcmode = mode_for_size (size, MODE_INT, 0);
50367 if (!TARGET_SSE4_1)
50369 dstmode = V16QImode;
50370 pinsr = gen_sse4_1_pinsrb;
50376 dstmode = V8HImode;
50377 pinsr = gen_sse2_pinsrw;
50381 if (!TARGET_SSE4_1)
50383 dstmode = V4SImode;
50384 pinsr = gen_sse4_1_pinsrd;
50388 gcc_assert (TARGET_64BIT);
50389 if (!TARGET_SSE4_1)
50391 dstmode = V2DImode;
50392 pinsr = gen_sse4_1_pinsrq;
50400 if (GET_MODE (dst) != dstmode)
50401 d = gen_reg_rtx (dstmode);
50402 src = gen_lowpart (srcmode, src);
50406 emit_insn (pinsr (d, gen_lowpart (dstmode, dst), src,
50407 GEN_INT (1 << pos)));
50409 emit_move_insn (dst, gen_lowpart (GET_MODE (dst), d));
50418 /* This function returns the calling abi specific va_list type node.
50419 It returns the FNDECL specific va_list type. */
50422 ix86_fn_abi_va_list (tree fndecl)
50425 return va_list_type_node;
50426 gcc_assert (fndecl != NULL_TREE);
50428 if (ix86_function_abi ((const_tree) fndecl) == MS_ABI)
50429 return ms_va_list_type_node;
50431 return sysv_va_list_type_node;
50434 /* Returns the canonical va_list type specified by TYPE. If there
50435 is no valid TYPE provided, it return NULL_TREE. */
50438 ix86_canonical_va_list_type (tree type)
50442 /* Resolve references and pointers to va_list type. */
50443 if (TREE_CODE (type) == MEM_REF)
50444 type = TREE_TYPE (type);
50445 else if (POINTER_TYPE_P (type) && POINTER_TYPE_P (TREE_TYPE(type)))
50446 type = TREE_TYPE (type);
50447 else if (POINTER_TYPE_P (type) && TREE_CODE (TREE_TYPE (type)) == ARRAY_TYPE)
50448 type = TREE_TYPE (type);
50450 if (TARGET_64BIT && va_list_type_node != NULL_TREE)
50452 wtype = va_list_type_node;
50453 gcc_assert (wtype != NULL_TREE);
50455 if (TREE_CODE (wtype) == ARRAY_TYPE)
50457 /* If va_list is an array type, the argument may have decayed
50458 to a pointer type, e.g. by being passed to another function.
50459 In that case, unwrap both types so that we can compare the
50460 underlying records. */
50461 if (TREE_CODE (htype) == ARRAY_TYPE
50462 || POINTER_TYPE_P (htype))
50464 wtype = TREE_TYPE (wtype);
50465 htype = TREE_TYPE (htype);
50468 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
50469 return va_list_type_node;
50470 wtype = sysv_va_list_type_node;
50471 gcc_assert (wtype != NULL_TREE);
50473 if (TREE_CODE (wtype) == ARRAY_TYPE)
50475 /* If va_list is an array type, the argument may have decayed
50476 to a pointer type, e.g. by being passed to another function.
50477 In that case, unwrap both types so that we can compare the
50478 underlying records. */
50479 if (TREE_CODE (htype) == ARRAY_TYPE
50480 || POINTER_TYPE_P (htype))
50482 wtype = TREE_TYPE (wtype);
50483 htype = TREE_TYPE (htype);
50486 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
50487 return sysv_va_list_type_node;
50488 wtype = ms_va_list_type_node;
50489 gcc_assert (wtype != NULL_TREE);
50491 if (TREE_CODE (wtype) == ARRAY_TYPE)
50493 /* If va_list is an array type, the argument may have decayed
50494 to a pointer type, e.g. by being passed to another function.
50495 In that case, unwrap both types so that we can compare the
50496 underlying records. */
50497 if (TREE_CODE (htype) == ARRAY_TYPE
50498 || POINTER_TYPE_P (htype))
50500 wtype = TREE_TYPE (wtype);
50501 htype = TREE_TYPE (htype);
50504 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
50505 return ms_va_list_type_node;
50508 return std_canonical_va_list_type (type);
50511 /* Iterate through the target-specific builtin types for va_list.
50512 IDX denotes the iterator, *PTREE is set to the result type of
50513 the va_list builtin, and *PNAME to its internal type.
50514 Returns zero if there is no element for this index, otherwise
50515 IDX should be increased upon the next call.
50516 Note, do not iterate a base builtin's name like __builtin_va_list.
50517 Used from c_common_nodes_and_builtins. */
50520 ix86_enum_va_list (int idx, const char **pname, tree *ptree)
50530 *ptree = ms_va_list_type_node;
50531 *pname = "__builtin_ms_va_list";
50535 *ptree = sysv_va_list_type_node;
50536 *pname = "__builtin_sysv_va_list";
50544 #undef TARGET_SCHED_DISPATCH
50545 #define TARGET_SCHED_DISPATCH has_dispatch
50546 #undef TARGET_SCHED_DISPATCH_DO
50547 #define TARGET_SCHED_DISPATCH_DO do_dispatch
50548 #undef TARGET_SCHED_REASSOCIATION_WIDTH
50549 #define TARGET_SCHED_REASSOCIATION_WIDTH ix86_reassociation_width
50550 #undef TARGET_SCHED_REORDER
50551 #define TARGET_SCHED_REORDER ix86_sched_reorder
50552 #undef TARGET_SCHED_ADJUST_PRIORITY
50553 #define TARGET_SCHED_ADJUST_PRIORITY ix86_adjust_priority
50554 #undef TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK
50555 #define TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK \
50556 ix86_dependencies_evaluation_hook
50558 /* The size of the dispatch window is the total number of bytes of
50559 object code allowed in a window. */
50560 #define DISPATCH_WINDOW_SIZE 16
50562 /* Number of dispatch windows considered for scheduling. */
50563 #define MAX_DISPATCH_WINDOWS 3
50565 /* Maximum number of instructions in a window. */
50568 /* Maximum number of immediate operands in a window. */
50571 /* Maximum number of immediate bits allowed in a window. */
50572 #define MAX_IMM_SIZE 128
50574 /* Maximum number of 32 bit immediates allowed in a window. */
50575 #define MAX_IMM_32 4
50577 /* Maximum number of 64 bit immediates allowed in a window. */
50578 #define MAX_IMM_64 2
50580 /* Maximum total of loads or prefetches allowed in a window. */
50583 /* Maximum total of stores allowed in a window. */
50584 #define MAX_STORE 1
50590 /* Dispatch groups. Istructions that affect the mix in a dispatch window. */
50591 enum dispatch_group {
50606 /* Number of allowable groups in a dispatch window. It is an array
50607 indexed by dispatch_group enum. 100 is used as a big number,
50608 because the number of these kind of operations does not have any
50609 effect in dispatch window, but we need them for other reasons in
50611 static unsigned int num_allowable_groups[disp_last] = {
50612 0, 2, 1, 1, 2, 4, 4, 2, 1, BIG, BIG
50615 char group_name[disp_last + 1][16] = {
50616 "disp_no_group", "disp_load", "disp_store", "disp_load_store",
50617 "disp_prefetch", "disp_imm", "disp_imm_32", "disp_imm_64",
50618 "disp_branch", "disp_cmp", "disp_jcc", "disp_last"
50621 /* Instruction path. */
50624 path_single, /* Single micro op. */
50625 path_double, /* Double micro op. */
50626 path_multi, /* Instructions with more than 2 micro op.. */
50630 /* sched_insn_info defines a window to the instructions scheduled in
50631 the basic block. It contains a pointer to the insn_info table and
50632 the instruction scheduled.
50634 Windows are allocated for each basic block and are linked
50636 typedef struct sched_insn_info_s {
50638 enum dispatch_group group;
50639 enum insn_path path;
50644 /* Linked list of dispatch windows. This is a two way list of
50645 dispatch windows of a basic block. It contains information about
50646 the number of uops in the window and the total number of
50647 instructions and of bytes in the object code for this dispatch
50649 typedef struct dispatch_windows_s {
50650 int num_insn; /* Number of insn in the window. */
50651 int num_uops; /* Number of uops in the window. */
50652 int window_size; /* Number of bytes in the window. */
50653 int window_num; /* Window number between 0 or 1. */
50654 int num_imm; /* Number of immediates in an insn. */
50655 int num_imm_32; /* Number of 32 bit immediates in an insn. */
50656 int num_imm_64; /* Number of 64 bit immediates in an insn. */
50657 int imm_size; /* Total immediates in the window. */
50658 int num_loads; /* Total memory loads in the window. */
50659 int num_stores; /* Total memory stores in the window. */
50660 int violation; /* Violation exists in window. */
50661 sched_insn_info *window; /* Pointer to the window. */
50662 struct dispatch_windows_s *next;
50663 struct dispatch_windows_s *prev;
50664 } dispatch_windows;
50666 /* Immediate valuse used in an insn. */
50667 typedef struct imm_info_s
50674 static dispatch_windows *dispatch_window_list;
50675 static dispatch_windows *dispatch_window_list1;
50677 /* Get dispatch group of insn. */
50679 static enum dispatch_group
50680 get_mem_group (rtx_insn *insn)
50682 enum attr_memory memory;
50684 if (INSN_CODE (insn) < 0)
50685 return disp_no_group;
50686 memory = get_attr_memory (insn);
50687 if (memory == MEMORY_STORE)
50690 if (memory == MEMORY_LOAD)
50693 if (memory == MEMORY_BOTH)
50694 return disp_load_store;
50696 return disp_no_group;
50699 /* Return true if insn is a compare instruction. */
50702 is_cmp (rtx_insn *insn)
50704 enum attr_type type;
50706 type = get_attr_type (insn);
50707 return (type == TYPE_TEST
50708 || type == TYPE_ICMP
50709 || type == TYPE_FCMP
50710 || GET_CODE (PATTERN (insn)) == COMPARE);
50713 /* Return true if a dispatch violation encountered. */
50716 dispatch_violation (void)
50718 if (dispatch_window_list->next)
50719 return dispatch_window_list->next->violation;
50720 return dispatch_window_list->violation;
50723 /* Return true if insn is a branch instruction. */
50726 is_branch (rtx insn)
50728 return (CALL_P (insn) || JUMP_P (insn));
50731 /* Return true if insn is a prefetch instruction. */
50734 is_prefetch (rtx insn)
50736 return NONJUMP_INSN_P (insn) && GET_CODE (PATTERN (insn)) == PREFETCH;
50739 /* This function initializes a dispatch window and the list container holding a
50740 pointer to the window. */
50743 init_window (int window_num)
50746 dispatch_windows *new_list;
50748 if (window_num == 0)
50749 new_list = dispatch_window_list;
50751 new_list = dispatch_window_list1;
50753 new_list->num_insn = 0;
50754 new_list->num_uops = 0;
50755 new_list->window_size = 0;
50756 new_list->next = NULL;
50757 new_list->prev = NULL;
50758 new_list->window_num = window_num;
50759 new_list->num_imm = 0;
50760 new_list->num_imm_32 = 0;
50761 new_list->num_imm_64 = 0;
50762 new_list->imm_size = 0;
50763 new_list->num_loads = 0;
50764 new_list->num_stores = 0;
50765 new_list->violation = false;
50767 for (i = 0; i < MAX_INSN; i++)
50769 new_list->window[i].insn = NULL;
50770 new_list->window[i].group = disp_no_group;
50771 new_list->window[i].path = no_path;
50772 new_list->window[i].byte_len = 0;
50773 new_list->window[i].imm_bytes = 0;
50778 /* This function allocates and initializes a dispatch window and the
50779 list container holding a pointer to the window. */
50781 static dispatch_windows *
50782 allocate_window (void)
50784 dispatch_windows *new_list = XNEW (struct dispatch_windows_s);
50785 new_list->window = XNEWVEC (struct sched_insn_info_s, MAX_INSN + 1);
50790 /* This routine initializes the dispatch scheduling information. It
50791 initiates building dispatch scheduler tables and constructs the
50792 first dispatch window. */
50795 init_dispatch_sched (void)
50797 /* Allocate a dispatch list and a window. */
50798 dispatch_window_list = allocate_window ();
50799 dispatch_window_list1 = allocate_window ();
50804 /* This function returns true if a branch is detected. End of a basic block
50805 does not have to be a branch, but here we assume only branches end a
50809 is_end_basic_block (enum dispatch_group group)
50811 return group == disp_branch;
50814 /* This function is called when the end of a window processing is reached. */
50817 process_end_window (void)
50819 gcc_assert (dispatch_window_list->num_insn <= MAX_INSN);
50820 if (dispatch_window_list->next)
50822 gcc_assert (dispatch_window_list1->num_insn <= MAX_INSN);
50823 gcc_assert (dispatch_window_list->window_size
50824 + dispatch_window_list1->window_size <= 48);
50830 /* Allocates a new dispatch window and adds it to WINDOW_LIST.
50831 WINDOW_NUM is either 0 or 1. A maximum of two windows are generated
50832 for 48 bytes of instructions. Note that these windows are not dispatch
50833 windows that their sizes are DISPATCH_WINDOW_SIZE. */
50835 static dispatch_windows *
50836 allocate_next_window (int window_num)
50838 if (window_num == 0)
50840 if (dispatch_window_list->next)
50843 return dispatch_window_list;
50846 dispatch_window_list->next = dispatch_window_list1;
50847 dispatch_window_list1->prev = dispatch_window_list;
50849 return dispatch_window_list1;
50852 /* Compute number of immediate operands of an instruction. */
50855 find_constant (rtx in_rtx, imm_info *imm_values)
50857 if (INSN_P (in_rtx))
50858 in_rtx = PATTERN (in_rtx);
50859 subrtx_iterator::array_type array;
50860 FOR_EACH_SUBRTX (iter, array, in_rtx, ALL)
50861 if (const_rtx x = *iter)
50862 switch (GET_CODE (x))
50867 (imm_values->imm)++;
50868 if (x86_64_immediate_operand (CONST_CAST_RTX (x), SImode))
50869 (imm_values->imm32)++;
50871 (imm_values->imm64)++;
50875 (imm_values->imm)++;
50876 (imm_values->imm64)++;
50880 if (LABEL_KIND (x) == LABEL_NORMAL)
50882 (imm_values->imm)++;
50883 (imm_values->imm32)++;
50892 /* Return total size of immediate operands of an instruction along with number
50893 of corresponding immediate-operands. It initializes its parameters to zero
50894 befor calling FIND_CONSTANT.
50895 INSN is the input instruction. IMM is the total of immediates.
50896 IMM32 is the number of 32 bit immediates. IMM64 is the number of 64
50900 get_num_immediates (rtx insn, int *imm, int *imm32, int *imm64)
50902 imm_info imm_values = {0, 0, 0};
50904 find_constant (insn, &imm_values);
50905 *imm = imm_values.imm;
50906 *imm32 = imm_values.imm32;
50907 *imm64 = imm_values.imm64;
50908 return imm_values.imm32 * 4 + imm_values.imm64 * 8;
50911 /* This function indicates if an operand of an instruction is an
50915 has_immediate (rtx insn)
50917 int num_imm_operand;
50918 int num_imm32_operand;
50919 int num_imm64_operand;
50922 return get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
50923 &num_imm64_operand);
50927 /* Return single or double path for instructions. */
50929 static enum insn_path
50930 get_insn_path (rtx_insn *insn)
50932 enum attr_amdfam10_decode path = get_attr_amdfam10_decode (insn);
50934 if ((int)path == 0)
50935 return path_single;
50937 if ((int)path == 1)
50938 return path_double;
50943 /* Return insn dispatch group. */
50945 static enum dispatch_group
50946 get_insn_group (rtx_insn *insn)
50948 enum dispatch_group group = get_mem_group (insn);
50952 if (is_branch (insn))
50953 return disp_branch;
50958 if (has_immediate (insn))
50961 if (is_prefetch (insn))
50962 return disp_prefetch;
50964 return disp_no_group;
50967 /* Count number of GROUP restricted instructions in a dispatch
50968 window WINDOW_LIST. */
50971 count_num_restricted (rtx_insn *insn, dispatch_windows *window_list)
50973 enum dispatch_group group = get_insn_group (insn);
50975 int num_imm_operand;
50976 int num_imm32_operand;
50977 int num_imm64_operand;
50979 if (group == disp_no_group)
50982 if (group == disp_imm)
50984 imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
50985 &num_imm64_operand);
50986 if (window_list->imm_size + imm_size > MAX_IMM_SIZE
50987 || num_imm_operand + window_list->num_imm > MAX_IMM
50988 || (num_imm32_operand > 0
50989 && (window_list->num_imm_32 + num_imm32_operand > MAX_IMM_32
50990 || window_list->num_imm_64 * 2 + num_imm32_operand > MAX_IMM_32))
50991 || (num_imm64_operand > 0
50992 && (window_list->num_imm_64 + num_imm64_operand > MAX_IMM_64
50993 || window_list->num_imm_32 + num_imm64_operand * 2 > MAX_IMM_32))
50994 || (window_list->imm_size + imm_size == MAX_IMM_SIZE
50995 && num_imm64_operand > 0
50996 && ((window_list->num_imm_64 > 0
50997 && window_list->num_insn >= 2)
50998 || window_list->num_insn >= 3)))
51004 if ((group == disp_load_store
51005 && (window_list->num_loads >= MAX_LOAD
51006 || window_list->num_stores >= MAX_STORE))
51007 || ((group == disp_load
51008 || group == disp_prefetch)
51009 && window_list->num_loads >= MAX_LOAD)
51010 || (group == disp_store
51011 && window_list->num_stores >= MAX_STORE))
51017 /* This function returns true if insn satisfies dispatch rules on the
51018 last window scheduled. */
51021 fits_dispatch_window (rtx_insn *insn)
51023 dispatch_windows *window_list = dispatch_window_list;
51024 dispatch_windows *window_list_next = dispatch_window_list->next;
51025 unsigned int num_restrict;
51026 enum dispatch_group group = get_insn_group (insn);
51027 enum insn_path path = get_insn_path (insn);
51030 /* Make disp_cmp and disp_jcc get scheduled at the latest. These
51031 instructions should be given the lowest priority in the
51032 scheduling process in Haifa scheduler to make sure they will be
51033 scheduled in the same dispatch window as the reference to them. */
51034 if (group == disp_jcc || group == disp_cmp)
51037 /* Check nonrestricted. */
51038 if (group == disp_no_group || group == disp_branch)
51041 /* Get last dispatch window. */
51042 if (window_list_next)
51043 window_list = window_list_next;
51045 if (window_list->window_num == 1)
51047 sum = window_list->prev->window_size + window_list->window_size;
51050 || (min_insn_size (insn) + sum) >= 48)
51051 /* Window 1 is full. Go for next window. */
51055 num_restrict = count_num_restricted (insn, window_list);
51057 if (num_restrict > num_allowable_groups[group])
51060 /* See if it fits in the first window. */
51061 if (window_list->window_num == 0)
51063 /* The first widow should have only single and double path
51065 if (path == path_double
51066 && (window_list->num_uops + 2) > MAX_INSN)
51068 else if (path != path_single)
51074 /* Add an instruction INSN with NUM_UOPS micro-operations to the
51075 dispatch window WINDOW_LIST. */
51078 add_insn_window (rtx_insn *insn, dispatch_windows *window_list, int num_uops)
51080 int byte_len = min_insn_size (insn);
51081 int num_insn = window_list->num_insn;
51083 sched_insn_info *window = window_list->window;
51084 enum dispatch_group group = get_insn_group (insn);
51085 enum insn_path path = get_insn_path (insn);
51086 int num_imm_operand;
51087 int num_imm32_operand;
51088 int num_imm64_operand;
51090 if (!window_list->violation && group != disp_cmp
51091 && !fits_dispatch_window (insn))
51092 window_list->violation = true;
51094 imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
51095 &num_imm64_operand);
51097 /* Initialize window with new instruction. */
51098 window[num_insn].insn = insn;
51099 window[num_insn].byte_len = byte_len;
51100 window[num_insn].group = group;
51101 window[num_insn].path = path;
51102 window[num_insn].imm_bytes = imm_size;
51104 window_list->window_size += byte_len;
51105 window_list->num_insn = num_insn + 1;
51106 window_list->num_uops = window_list->num_uops + num_uops;
51107 window_list->imm_size += imm_size;
51108 window_list->num_imm += num_imm_operand;
51109 window_list->num_imm_32 += num_imm32_operand;
51110 window_list->num_imm_64 += num_imm64_operand;
51112 if (group == disp_store)
51113 window_list->num_stores += 1;
51114 else if (group == disp_load
51115 || group == disp_prefetch)
51116 window_list->num_loads += 1;
51117 else if (group == disp_load_store)
51119 window_list->num_stores += 1;
51120 window_list->num_loads += 1;
51124 /* Adds a scheduled instruction, INSN, to the current dispatch window.
51125 If the total bytes of instructions or the number of instructions in
51126 the window exceed allowable, it allocates a new window. */
51129 add_to_dispatch_window (rtx_insn *insn)
51132 dispatch_windows *window_list;
51133 dispatch_windows *next_list;
51134 dispatch_windows *window0_list;
51135 enum insn_path path;
51136 enum dispatch_group insn_group;
51144 if (INSN_CODE (insn) < 0)
51147 byte_len = min_insn_size (insn);
51148 window_list = dispatch_window_list;
51149 next_list = window_list->next;
51150 path = get_insn_path (insn);
51151 insn_group = get_insn_group (insn);
51153 /* Get the last dispatch window. */
51155 window_list = dispatch_window_list->next;
51157 if (path == path_single)
51159 else if (path == path_double)
51162 insn_num_uops = (int) path;
51164 /* If current window is full, get a new window.
51165 Window number zero is full, if MAX_INSN uops are scheduled in it.
51166 Window number one is full, if window zero's bytes plus window
51167 one's bytes is 32, or if the bytes of the new instruction added
51168 to the total makes it greater than 48, or it has already MAX_INSN
51169 instructions in it. */
51170 num_insn = window_list->num_insn;
51171 num_uops = window_list->num_uops;
51172 window_num = window_list->window_num;
51173 insn_fits = fits_dispatch_window (insn);
51175 if (num_insn >= MAX_INSN
51176 || num_uops + insn_num_uops > MAX_INSN
51179 window_num = ~window_num & 1;
51180 window_list = allocate_next_window (window_num);
51183 if (window_num == 0)
51185 add_insn_window (insn, window_list, insn_num_uops);
51186 if (window_list->num_insn >= MAX_INSN
51187 && insn_group == disp_branch)
51189 process_end_window ();
51193 else if (window_num == 1)
51195 window0_list = window_list->prev;
51196 sum = window0_list->window_size + window_list->window_size;
51198 || (byte_len + sum) >= 48)
51200 process_end_window ();
51201 window_list = dispatch_window_list;
51204 add_insn_window (insn, window_list, insn_num_uops);
51207 gcc_unreachable ();
51209 if (is_end_basic_block (insn_group))
51211 /* End of basic block is reached do end-basic-block process. */
51212 process_end_window ();
51217 /* Print the dispatch window, WINDOW_NUM, to FILE. */
51219 DEBUG_FUNCTION static void
51220 debug_dispatch_window_file (FILE *file, int window_num)
51222 dispatch_windows *list;
51225 if (window_num == 0)
51226 list = dispatch_window_list;
51228 list = dispatch_window_list1;
51230 fprintf (file, "Window #%d:\n", list->window_num);
51231 fprintf (file, " num_insn = %d, num_uops = %d, window_size = %d\n",
51232 list->num_insn, list->num_uops, list->window_size);
51233 fprintf (file, " num_imm = %d, num_imm_32 = %d, num_imm_64 = %d, imm_size = %d\n",
51234 list->num_imm, list->num_imm_32, list->num_imm_64, list->imm_size);
51236 fprintf (file, " num_loads = %d, num_stores = %d\n", list->num_loads,
51238 fprintf (file, " insn info:\n");
51240 for (i = 0; i < MAX_INSN; i++)
51242 if (!list->window[i].insn)
51244 fprintf (file, " group[%d] = %s, insn[%d] = %p, path[%d] = %d byte_len[%d] = %d, imm_bytes[%d] = %d\n",
51245 i, group_name[list->window[i].group],
51246 i, (void *)list->window[i].insn,
51247 i, list->window[i].path,
51248 i, list->window[i].byte_len,
51249 i, list->window[i].imm_bytes);
51253 /* Print to stdout a dispatch window. */
51255 DEBUG_FUNCTION void
51256 debug_dispatch_window (int window_num)
51258 debug_dispatch_window_file (stdout, window_num);
51261 /* Print INSN dispatch information to FILE. */
51263 DEBUG_FUNCTION static void
51264 debug_insn_dispatch_info_file (FILE *file, rtx_insn *insn)
51267 enum insn_path path;
51268 enum dispatch_group group;
51270 int num_imm_operand;
51271 int num_imm32_operand;
51272 int num_imm64_operand;
51274 if (INSN_CODE (insn) < 0)
51277 byte_len = min_insn_size (insn);
51278 path = get_insn_path (insn);
51279 group = get_insn_group (insn);
51280 imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
51281 &num_imm64_operand);
51283 fprintf (file, " insn info:\n");
51284 fprintf (file, " group = %s, path = %d, byte_len = %d\n",
51285 group_name[group], path, byte_len);
51286 fprintf (file, " num_imm = %d, num_imm_32 = %d, num_imm_64 = %d, imm_size = %d\n",
51287 num_imm_operand, num_imm32_operand, num_imm64_operand, imm_size);
51290 /* Print to STDERR the status of the ready list with respect to
51291 dispatch windows. */
51293 DEBUG_FUNCTION void
51294 debug_ready_dispatch (void)
51297 int no_ready = number_in_ready ();
51299 fprintf (stdout, "Number of ready: %d\n", no_ready);
51301 for (i = 0; i < no_ready; i++)
51302 debug_insn_dispatch_info_file (stdout, get_ready_element (i));
51305 /* This routine is the driver of the dispatch scheduler. */
51308 do_dispatch (rtx_insn *insn, int mode)
51310 if (mode == DISPATCH_INIT)
51311 init_dispatch_sched ();
51312 else if (mode == ADD_TO_DISPATCH_WINDOW)
51313 add_to_dispatch_window (insn);
51316 /* Return TRUE if Dispatch Scheduling is supported. */
51319 has_dispatch (rtx_insn *insn, int action)
51321 if ((TARGET_BDVER1 || TARGET_BDVER2 || TARGET_BDVER3 || TARGET_BDVER4)
51322 && flag_dispatch_scheduler)
51328 case IS_DISPATCH_ON:
51333 return is_cmp (insn);
51335 case DISPATCH_VIOLATION:
51336 return dispatch_violation ();
51338 case FITS_DISPATCH_WINDOW:
51339 return fits_dispatch_window (insn);
51345 /* Implementation of reassociation_width target hook used by
51346 reassoc phase to identify parallelism level in reassociated
51347 tree. Statements tree_code is passed in OPC. Arguments type
51350 Currently parallel reassociation is enabled for Atom
51351 processors only and we set reassociation width to be 2
51352 because Atom may issue up to 2 instructions per cycle.
51354 Return value should be fixed if parallel reassociation is
51355 enabled for other processors. */
51358 ix86_reassociation_width (unsigned int, machine_mode mode)
51361 if (VECTOR_MODE_P (mode))
51363 if (TARGET_VECTOR_PARALLEL_EXECUTION)
51370 if (INTEGRAL_MODE_P (mode) && TARGET_REASSOC_INT_TO_PARALLEL)
51372 else if (FLOAT_MODE_P (mode) && TARGET_REASSOC_FP_TO_PARALLEL)
51378 /* ??? No autovectorization into MMX or 3DNOW until we can reliably
51379 place emms and femms instructions. */
51381 static machine_mode
51382 ix86_preferred_simd_mode (machine_mode mode)
51390 return TARGET_AVX512BW ? V64QImode :
51391 (TARGET_AVX && !TARGET_PREFER_AVX128) ? V32QImode : V16QImode;
51393 return TARGET_AVX512BW ? V32HImode :
51394 (TARGET_AVX && !TARGET_PREFER_AVX128) ? V16HImode : V8HImode;
51396 return TARGET_AVX512F ? V16SImode :
51397 (TARGET_AVX && !TARGET_PREFER_AVX128) ? V8SImode : V4SImode;
51399 return TARGET_AVX512F ? V8DImode :
51400 (TARGET_AVX && !TARGET_PREFER_AVX128) ? V4DImode : V2DImode;
51403 if (TARGET_AVX512F)
51405 else if (TARGET_AVX && !TARGET_PREFER_AVX128)
51411 if (!TARGET_VECTORIZE_DOUBLE)
51413 else if (TARGET_AVX512F)
51415 else if (TARGET_AVX && !TARGET_PREFER_AVX128)
51417 else if (TARGET_SSE2)
51426 /* If AVX is enabled then try vectorizing with both 256bit and 128bit
51427 vectors. If AVX512F is enabled then try vectorizing with 512bit,
51428 256bit and 128bit vectors. */
51430 static unsigned int
51431 ix86_autovectorize_vector_sizes (void)
51433 return TARGET_AVX512F ? 64 | 32 | 16 :
51434 (TARGET_AVX && !TARGET_PREFER_AVX128) ? 32 | 16 : 0;
51439 /* Return class of registers which could be used for pseudo of MODE
51440 and of class RCLASS for spilling instead of memory. Return NO_REGS
51441 if it is not possible or non-profitable. */
51443 ix86_spill_class (reg_class_t rclass, machine_mode mode)
51445 if (TARGET_SSE && TARGET_GENERAL_REGS_SSE_SPILL && ! TARGET_MMX
51446 && (mode == SImode || (TARGET_64BIT && mode == DImode))
51447 && rclass != NO_REGS && INTEGER_CLASS_P (rclass))
51448 return ALL_SSE_REGS;
51452 /* Implement targetm.vectorize.init_cost. */
51455 ix86_init_cost (struct loop *)
51457 unsigned *cost = XNEWVEC (unsigned, 3);
51458 cost[vect_prologue] = cost[vect_body] = cost[vect_epilogue] = 0;
51462 /* Implement targetm.vectorize.add_stmt_cost. */
51465 ix86_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
51466 struct _stmt_vec_info *stmt_info, int misalign,
51467 enum vect_cost_model_location where)
51469 unsigned *cost = (unsigned *) data;
51470 unsigned retval = 0;
51472 tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
51473 int stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign);
51475 /* Statements in an inner loop relative to the loop being
51476 vectorized are weighted more heavily. The value here is
51477 arbitrary and could potentially be improved with analysis. */
51478 if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
51479 count *= 50; /* FIXME. */
51481 retval = (unsigned) (count * stmt_cost);
51483 /* We need to multiply all vector stmt cost by 1.7 (estimated cost)
51484 for Silvermont as it has out of order integer pipeline and can execute
51485 2 scalar instruction per tick, but has in order SIMD pipeline. */
51486 if (TARGET_SILVERMONT || TARGET_INTEL)
51487 if (stmt_info && stmt_info->stmt)
51489 tree lhs_op = gimple_get_lhs (stmt_info->stmt);
51490 if (lhs_op && TREE_CODE (TREE_TYPE (lhs_op)) == INTEGER_TYPE)
51491 retval = (retval * 17) / 10;
51494 cost[where] += retval;
51499 /* Implement targetm.vectorize.finish_cost. */
51502 ix86_finish_cost (void *data, unsigned *prologue_cost,
51503 unsigned *body_cost, unsigned *epilogue_cost)
51505 unsigned *cost = (unsigned *) data;
51506 *prologue_cost = cost[vect_prologue];
51507 *body_cost = cost[vect_body];
51508 *epilogue_cost = cost[vect_epilogue];
51511 /* Implement targetm.vectorize.destroy_cost_data. */
51514 ix86_destroy_cost_data (void *data)
51519 /* Validate target specific memory model bits in VAL. */
51521 static unsigned HOST_WIDE_INT
51522 ix86_memmodel_check (unsigned HOST_WIDE_INT val)
51524 unsigned HOST_WIDE_INT model = val & MEMMODEL_MASK;
51527 if (val & ~(unsigned HOST_WIDE_INT)(IX86_HLE_ACQUIRE|IX86_HLE_RELEASE
51529 || ((val & IX86_HLE_ACQUIRE) && (val & IX86_HLE_RELEASE)))
51531 warning (OPT_Winvalid_memory_model,
51532 "Unknown architecture specific memory model");
51533 return MEMMODEL_SEQ_CST;
51535 strong = (model == MEMMODEL_ACQ_REL || model == MEMMODEL_SEQ_CST);
51536 if (val & IX86_HLE_ACQUIRE && !(model == MEMMODEL_ACQUIRE || strong))
51538 warning (OPT_Winvalid_memory_model,
51539 "HLE_ACQUIRE not used with ACQUIRE or stronger memory model");
51540 return MEMMODEL_SEQ_CST | IX86_HLE_ACQUIRE;
51542 if (val & IX86_HLE_RELEASE && !(model == MEMMODEL_RELEASE || strong))
51544 warning (OPT_Winvalid_memory_model,
51545 "HLE_RELEASE not used with RELEASE or stronger memory model");
51546 return MEMMODEL_SEQ_CST | IX86_HLE_RELEASE;
51551 /* Set CLONEI->vecsize_mangle, CLONEI->vecsize_int,
51552 CLONEI->vecsize_float and if CLONEI->simdlen is 0, also
51553 CLONEI->simdlen. Return 0 if SIMD clones shouldn't be emitted,
51554 or number of vecsize_mangle variants that should be emitted. */
51557 ix86_simd_clone_compute_vecsize_and_simdlen (struct cgraph_node *node,
51558 struct cgraph_simd_clone *clonei,
51559 tree base_type, int num)
51563 if (clonei->simdlen
51564 && (clonei->simdlen < 2
51565 || clonei->simdlen > 16
51566 || (clonei->simdlen & (clonei->simdlen - 1)) != 0))
51568 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
51569 "unsupported simdlen %d", clonei->simdlen);
51573 tree ret_type = TREE_TYPE (TREE_TYPE (node->decl));
51574 if (TREE_CODE (ret_type) != VOID_TYPE)
51575 switch (TYPE_MODE (ret_type))
51587 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
51588 "unsupported return type %qT for simd\n", ret_type);
51595 for (t = DECL_ARGUMENTS (node->decl), i = 0; t; t = DECL_CHAIN (t), i++)
51596 /* FIXME: Shouldn't we allow such arguments if they are uniform? */
51597 switch (TYPE_MODE (TREE_TYPE (t)))
51609 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
51610 "unsupported argument type %qT for simd\n", TREE_TYPE (t));
51614 if (clonei->cilk_elemental)
51616 /* Parse here processor clause. If not present, default to 'b'. */
51617 clonei->vecsize_mangle = 'b';
51619 else if (!TREE_PUBLIC (node->decl))
51621 /* If the function isn't exported, we can pick up just one ISA
51624 clonei->vecsize_mangle = 'd';
51625 else if (TARGET_AVX)
51626 clonei->vecsize_mangle = 'c';
51628 clonei->vecsize_mangle = 'b';
51633 clonei->vecsize_mangle = "bcd"[num];
51636 switch (clonei->vecsize_mangle)
51639 clonei->vecsize_int = 128;
51640 clonei->vecsize_float = 128;
51643 clonei->vecsize_int = 128;
51644 clonei->vecsize_float = 256;
51647 clonei->vecsize_int = 256;
51648 clonei->vecsize_float = 256;
51651 if (clonei->simdlen == 0)
51653 if (SCALAR_INT_MODE_P (TYPE_MODE (base_type)))
51654 clonei->simdlen = clonei->vecsize_int;
51656 clonei->simdlen = clonei->vecsize_float;
51657 clonei->simdlen /= GET_MODE_BITSIZE (TYPE_MODE (base_type));
51658 if (clonei->simdlen > 16)
51659 clonei->simdlen = 16;
51664 /* Add target attribute to SIMD clone NODE if needed. */
51667 ix86_simd_clone_adjust (struct cgraph_node *node)
51669 const char *str = NULL;
51670 gcc_assert (node->decl == cfun->decl);
51671 switch (node->simdclone->vecsize_mangle)
51686 gcc_unreachable ();
51691 tree args = build_tree_list (NULL_TREE, build_string (strlen (str), str));
51692 bool ok = ix86_valid_target_attribute_p (node->decl, NULL, args, 0);
51695 ix86_reset_previous_fndecl ();
51696 ix86_set_current_function (node->decl);
51699 /* If SIMD clone NODE can't be used in a vectorized loop
51700 in current function, return -1, otherwise return a badness of using it
51701 (0 if it is most desirable from vecsize_mangle point of view, 1
51702 slightly less desirable, etc.). */
51705 ix86_simd_clone_usable (struct cgraph_node *node)
51707 switch (node->simdclone->vecsize_mangle)
51714 return TARGET_AVX2 ? 2 : 1;
51718 return TARGET_AVX2 ? 1 : 0;
51725 gcc_unreachable ();
51729 /* This function adjusts the unroll factor based on
51730 the hardware capabilities. For ex, bdver3 has
51731 a loop buffer which makes unrolling of smaller
51732 loops less important. This function decides the
51733 unroll factor using number of memory references
51734 (value 32 is used) as a heuristic. */
51737 ix86_loop_unroll_adjust (unsigned nunroll, struct loop *loop)
51742 unsigned mem_count = 0;
51744 if (!TARGET_ADJUST_UNROLL)
51747 /* Count the number of memory references within the loop body.
51748 This value determines the unrolling factor for bdver3 and bdver4
51750 subrtx_iterator::array_type array;
51751 bbs = get_loop_body (loop);
51752 for (i = 0; i < loop->num_nodes; i++)
51753 FOR_BB_INSNS (bbs[i], insn)
51754 if (NONDEBUG_INSN_P (insn))
51755 FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
51756 if (const_rtx x = *iter)
51759 machine_mode mode = GET_MODE (x);
51760 unsigned int n_words = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
51768 if (mem_count && mem_count <=32)
51769 return 32/mem_count;
51775 /* Implement TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P. */
51778 ix86_float_exceptions_rounding_supported_p (void)
51780 /* For x87 floating point with standard excess precision handling,
51781 there is no adddf3 pattern (since x87 floating point only has
51782 XFmode operations) so the default hook implementation gets this
51784 return TARGET_80387 || TARGET_SSE_MATH;
51787 /* Implement TARGET_ATOMIC_ASSIGN_EXPAND_FENV. */
51790 ix86_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update)
51792 if (!TARGET_80387 && !TARGET_SSE_MATH)
51794 tree exceptions_var = create_tmp_var (integer_type_node);
51797 tree fenv_index_type = build_index_type (size_int (6));
51798 tree fenv_type = build_array_type (unsigned_type_node, fenv_index_type);
51799 tree fenv_var = create_tmp_var (fenv_type);
51800 mark_addressable (fenv_var);
51801 tree fenv_ptr = build_pointer_type (fenv_type);
51802 tree fenv_addr = build1 (ADDR_EXPR, fenv_ptr, fenv_var);
51803 fenv_addr = fold_convert (ptr_type_node, fenv_addr);
51804 tree fnstenv = ix86_builtins[IX86_BUILTIN_FNSTENV];
51805 tree fldenv = ix86_builtins[IX86_BUILTIN_FLDENV];
51806 tree fnstsw = ix86_builtins[IX86_BUILTIN_FNSTSW];
51807 tree fnclex = ix86_builtins[IX86_BUILTIN_FNCLEX];
51808 tree hold_fnstenv = build_call_expr (fnstenv, 1, fenv_addr);
51809 tree hold_fnclex = build_call_expr (fnclex, 0);
51810 *hold = build2 (COMPOUND_EXPR, void_type_node, hold_fnstenv,
51812 *clear = build_call_expr (fnclex, 0);
51813 tree sw_var = create_tmp_var (short_unsigned_type_node);
51814 tree fnstsw_call = build_call_expr (fnstsw, 0);
51815 tree sw_mod = build2 (MODIFY_EXPR, short_unsigned_type_node,
51816 sw_var, fnstsw_call);
51817 tree exceptions_x87 = fold_convert (integer_type_node, sw_var);
51818 tree update_mod = build2 (MODIFY_EXPR, integer_type_node,
51819 exceptions_var, exceptions_x87);
51820 *update = build2 (COMPOUND_EXPR, integer_type_node,
51821 sw_mod, update_mod);
51822 tree update_fldenv = build_call_expr (fldenv, 1, fenv_addr);
51823 *update = build2 (COMPOUND_EXPR, void_type_node, *update, update_fldenv);
51825 if (TARGET_SSE_MATH)
51827 tree mxcsr_orig_var = create_tmp_var (unsigned_type_node);
51828 tree mxcsr_mod_var = create_tmp_var (unsigned_type_node);
51829 tree stmxcsr = ix86_builtins[IX86_BUILTIN_STMXCSR];
51830 tree ldmxcsr = ix86_builtins[IX86_BUILTIN_LDMXCSR];
51831 tree stmxcsr_hold_call = build_call_expr (stmxcsr, 0);
51832 tree hold_assign_orig = build2 (MODIFY_EXPR, unsigned_type_node,
51833 mxcsr_orig_var, stmxcsr_hold_call);
51834 tree hold_mod_val = build2 (BIT_IOR_EXPR, unsigned_type_node,
51836 build_int_cst (unsigned_type_node, 0x1f80));
51837 hold_mod_val = build2 (BIT_AND_EXPR, unsigned_type_node, hold_mod_val,
51838 build_int_cst (unsigned_type_node, 0xffffffc0));
51839 tree hold_assign_mod = build2 (MODIFY_EXPR, unsigned_type_node,
51840 mxcsr_mod_var, hold_mod_val);
51841 tree ldmxcsr_hold_call = build_call_expr (ldmxcsr, 1, mxcsr_mod_var);
51842 tree hold_all = build2 (COMPOUND_EXPR, unsigned_type_node,
51843 hold_assign_orig, hold_assign_mod);
51844 hold_all = build2 (COMPOUND_EXPR, void_type_node, hold_all,
51845 ldmxcsr_hold_call);
51847 *hold = build2 (COMPOUND_EXPR, void_type_node, *hold, hold_all);
51850 tree ldmxcsr_clear_call = build_call_expr (ldmxcsr, 1, mxcsr_mod_var);
51852 *clear = build2 (COMPOUND_EXPR, void_type_node, *clear,
51853 ldmxcsr_clear_call);
51855 *clear = ldmxcsr_clear_call;
51856 tree stxmcsr_update_call = build_call_expr (stmxcsr, 0);
51857 tree exceptions_sse = fold_convert (integer_type_node,
51858 stxmcsr_update_call);
51861 tree exceptions_mod = build2 (BIT_IOR_EXPR, integer_type_node,
51862 exceptions_var, exceptions_sse);
51863 tree exceptions_assign = build2 (MODIFY_EXPR, integer_type_node,
51864 exceptions_var, exceptions_mod);
51865 *update = build2 (COMPOUND_EXPR, integer_type_node, *update,
51866 exceptions_assign);
51869 *update = build2 (MODIFY_EXPR, integer_type_node,
51870 exceptions_var, exceptions_sse);
51871 tree ldmxcsr_update_call = build_call_expr (ldmxcsr, 1, mxcsr_orig_var);
51872 *update = build2 (COMPOUND_EXPR, void_type_node, *update,
51873 ldmxcsr_update_call);
51875 tree atomic_feraiseexcept
51876 = builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT);
51877 tree atomic_feraiseexcept_call = build_call_expr (atomic_feraiseexcept,
51878 1, exceptions_var);
51879 *update = build2 (COMPOUND_EXPR, void_type_node, *update,
51880 atomic_feraiseexcept_call);
51883 /* Return mode to be used for bounds or VOIDmode
51884 if bounds are not supported. */
51886 static enum machine_mode
51887 ix86_mpx_bound_mode ()
51889 /* Do not support pointer checker if MPX
51893 if (flag_check_pointer_bounds)
51894 warning (0, "Pointer Checker requires MPX support on this target."
51895 " Use -mmpx options to enable MPX.");
51902 /* Return constant used to statically initialize constant bounds.
51904 This function is used to create special bound values. For now
51905 only INIT bounds and NONE bounds are expected. More special
51906 values may be added later. */
51909 ix86_make_bounds_constant (HOST_WIDE_INT lb, HOST_WIDE_INT ub)
51911 tree low = lb ? build_minus_one_cst (pointer_sized_int_node)
51912 : build_zero_cst (pointer_sized_int_node);
51913 tree high = ub ? build_zero_cst (pointer_sized_int_node)
51914 : build_minus_one_cst (pointer_sized_int_node);
51916 /* This function is supposed to be used to create INIT and
51917 NONE bounds only. */
51918 gcc_assert ((lb == 0 && ub == -1)
51919 || (lb == -1 && ub == 0));
51921 return build_complex (NULL, low, high);
51924 /* Generate a list of statements STMTS to initialize pointer bounds
51925 variable VAR with bounds LB and UB. Return the number of generated
51929 ix86_initialize_bounds (tree var, tree lb, tree ub, tree *stmts)
51931 tree bnd_ptr = build_pointer_type (pointer_sized_int_node);
51932 tree lhs, modify, var_p;
51934 ub = build1 (BIT_NOT_EXPR, pointer_sized_int_node, ub);
51935 var_p = fold_convert (bnd_ptr, build_fold_addr_expr (var));
51937 lhs = build1 (INDIRECT_REF, pointer_sized_int_node, var_p);
51938 modify = build2 (MODIFY_EXPR, TREE_TYPE (lhs), lhs, lb);
51939 append_to_statement_list (modify, stmts);
51941 lhs = build1 (INDIRECT_REF, pointer_sized_int_node,
51942 build2 (POINTER_PLUS_EXPR, bnd_ptr, var_p,
51943 TYPE_SIZE_UNIT (pointer_sized_int_node)));
51944 modify = build2 (MODIFY_EXPR, TREE_TYPE (lhs), lhs, ub);
51945 append_to_statement_list (modify, stmts);
51950 #if !TARGET_MACHO && !TARGET_DLLIMPORT_DECL_ATTRIBUTES
51951 /* For i386, common symbol is local only for non-PIE binaries. For
51952 x86-64, common symbol is local only for non-PIE binaries or linker
51953 supports copy reloc in PIE binaries. */
51956 ix86_binds_local_p (const_tree exp)
51958 return default_binds_local_p_3 (exp, flag_shlib != 0, true, true,
51961 && HAVE_LD_PIE_COPYRELOC != 0)));
51965 /* Initialize the GCC target structure. */
51966 #undef TARGET_RETURN_IN_MEMORY
51967 #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
51969 #undef TARGET_LEGITIMIZE_ADDRESS
51970 #define TARGET_LEGITIMIZE_ADDRESS ix86_legitimize_address
51972 #undef TARGET_ATTRIBUTE_TABLE
51973 #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
51974 #undef TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P
51975 #define TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P hook_bool_const_tree_true
51976 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
51977 # undef TARGET_MERGE_DECL_ATTRIBUTES
51978 # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
51981 #undef TARGET_COMP_TYPE_ATTRIBUTES
51982 #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
51984 #undef TARGET_INIT_BUILTINS
51985 #define TARGET_INIT_BUILTINS ix86_init_builtins
51986 #undef TARGET_BUILTIN_DECL
51987 #define TARGET_BUILTIN_DECL ix86_builtin_decl
51988 #undef TARGET_EXPAND_BUILTIN
51989 #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
51991 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
51992 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
51993 ix86_builtin_vectorized_function
51995 #undef TARGET_VECTORIZE_BUILTIN_TM_LOAD
51996 #define TARGET_VECTORIZE_BUILTIN_TM_LOAD ix86_builtin_tm_load
51998 #undef TARGET_VECTORIZE_BUILTIN_TM_STORE
51999 #define TARGET_VECTORIZE_BUILTIN_TM_STORE ix86_builtin_tm_store
52001 #undef TARGET_VECTORIZE_BUILTIN_GATHER
52002 #define TARGET_VECTORIZE_BUILTIN_GATHER ix86_vectorize_builtin_gather
52004 #undef TARGET_BUILTIN_RECIPROCAL
52005 #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
52007 #undef TARGET_ASM_FUNCTION_EPILOGUE
52008 #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
52010 #undef TARGET_ENCODE_SECTION_INFO
52011 #ifndef SUBTARGET_ENCODE_SECTION_INFO
52012 #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
52014 #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
52017 #undef TARGET_ASM_OPEN_PAREN
52018 #define TARGET_ASM_OPEN_PAREN ""
52019 #undef TARGET_ASM_CLOSE_PAREN
52020 #define TARGET_ASM_CLOSE_PAREN ""
52022 #undef TARGET_ASM_BYTE_OP
52023 #define TARGET_ASM_BYTE_OP ASM_BYTE
52025 #undef TARGET_ASM_ALIGNED_HI_OP
52026 #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
52027 #undef TARGET_ASM_ALIGNED_SI_OP
52028 #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
52030 #undef TARGET_ASM_ALIGNED_DI_OP
52031 #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
52034 #undef TARGET_PROFILE_BEFORE_PROLOGUE
52035 #define TARGET_PROFILE_BEFORE_PROLOGUE ix86_profile_before_prologue
52037 #undef TARGET_MANGLE_DECL_ASSEMBLER_NAME
52038 #define TARGET_MANGLE_DECL_ASSEMBLER_NAME ix86_mangle_decl_assembler_name
52040 #undef TARGET_ASM_UNALIGNED_HI_OP
52041 #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
52042 #undef TARGET_ASM_UNALIGNED_SI_OP
52043 #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
52044 #undef TARGET_ASM_UNALIGNED_DI_OP
52045 #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
52047 #undef TARGET_PRINT_OPERAND
52048 #define TARGET_PRINT_OPERAND ix86_print_operand
52049 #undef TARGET_PRINT_OPERAND_ADDRESS
52050 #define TARGET_PRINT_OPERAND_ADDRESS ix86_print_operand_address
52051 #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
52052 #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ix86_print_operand_punct_valid_p
52053 #undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
52054 #define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra
52056 #undef TARGET_SCHED_INIT_GLOBAL
52057 #define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global
52058 #undef TARGET_SCHED_ADJUST_COST
52059 #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
52060 #undef TARGET_SCHED_ISSUE_RATE
52061 #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
52062 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
52063 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
52064 ia32_multipass_dfa_lookahead
52065 #undef TARGET_SCHED_MACRO_FUSION_P
52066 #define TARGET_SCHED_MACRO_FUSION_P ix86_macro_fusion_p
52067 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
52068 #define TARGET_SCHED_MACRO_FUSION_PAIR_P ix86_macro_fusion_pair_p
52070 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
52071 #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
52073 #undef TARGET_MEMMODEL_CHECK
52074 #define TARGET_MEMMODEL_CHECK ix86_memmodel_check
52076 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
52077 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV ix86_atomic_assign_expand_fenv
52080 #undef TARGET_HAVE_TLS
52081 #define TARGET_HAVE_TLS true
52083 #undef TARGET_CANNOT_FORCE_CONST_MEM
52084 #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
52085 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
52086 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
52088 #undef TARGET_DELEGITIMIZE_ADDRESS
52089 #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
52091 #undef TARGET_MS_BITFIELD_LAYOUT_P
52092 #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
52095 #undef TARGET_BINDS_LOCAL_P
52096 #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
52098 #undef TARGET_BINDS_LOCAL_P
52099 #define TARGET_BINDS_LOCAL_P ix86_binds_local_p
52101 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
52102 #undef TARGET_BINDS_LOCAL_P
52103 #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
52106 #undef TARGET_ASM_OUTPUT_MI_THUNK
52107 #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
52108 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
52109 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
52111 #undef TARGET_ASM_FILE_START
52112 #define TARGET_ASM_FILE_START x86_file_start
52114 #undef TARGET_OPTION_OVERRIDE
52115 #define TARGET_OPTION_OVERRIDE ix86_option_override
52117 #undef TARGET_REGISTER_MOVE_COST
52118 #define TARGET_REGISTER_MOVE_COST ix86_register_move_cost
52119 #undef TARGET_MEMORY_MOVE_COST
52120 #define TARGET_MEMORY_MOVE_COST ix86_memory_move_cost
52121 #undef TARGET_RTX_COSTS
52122 #define TARGET_RTX_COSTS ix86_rtx_costs
52123 #undef TARGET_ADDRESS_COST
52124 #define TARGET_ADDRESS_COST ix86_address_cost
52126 #undef TARGET_FIXED_CONDITION_CODE_REGS
52127 #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
52128 #undef TARGET_CC_MODES_COMPATIBLE
52129 #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
52131 #undef TARGET_MACHINE_DEPENDENT_REORG
52132 #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
52134 #undef TARGET_BUILTIN_SETJMP_FRAME_VALUE
52135 #define TARGET_BUILTIN_SETJMP_FRAME_VALUE ix86_builtin_setjmp_frame_value
52137 #undef TARGET_BUILD_BUILTIN_VA_LIST
52138 #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
52140 #undef TARGET_FOLD_BUILTIN
52141 #define TARGET_FOLD_BUILTIN ix86_fold_builtin
52143 #undef TARGET_COMPARE_VERSION_PRIORITY
52144 #define TARGET_COMPARE_VERSION_PRIORITY ix86_compare_version_priority
52146 #undef TARGET_GENERATE_VERSION_DISPATCHER_BODY
52147 #define TARGET_GENERATE_VERSION_DISPATCHER_BODY \
52148 ix86_generate_version_dispatcher_body
52150 #undef TARGET_GET_FUNCTION_VERSIONS_DISPATCHER
52151 #define TARGET_GET_FUNCTION_VERSIONS_DISPATCHER \
52152 ix86_get_function_versions_dispatcher
52154 #undef TARGET_ENUM_VA_LIST_P
52155 #define TARGET_ENUM_VA_LIST_P ix86_enum_va_list
52157 #undef TARGET_FN_ABI_VA_LIST
52158 #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
52160 #undef TARGET_CANONICAL_VA_LIST_TYPE
52161 #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
52163 #undef TARGET_EXPAND_BUILTIN_VA_START
52164 #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
52166 #undef TARGET_MD_ASM_CLOBBERS
52167 #define TARGET_MD_ASM_CLOBBERS ix86_md_asm_clobbers
52169 #undef TARGET_PROMOTE_PROTOTYPES
52170 #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
52171 #undef TARGET_SETUP_INCOMING_VARARGS
52172 #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
52173 #undef TARGET_MUST_PASS_IN_STACK
52174 #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
52175 #undef TARGET_FUNCTION_ARG_ADVANCE
52176 #define TARGET_FUNCTION_ARG_ADVANCE ix86_function_arg_advance
52177 #undef TARGET_FUNCTION_ARG
52178 #define TARGET_FUNCTION_ARG ix86_function_arg
52179 #undef TARGET_INIT_PIC_REG
52180 #define TARGET_INIT_PIC_REG ix86_init_pic_reg
52181 #undef TARGET_USE_PSEUDO_PIC_REG
52182 #define TARGET_USE_PSEUDO_PIC_REG ix86_use_pseudo_pic_reg
52183 #undef TARGET_FUNCTION_ARG_BOUNDARY
52184 #define TARGET_FUNCTION_ARG_BOUNDARY ix86_function_arg_boundary
52185 #undef TARGET_PASS_BY_REFERENCE
52186 #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
52187 #undef TARGET_INTERNAL_ARG_POINTER
52188 #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
52189 #undef TARGET_UPDATE_STACK_BOUNDARY
52190 #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
52191 #undef TARGET_GET_DRAP_RTX
52192 #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
52193 #undef TARGET_STRICT_ARGUMENT_NAMING
52194 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
52195 #undef TARGET_STATIC_CHAIN
52196 #define TARGET_STATIC_CHAIN ix86_static_chain
52197 #undef TARGET_TRAMPOLINE_INIT
52198 #define TARGET_TRAMPOLINE_INIT ix86_trampoline_init
52199 #undef TARGET_RETURN_POPS_ARGS
52200 #define TARGET_RETURN_POPS_ARGS ix86_return_pops_args
52202 #undef TARGET_LEGITIMATE_COMBINED_INSN
52203 #define TARGET_LEGITIMATE_COMBINED_INSN ix86_legitimate_combined_insn
52205 #undef TARGET_ASAN_SHADOW_OFFSET
52206 #define TARGET_ASAN_SHADOW_OFFSET ix86_asan_shadow_offset
52208 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
52209 #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
52211 #undef TARGET_SCALAR_MODE_SUPPORTED_P
52212 #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
52214 #undef TARGET_VECTOR_MODE_SUPPORTED_P
52215 #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
52217 #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
52218 #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
52219 ix86_libgcc_floating_mode_supported_p
52221 #undef TARGET_C_MODE_FOR_SUFFIX
52222 #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
52225 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
52226 #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
52229 #ifdef SUBTARGET_INSERT_ATTRIBUTES
52230 #undef TARGET_INSERT_ATTRIBUTES
52231 #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
52234 #undef TARGET_MANGLE_TYPE
52235 #define TARGET_MANGLE_TYPE ix86_mangle_type
52238 #undef TARGET_STACK_PROTECT_FAIL
52239 #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
52242 #undef TARGET_FUNCTION_VALUE
52243 #define TARGET_FUNCTION_VALUE ix86_function_value
52245 #undef TARGET_FUNCTION_VALUE_REGNO_P
52246 #define TARGET_FUNCTION_VALUE_REGNO_P ix86_function_value_regno_p
52248 #undef TARGET_PROMOTE_FUNCTION_MODE
52249 #define TARGET_PROMOTE_FUNCTION_MODE ix86_promote_function_mode
52251 #undef TARGET_MEMBER_TYPE_FORCES_BLK
52252 #define TARGET_MEMBER_TYPE_FORCES_BLK ix86_member_type_forces_blk
52254 #undef TARGET_INSTANTIATE_DECLS
52255 #define TARGET_INSTANTIATE_DECLS ix86_instantiate_decls
52257 #undef TARGET_SECONDARY_RELOAD
52258 #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
52260 #undef TARGET_CLASS_MAX_NREGS
52261 #define TARGET_CLASS_MAX_NREGS ix86_class_max_nregs
52263 #undef TARGET_PREFERRED_RELOAD_CLASS
52264 #define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class
52265 #undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS
52266 #define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
52267 #undef TARGET_CLASS_LIKELY_SPILLED_P
52268 #define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
52270 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
52271 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
52272 ix86_builtin_vectorization_cost
52273 #undef TARGET_VECTORIZE_VEC_PERM_CONST_OK
52274 #define TARGET_VECTORIZE_VEC_PERM_CONST_OK \
52275 ix86_vectorize_vec_perm_const_ok
52276 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
52277 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
52278 ix86_preferred_simd_mode
52279 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
52280 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
52281 ix86_autovectorize_vector_sizes
52282 #undef TARGET_VECTORIZE_INIT_COST
52283 #define TARGET_VECTORIZE_INIT_COST ix86_init_cost
52284 #undef TARGET_VECTORIZE_ADD_STMT_COST
52285 #define TARGET_VECTORIZE_ADD_STMT_COST ix86_add_stmt_cost
52286 #undef TARGET_VECTORIZE_FINISH_COST
52287 #define TARGET_VECTORIZE_FINISH_COST ix86_finish_cost
52288 #undef TARGET_VECTORIZE_DESTROY_COST_DATA
52289 #define TARGET_VECTORIZE_DESTROY_COST_DATA ix86_destroy_cost_data
52291 #undef TARGET_SET_CURRENT_FUNCTION
52292 #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
52294 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
52295 #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
52297 #undef TARGET_OPTION_SAVE
52298 #define TARGET_OPTION_SAVE ix86_function_specific_save
52300 #undef TARGET_OPTION_RESTORE
52301 #define TARGET_OPTION_RESTORE ix86_function_specific_restore
52303 #undef TARGET_OPTION_POST_STREAM_IN
52304 #define TARGET_OPTION_POST_STREAM_IN ix86_function_specific_post_stream_in
52306 #undef TARGET_OPTION_PRINT
52307 #define TARGET_OPTION_PRINT ix86_function_specific_print
52309 #undef TARGET_OPTION_FUNCTION_VERSIONS
52310 #define TARGET_OPTION_FUNCTION_VERSIONS ix86_function_versions
52312 #undef TARGET_CAN_INLINE_P
52313 #define TARGET_CAN_INLINE_P ix86_can_inline_p
52315 #undef TARGET_EXPAND_TO_RTL_HOOK
52316 #define TARGET_EXPAND_TO_RTL_HOOK ix86_maybe_switch_abi
52318 #undef TARGET_LEGITIMATE_ADDRESS_P
52319 #define TARGET_LEGITIMATE_ADDRESS_P ix86_legitimate_address_p
52321 #undef TARGET_LRA_P
52322 #define TARGET_LRA_P hook_bool_void_true
52324 #undef TARGET_REGISTER_PRIORITY
52325 #define TARGET_REGISTER_PRIORITY ix86_register_priority
52327 #undef TARGET_REGISTER_USAGE_LEVELING_P
52328 #define TARGET_REGISTER_USAGE_LEVELING_P hook_bool_void_true
52330 #undef TARGET_LEGITIMATE_CONSTANT_P
52331 #define TARGET_LEGITIMATE_CONSTANT_P ix86_legitimate_constant_p
52333 #undef TARGET_FRAME_POINTER_REQUIRED
52334 #define TARGET_FRAME_POINTER_REQUIRED ix86_frame_pointer_required
52336 #undef TARGET_CAN_ELIMINATE
52337 #define TARGET_CAN_ELIMINATE ix86_can_eliminate
52339 #undef TARGET_EXTRA_LIVE_ON_ENTRY
52340 #define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry
52342 #undef TARGET_ASM_CODE_END
52343 #define TARGET_ASM_CODE_END ix86_code_end
52345 #undef TARGET_CONDITIONAL_REGISTER_USAGE
52346 #define TARGET_CONDITIONAL_REGISTER_USAGE ix86_conditional_register_usage
52349 #undef TARGET_INIT_LIBFUNCS
52350 #define TARGET_INIT_LIBFUNCS darwin_rename_builtins
52353 #undef TARGET_LOOP_UNROLL_ADJUST
52354 #define TARGET_LOOP_UNROLL_ADJUST ix86_loop_unroll_adjust
52356 #undef TARGET_SPILL_CLASS
52357 #define TARGET_SPILL_CLASS ix86_spill_class
52359 #undef TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN
52360 #define TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN \
52361 ix86_simd_clone_compute_vecsize_and_simdlen
52363 #undef TARGET_SIMD_CLONE_ADJUST
52364 #define TARGET_SIMD_CLONE_ADJUST \
52365 ix86_simd_clone_adjust
52367 #undef TARGET_SIMD_CLONE_USABLE
52368 #define TARGET_SIMD_CLONE_USABLE \
52369 ix86_simd_clone_usable
52371 #undef TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P
52372 #define TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P \
52373 ix86_float_exceptions_rounding_supported_p
52375 #undef TARGET_MODE_EMIT
52376 #define TARGET_MODE_EMIT ix86_emit_mode_set
52378 #undef TARGET_MODE_NEEDED
52379 #define TARGET_MODE_NEEDED ix86_mode_needed
52381 #undef TARGET_MODE_AFTER
52382 #define TARGET_MODE_AFTER ix86_mode_after
52384 #undef TARGET_MODE_ENTRY
52385 #define TARGET_MODE_ENTRY ix86_mode_entry
52387 #undef TARGET_MODE_EXIT
52388 #define TARGET_MODE_EXIT ix86_mode_exit
52390 #undef TARGET_MODE_PRIORITY
52391 #define TARGET_MODE_PRIORITY ix86_mode_priority
52393 #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
52394 #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
52396 #undef TARGET_LOAD_BOUNDS_FOR_ARG
52397 #define TARGET_LOAD_BOUNDS_FOR_ARG ix86_load_bounds
52399 #undef TARGET_STORE_BOUNDS_FOR_ARG
52400 #define TARGET_STORE_BOUNDS_FOR_ARG ix86_store_bounds
52402 #undef TARGET_LOAD_RETURNED_BOUNDS
52403 #define TARGET_LOAD_RETURNED_BOUNDS ix86_load_returned_bounds
52405 #undef TARGET_STORE_RETURNED_BOUNDS
52406 #define TARGET_STORE_RETURNED_BOUNDS ix86_store_returned_bounds
52408 #undef TARGET_CHKP_BOUND_MODE
52409 #define TARGET_CHKP_BOUND_MODE ix86_mpx_bound_mode
52411 #undef TARGET_BUILTIN_CHKP_FUNCTION
52412 #define TARGET_BUILTIN_CHKP_FUNCTION ix86_builtin_mpx_function
52414 #undef TARGET_CHKP_FUNCTION_VALUE_BOUNDS
52415 #define TARGET_CHKP_FUNCTION_VALUE_BOUNDS ix86_function_value_bounds
52417 #undef TARGET_CHKP_MAKE_BOUNDS_CONSTANT
52418 #define TARGET_CHKP_MAKE_BOUNDS_CONSTANT ix86_make_bounds_constant
52420 #undef TARGET_CHKP_INITIALIZE_BOUNDS
52421 #define TARGET_CHKP_INITIALIZE_BOUNDS ix86_initialize_bounds
52423 #undef TARGET_SETUP_INCOMING_VARARG_BOUNDS
52424 #define TARGET_SETUP_INCOMING_VARARG_BOUNDS ix86_setup_incoming_vararg_bounds
52426 #undef TARGET_OFFLOAD_OPTIONS
52427 #define TARGET_OFFLOAD_OPTIONS \
52428 ix86_offload_options
52430 #undef TARGET_ABSOLUTE_BIGGEST_ALIGNMENT
52431 #define TARGET_ABSOLUTE_BIGGEST_ALIGNMENT 512
52433 struct gcc_target targetm = TARGET_INITIALIZER;
52435 #include "gt-i386.h"