1 /* Scalar Replacement of Aggregates (SRA) converts some structure
2 references into scalar references, exposing them to the scalar
4 Copyright (C) 2008-2015 Free Software Foundation, Inc.
5 Contributed by Martin Jambor <mjambor@suse.cz>
7 This file is part of GCC.
9 GCC is free software; you can redistribute it and/or modify it under
10 the terms of the GNU General Public License as published by the Free
11 Software Foundation; either version 3, or (at your option) any later
14 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 You should have received a copy of the GNU General Public License
20 along with GCC; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
23 /* This file implements Scalar Reduction of Aggregates (SRA). SRA is run
24 twice, once in the early stages of compilation (early SRA) and once in the
25 late stages (late SRA). The aim of both is to turn references to scalar
26 parts of aggregates into uses of independent scalar variables.
28 The two passes are nearly identical, the only difference is that early SRA
29 does not scalarize unions which are used as the result in a GIMPLE_RETURN
30 statement because together with inlining this can lead to weird type
33 Both passes operate in four stages:
35 1. The declarations that have properties which make them candidates for
36 scalarization are identified in function find_var_candidates(). The
37 candidates are stored in candidate_bitmap.
39 2. The function body is scanned. In the process, declarations which are
40 used in a manner that prevent their scalarization are removed from the
41 candidate bitmap. More importantly, for every access into an aggregate,
42 an access structure (struct access) is created by create_access() and
43 stored in a vector associated with the aggregate. Among other
44 information, the aggregate declaration, the offset and size of the access
45 and its type are stored in the structure.
47 On a related note, assign_link structures are created for every assign
48 statement between candidate aggregates and attached to the related
51 3. The vectors of accesses are analyzed. They are first sorted according to
52 their offset and size and then scanned for partially overlapping accesses
53 (i.e. those which overlap but one is not entirely within another). Such
54 an access disqualifies the whole aggregate from being scalarized.
56 If there is no such inhibiting overlap, a representative access structure
57 is chosen for every unique combination of offset and size. Afterwards,
58 the pass builds a set of trees from these structures, in which children
59 of an access are within their parent (in terms of offset and size).
61 Then accesses are propagated whenever possible (i.e. in cases when it
62 does not create a partially overlapping access) across assign_links from
63 the right hand side to the left hand side.
65 Then the set of trees for each declaration is traversed again and those
66 accesses which should be replaced by a scalar are identified.
68 4. The function is traversed again, and for every reference into an
69 aggregate that has some component which is about to be scalarized,
70 statements are amended and new statements are created as necessary.
71 Finally, if a parameter got scalarized, the scalar replacements are
72 initialized with values from respective parameter aggregates. */
76 #include "coretypes.h"
78 #include "hash-table.h"
79 #include "alloc-pool.h"
84 #include "double-int.h"
91 #include "fold-const.h"
93 #include "hard-reg-set.h"
95 #include "dominance.h"
97 #include "basic-block.h"
98 #include "tree-ssa-alias.h"
99 #include "internal-fn.h"
101 #include "gimple-expr.h"
104 #include "stor-layout.h"
105 #include "gimplify.h"
106 #include "gimple-iterator.h"
107 #include "gimplify-me.h"
108 #include "gimple-walk.h"
110 #include "gimple-ssa.h"
111 #include "tree-cfg.h"
112 #include "tree-phinodes.h"
113 #include "ssa-iterators.h"
114 #include "stringpool.h"
115 #include "tree-ssanames.h"
119 #include "statistics.h"
121 #include "fixed-value.h"
122 #include "insn-config.h"
127 #include "emit-rtl.h"
131 #include "tree-dfa.h"
132 #include "tree-ssa.h"
133 #include "tree-pass.h"
134 #include "plugin-api.h"
137 #include "symbol-summary.h"
138 #include "ipa-prop.h"
142 #include "tree-inline.h"
143 #include "gimple-pretty-print.h"
144 #include "ipa-inline.h"
145 #include "ipa-utils.h"
146 #include "builtins.h"
148 /* Enumeration of all aggregate reductions we can do. */
149 enum sra_mode { SRA_MODE_EARLY_IPA, /* early call regularization */
150 SRA_MODE_EARLY_INTRA, /* early intraprocedural SRA */
151 SRA_MODE_INTRA }; /* late intraprocedural SRA */
153 /* Global variable describing which aggregate reduction we are performing at
155 static enum sra_mode sra_mode;
159 /* ACCESS represents each access to an aggregate variable (as a whole or a
160 part). It can also represent a group of accesses that refer to exactly the
161 same fragment of an aggregate (i.e. those that have exactly the same offset
162 and size). Such representatives for a single aggregate, once determined,
163 are linked in a linked list and have the group fields set.
165 Moreover, when doing intraprocedural SRA, a tree is built from those
166 representatives (by the means of first_child and next_sibling pointers), in
167 which all items in a subtree are "within" the root, i.e. their offset is
168 greater or equal to offset of the root and offset+size is smaller or equal
169 to offset+size of the root. Children of an access are sorted by offset.
171 Note that accesses to parts of vector and complex number types always
172 represented by an access to the whole complex number or a vector. It is a
173 duty of the modifying functions to replace them appropriately. */
177 /* Values returned by `get_ref_base_and_extent' for each component reference
178 If EXPR isn't a component reference just set `BASE = EXPR', `OFFSET = 0',
179 `SIZE = TREE_SIZE (TREE_TYPE (expr))'. */
180 HOST_WIDE_INT offset;
184 /* Expression. It is context dependent so do not use it to create new
185 expressions to access the original aggregate. See PR 42154 for a
191 /* The statement this access belongs to. */
194 /* Next group representative for this aggregate. */
195 struct access *next_grp;
197 /* Pointer to the group representative. Pointer to itself if the struct is
198 the representative. */
199 struct access *group_representative;
201 /* If this access has any children (in terms of the definition above), this
202 points to the first one. */
203 struct access *first_child;
205 /* In intraprocedural SRA, pointer to the next sibling in the access tree as
206 described above. In IPA-SRA this is a pointer to the next access
207 belonging to the same group (having the same representative). */
208 struct access *next_sibling;
210 /* Pointers to the first and last element in the linked list of assign
212 struct assign_link *first_link, *last_link;
214 /* Pointer to the next access in the work queue. */
215 struct access *next_queued;
217 /* Replacement variable for this access "region." Never to be accessed
218 directly, always only by the means of get_access_replacement() and only
219 when grp_to_be_replaced flag is set. */
220 tree replacement_decl;
222 /* Is this particular access write access? */
225 /* Is this access an access to a non-addressable field? */
226 unsigned non_addressable : 1;
228 /* Is this access currently in the work queue? */
229 unsigned grp_queued : 1;
231 /* Does this group contain a write access? This flag is propagated down the
233 unsigned grp_write : 1;
235 /* Does this group contain a read access? This flag is propagated down the
237 unsigned grp_read : 1;
239 /* Does this group contain a read access that comes from an assignment
240 statement? This flag is propagated down the access tree. */
241 unsigned grp_assignment_read : 1;
243 /* Does this group contain a write access that comes from an assignment
244 statement? This flag is propagated down the access tree. */
245 unsigned grp_assignment_write : 1;
247 /* Does this group contain a read access through a scalar type? This flag is
248 not propagated in the access tree in any direction. */
249 unsigned grp_scalar_read : 1;
251 /* Does this group contain a write access through a scalar type? This flag
252 is not propagated in the access tree in any direction. */
253 unsigned grp_scalar_write : 1;
255 /* Is this access an artificial one created to scalarize some record
257 unsigned grp_total_scalarization : 1;
259 /* Other passes of the analysis use this bit to make function
260 analyze_access_subtree create scalar replacements for this group if
262 unsigned grp_hint : 1;
264 /* Is the subtree rooted in this access fully covered by scalar
266 unsigned grp_covered : 1;
268 /* If set to true, this access and all below it in an access tree must not be
270 unsigned grp_unscalarizable_region : 1;
272 /* Whether data have been written to parts of the aggregate covered by this
273 access which is not to be scalarized. This flag is propagated up in the
275 unsigned grp_unscalarized_data : 1;
277 /* Does this access and/or group contain a write access through a
279 unsigned grp_partial_lhs : 1;
281 /* Set when a scalar replacement should be created for this variable. */
282 unsigned grp_to_be_replaced : 1;
284 /* Set when we want a replacement for the sole purpose of having it in
285 generated debug statements. */
286 unsigned grp_to_be_debug_replaced : 1;
288 /* Should TREE_NO_WARNING of a replacement be set? */
289 unsigned grp_no_warning : 1;
291 /* Is it possible that the group refers to data which might be (directly or
292 otherwise) modified? */
293 unsigned grp_maybe_modified : 1;
295 /* Set when this is a representative of a pointer to scalar (i.e. by
296 reference) parameter which we consider for turning into a plain scalar
297 (i.e. a by value parameter). */
298 unsigned grp_scalar_ptr : 1;
300 /* Set when we discover that this pointer is not safe to dereference in the
302 unsigned grp_not_necessarilly_dereferenced : 1;
305 typedef struct access *access_p;
308 /* Alloc pool for allocating access structures. */
309 static alloc_pool access_pool;
311 /* A structure linking lhs and rhs accesses from an aggregate assignment. They
312 are used to propagate subaccesses from rhs to lhs as long as they don't
313 conflict with what is already there. */
316 struct access *lacc, *racc;
317 struct assign_link *next;
320 /* Alloc pool for allocating assign link structures. */
321 static alloc_pool link_pool;
323 /* Base (tree) -> Vector (vec<access_p> *) map. */
324 static hash_map<tree, auto_vec<access_p> > *base_access_vec;
326 /* Candidate hash table helpers. */
328 struct uid_decl_hasher : typed_noop_remove <tree_node>
330 typedef tree_node value_type;
331 typedef tree_node compare_type;
332 static inline hashval_t hash (const value_type *);
333 static inline bool equal (const value_type *, const compare_type *);
336 /* Hash a tree in a uid_decl_map. */
339 uid_decl_hasher::hash (const value_type *item)
341 return item->decl_minimal.uid;
344 /* Return true if the DECL_UID in both trees are equal. */
347 uid_decl_hasher::equal (const value_type *a, const compare_type *b)
349 return (a->decl_minimal.uid == b->decl_minimal.uid);
352 /* Set of candidates. */
353 static bitmap candidate_bitmap;
354 static hash_table<uid_decl_hasher> *candidates;
356 /* For a candidate UID return the candidates decl. */
359 candidate (unsigned uid)
362 t.decl_minimal.uid = uid;
363 return candidates->find_with_hash (&t, static_cast <hashval_t> (uid));
366 /* Bitmap of candidates which we should try to entirely scalarize away and
367 those which cannot be (because they are and need be used as a whole). */
368 static bitmap should_scalarize_away_bitmap, cannot_scalarize_away_bitmap;
370 /* Obstack for creation of fancy names. */
371 static struct obstack name_obstack;
373 /* Head of a linked list of accesses that need to have its subaccesses
374 propagated to their assignment counterparts. */
375 static struct access *work_queue_head;
377 /* Number of parameters of the analyzed function when doing early ipa SRA. */
378 static int func_param_count;
380 /* scan_function sets the following to true if it encounters a call to
381 __builtin_apply_args. */
382 static bool encountered_apply_args;
384 /* Set by scan_function when it finds a recursive call. */
385 static bool encountered_recursive_call;
387 /* Set by scan_function when it finds a recursive call with less actual
388 arguments than formal parameters.. */
389 static bool encountered_unchangable_recursive_call;
391 /* This is a table in which for each basic block and parameter there is a
392 distance (offset + size) in that parameter which is dereferenced and
393 accessed in that BB. */
394 static HOST_WIDE_INT *bb_dereferences;
395 /* Bitmap of BBs that can cause the function to "stop" progressing by
396 returning, throwing externally, looping infinitely or calling a function
397 which might abort etc.. */
398 static bitmap final_bbs;
400 /* Representative of no accesses at all. */
401 static struct access no_accesses_representant;
403 /* Predicate to test the special value. */
406 no_accesses_p (struct access *access)
408 return access == &no_accesses_representant;
411 /* Dump contents of ACCESS to file F in a human friendly way. If GRP is true,
412 representative fields are dumped, otherwise those which only describe the
413 individual access are. */
417 /* Number of processed aggregates is readily available in
418 analyze_all_variable_accesses and so is not stored here. */
420 /* Number of created scalar replacements. */
423 /* Number of times sra_modify_expr or sra_modify_assign themselves changed an
427 /* Number of statements created by generate_subtree_copies. */
430 /* Number of statements created by load_assign_lhs_subreplacements. */
433 /* Number of times sra_modify_assign has deleted a statement. */
436 /* Number of times sra_modify_assign has to deal with subaccesses of LHS and
437 RHS reparately due to type conversions or nonexistent matching
439 int separate_lhs_rhs_handling;
441 /* Number of parameters that were removed because they were unused. */
442 int deleted_unused_parameters;
444 /* Number of scalars passed as parameters by reference that have been
445 converted to be passed by value. */
446 int scalar_by_ref_to_by_val;
448 /* Number of aggregate parameters that were replaced by one or more of their
450 int aggregate_params_reduced;
452 /* Numbber of components created when splitting aggregate parameters. */
453 int param_reductions_created;
457 dump_access (FILE *f, struct access *access, bool grp)
459 fprintf (f, "access { ");
460 fprintf (f, "base = (%d)'", DECL_UID (access->base));
461 print_generic_expr (f, access->base, 0);
462 fprintf (f, "', offset = " HOST_WIDE_INT_PRINT_DEC, access->offset);
463 fprintf (f, ", size = " HOST_WIDE_INT_PRINT_DEC, access->size);
464 fprintf (f, ", expr = ");
465 print_generic_expr (f, access->expr, 0);
466 fprintf (f, ", type = ");
467 print_generic_expr (f, access->type, 0);
469 fprintf (f, ", grp_read = %d, grp_write = %d, grp_assignment_read = %d, "
470 "grp_assignment_write = %d, grp_scalar_read = %d, "
471 "grp_scalar_write = %d, grp_total_scalarization = %d, "
472 "grp_hint = %d, grp_covered = %d, "
473 "grp_unscalarizable_region = %d, grp_unscalarized_data = %d, "
474 "grp_partial_lhs = %d, grp_to_be_replaced = %d, "
475 "grp_to_be_debug_replaced = %d, grp_maybe_modified = %d, "
476 "grp_not_necessarilly_dereferenced = %d\n",
477 access->grp_read, access->grp_write, access->grp_assignment_read,
478 access->grp_assignment_write, access->grp_scalar_read,
479 access->grp_scalar_write, access->grp_total_scalarization,
480 access->grp_hint, access->grp_covered,
481 access->grp_unscalarizable_region, access->grp_unscalarized_data,
482 access->grp_partial_lhs, access->grp_to_be_replaced,
483 access->grp_to_be_debug_replaced, access->grp_maybe_modified,
484 access->grp_not_necessarilly_dereferenced);
486 fprintf (f, ", write = %d, grp_total_scalarization = %d, "
487 "grp_partial_lhs = %d\n",
488 access->write, access->grp_total_scalarization,
489 access->grp_partial_lhs);
492 /* Dump a subtree rooted in ACCESS to file F, indent by LEVEL. */
495 dump_access_tree_1 (FILE *f, struct access *access, int level)
501 for (i = 0; i < level; i++)
502 fputs ("* ", dump_file);
504 dump_access (f, access, true);
506 if (access->first_child)
507 dump_access_tree_1 (f, access->first_child, level + 1);
509 access = access->next_sibling;
514 /* Dump all access trees for a variable, given the pointer to the first root in
518 dump_access_tree (FILE *f, struct access *access)
520 for (; access; access = access->next_grp)
521 dump_access_tree_1 (f, access, 0);
524 /* Return true iff ACC is non-NULL and has subaccesses. */
527 access_has_children_p (struct access *acc)
529 return acc && acc->first_child;
532 /* Return true iff ACC is (partly) covered by at least one replacement. */
535 access_has_replacements_p (struct access *acc)
537 struct access *child;
538 if (acc->grp_to_be_replaced)
540 for (child = acc->first_child; child; child = child->next_sibling)
541 if (access_has_replacements_p (child))
546 /* Return a vector of pointers to accesses for the variable given in BASE or
547 NULL if there is none. */
549 static vec<access_p> *
550 get_base_access_vector (tree base)
552 return base_access_vec->get (base);
555 /* Find an access with required OFFSET and SIZE in a subtree of accesses rooted
556 in ACCESS. Return NULL if it cannot be found. */
558 static struct access *
559 find_access_in_subtree (struct access *access, HOST_WIDE_INT offset,
562 while (access && (access->offset != offset || access->size != size))
564 struct access *child = access->first_child;
566 while (child && (child->offset + child->size <= offset))
567 child = child->next_sibling;
574 /* Return the first group representative for DECL or NULL if none exists. */
576 static struct access *
577 get_first_repr_for_decl (tree base)
579 vec<access_p> *access_vec;
581 access_vec = get_base_access_vector (base);
585 return (*access_vec)[0];
588 /* Find an access representative for the variable BASE and given OFFSET and
589 SIZE. Requires that access trees have already been built. Return NULL if
590 it cannot be found. */
592 static struct access *
593 get_var_base_offset_size_access (tree base, HOST_WIDE_INT offset,
596 struct access *access;
598 access = get_first_repr_for_decl (base);
599 while (access && (access->offset + access->size <= offset))
600 access = access->next_grp;
604 return find_access_in_subtree (access, offset, size);
607 /* Add LINK to the linked list of assign links of RACC. */
609 add_link_to_rhs (struct access *racc, struct assign_link *link)
611 gcc_assert (link->racc == racc);
613 if (!racc->first_link)
615 gcc_assert (!racc->last_link);
616 racc->first_link = link;
619 racc->last_link->next = link;
621 racc->last_link = link;
625 /* Move all link structures in their linked list in OLD_RACC to the linked list
628 relink_to_new_repr (struct access *new_racc, struct access *old_racc)
630 if (!old_racc->first_link)
632 gcc_assert (!old_racc->last_link);
636 if (new_racc->first_link)
638 gcc_assert (!new_racc->last_link->next);
639 gcc_assert (!old_racc->last_link || !old_racc->last_link->next);
641 new_racc->last_link->next = old_racc->first_link;
642 new_racc->last_link = old_racc->last_link;
646 gcc_assert (!new_racc->last_link);
648 new_racc->first_link = old_racc->first_link;
649 new_racc->last_link = old_racc->last_link;
651 old_racc->first_link = old_racc->last_link = NULL;
654 /* Add ACCESS to the work queue (which is actually a stack). */
657 add_access_to_work_queue (struct access *access)
659 if (!access->grp_queued)
661 gcc_assert (!access->next_queued);
662 access->next_queued = work_queue_head;
663 access->grp_queued = 1;
664 work_queue_head = access;
668 /* Pop an access from the work queue, and return it, assuming there is one. */
670 static struct access *
671 pop_access_from_work_queue (void)
673 struct access *access = work_queue_head;
675 work_queue_head = access->next_queued;
676 access->next_queued = NULL;
677 access->grp_queued = 0;
682 /* Allocate necessary structures. */
685 sra_initialize (void)
687 candidate_bitmap = BITMAP_ALLOC (NULL);
688 candidates = new hash_table<uid_decl_hasher>
689 (vec_safe_length (cfun->local_decls) / 2);
690 should_scalarize_away_bitmap = BITMAP_ALLOC (NULL);
691 cannot_scalarize_away_bitmap = BITMAP_ALLOC (NULL);
692 gcc_obstack_init (&name_obstack);
693 access_pool = create_alloc_pool ("SRA accesses", sizeof (struct access), 16);
694 link_pool = create_alloc_pool ("SRA links", sizeof (struct assign_link), 16);
695 base_access_vec = new hash_map<tree, auto_vec<access_p> >;
696 memset (&sra_stats, 0, sizeof (sra_stats));
697 encountered_apply_args = false;
698 encountered_recursive_call = false;
699 encountered_unchangable_recursive_call = false;
702 /* Deallocate all general structures. */
705 sra_deinitialize (void)
707 BITMAP_FREE (candidate_bitmap);
710 BITMAP_FREE (should_scalarize_away_bitmap);
711 BITMAP_FREE (cannot_scalarize_away_bitmap);
712 free_alloc_pool (access_pool);
713 free_alloc_pool (link_pool);
714 obstack_free (&name_obstack, NULL);
716 delete base_access_vec;
719 /* Remove DECL from candidates for SRA and write REASON to the dump file if
722 disqualify_candidate (tree decl, const char *reason)
724 if (bitmap_clear_bit (candidate_bitmap, DECL_UID (decl)))
725 candidates->remove_elt_with_hash (decl, DECL_UID (decl));
727 if (dump_file && (dump_flags & TDF_DETAILS))
729 fprintf (dump_file, "! Disqualifying ");
730 print_generic_expr (dump_file, decl, 0);
731 fprintf (dump_file, " - %s\n", reason);
735 /* Return true iff the type contains a field or an element which does not allow
739 type_internals_preclude_sra_p (tree type, const char **msg)
744 switch (TREE_CODE (type))
748 case QUAL_UNION_TYPE:
749 for (fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
750 if (TREE_CODE (fld) == FIELD_DECL)
752 tree ft = TREE_TYPE (fld);
754 if (TREE_THIS_VOLATILE (fld))
756 *msg = "volatile structure field";
759 if (!DECL_FIELD_OFFSET (fld))
761 *msg = "no structure field offset";
764 if (!DECL_SIZE (fld))
766 *msg = "zero structure field size";
769 if (!tree_fits_uhwi_p (DECL_FIELD_OFFSET (fld)))
771 *msg = "structure field offset not fixed";
774 if (!tree_fits_uhwi_p (DECL_SIZE (fld)))
776 *msg = "structure field size not fixed";
779 if (!tree_fits_shwi_p (bit_position (fld)))
781 *msg = "structure field size too big";
784 if (AGGREGATE_TYPE_P (ft)
785 && int_bit_position (fld) % BITS_PER_UNIT != 0)
787 *msg = "structure field is bit field";
791 if (AGGREGATE_TYPE_P (ft) && type_internals_preclude_sra_p (ft, msg))
798 et = TREE_TYPE (type);
800 if (TYPE_VOLATILE (et))
802 *msg = "element type is volatile";
806 if (AGGREGATE_TYPE_P (et) && type_internals_preclude_sra_p (et, msg))
816 /* If T is an SSA_NAME, return NULL if it is not a default def or return its
817 base variable if it is. Return T if it is not an SSA_NAME. */
820 get_ssa_base_param (tree t)
822 if (TREE_CODE (t) == SSA_NAME)
824 if (SSA_NAME_IS_DEFAULT_DEF (t))
825 return SSA_NAME_VAR (t);
832 /* Mark a dereference of BASE of distance DIST in a basic block tht STMT
833 belongs to, unless the BB has already been marked as a potentially
837 mark_parm_dereference (tree base, HOST_WIDE_INT dist, gimple stmt)
839 basic_block bb = gimple_bb (stmt);
840 int idx, parm_index = 0;
843 if (bitmap_bit_p (final_bbs, bb->index))
846 for (parm = DECL_ARGUMENTS (current_function_decl);
847 parm && parm != base;
848 parm = DECL_CHAIN (parm))
851 gcc_assert (parm_index < func_param_count);
853 idx = bb->index * func_param_count + parm_index;
854 if (bb_dereferences[idx] < dist)
855 bb_dereferences[idx] = dist;
858 /* Allocate an access structure for BASE, OFFSET and SIZE, clear it, fill in
859 the three fields. Also add it to the vector of accesses corresponding to
860 the base. Finally, return the new access. */
862 static struct access *
863 create_access_1 (tree base, HOST_WIDE_INT offset, HOST_WIDE_INT size)
865 struct access *access;
867 access = (struct access *) pool_alloc (access_pool);
868 memset (access, 0, sizeof (struct access));
870 access->offset = offset;
873 base_access_vec->get_or_insert (base).safe_push (access);
878 /* Create and insert access for EXPR. Return created access, or NULL if it is
881 static struct access *
882 create_access (tree expr, gimple stmt, bool write)
884 struct access *access;
885 HOST_WIDE_INT offset, size, max_size;
887 bool ptr, unscalarizable_region = false;
889 base = get_ref_base_and_extent (expr, &offset, &size, &max_size);
891 if (sra_mode == SRA_MODE_EARLY_IPA
892 && TREE_CODE (base) == MEM_REF)
894 base = get_ssa_base_param (TREE_OPERAND (base, 0));
902 if (!DECL_P (base) || !bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
905 if (sra_mode == SRA_MODE_EARLY_IPA)
907 if (size < 0 || size != max_size)
909 disqualify_candidate (base, "Encountered a variable sized access.");
912 if (TREE_CODE (expr) == COMPONENT_REF
913 && DECL_BIT_FIELD (TREE_OPERAND (expr, 1)))
915 disqualify_candidate (base, "Encountered a bit-field access.");
918 gcc_checking_assert ((offset % BITS_PER_UNIT) == 0);
921 mark_parm_dereference (base, offset + size, stmt);
925 if (size != max_size)
928 unscalarizable_region = true;
932 disqualify_candidate (base, "Encountered an unconstrained access.");
937 access = create_access_1 (base, offset, size);
939 access->type = TREE_TYPE (expr);
940 access->write = write;
941 access->grp_unscalarizable_region = unscalarizable_region;
944 if (TREE_CODE (expr) == COMPONENT_REF
945 && DECL_NONADDRESSABLE_P (TREE_OPERAND (expr, 1)))
946 access->non_addressable = 1;
952 /* Return true iff TYPE is a RECORD_TYPE with fields that are either of gimple
953 register types or (recursively) records with only these two kinds of fields.
954 It also returns false if any of these records contains a bit-field. */
957 type_consists_of_records_p (tree type)
961 if (TREE_CODE (type) != RECORD_TYPE)
964 for (fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
965 if (TREE_CODE (fld) == FIELD_DECL)
967 tree ft = TREE_TYPE (fld);
969 if (DECL_BIT_FIELD (fld))
972 if (!is_gimple_reg_type (ft)
973 && !type_consists_of_records_p (ft))
980 /* Create total_scalarization accesses for all scalar type fields in DECL that
981 must be of a RECORD_TYPE conforming to type_consists_of_records_p. BASE
982 must be the top-most VAR_DECL representing the variable, OFFSET must be the
983 offset of DECL within BASE. REF must be the memory reference expression for
987 completely_scalarize_record (tree base, tree decl, HOST_WIDE_INT offset,
990 tree fld, decl_type = TREE_TYPE (decl);
992 for (fld = TYPE_FIELDS (decl_type); fld; fld = DECL_CHAIN (fld))
993 if (TREE_CODE (fld) == FIELD_DECL)
995 HOST_WIDE_INT pos = offset + int_bit_position (fld);
996 tree ft = TREE_TYPE (fld);
997 tree nref = build3 (COMPONENT_REF, TREE_TYPE (fld), ref, fld,
1000 if (is_gimple_reg_type (ft))
1002 struct access *access;
1005 size = tree_to_uhwi (DECL_SIZE (fld));
1006 access = create_access_1 (base, pos, size);
1007 access->expr = nref;
1009 access->grp_total_scalarization = 1;
1010 /* Accesses for intraprocedural SRA can have their stmt NULL. */
1013 completely_scalarize_record (base, fld, pos, nref);
1017 /* Create total_scalarization accesses for all scalar type fields in VAR and
1018 for VAR a a whole. VAR must be of a RECORD_TYPE conforming to
1019 type_consists_of_records_p. */
1022 completely_scalarize_var (tree var)
1024 HOST_WIDE_INT size = tree_to_uhwi (DECL_SIZE (var));
1025 struct access *access;
1027 access = create_access_1 (var, 0, size);
1029 access->type = TREE_TYPE (var);
1030 access->grp_total_scalarization = 1;
1032 completely_scalarize_record (var, var, 0, var);
1035 /* Return true if REF has an VIEW_CONVERT_EXPR somewhere in it. */
1038 contains_view_convert_expr_p (const_tree ref)
1040 while (handled_component_p (ref))
1042 if (TREE_CODE (ref) == VIEW_CONVERT_EXPR)
1044 ref = TREE_OPERAND (ref, 0);
1050 /* Search the given tree for a declaration by skipping handled components and
1051 exclude it from the candidates. */
1054 disqualify_base_of_expr (tree t, const char *reason)
1056 t = get_base_address (t);
1057 if (sra_mode == SRA_MODE_EARLY_IPA
1058 && TREE_CODE (t) == MEM_REF)
1059 t = get_ssa_base_param (TREE_OPERAND (t, 0));
1061 if (t && DECL_P (t))
1062 disqualify_candidate (t, reason);
1065 /* Scan expression EXPR and create access structures for all accesses to
1066 candidates for scalarization. Return the created access or NULL if none is
1069 static struct access *
1070 build_access_from_expr_1 (tree expr, gimple stmt, bool write)
1072 struct access *ret = NULL;
1075 if (TREE_CODE (expr) == BIT_FIELD_REF
1076 || TREE_CODE (expr) == IMAGPART_EXPR
1077 || TREE_CODE (expr) == REALPART_EXPR)
1079 expr = TREE_OPERAND (expr, 0);
1083 partial_ref = false;
1085 /* We need to dive through V_C_Es in order to get the size of its parameter
1086 and not the result type. Ada produces such statements. We are also
1087 capable of handling the topmost V_C_E but not any of those buried in other
1088 handled components. */
1089 if (TREE_CODE (expr) == VIEW_CONVERT_EXPR)
1090 expr = TREE_OPERAND (expr, 0);
1092 if (contains_view_convert_expr_p (expr))
1094 disqualify_base_of_expr (expr, "V_C_E under a different handled "
1098 if (TREE_THIS_VOLATILE (expr))
1100 disqualify_base_of_expr (expr, "part of a volatile reference.");
1104 switch (TREE_CODE (expr))
1107 if (TREE_CODE (TREE_OPERAND (expr, 0)) != ADDR_EXPR
1108 && sra_mode != SRA_MODE_EARLY_IPA)
1116 case ARRAY_RANGE_REF:
1117 ret = create_access (expr, stmt, write);
1124 if (write && partial_ref && ret)
1125 ret->grp_partial_lhs = 1;
1130 /* Scan expression EXPR and create access structures for all accesses to
1131 candidates for scalarization. Return true if any access has been inserted.
1132 STMT must be the statement from which the expression is taken, WRITE must be
1133 true if the expression is a store and false otherwise. */
1136 build_access_from_expr (tree expr, gimple stmt, bool write)
1138 struct access *access;
1140 access = build_access_from_expr_1 (expr, stmt, write);
1143 /* This means the aggregate is accesses as a whole in a way other than an
1144 assign statement and thus cannot be removed even if we had a scalar
1145 replacement for everything. */
1146 if (cannot_scalarize_away_bitmap)
1147 bitmap_set_bit (cannot_scalarize_away_bitmap, DECL_UID (access->base));
1153 /* Return the single non-EH successor edge of BB or NULL if there is none or
1157 single_non_eh_succ (basic_block bb)
1162 FOR_EACH_EDGE (e, ei, bb->succs)
1163 if (!(e->flags & EDGE_EH))
1173 /* Disqualify LHS and RHS for scalarization if STMT has to terminate its BB and
1174 there is no alternative spot where to put statements SRA might need to
1175 generate after it. The spot we are looking for is an edge leading to a
1176 single non-EH successor, if it exists and is indeed single. RHS may be
1177 NULL, in that case ignore it. */
1180 disqualify_if_bad_bb_terminating_stmt (gimple stmt, tree lhs, tree rhs)
1182 if ((sra_mode == SRA_MODE_EARLY_INTRA || sra_mode == SRA_MODE_INTRA)
1183 && stmt_ends_bb_p (stmt))
1185 if (single_non_eh_succ (gimple_bb (stmt)))
1188 disqualify_base_of_expr (lhs, "LHS of a throwing stmt.");
1190 disqualify_base_of_expr (rhs, "RHS of a throwing stmt.");
1196 /* Scan expressions occurring in STMT, create access structures for all accesses
1197 to candidates for scalarization and remove those candidates which occur in
1198 statements or expressions that prevent them from being split apart. Return
1199 true if any access has been inserted. */
1202 build_accesses_from_assign (gimple stmt)
1205 struct access *lacc, *racc;
1207 if (!gimple_assign_single_p (stmt)
1208 /* Scope clobbers don't influence scalarization. */
1209 || gimple_clobber_p (stmt))
1212 lhs = gimple_assign_lhs (stmt);
1213 rhs = gimple_assign_rhs1 (stmt);
1215 if (disqualify_if_bad_bb_terminating_stmt (stmt, lhs, rhs))
1218 racc = build_access_from_expr_1 (rhs, stmt, false);
1219 lacc = build_access_from_expr_1 (lhs, stmt, true);
1222 lacc->grp_assignment_write = 1;
1226 racc->grp_assignment_read = 1;
1227 if (should_scalarize_away_bitmap && !gimple_has_volatile_ops (stmt)
1228 && !is_gimple_reg_type (racc->type))
1229 bitmap_set_bit (should_scalarize_away_bitmap, DECL_UID (racc->base));
1233 && (sra_mode == SRA_MODE_EARLY_INTRA || sra_mode == SRA_MODE_INTRA)
1234 && !lacc->grp_unscalarizable_region
1235 && !racc->grp_unscalarizable_region
1236 && AGGREGATE_TYPE_P (TREE_TYPE (lhs))
1237 && lacc->size == racc->size
1238 && useless_type_conversion_p (lacc->type, racc->type))
1240 struct assign_link *link;
1242 link = (struct assign_link *) pool_alloc (link_pool);
1243 memset (link, 0, sizeof (struct assign_link));
1248 add_link_to_rhs (racc, link);
1251 return lacc || racc;
1254 /* Callback of walk_stmt_load_store_addr_ops visit_addr used to determine
1255 GIMPLE_ASM operands with memory constrains which cannot be scalarized. */
1258 asm_visit_addr (gimple, tree op, tree, void *)
1260 op = get_base_address (op);
1263 disqualify_candidate (op, "Non-scalarizable GIMPLE_ASM operand.");
1268 /* Return true iff callsite CALL has at least as many actual arguments as there
1269 are formal parameters of the function currently processed by IPA-SRA and
1270 that their types match. */
1273 callsite_arguments_match_p (gimple call)
1275 if (gimple_call_num_args (call) < (unsigned) func_param_count)
1280 for (parm = DECL_ARGUMENTS (current_function_decl), i = 0;
1282 parm = DECL_CHAIN (parm), i++)
1284 tree arg = gimple_call_arg (call, i);
1285 if (!useless_type_conversion_p (TREE_TYPE (parm), TREE_TYPE (arg)))
1291 /* Scan function and look for interesting expressions and create access
1292 structures for them. Return true iff any access is created. */
1295 scan_function (void)
1300 FOR_EACH_BB_FN (bb, cfun)
1302 gimple_stmt_iterator gsi;
1303 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1305 gimple stmt = gsi_stmt (gsi);
1309 if (final_bbs && stmt_can_throw_external (stmt))
1310 bitmap_set_bit (final_bbs, bb->index);
1311 switch (gimple_code (stmt))
1314 t = gimple_return_retval (as_a <greturn *> (stmt));
1316 ret |= build_access_from_expr (t, stmt, false);
1318 bitmap_set_bit (final_bbs, bb->index);
1322 ret |= build_accesses_from_assign (stmt);
1326 for (i = 0; i < gimple_call_num_args (stmt); i++)
1327 ret |= build_access_from_expr (gimple_call_arg (stmt, i),
1330 if (sra_mode == SRA_MODE_EARLY_IPA)
1332 tree dest = gimple_call_fndecl (stmt);
1333 int flags = gimple_call_flags (stmt);
1337 if (DECL_BUILT_IN_CLASS (dest) == BUILT_IN_NORMAL
1338 && DECL_FUNCTION_CODE (dest) == BUILT_IN_APPLY_ARGS)
1339 encountered_apply_args = true;
1340 if (recursive_call_p (current_function_decl, dest))
1342 encountered_recursive_call = true;
1343 if (!callsite_arguments_match_p (stmt))
1344 encountered_unchangable_recursive_call = true;
1349 && (flags & (ECF_CONST | ECF_PURE)) == 0)
1350 bitmap_set_bit (final_bbs, bb->index);
1353 t = gimple_call_lhs (stmt);
1354 if (t && !disqualify_if_bad_bb_terminating_stmt (stmt, t, NULL))
1355 ret |= build_access_from_expr (t, stmt, true);
1360 gasm *asm_stmt = as_a <gasm *> (stmt);
1361 walk_stmt_load_store_addr_ops (asm_stmt, NULL, NULL, NULL,
1364 bitmap_set_bit (final_bbs, bb->index);
1366 for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
1368 t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
1369 ret |= build_access_from_expr (t, asm_stmt, false);
1371 for (i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
1373 t = TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
1374 ret |= build_access_from_expr (t, asm_stmt, true);
1388 /* Helper of QSORT function. There are pointers to accesses in the array. An
1389 access is considered smaller than another if it has smaller offset or if the
1390 offsets are the same but is size is bigger. */
1393 compare_access_positions (const void *a, const void *b)
1395 const access_p *fp1 = (const access_p *) a;
1396 const access_p *fp2 = (const access_p *) b;
1397 const access_p f1 = *fp1;
1398 const access_p f2 = *fp2;
1400 if (f1->offset != f2->offset)
1401 return f1->offset < f2->offset ? -1 : 1;
1403 if (f1->size == f2->size)
1405 if (f1->type == f2->type)
1407 /* Put any non-aggregate type before any aggregate type. */
1408 else if (!is_gimple_reg_type (f1->type)
1409 && is_gimple_reg_type (f2->type))
1411 else if (is_gimple_reg_type (f1->type)
1412 && !is_gimple_reg_type (f2->type))
1414 /* Put any complex or vector type before any other scalar type. */
1415 else if (TREE_CODE (f1->type) != COMPLEX_TYPE
1416 && TREE_CODE (f1->type) != VECTOR_TYPE
1417 && (TREE_CODE (f2->type) == COMPLEX_TYPE
1418 || TREE_CODE (f2->type) == VECTOR_TYPE))
1420 else if ((TREE_CODE (f1->type) == COMPLEX_TYPE
1421 || TREE_CODE (f1->type) == VECTOR_TYPE)
1422 && TREE_CODE (f2->type) != COMPLEX_TYPE
1423 && TREE_CODE (f2->type) != VECTOR_TYPE)
1425 /* Put the integral type with the bigger precision first. */
1426 else if (INTEGRAL_TYPE_P (f1->type)
1427 && INTEGRAL_TYPE_P (f2->type))
1428 return TYPE_PRECISION (f2->type) - TYPE_PRECISION (f1->type);
1429 /* Put any integral type with non-full precision last. */
1430 else if (INTEGRAL_TYPE_P (f1->type)
1431 && (TREE_INT_CST_LOW (TYPE_SIZE (f1->type))
1432 != TYPE_PRECISION (f1->type)))
1434 else if (INTEGRAL_TYPE_P (f2->type)
1435 && (TREE_INT_CST_LOW (TYPE_SIZE (f2->type))
1436 != TYPE_PRECISION (f2->type)))
1438 /* Stabilize the sort. */
1439 return TYPE_UID (f1->type) - TYPE_UID (f2->type);
1442 /* We want the bigger accesses first, thus the opposite operator in the next
1444 return f1->size > f2->size ? -1 : 1;
1448 /* Append a name of the declaration to the name obstack. A helper function for
1452 make_fancy_decl_name (tree decl)
1456 tree name = DECL_NAME (decl);
1458 obstack_grow (&name_obstack, IDENTIFIER_POINTER (name),
1459 IDENTIFIER_LENGTH (name));
1462 sprintf (buffer, "D%u", DECL_UID (decl));
1463 obstack_grow (&name_obstack, buffer, strlen (buffer));
1467 /* Helper for make_fancy_name. */
1470 make_fancy_name_1 (tree expr)
1477 make_fancy_decl_name (expr);
1481 switch (TREE_CODE (expr))
1484 make_fancy_name_1 (TREE_OPERAND (expr, 0));
1485 obstack_1grow (&name_obstack, '$');
1486 make_fancy_decl_name (TREE_OPERAND (expr, 1));
1490 make_fancy_name_1 (TREE_OPERAND (expr, 0));
1491 obstack_1grow (&name_obstack, '$');
1492 /* Arrays with only one element may not have a constant as their
1494 index = TREE_OPERAND (expr, 1);
1495 if (TREE_CODE (index) != INTEGER_CST)
1497 sprintf (buffer, HOST_WIDE_INT_PRINT_DEC, TREE_INT_CST_LOW (index));
1498 obstack_grow (&name_obstack, buffer, strlen (buffer));
1502 make_fancy_name_1 (TREE_OPERAND (expr, 0));
1506 make_fancy_name_1 (TREE_OPERAND (expr, 0));
1507 if (!integer_zerop (TREE_OPERAND (expr, 1)))
1509 obstack_1grow (&name_obstack, '$');
1510 sprintf (buffer, HOST_WIDE_INT_PRINT_DEC,
1511 TREE_INT_CST_LOW (TREE_OPERAND (expr, 1)));
1512 obstack_grow (&name_obstack, buffer, strlen (buffer));
1519 gcc_unreachable (); /* we treat these as scalars. */
1526 /* Create a human readable name for replacement variable of ACCESS. */
1529 make_fancy_name (tree expr)
1531 make_fancy_name_1 (expr);
1532 obstack_1grow (&name_obstack, '\0');
1533 return XOBFINISH (&name_obstack, char *);
1536 /* Construct a MEM_REF that would reference a part of aggregate BASE of type
1537 EXP_TYPE at the given OFFSET. If BASE is something for which
1538 get_addr_base_and_unit_offset returns NULL, gsi must be non-NULL and is used
1539 to insert new statements either before or below the current one as specified
1540 by INSERT_AFTER. This function is not capable of handling bitfields.
1542 BASE must be either a declaration or a memory reference that has correct
1543 alignment ifformation embeded in it (e.g. a pre-existing one in SRA). */
1546 build_ref_for_offset (location_t loc, tree base, HOST_WIDE_INT offset,
1547 tree exp_type, gimple_stmt_iterator *gsi,
1550 tree prev_base = base;
1553 HOST_WIDE_INT base_offset;
1554 unsigned HOST_WIDE_INT misalign;
1557 gcc_checking_assert (offset % BITS_PER_UNIT == 0);
1558 get_object_alignment_1 (base, &align, &misalign);
1559 base = get_addr_base_and_unit_offset (base, &base_offset);
1561 /* get_addr_base_and_unit_offset returns NULL for references with a variable
1562 offset such as array[var_index]. */
1568 gcc_checking_assert (gsi);
1569 tmp = make_ssa_name (build_pointer_type (TREE_TYPE (prev_base)));
1570 addr = build_fold_addr_expr (unshare_expr (prev_base));
1571 STRIP_USELESS_TYPE_CONVERSION (addr);
1572 stmt = gimple_build_assign (tmp, addr);
1573 gimple_set_location (stmt, loc);
1575 gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
1577 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1579 off = build_int_cst (reference_alias_ptr_type (prev_base),
1580 offset / BITS_PER_UNIT);
1583 else if (TREE_CODE (base) == MEM_REF)
1585 off = build_int_cst (TREE_TYPE (TREE_OPERAND (base, 1)),
1586 base_offset + offset / BITS_PER_UNIT);
1587 off = int_const_binop (PLUS_EXPR, TREE_OPERAND (base, 1), off);
1588 base = unshare_expr (TREE_OPERAND (base, 0));
1592 off = build_int_cst (reference_alias_ptr_type (prev_base),
1593 base_offset + offset / BITS_PER_UNIT);
1594 base = build_fold_addr_expr (unshare_expr (base));
1597 misalign = (misalign + offset) & (align - 1);
1599 align = (misalign & -misalign);
1600 if (align != TYPE_ALIGN (exp_type))
1601 exp_type = build_aligned_type (exp_type, align);
1603 mem_ref = fold_build2_loc (loc, MEM_REF, exp_type, base, off);
1604 if (TREE_THIS_VOLATILE (prev_base))
1605 TREE_THIS_VOLATILE (mem_ref) = 1;
1606 if (TREE_SIDE_EFFECTS (prev_base))
1607 TREE_SIDE_EFFECTS (mem_ref) = 1;
1611 /* Construct a memory reference to a part of an aggregate BASE at the given
1612 OFFSET and of the same type as MODEL. In case this is a reference to a
1613 bit-field, the function will replicate the last component_ref of model's
1614 expr to access it. GSI and INSERT_AFTER have the same meaning as in
1615 build_ref_for_offset. */
1618 build_ref_for_model (location_t loc, tree base, HOST_WIDE_INT offset,
1619 struct access *model, gimple_stmt_iterator *gsi,
1622 if (TREE_CODE (model->expr) == COMPONENT_REF
1623 && DECL_BIT_FIELD (TREE_OPERAND (model->expr, 1)))
1625 /* This access represents a bit-field. */
1626 tree t, exp_type, fld = TREE_OPERAND (model->expr, 1);
1628 offset -= int_bit_position (fld);
1629 exp_type = TREE_TYPE (TREE_OPERAND (model->expr, 0));
1630 t = build_ref_for_offset (loc, base, offset, exp_type, gsi, insert_after);
1631 return fold_build3_loc (loc, COMPONENT_REF, TREE_TYPE (fld), t, fld,
1635 return build_ref_for_offset (loc, base, offset, model->type,
1639 /* Attempt to build a memory reference that we could but into a gimple
1640 debug_bind statement. Similar to build_ref_for_model but punts if it has to
1641 create statements and return s NULL instead. This function also ignores
1642 alignment issues and so its results should never end up in non-debug
1646 build_debug_ref_for_model (location_t loc, tree base, HOST_WIDE_INT offset,
1647 struct access *model)
1649 HOST_WIDE_INT base_offset;
1652 if (TREE_CODE (model->expr) == COMPONENT_REF
1653 && DECL_BIT_FIELD (TREE_OPERAND (model->expr, 1)))
1656 base = get_addr_base_and_unit_offset (base, &base_offset);
1659 if (TREE_CODE (base) == MEM_REF)
1661 off = build_int_cst (TREE_TYPE (TREE_OPERAND (base, 1)),
1662 base_offset + offset / BITS_PER_UNIT);
1663 off = int_const_binop (PLUS_EXPR, TREE_OPERAND (base, 1), off);
1664 base = unshare_expr (TREE_OPERAND (base, 0));
1668 off = build_int_cst (reference_alias_ptr_type (base),
1669 base_offset + offset / BITS_PER_UNIT);
1670 base = build_fold_addr_expr (unshare_expr (base));
1673 return fold_build2_loc (loc, MEM_REF, model->type, base, off);
1676 /* Construct a memory reference consisting of component_refs and array_refs to
1677 a part of an aggregate *RES (which is of type TYPE). The requested part
1678 should have type EXP_TYPE at be the given OFFSET. This function might not
1679 succeed, it returns true when it does and only then *RES points to something
1680 meaningful. This function should be used only to build expressions that we
1681 might need to present to user (e.g. in warnings). In all other situations,
1682 build_ref_for_model or build_ref_for_offset should be used instead. */
1685 build_user_friendly_ref_for_offset (tree *res, tree type, HOST_WIDE_INT offset,
1691 tree tr_size, index, minidx;
1692 HOST_WIDE_INT el_size;
1694 if (offset == 0 && exp_type
1695 && types_compatible_p (exp_type, type))
1698 switch (TREE_CODE (type))
1701 case QUAL_UNION_TYPE:
1703 for (fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
1705 HOST_WIDE_INT pos, size;
1706 tree tr_pos, expr, *expr_ptr;
1708 if (TREE_CODE (fld) != FIELD_DECL)
1711 tr_pos = bit_position (fld);
1712 if (!tr_pos || !tree_fits_uhwi_p (tr_pos))
1714 pos = tree_to_uhwi (tr_pos);
1715 gcc_assert (TREE_CODE (type) == RECORD_TYPE || pos == 0);
1716 tr_size = DECL_SIZE (fld);
1717 if (!tr_size || !tree_fits_uhwi_p (tr_size))
1719 size = tree_to_uhwi (tr_size);
1725 else if (pos > offset || (pos + size) <= offset)
1728 expr = build3 (COMPONENT_REF, TREE_TYPE (fld), *res, fld,
1731 if (build_user_friendly_ref_for_offset (expr_ptr, TREE_TYPE (fld),
1732 offset - pos, exp_type))
1741 tr_size = TYPE_SIZE (TREE_TYPE (type));
1742 if (!tr_size || !tree_fits_uhwi_p (tr_size))
1744 el_size = tree_to_uhwi (tr_size);
1746 minidx = TYPE_MIN_VALUE (TYPE_DOMAIN (type));
1747 if (TREE_CODE (minidx) != INTEGER_CST || el_size == 0)
1749 index = build_int_cst (TYPE_DOMAIN (type), offset / el_size);
1750 if (!integer_zerop (minidx))
1751 index = int_const_binop (PLUS_EXPR, index, minidx);
1752 *res = build4 (ARRAY_REF, TREE_TYPE (type), *res, index,
1753 NULL_TREE, NULL_TREE);
1754 offset = offset % el_size;
1755 type = TREE_TYPE (type);
1770 /* Return true iff TYPE is stdarg va_list type. */
1773 is_va_list_type (tree type)
1775 return TYPE_MAIN_VARIANT (type) == TYPE_MAIN_VARIANT (va_list_type_node);
1778 /* Print message to dump file why a variable was rejected. */
1781 reject (tree var, const char *msg)
1783 if (dump_file && (dump_flags & TDF_DETAILS))
1785 fprintf (dump_file, "Rejected (%d): %s: ", DECL_UID (var), msg);
1786 print_generic_expr (dump_file, var, 0);
1787 fprintf (dump_file, "\n");
1791 /* Return true if VAR is a candidate for SRA. */
1794 maybe_add_sra_candidate (tree var)
1796 tree type = TREE_TYPE (var);
1800 if (!AGGREGATE_TYPE_P (type))
1802 reject (var, "not aggregate");
1805 if (needs_to_live_in_memory (var))
1807 reject (var, "needs to live in memory");
1810 if (TREE_THIS_VOLATILE (var))
1812 reject (var, "is volatile");
1815 if (!COMPLETE_TYPE_P (type))
1817 reject (var, "has incomplete type");
1820 if (!tree_fits_uhwi_p (TYPE_SIZE (type)))
1822 reject (var, "type size not fixed");
1825 if (tree_to_uhwi (TYPE_SIZE (type)) == 0)
1827 reject (var, "type size is zero");
1830 if (type_internals_preclude_sra_p (type, &msg))
1835 if (/* Fix for PR 41089. tree-stdarg.c needs to have va_lists intact but
1836 we also want to schedule it rather late. Thus we ignore it in
1838 (sra_mode == SRA_MODE_EARLY_INTRA
1839 && is_va_list_type (type)))
1841 reject (var, "is va_list");
1845 bitmap_set_bit (candidate_bitmap, DECL_UID (var));
1846 slot = candidates->find_slot_with_hash (var, DECL_UID (var), INSERT);
1849 if (dump_file && (dump_flags & TDF_DETAILS))
1851 fprintf (dump_file, "Candidate (%d): ", DECL_UID (var));
1852 print_generic_expr (dump_file, var, 0);
1853 fprintf (dump_file, "\n");
1859 /* The very first phase of intraprocedural SRA. It marks in candidate_bitmap
1860 those with type which is suitable for scalarization. */
1863 find_var_candidates (void)
1869 for (parm = DECL_ARGUMENTS (current_function_decl);
1871 parm = DECL_CHAIN (parm))
1872 ret |= maybe_add_sra_candidate (parm);
1874 FOR_EACH_LOCAL_DECL (cfun, i, var)
1876 if (TREE_CODE (var) != VAR_DECL)
1879 ret |= maybe_add_sra_candidate (var);
1885 /* Sort all accesses for the given variable, check for partial overlaps and
1886 return NULL if there are any. If there are none, pick a representative for
1887 each combination of offset and size and create a linked list out of them.
1888 Return the pointer to the first representative and make sure it is the first
1889 one in the vector of accesses. */
1891 static struct access *
1892 sort_and_splice_var_accesses (tree var)
1894 int i, j, access_count;
1895 struct access *res, **prev_acc_ptr = &res;
1896 vec<access_p> *access_vec;
1898 HOST_WIDE_INT low = -1, high = 0;
1900 access_vec = get_base_access_vector (var);
1903 access_count = access_vec->length ();
1905 /* Sort by <OFFSET, SIZE>. */
1906 access_vec->qsort (compare_access_positions);
1909 while (i < access_count)
1911 struct access *access = (*access_vec)[i];
1912 bool grp_write = access->write;
1913 bool grp_read = !access->write;
1914 bool grp_scalar_write = access->write
1915 && is_gimple_reg_type (access->type);
1916 bool grp_scalar_read = !access->write
1917 && is_gimple_reg_type (access->type);
1918 bool grp_assignment_read = access->grp_assignment_read;
1919 bool grp_assignment_write = access->grp_assignment_write;
1920 bool multiple_scalar_reads = false;
1921 bool total_scalarization = access->grp_total_scalarization;
1922 bool grp_partial_lhs = access->grp_partial_lhs;
1923 bool first_scalar = is_gimple_reg_type (access->type);
1924 bool unscalarizable_region = access->grp_unscalarizable_region;
1926 if (first || access->offset >= high)
1929 low = access->offset;
1930 high = access->offset + access->size;
1932 else if (access->offset > low && access->offset + access->size > high)
1935 gcc_assert (access->offset >= low
1936 && access->offset + access->size <= high);
1939 while (j < access_count)
1941 struct access *ac2 = (*access_vec)[j];
1942 if (ac2->offset != access->offset || ac2->size != access->size)
1947 grp_scalar_write = (grp_scalar_write
1948 || is_gimple_reg_type (ac2->type));
1953 if (is_gimple_reg_type (ac2->type))
1955 if (grp_scalar_read)
1956 multiple_scalar_reads = true;
1958 grp_scalar_read = true;
1961 grp_assignment_read |= ac2->grp_assignment_read;
1962 grp_assignment_write |= ac2->grp_assignment_write;
1963 grp_partial_lhs |= ac2->grp_partial_lhs;
1964 unscalarizable_region |= ac2->grp_unscalarizable_region;
1965 total_scalarization |= ac2->grp_total_scalarization;
1966 relink_to_new_repr (access, ac2);
1968 /* If there are both aggregate-type and scalar-type accesses with
1969 this combination of size and offset, the comparison function
1970 should have put the scalars first. */
1971 gcc_assert (first_scalar || !is_gimple_reg_type (ac2->type));
1972 ac2->group_representative = access;
1978 access->group_representative = access;
1979 access->grp_write = grp_write;
1980 access->grp_read = grp_read;
1981 access->grp_scalar_read = grp_scalar_read;
1982 access->grp_scalar_write = grp_scalar_write;
1983 access->grp_assignment_read = grp_assignment_read;
1984 access->grp_assignment_write = grp_assignment_write;
1985 access->grp_hint = multiple_scalar_reads || total_scalarization;
1986 access->grp_total_scalarization = total_scalarization;
1987 access->grp_partial_lhs = grp_partial_lhs;
1988 access->grp_unscalarizable_region = unscalarizable_region;
1989 if (access->first_link)
1990 add_access_to_work_queue (access);
1992 *prev_acc_ptr = access;
1993 prev_acc_ptr = &access->next_grp;
1996 gcc_assert (res == (*access_vec)[0]);
2000 /* Create a variable for the given ACCESS which determines the type, name and a
2001 few other properties. Return the variable declaration and store it also to
2002 ACCESS->replacement. */
2005 create_access_replacement (struct access *access)
2009 if (access->grp_to_be_debug_replaced)
2011 repl = create_tmp_var_raw (access->type);
2012 DECL_CONTEXT (repl) = current_function_decl;
2015 /* Drop any special alignment on the type if it's not on the main
2016 variant. This avoids issues with weirdo ABIs like AAPCS. */
2017 repl = create_tmp_var (build_qualified_type
2018 (TYPE_MAIN_VARIANT (access->type),
2019 TYPE_QUALS (access->type)), "SR");
2020 if (TREE_CODE (access->type) == COMPLEX_TYPE
2021 || TREE_CODE (access->type) == VECTOR_TYPE)
2023 if (!access->grp_partial_lhs)
2024 DECL_GIMPLE_REG_P (repl) = 1;
2026 else if (access->grp_partial_lhs
2027 && is_gimple_reg_type (access->type))
2028 TREE_ADDRESSABLE (repl) = 1;
2030 DECL_SOURCE_LOCATION (repl) = DECL_SOURCE_LOCATION (access->base);
2031 DECL_ARTIFICIAL (repl) = 1;
2032 DECL_IGNORED_P (repl) = DECL_IGNORED_P (access->base);
2034 if (DECL_NAME (access->base)
2035 && !DECL_IGNORED_P (access->base)
2036 && !DECL_ARTIFICIAL (access->base))
2038 char *pretty_name = make_fancy_name (access->expr);
2039 tree debug_expr = unshare_expr_without_location (access->expr), d;
2042 DECL_NAME (repl) = get_identifier (pretty_name);
2043 obstack_free (&name_obstack, pretty_name);
2045 /* Get rid of any SSA_NAMEs embedded in debug_expr,
2046 as DECL_DEBUG_EXPR isn't considered when looking for still
2047 used SSA_NAMEs and thus they could be freed. All debug info
2048 generation cares is whether something is constant or variable
2049 and that get_ref_base_and_extent works properly on the
2050 expression. It cannot handle accesses at a non-constant offset
2051 though, so just give up in those cases. */
2052 for (d = debug_expr;
2053 !fail && (handled_component_p (d) || TREE_CODE (d) == MEM_REF);
2054 d = TREE_OPERAND (d, 0))
2055 switch (TREE_CODE (d))
2058 case ARRAY_RANGE_REF:
2059 if (TREE_OPERAND (d, 1)
2060 && TREE_CODE (TREE_OPERAND (d, 1)) != INTEGER_CST)
2062 if (TREE_OPERAND (d, 3)
2063 && TREE_CODE (TREE_OPERAND (d, 3)) != INTEGER_CST)
2067 if (TREE_OPERAND (d, 2)
2068 && TREE_CODE (TREE_OPERAND (d, 2)) != INTEGER_CST)
2072 if (TREE_CODE (TREE_OPERAND (d, 0)) != ADDR_EXPR)
2075 d = TREE_OPERAND (d, 0);
2082 SET_DECL_DEBUG_EXPR (repl, debug_expr);
2083 DECL_HAS_DEBUG_EXPR_P (repl) = 1;
2085 if (access->grp_no_warning)
2086 TREE_NO_WARNING (repl) = 1;
2088 TREE_NO_WARNING (repl) = TREE_NO_WARNING (access->base);
2091 TREE_NO_WARNING (repl) = 1;
2095 if (access->grp_to_be_debug_replaced)
2097 fprintf (dump_file, "Created a debug-only replacement for ");
2098 print_generic_expr (dump_file, access->base, 0);
2099 fprintf (dump_file, " offset: %u, size: %u\n",
2100 (unsigned) access->offset, (unsigned) access->size);
2104 fprintf (dump_file, "Created a replacement for ");
2105 print_generic_expr (dump_file, access->base, 0);
2106 fprintf (dump_file, " offset: %u, size: %u: ",
2107 (unsigned) access->offset, (unsigned) access->size);
2108 print_generic_expr (dump_file, repl, 0);
2109 fprintf (dump_file, "\n");
2112 sra_stats.replacements++;
2117 /* Return ACCESS scalar replacement, create it if it does not exist yet. */
2120 get_access_replacement (struct access *access)
2122 gcc_checking_assert (access->replacement_decl);
2123 return access->replacement_decl;
2127 /* Build a subtree of accesses rooted in *ACCESS, and move the pointer in the
2128 linked list along the way. Stop when *ACCESS is NULL or the access pointed
2129 to it is not "within" the root. Return false iff some accesses partially
2133 build_access_subtree (struct access **access)
2135 struct access *root = *access, *last_child = NULL;
2136 HOST_WIDE_INT limit = root->offset + root->size;
2138 *access = (*access)->next_grp;
2139 while (*access && (*access)->offset + (*access)->size <= limit)
2142 root->first_child = *access;
2144 last_child->next_sibling = *access;
2145 last_child = *access;
2147 if (!build_access_subtree (access))
2151 if (*access && (*access)->offset < limit)
2157 /* Build a tree of access representatives, ACCESS is the pointer to the first
2158 one, others are linked in a list by the next_grp field. Return false iff
2159 some accesses partially overlap. */
2162 build_access_trees (struct access *access)
2166 struct access *root = access;
2168 if (!build_access_subtree (&access))
2170 root->next_grp = access;
2175 /* Return true if expr contains some ARRAY_REFs into a variable bounded
2179 expr_with_var_bounded_array_refs_p (tree expr)
2181 while (handled_component_p (expr))
2183 if (TREE_CODE (expr) == ARRAY_REF
2184 && !tree_fits_shwi_p (array_ref_low_bound (expr)))
2186 expr = TREE_OPERAND (expr, 0);
2191 /* Analyze the subtree of accesses rooted in ROOT, scheduling replacements when
2192 both seeming beneficial and when ALLOW_REPLACEMENTS allows it. Also set all
2193 sorts of access flags appropriately along the way, notably always set
2194 grp_read and grp_assign_read according to MARK_READ and grp_write when
2197 Creating a replacement for a scalar access is considered beneficial if its
2198 grp_hint is set (this means we are either attempting total scalarization or
2199 there is more than one direct read access) or according to the following
2202 Access written to through a scalar type (once or more times)
2204 | Written to in an assignment statement
2206 | | Access read as scalar _once_
2208 | | | Read in an assignment statement
2210 | | | | Scalarize Comment
2211 -----------------------------------------------------------------------------
2212 0 0 0 0 No access for the scalar
2213 0 0 0 1 No access for the scalar
2214 0 0 1 0 No Single read - won't help
2215 0 0 1 1 No The same case
2216 0 1 0 0 No access for the scalar
2217 0 1 0 1 No access for the scalar
2218 0 1 1 0 Yes s = *g; return s.i;
2219 0 1 1 1 Yes The same case as above
2220 1 0 0 0 No Won't help
2221 1 0 0 1 Yes s.i = 1; *g = s;
2222 1 0 1 0 Yes s.i = 5; g = s.i;
2223 1 0 1 1 Yes The same case as above
2224 1 1 0 0 No Won't help.
2225 1 1 0 1 Yes s.i = 1; *g = s;
2226 1 1 1 0 Yes s = *g; return s.i;
2227 1 1 1 1 Yes Any of the above yeses */
2230 analyze_access_subtree (struct access *root, struct access *parent,
2231 bool allow_replacements)
2233 struct access *child;
2234 HOST_WIDE_INT limit = root->offset + root->size;
2235 HOST_WIDE_INT covered_to = root->offset;
2236 bool scalar = is_gimple_reg_type (root->type);
2237 bool hole = false, sth_created = false;
2241 if (parent->grp_read)
2243 if (parent->grp_assignment_read)
2244 root->grp_assignment_read = 1;
2245 if (parent->grp_write)
2246 root->grp_write = 1;
2247 if (parent->grp_assignment_write)
2248 root->grp_assignment_write = 1;
2249 if (parent->grp_total_scalarization)
2250 root->grp_total_scalarization = 1;
2253 if (root->grp_unscalarizable_region)
2254 allow_replacements = false;
2256 if (allow_replacements && expr_with_var_bounded_array_refs_p (root->expr))
2257 allow_replacements = false;
2259 for (child = root->first_child; child; child = child->next_sibling)
2261 hole |= covered_to < child->offset;
2262 sth_created |= analyze_access_subtree (child, root,
2263 allow_replacements && !scalar);
2265 root->grp_unscalarized_data |= child->grp_unscalarized_data;
2266 root->grp_total_scalarization &= child->grp_total_scalarization;
2267 if (child->grp_covered)
2268 covered_to += child->size;
2273 if (allow_replacements && scalar && !root->first_child
2275 || ((root->grp_scalar_read || root->grp_assignment_read)
2276 && (root->grp_scalar_write || root->grp_assignment_write))))
2278 /* Always create access replacements that cover the whole access.
2279 For integral types this means the precision has to match.
2280 Avoid assumptions based on the integral type kind, too. */
2281 if (INTEGRAL_TYPE_P (root->type)
2282 && (TREE_CODE (root->type) != INTEGER_TYPE
2283 || TYPE_PRECISION (root->type) != root->size)
2284 /* But leave bitfield accesses alone. */
2285 && (TREE_CODE (root->expr) != COMPONENT_REF
2286 || !DECL_BIT_FIELD (TREE_OPERAND (root->expr, 1))))
2288 tree rt = root->type;
2289 gcc_assert ((root->offset % BITS_PER_UNIT) == 0
2290 && (root->size % BITS_PER_UNIT) == 0);
2291 root->type = build_nonstandard_integer_type (root->size,
2292 TYPE_UNSIGNED (rt));
2293 root->expr = build_ref_for_offset (UNKNOWN_LOCATION,
2294 root->base, root->offset,
2295 root->type, NULL, false);
2297 if (dump_file && (dump_flags & TDF_DETAILS))
2299 fprintf (dump_file, "Changing the type of a replacement for ");
2300 print_generic_expr (dump_file, root->base, 0);
2301 fprintf (dump_file, " offset: %u, size: %u ",
2302 (unsigned) root->offset, (unsigned) root->size);
2303 fprintf (dump_file, " to an integer.\n");
2307 root->grp_to_be_replaced = 1;
2308 root->replacement_decl = create_access_replacement (root);
2314 if (allow_replacements
2315 && scalar && !root->first_child
2316 && (root->grp_scalar_write || root->grp_assignment_write)
2317 && !bitmap_bit_p (cannot_scalarize_away_bitmap,
2318 DECL_UID (root->base)))
2320 gcc_checking_assert (!root->grp_scalar_read
2321 && !root->grp_assignment_read);
2323 if (MAY_HAVE_DEBUG_STMTS)
2325 root->grp_to_be_debug_replaced = 1;
2326 root->replacement_decl = create_access_replacement (root);
2330 if (covered_to < limit)
2332 if (scalar || !allow_replacements)
2333 root->grp_total_scalarization = 0;
2336 if (!hole || root->grp_total_scalarization)
2337 root->grp_covered = 1;
2338 else if (root->grp_write || TREE_CODE (root->base) == PARM_DECL)
2339 root->grp_unscalarized_data = 1; /* not covered and written to */
2343 /* Analyze all access trees linked by next_grp by the means of
2344 analyze_access_subtree. */
2346 analyze_access_trees (struct access *access)
2352 if (analyze_access_subtree (access, NULL, true))
2354 access = access->next_grp;
2360 /* Return true iff a potential new child of LACC at offset OFFSET and with size
2361 SIZE would conflict with an already existing one. If exactly such a child
2362 already exists in LACC, store a pointer to it in EXACT_MATCH. */
2365 child_would_conflict_in_lacc (struct access *lacc, HOST_WIDE_INT norm_offset,
2366 HOST_WIDE_INT size, struct access **exact_match)
2368 struct access *child;
2370 for (child = lacc->first_child; child; child = child->next_sibling)
2372 if (child->offset == norm_offset && child->size == size)
2374 *exact_match = child;
2378 if (child->offset < norm_offset + size
2379 && child->offset + child->size > norm_offset)
2386 /* Create a new child access of PARENT, with all properties just like MODEL
2387 except for its offset and with its grp_write false and grp_read true.
2388 Return the new access or NULL if it cannot be created. Note that this access
2389 is created long after all splicing and sorting, it's not located in any
2390 access vector and is automatically a representative of its group. */
2392 static struct access *
2393 create_artificial_child_access (struct access *parent, struct access *model,
2394 HOST_WIDE_INT new_offset)
2396 struct access *access;
2397 struct access **child;
2398 tree expr = parent->base;
2400 gcc_assert (!model->grp_unscalarizable_region);
2402 access = (struct access *) pool_alloc (access_pool);
2403 memset (access, 0, sizeof (struct access));
2404 if (!build_user_friendly_ref_for_offset (&expr, TREE_TYPE (expr), new_offset,
2407 access->grp_no_warning = true;
2408 expr = build_ref_for_model (EXPR_LOCATION (parent->base), parent->base,
2409 new_offset, model, NULL, false);
2412 access->base = parent->base;
2413 access->expr = expr;
2414 access->offset = new_offset;
2415 access->size = model->size;
2416 access->type = model->type;
2417 access->grp_write = true;
2418 access->grp_read = false;
2420 child = &parent->first_child;
2421 while (*child && (*child)->offset < new_offset)
2422 child = &(*child)->next_sibling;
2424 access->next_sibling = *child;
2431 /* Propagate all subaccesses of RACC across an assignment link to LACC. Return
2432 true if any new subaccess was created. Additionally, if RACC is a scalar
2433 access but LACC is not, change the type of the latter, if possible. */
2436 propagate_subaccesses_across_link (struct access *lacc, struct access *racc)
2438 struct access *rchild;
2439 HOST_WIDE_INT norm_delta = lacc->offset - racc->offset;
2442 if (is_gimple_reg_type (lacc->type)
2443 || lacc->grp_unscalarizable_region
2444 || racc->grp_unscalarizable_region)
2447 if (is_gimple_reg_type (racc->type))
2449 if (!lacc->first_child && !racc->first_child)
2451 tree t = lacc->base;
2453 lacc->type = racc->type;
2454 if (build_user_friendly_ref_for_offset (&t, TREE_TYPE (t),
2455 lacc->offset, racc->type))
2459 lacc->expr = build_ref_for_model (EXPR_LOCATION (lacc->base),
2460 lacc->base, lacc->offset,
2462 lacc->grp_no_warning = true;
2468 for (rchild = racc->first_child; rchild; rchild = rchild->next_sibling)
2470 struct access *new_acc = NULL;
2471 HOST_WIDE_INT norm_offset = rchild->offset + norm_delta;
2473 if (rchild->grp_unscalarizable_region)
2476 if (child_would_conflict_in_lacc (lacc, norm_offset, rchild->size,
2481 rchild->grp_hint = 1;
2482 new_acc->grp_hint |= new_acc->grp_read;
2483 if (rchild->first_child)
2484 ret |= propagate_subaccesses_across_link (new_acc, rchild);
2489 rchild->grp_hint = 1;
2490 new_acc = create_artificial_child_access (lacc, rchild, norm_offset);
2494 if (racc->first_child)
2495 propagate_subaccesses_across_link (new_acc, rchild);
2502 /* Propagate all subaccesses across assignment links. */
2505 propagate_all_subaccesses (void)
2507 while (work_queue_head)
2509 struct access *racc = pop_access_from_work_queue ();
2510 struct assign_link *link;
2512 gcc_assert (racc->first_link);
2514 for (link = racc->first_link; link; link = link->next)
2516 struct access *lacc = link->lacc;
2518 if (!bitmap_bit_p (candidate_bitmap, DECL_UID (lacc->base)))
2520 lacc = lacc->group_representative;
2521 if (propagate_subaccesses_across_link (lacc, racc)
2522 && lacc->first_link)
2523 add_access_to_work_queue (lacc);
2528 /* Go through all accesses collected throughout the (intraprocedural) analysis
2529 stage, exclude overlapping ones, identify representatives and build trees
2530 out of them, making decisions about scalarization on the way. Return true
2531 iff there are any to-be-scalarized variables after this stage. */
2534 analyze_all_variable_accesses (void)
2537 bitmap tmp = BITMAP_ALLOC (NULL);
2540 bool optimize_speed_p = !optimize_function_for_size_p (cfun);
2542 enum compiler_param param = optimize_speed_p
2543 ? PARAM_SRA_MAX_SCALARIZATION_SIZE_SPEED
2544 : PARAM_SRA_MAX_SCALARIZATION_SIZE_SIZE;
2546 /* If the user didn't set PARAM_SRA_MAX_SCALARIZATION_SIZE_<...>,
2547 fall back to a target default. */
2548 unsigned HOST_WIDE_INT max_scalarization_size
2549 = global_options_set.x_param_values[param]
2550 ? PARAM_VALUE (param)
2551 : get_move_ratio (optimize_speed_p) * UNITS_PER_WORD;
2553 max_scalarization_size *= BITS_PER_UNIT;
2555 EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
2556 if (bitmap_bit_p (should_scalarize_away_bitmap, i)
2557 && !bitmap_bit_p (cannot_scalarize_away_bitmap, i))
2559 tree var = candidate (i);
2561 if (TREE_CODE (var) == VAR_DECL
2562 && type_consists_of_records_p (TREE_TYPE (var)))
2564 if (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var)))
2565 <= max_scalarization_size)
2567 completely_scalarize_var (var);
2568 if (dump_file && (dump_flags & TDF_DETAILS))
2570 fprintf (dump_file, "Will attempt to totally scalarize ");
2571 print_generic_expr (dump_file, var, 0);
2572 fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
2575 else if (dump_file && (dump_flags & TDF_DETAILS))
2577 fprintf (dump_file, "Too big to totally scalarize: ");
2578 print_generic_expr (dump_file, var, 0);
2579 fprintf (dump_file, " (UID: %u)\n", DECL_UID (var));
2584 bitmap_copy (tmp, candidate_bitmap);
2585 EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
2587 tree var = candidate (i);
2588 struct access *access;
2590 access = sort_and_splice_var_accesses (var);
2591 if (!access || !build_access_trees (access))
2592 disqualify_candidate (var,
2593 "No or inhibitingly overlapping accesses.");
2596 propagate_all_subaccesses ();
2598 bitmap_copy (tmp, candidate_bitmap);
2599 EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
2601 tree var = candidate (i);
2602 struct access *access = get_first_repr_for_decl (var);
2604 if (analyze_access_trees (access))
2607 if (dump_file && (dump_flags & TDF_DETAILS))
2609 fprintf (dump_file, "\nAccess trees for ");
2610 print_generic_expr (dump_file, var, 0);
2611 fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
2612 dump_access_tree (dump_file, access);
2613 fprintf (dump_file, "\n");
2617 disqualify_candidate (var, "No scalar replacements to be created.");
2624 statistics_counter_event (cfun, "Scalarized aggregates", res);
2631 /* Generate statements copying scalar replacements of accesses within a subtree
2632 into or out of AGG. ACCESS, all its children, siblings and their children
2633 are to be processed. AGG is an aggregate type expression (can be a
2634 declaration but does not have to be, it can for example also be a mem_ref or
2635 a series of handled components). TOP_OFFSET is the offset of the processed
2636 subtree which has to be subtracted from offsets of individual accesses to
2637 get corresponding offsets for AGG. If CHUNK_SIZE is non-null, copy only
2638 replacements in the interval <start_offset, start_offset + chunk_size>,
2639 otherwise copy all. GSI is a statement iterator used to place the new
2640 statements. WRITE should be true when the statements should write from AGG
2641 to the replacement and false if vice versa. if INSERT_AFTER is true, new
2642 statements will be added after the current statement in GSI, they will be
2643 added before the statement otherwise. */
2646 generate_subtree_copies (struct access *access, tree agg,
2647 HOST_WIDE_INT top_offset,
2648 HOST_WIDE_INT start_offset, HOST_WIDE_INT chunk_size,
2649 gimple_stmt_iterator *gsi, bool write,
2650 bool insert_after, location_t loc)
2654 if (chunk_size && access->offset >= start_offset + chunk_size)
2657 if (access->grp_to_be_replaced
2659 || access->offset + access->size > start_offset))
2661 tree expr, repl = get_access_replacement (access);
2664 expr = build_ref_for_model (loc, agg, access->offset - top_offset,
2665 access, gsi, insert_after);
2669 if (access->grp_partial_lhs)
2670 expr = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE,
2672 insert_after ? GSI_NEW_STMT
2674 stmt = gimple_build_assign (repl, expr);
2678 TREE_NO_WARNING (repl) = 1;
2679 if (access->grp_partial_lhs)
2680 repl = force_gimple_operand_gsi (gsi, repl, true, NULL_TREE,
2682 insert_after ? GSI_NEW_STMT
2684 stmt = gimple_build_assign (expr, repl);
2686 gimple_set_location (stmt, loc);
2689 gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
2691 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
2693 sra_stats.subtree_copies++;
2696 && access->grp_to_be_debug_replaced
2698 || access->offset + access->size > start_offset))
2701 tree drhs = build_debug_ref_for_model (loc, agg,
2702 access->offset - top_offset,
2704 ds = gimple_build_debug_bind (get_access_replacement (access),
2705 drhs, gsi_stmt (*gsi));
2707 gsi_insert_after (gsi, ds, GSI_NEW_STMT);
2709 gsi_insert_before (gsi, ds, GSI_SAME_STMT);
2712 if (access->first_child)
2713 generate_subtree_copies (access->first_child, agg, top_offset,
2714 start_offset, chunk_size, gsi,
2715 write, insert_after, loc);
2717 access = access->next_sibling;
2722 /* Assign zero to all scalar replacements in an access subtree. ACCESS is the
2723 the root of the subtree to be processed. GSI is the statement iterator used
2724 for inserting statements which are added after the current statement if
2725 INSERT_AFTER is true or before it otherwise. */
2728 init_subtree_with_zero (struct access *access, gimple_stmt_iterator *gsi,
2729 bool insert_after, location_t loc)
2732 struct access *child;
2734 if (access->grp_to_be_replaced)
2738 stmt = gimple_build_assign (get_access_replacement (access),
2739 build_zero_cst (access->type));
2741 gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
2743 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
2745 gimple_set_location (stmt, loc);
2747 else if (access->grp_to_be_debug_replaced)
2750 = gimple_build_debug_bind (get_access_replacement (access),
2751 build_zero_cst (access->type),
2754 gsi_insert_after (gsi, ds, GSI_NEW_STMT);
2756 gsi_insert_before (gsi, ds, GSI_SAME_STMT);
2759 for (child = access->first_child; child; child = child->next_sibling)
2760 init_subtree_with_zero (child, gsi, insert_after, loc);
2763 /* Clobber all scalar replacements in an access subtree. ACCESS is the the
2764 root of the subtree to be processed. GSI is the statement iterator used
2765 for inserting statements which are added after the current statement if
2766 INSERT_AFTER is true or before it otherwise. */
2769 clobber_subtree (struct access *access, gimple_stmt_iterator *gsi,
2770 bool insert_after, location_t loc)
2773 struct access *child;
2775 if (access->grp_to_be_replaced)
2777 tree rep = get_access_replacement (access);
2778 tree clobber = build_constructor (access->type, NULL);
2779 TREE_THIS_VOLATILE (clobber) = 1;
2780 gimple stmt = gimple_build_assign (rep, clobber);
2783 gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
2785 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
2787 gimple_set_location (stmt, loc);
2790 for (child = access->first_child; child; child = child->next_sibling)
2791 clobber_subtree (child, gsi, insert_after, loc);
2794 /* Search for an access representative for the given expression EXPR and
2795 return it or NULL if it cannot be found. */
2797 static struct access *
2798 get_access_for_expr (tree expr)
2800 HOST_WIDE_INT offset, size, max_size;
2803 /* FIXME: This should not be necessary but Ada produces V_C_Es with a type of
2804 a different size than the size of its argument and we need the latter
2806 if (TREE_CODE (expr) == VIEW_CONVERT_EXPR)
2807 expr = TREE_OPERAND (expr, 0);
2809 base = get_ref_base_and_extent (expr, &offset, &size, &max_size);
2810 if (max_size == -1 || !DECL_P (base))
2813 if (!bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
2816 return get_var_base_offset_size_access (base, offset, max_size);
2819 /* Replace the expression EXPR with a scalar replacement if there is one and
2820 generate other statements to do type conversion or subtree copying if
2821 necessary. GSI is used to place newly created statements, WRITE is true if
2822 the expression is being written to (it is on a LHS of a statement or output
2823 in an assembly statement). */
2826 sra_modify_expr (tree *expr, gimple_stmt_iterator *gsi, bool write)
2829 struct access *access;
2830 tree type, bfr, orig_expr;
2832 if (TREE_CODE (*expr) == BIT_FIELD_REF)
2835 expr = &TREE_OPERAND (*expr, 0);
2840 if (TREE_CODE (*expr) == REALPART_EXPR || TREE_CODE (*expr) == IMAGPART_EXPR)
2841 expr = &TREE_OPERAND (*expr, 0);
2842 access = get_access_for_expr (*expr);
2845 type = TREE_TYPE (*expr);
2848 loc = gimple_location (gsi_stmt (*gsi));
2849 gimple_stmt_iterator alt_gsi = gsi_none ();
2850 if (write && stmt_ends_bb_p (gsi_stmt (*gsi)))
2852 alt_gsi = gsi_start_edge (single_non_eh_succ (gsi_bb (*gsi)));
2856 if (access->grp_to_be_replaced)
2858 tree repl = get_access_replacement (access);
2859 /* If we replace a non-register typed access simply use the original
2860 access expression to extract the scalar component afterwards.
2861 This happens if scalarizing a function return value or parameter
2862 like in gcc.c-torture/execute/20041124-1.c, 20050316-1.c and
2863 gcc.c-torture/compile/20011217-1.c.
2865 We also want to use this when accessing a complex or vector which can
2866 be accessed as a different type too, potentially creating a need for
2867 type conversion (see PR42196) and when scalarized unions are involved
2868 in assembler statements (see PR42398). */
2869 if (!useless_type_conversion_p (type, access->type))
2873 ref = build_ref_for_model (loc, orig_expr, 0, access, gsi, false);
2879 if (access->grp_partial_lhs)
2880 ref = force_gimple_operand_gsi (gsi, ref, true, NULL_TREE,
2881 false, GSI_NEW_STMT);
2882 stmt = gimple_build_assign (repl, ref);
2883 gimple_set_location (stmt, loc);
2884 gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
2890 if (access->grp_partial_lhs)
2891 repl = force_gimple_operand_gsi (gsi, repl, true, NULL_TREE,
2892 true, GSI_SAME_STMT);
2893 stmt = gimple_build_assign (ref, repl);
2894 gimple_set_location (stmt, loc);
2895 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
2902 else if (write && access->grp_to_be_debug_replaced)
2904 gdebug *ds = gimple_build_debug_bind (get_access_replacement (access),
2907 gsi_insert_after (gsi, ds, GSI_NEW_STMT);
2910 if (access->first_child)
2912 HOST_WIDE_INT start_offset, chunk_size;
2914 && tree_fits_uhwi_p (TREE_OPERAND (bfr, 1))
2915 && tree_fits_uhwi_p (TREE_OPERAND (bfr, 2)))
2917 chunk_size = tree_to_uhwi (TREE_OPERAND (bfr, 1));
2918 start_offset = access->offset
2919 + tree_to_uhwi (TREE_OPERAND (bfr, 2));
2922 start_offset = chunk_size = 0;
2924 generate_subtree_copies (access->first_child, orig_expr, access->offset,
2925 start_offset, chunk_size, gsi, write, write,
2931 /* Where scalar replacements of the RHS have been written to when a replacement
2932 of a LHS of an assigments cannot be direclty loaded from a replacement of
2934 enum unscalarized_data_handling { SRA_UDH_NONE, /* Nothing done so far. */
2935 SRA_UDH_RIGHT, /* Data flushed to the RHS. */
2936 SRA_UDH_LEFT }; /* Data flushed to the LHS. */
2938 struct subreplacement_assignment_data
2940 /* Offset of the access representing the lhs of the assignment. */
2941 HOST_WIDE_INT left_offset;
2943 /* LHS and RHS of the original assignment. */
2944 tree assignment_lhs, assignment_rhs;
2946 /* Access representing the rhs of the whole assignment. */
2947 struct access *top_racc;
2949 /* Stmt iterator used for statement insertions after the original assignment.
2950 It points to the main GSI used to traverse a BB during function body
2952 gimple_stmt_iterator *new_gsi;
2954 /* Stmt iterator used for statement insertions before the original
2955 assignment. Keeps on pointing to the original statement. */
2956 gimple_stmt_iterator old_gsi;
2958 /* Location of the assignment. */
2961 /* Keeps the information whether we have needed to refresh replacements of
2962 the LHS and from which side of the assignments this takes place. */
2963 enum unscalarized_data_handling refreshed;
2966 /* Store all replacements in the access tree rooted in TOP_RACC either to their
2967 base aggregate if there are unscalarized data or directly to LHS of the
2968 statement that is pointed to by GSI otherwise. */
2971 handle_unscalarized_data_in_subtree (struct subreplacement_assignment_data *sad)
2974 if (sad->top_racc->grp_unscalarized_data)
2976 src = sad->assignment_rhs;
2977 sad->refreshed = SRA_UDH_RIGHT;
2981 src = sad->assignment_lhs;
2982 sad->refreshed = SRA_UDH_LEFT;
2984 generate_subtree_copies (sad->top_racc->first_child, src,
2985 sad->top_racc->offset, 0, 0,
2986 &sad->old_gsi, false, false, sad->loc);
2989 /* Try to generate statements to load all sub-replacements in an access subtree
2990 formed by children of LACC from scalar replacements in the SAD->top_racc
2991 subtree. If that is not possible, refresh the SAD->top_racc base aggregate
2992 and load the accesses from it. */
2995 load_assign_lhs_subreplacements (struct access *lacc,
2996 struct subreplacement_assignment_data *sad)
2998 for (lacc = lacc->first_child; lacc; lacc = lacc->next_sibling)
3000 HOST_WIDE_INT offset;
3001 offset = lacc->offset - sad->left_offset + sad->top_racc->offset;
3003 if (lacc->grp_to_be_replaced)
3005 struct access *racc;
3009 racc = find_access_in_subtree (sad->top_racc, offset, lacc->size);
3010 if (racc && racc->grp_to_be_replaced)
3012 rhs = get_access_replacement (racc);
3013 if (!useless_type_conversion_p (lacc->type, racc->type))
3014 rhs = fold_build1_loc (sad->loc, VIEW_CONVERT_EXPR,
3017 if (racc->grp_partial_lhs && lacc->grp_partial_lhs)
3018 rhs = force_gimple_operand_gsi (&sad->old_gsi, rhs, true,
3019 NULL_TREE, true, GSI_SAME_STMT);
3023 /* No suitable access on the right hand side, need to load from
3024 the aggregate. See if we have to update it first... */
3025 if (sad->refreshed == SRA_UDH_NONE)
3026 handle_unscalarized_data_in_subtree (sad);
3028 if (sad->refreshed == SRA_UDH_LEFT)
3029 rhs = build_ref_for_model (sad->loc, sad->assignment_lhs,
3030 lacc->offset - sad->left_offset,
3031 lacc, sad->new_gsi, true);
3033 rhs = build_ref_for_model (sad->loc, sad->assignment_rhs,
3034 lacc->offset - sad->left_offset,
3035 lacc, sad->new_gsi, true);
3036 if (lacc->grp_partial_lhs)
3037 rhs = force_gimple_operand_gsi (sad->new_gsi,
3038 rhs, true, NULL_TREE,
3039 false, GSI_NEW_STMT);
3042 stmt = gimple_build_assign (get_access_replacement (lacc), rhs);
3043 gsi_insert_after (sad->new_gsi, stmt, GSI_NEW_STMT);
3044 gimple_set_location (stmt, sad->loc);
3046 sra_stats.subreplacements++;
3050 if (sad->refreshed == SRA_UDH_NONE
3051 && lacc->grp_read && !lacc->grp_covered)
3052 handle_unscalarized_data_in_subtree (sad);
3054 if (lacc && lacc->grp_to_be_debug_replaced)
3058 struct access *racc = find_access_in_subtree (sad->top_racc,
3062 if (racc && racc->grp_to_be_replaced)
3064 if (racc->grp_write)
3065 drhs = get_access_replacement (racc);
3069 else if (sad->refreshed == SRA_UDH_LEFT)
3070 drhs = build_debug_ref_for_model (sad->loc, lacc->base,
3071 lacc->offset, lacc);
3072 else if (sad->refreshed == SRA_UDH_RIGHT)
3073 drhs = build_debug_ref_for_model (sad->loc, sad->top_racc->base,
3078 && !useless_type_conversion_p (lacc->type, TREE_TYPE (drhs)))
3079 drhs = fold_build1_loc (sad->loc, VIEW_CONVERT_EXPR,
3081 ds = gimple_build_debug_bind (get_access_replacement (lacc),
3082 drhs, gsi_stmt (sad->old_gsi));
3083 gsi_insert_after (sad->new_gsi, ds, GSI_NEW_STMT);
3087 if (lacc->first_child)
3088 load_assign_lhs_subreplacements (lacc, sad);
3092 /* Result code for SRA assignment modification. */
3093 enum assignment_mod_result { SRA_AM_NONE, /* nothing done for the stmt */
3094 SRA_AM_MODIFIED, /* stmt changed but not
3096 SRA_AM_REMOVED }; /* stmt eliminated */
3098 /* Modify assignments with a CONSTRUCTOR on their RHS. STMT contains a pointer
3099 to the assignment and GSI is the statement iterator pointing at it. Returns
3100 the same values as sra_modify_assign. */
3102 static enum assignment_mod_result
3103 sra_modify_constructor_assign (gimple stmt, gimple_stmt_iterator *gsi)
3105 tree lhs = gimple_assign_lhs (stmt);
3106 struct access *acc = get_access_for_expr (lhs);
3109 location_t loc = gimple_location (stmt);
3111 if (gimple_clobber_p (stmt))
3113 /* Clobber the replacement variable. */
3114 clobber_subtree (acc, gsi, !acc->grp_covered, loc);
3115 /* Remove clobbers of fully scalarized variables, they are dead. */
3116 if (acc->grp_covered)
3118 unlink_stmt_vdef (stmt);
3119 gsi_remove (gsi, true);
3120 release_defs (stmt);
3121 return SRA_AM_REMOVED;
3124 return SRA_AM_MODIFIED;
3127 if (vec_safe_length (CONSTRUCTOR_ELTS (gimple_assign_rhs1 (stmt))) > 0)
3129 /* I have never seen this code path trigger but if it can happen the
3130 following should handle it gracefully. */
3131 if (access_has_children_p (acc))
3132 generate_subtree_copies (acc->first_child, lhs, acc->offset, 0, 0, gsi,
3134 return SRA_AM_MODIFIED;
3137 if (acc->grp_covered)
3139 init_subtree_with_zero (acc, gsi, false, loc);
3140 unlink_stmt_vdef (stmt);
3141 gsi_remove (gsi, true);
3142 release_defs (stmt);
3143 return SRA_AM_REMOVED;
3147 init_subtree_with_zero (acc, gsi, true, loc);
3148 return SRA_AM_MODIFIED;
3152 /* Create and return a new suitable default definition SSA_NAME for RACC which
3153 is an access describing an uninitialized part of an aggregate that is being
3157 get_repl_default_def_ssa_name (struct access *racc)
3159 gcc_checking_assert (!racc->grp_to_be_replaced
3160 && !racc->grp_to_be_debug_replaced);
3161 if (!racc->replacement_decl)
3162 racc->replacement_decl = create_access_replacement (racc);
3163 return get_or_create_ssa_default_def (cfun, racc->replacement_decl);
3166 /* Return true if REF has an VIEW_CONVERT_EXPR or a COMPONENT_REF with a
3167 bit-field field declaration somewhere in it. */
3170 contains_vce_or_bfcref_p (const_tree ref)
3172 while (handled_component_p (ref))
3174 if (TREE_CODE (ref) == VIEW_CONVERT_EXPR
3175 || (TREE_CODE (ref) == COMPONENT_REF
3176 && DECL_BIT_FIELD (TREE_OPERAND (ref, 1))))
3178 ref = TREE_OPERAND (ref, 0);
3184 /* Examine both sides of the assignment statement pointed to by STMT, replace
3185 them with a scalare replacement if there is one and generate copying of
3186 replacements if scalarized aggregates have been used in the assignment. GSI
3187 is used to hold generated statements for type conversions and subtree
3190 static enum assignment_mod_result
3191 sra_modify_assign (gimple stmt, gimple_stmt_iterator *gsi)
3193 struct access *lacc, *racc;
3195 bool modify_this_stmt = false;
3196 bool force_gimple_rhs = false;
3198 gimple_stmt_iterator orig_gsi = *gsi;
3200 if (!gimple_assign_single_p (stmt))
3202 lhs = gimple_assign_lhs (stmt);
3203 rhs = gimple_assign_rhs1 (stmt);
3205 if (TREE_CODE (rhs) == CONSTRUCTOR)
3206 return sra_modify_constructor_assign (stmt, gsi);
3208 if (TREE_CODE (rhs) == REALPART_EXPR || TREE_CODE (lhs) == REALPART_EXPR
3209 || TREE_CODE (rhs) == IMAGPART_EXPR || TREE_CODE (lhs) == IMAGPART_EXPR
3210 || TREE_CODE (rhs) == BIT_FIELD_REF || TREE_CODE (lhs) == BIT_FIELD_REF)
3212 modify_this_stmt = sra_modify_expr (gimple_assign_rhs1_ptr (stmt),
3214 modify_this_stmt |= sra_modify_expr (gimple_assign_lhs_ptr (stmt),
3216 return modify_this_stmt ? SRA_AM_MODIFIED : SRA_AM_NONE;
3219 lacc = get_access_for_expr (lhs);
3220 racc = get_access_for_expr (rhs);
3224 loc = gimple_location (stmt);
3225 if (lacc && lacc->grp_to_be_replaced)
3227 lhs = get_access_replacement (lacc);
3228 gimple_assign_set_lhs (stmt, lhs);
3229 modify_this_stmt = true;
3230 if (lacc->grp_partial_lhs)
3231 force_gimple_rhs = true;
3235 if (racc && racc->grp_to_be_replaced)
3237 rhs = get_access_replacement (racc);
3238 modify_this_stmt = true;
3239 if (racc->grp_partial_lhs)
3240 force_gimple_rhs = true;
3244 && !racc->grp_unscalarized_data
3245 && !racc->grp_unscalarizable_region
3246 && TREE_CODE (lhs) == SSA_NAME
3247 && !access_has_replacements_p (racc))
3249 rhs = get_repl_default_def_ssa_name (racc);
3250 modify_this_stmt = true;
3254 if (modify_this_stmt)
3256 if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
3258 /* If we can avoid creating a VIEW_CONVERT_EXPR do so.
3259 ??? This should move to fold_stmt which we simply should
3260 call after building a VIEW_CONVERT_EXPR here. */
3261 if (AGGREGATE_TYPE_P (TREE_TYPE (lhs))
3262 && !contains_bitfld_component_ref_p (lhs))
3264 lhs = build_ref_for_model (loc, lhs, 0, racc, gsi, false);
3265 gimple_assign_set_lhs (stmt, lhs);
3267 else if (AGGREGATE_TYPE_P (TREE_TYPE (rhs))
3268 && !contains_vce_or_bfcref_p (rhs))
3269 rhs = build_ref_for_model (loc, rhs, 0, lacc, gsi, false);
3271 if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
3273 rhs = fold_build1_loc (loc, VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
3275 if (is_gimple_reg_type (TREE_TYPE (lhs))
3276 && TREE_CODE (lhs) != SSA_NAME)
3277 force_gimple_rhs = true;
3282 if (lacc && lacc->grp_to_be_debug_replaced)
3284 tree dlhs = get_access_replacement (lacc);
3285 tree drhs = unshare_expr (rhs);
3286 if (!useless_type_conversion_p (TREE_TYPE (dlhs), TREE_TYPE (drhs)))
3288 if (AGGREGATE_TYPE_P (TREE_TYPE (drhs))
3289 && !contains_vce_or_bfcref_p (drhs))
3290 drhs = build_debug_ref_for_model (loc, drhs, 0, lacc);
3292 && !useless_type_conversion_p (TREE_TYPE (dlhs),
3294 drhs = fold_build1_loc (loc, VIEW_CONVERT_EXPR,
3295 TREE_TYPE (dlhs), drhs);
3297 gdebug *ds = gimple_build_debug_bind (dlhs, drhs, stmt);
3298 gsi_insert_before (gsi, ds, GSI_SAME_STMT);
3301 /* From this point on, the function deals with assignments in between
3302 aggregates when at least one has scalar reductions of some of its
3303 components. There are three possible scenarios: Both the LHS and RHS have
3304 to-be-scalarized components, 2) only the RHS has or 3) only the LHS has.
3306 In the first case, we would like to load the LHS components from RHS
3307 components whenever possible. If that is not possible, we would like to
3308 read it directly from the RHS (after updating it by storing in it its own
3309 components). If there are some necessary unscalarized data in the LHS,
3310 those will be loaded by the original assignment too. If neither of these
3311 cases happen, the original statement can be removed. Most of this is done
3312 by load_assign_lhs_subreplacements.
3314 In the second case, we would like to store all RHS scalarized components
3315 directly into LHS and if they cover the aggregate completely, remove the
3316 statement too. In the third case, we want the LHS components to be loaded
3317 directly from the RHS (DSE will remove the original statement if it
3320 This is a bit complex but manageable when types match and when unions do
3321 not cause confusion in a way that we cannot really load a component of LHS
3322 from the RHS or vice versa (the access representing this level can have
3323 subaccesses that are accessible only through a different union field at a
3324 higher level - different from the one used in the examined expression).
3327 Therefore, I specially handle a fourth case, happening when there is a
3328 specific type cast or it is impossible to locate a scalarized subaccess on
3329 the other side of the expression. If that happens, I simply "refresh" the
3330 RHS by storing in it is scalarized components leave the original statement
3331 there to do the copying and then load the scalar replacements of the LHS.
3332 This is what the first branch does. */
3334 if (modify_this_stmt
3335 || gimple_has_volatile_ops (stmt)
3336 || contains_vce_or_bfcref_p (rhs)
3337 || contains_vce_or_bfcref_p (lhs)
3338 || stmt_ends_bb_p (stmt))
3340 if (access_has_children_p (racc))
3341 generate_subtree_copies (racc->first_child, rhs, racc->offset, 0, 0,
3342 gsi, false, false, loc);
3343 if (access_has_children_p (lacc))
3345 gimple_stmt_iterator alt_gsi = gsi_none ();
3346 if (stmt_ends_bb_p (stmt))
3348 alt_gsi = gsi_start_edge (single_non_eh_succ (gsi_bb (*gsi)));
3351 generate_subtree_copies (lacc->first_child, lhs, lacc->offset, 0, 0,
3352 gsi, true, true, loc);
3354 sra_stats.separate_lhs_rhs_handling++;
3356 /* This gimplification must be done after generate_subtree_copies,
3357 lest we insert the subtree copies in the middle of the gimplified
3359 if (force_gimple_rhs)
3360 rhs = force_gimple_operand_gsi (&orig_gsi, rhs, true, NULL_TREE,
3361 true, GSI_SAME_STMT);
3362 if (gimple_assign_rhs1 (stmt) != rhs)
3364 modify_this_stmt = true;
3365 gimple_assign_set_rhs_from_tree (&orig_gsi, rhs);
3366 gcc_assert (stmt == gsi_stmt (orig_gsi));
3369 return modify_this_stmt ? SRA_AM_MODIFIED : SRA_AM_NONE;
3373 if (access_has_children_p (lacc)
3374 && access_has_children_p (racc)
3375 /* When an access represents an unscalarizable region, it usually
3376 represents accesses with variable offset and thus must not be used
3377 to generate new memory accesses. */
3378 && !lacc->grp_unscalarizable_region
3379 && !racc->grp_unscalarizable_region)
3381 struct subreplacement_assignment_data sad;
3383 sad.left_offset = lacc->offset;
3384 sad.assignment_lhs = lhs;
3385 sad.assignment_rhs = rhs;
3386 sad.top_racc = racc;
3389 sad.loc = gimple_location (stmt);
3390 sad.refreshed = SRA_UDH_NONE;
3392 if (lacc->grp_read && !lacc->grp_covered)
3393 handle_unscalarized_data_in_subtree (&sad);
3395 load_assign_lhs_subreplacements (lacc, &sad);
3396 if (sad.refreshed != SRA_UDH_RIGHT)
3399 unlink_stmt_vdef (stmt);
3400 gsi_remove (&sad.old_gsi, true);
3401 release_defs (stmt);
3402 sra_stats.deleted++;
3403 return SRA_AM_REMOVED;
3408 if (access_has_children_p (racc)
3409 && !racc->grp_unscalarized_data
3410 && TREE_CODE (lhs) != SSA_NAME)
3414 fprintf (dump_file, "Removing load: ");
3415 print_gimple_stmt (dump_file, stmt, 0, 0);
3417 generate_subtree_copies (racc->first_child, lhs,
3418 racc->offset, 0, 0, gsi,
3420 gcc_assert (stmt == gsi_stmt (*gsi));
3421 unlink_stmt_vdef (stmt);
3422 gsi_remove (gsi, true);
3423 release_defs (stmt);
3424 sra_stats.deleted++;
3425 return SRA_AM_REMOVED;
3427 /* Restore the aggregate RHS from its components so the
3428 prevailing aggregate copy does the right thing. */
3429 if (access_has_children_p (racc))
3430 generate_subtree_copies (racc->first_child, rhs, racc->offset, 0, 0,
3431 gsi, false, false, loc);
3432 /* Re-load the components of the aggregate copy destination.
3433 But use the RHS aggregate to load from to expose more
3434 optimization opportunities. */
3435 if (access_has_children_p (lacc))
3436 generate_subtree_copies (lacc->first_child, rhs, lacc->offset,
3437 0, 0, gsi, true, true, loc);
3444 /* Traverse the function body and all modifications as decided in
3445 analyze_all_variable_accesses. Return true iff the CFG has been
3449 sra_modify_function_body (void)
3451 bool cfg_changed = false;
3454 FOR_EACH_BB_FN (bb, cfun)
3456 gimple_stmt_iterator gsi = gsi_start_bb (bb);
3457 while (!gsi_end_p (gsi))
3459 gimple stmt = gsi_stmt (gsi);
3460 enum assignment_mod_result assign_result;
3461 bool modified = false, deleted = false;
3465 switch (gimple_code (stmt))
3468 t = gimple_return_retval_ptr (as_a <greturn *> (stmt));
3469 if (*t != NULL_TREE)
3470 modified |= sra_modify_expr (t, &gsi, false);
3474 assign_result = sra_modify_assign (stmt, &gsi);
3475 modified |= assign_result == SRA_AM_MODIFIED;
3476 deleted = assign_result == SRA_AM_REMOVED;
3480 /* Operands must be processed before the lhs. */
3481 for (i = 0; i < gimple_call_num_args (stmt); i++)
3483 t = gimple_call_arg_ptr (stmt, i);
3484 modified |= sra_modify_expr (t, &gsi, false);
3487 if (gimple_call_lhs (stmt))
3489 t = gimple_call_lhs_ptr (stmt);
3490 modified |= sra_modify_expr (t, &gsi, true);
3496 gasm *asm_stmt = as_a <gasm *> (stmt);
3497 for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
3499 t = &TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
3500 modified |= sra_modify_expr (t, &gsi, false);
3502 for (i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
3504 t = &TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
3505 modified |= sra_modify_expr (t, &gsi, true);
3517 if (maybe_clean_eh_stmt (stmt)
3518 && gimple_purge_dead_eh_edges (gimple_bb (stmt)))
3526 gsi_commit_edge_inserts ();
3530 /* Generate statements initializing scalar replacements of parts of function
3534 initialize_parameter_reductions (void)
3536 gimple_stmt_iterator gsi;
3537 gimple_seq seq = NULL;
3540 gsi = gsi_start (seq);
3541 for (parm = DECL_ARGUMENTS (current_function_decl);
3543 parm = DECL_CHAIN (parm))
3545 vec<access_p> *access_vec;
3546 struct access *access;
3548 if (!bitmap_bit_p (candidate_bitmap, DECL_UID (parm)))
3550 access_vec = get_base_access_vector (parm);
3554 for (access = (*access_vec)[0];
3556 access = access->next_grp)
3557 generate_subtree_copies (access, parm, 0, 0, 0, &gsi, true, true,
3558 EXPR_LOCATION (parm));
3561 seq = gsi_seq (gsi);
3563 gsi_insert_seq_on_edge_immediate (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
3566 /* The "main" function of intraprocedural SRA passes. Runs the analysis and if
3567 it reveals there are components of some aggregates to be scalarized, it runs
3568 the required transformations. */
3570 perform_intra_sra (void)
3575 if (!find_var_candidates ())
3578 if (!scan_function ())
3581 if (!analyze_all_variable_accesses ())
3584 if (sra_modify_function_body ())
3585 ret = TODO_update_ssa | TODO_cleanup_cfg;
3587 ret = TODO_update_ssa;
3588 initialize_parameter_reductions ();
3590 statistics_counter_event (cfun, "Scalar replacements created",
3591 sra_stats.replacements);
3592 statistics_counter_event (cfun, "Modified expressions", sra_stats.exprs);
3593 statistics_counter_event (cfun, "Subtree copy stmts",
3594 sra_stats.subtree_copies);
3595 statistics_counter_event (cfun, "Subreplacement stmts",
3596 sra_stats.subreplacements);
3597 statistics_counter_event (cfun, "Deleted stmts", sra_stats.deleted);
3598 statistics_counter_event (cfun, "Separate LHS and RHS handling",
3599 sra_stats.separate_lhs_rhs_handling);
3602 sra_deinitialize ();
3606 /* Perform early intraprocedural SRA. */
3608 early_intra_sra (void)
3610 sra_mode = SRA_MODE_EARLY_INTRA;
3611 return perform_intra_sra ();
3614 /* Perform "late" intraprocedural SRA. */
3616 late_intra_sra (void)
3618 sra_mode = SRA_MODE_INTRA;
3619 return perform_intra_sra ();
3624 gate_intra_sra (void)
3626 return flag_tree_sra != 0 && dbg_cnt (tree_sra);
3632 const pass_data pass_data_sra_early =
3634 GIMPLE_PASS, /* type */
3636 OPTGROUP_NONE, /* optinfo_flags */
3637 TV_TREE_SRA, /* tv_id */
3638 ( PROP_cfg | PROP_ssa ), /* properties_required */
3639 0, /* properties_provided */
3640 0, /* properties_destroyed */
3641 0, /* todo_flags_start */
3642 TODO_update_ssa, /* todo_flags_finish */
3645 class pass_sra_early : public gimple_opt_pass
3648 pass_sra_early (gcc::context *ctxt)
3649 : gimple_opt_pass (pass_data_sra_early, ctxt)
3652 /* opt_pass methods: */
3653 virtual bool gate (function *) { return gate_intra_sra (); }
3654 virtual unsigned int execute (function *) { return early_intra_sra (); }
3656 }; // class pass_sra_early
3661 make_pass_sra_early (gcc::context *ctxt)
3663 return new pass_sra_early (ctxt);
3668 const pass_data pass_data_sra =
3670 GIMPLE_PASS, /* type */
3672 OPTGROUP_NONE, /* optinfo_flags */
3673 TV_TREE_SRA, /* tv_id */
3674 ( PROP_cfg | PROP_ssa ), /* properties_required */
3675 0, /* properties_provided */
3676 0, /* properties_destroyed */
3677 TODO_update_address_taken, /* todo_flags_start */
3678 TODO_update_ssa, /* todo_flags_finish */
3681 class pass_sra : public gimple_opt_pass
3684 pass_sra (gcc::context *ctxt)
3685 : gimple_opt_pass (pass_data_sra, ctxt)
3688 /* opt_pass methods: */
3689 virtual bool gate (function *) { return gate_intra_sra (); }
3690 virtual unsigned int execute (function *) { return late_intra_sra (); }
3692 }; // class pass_sra
3697 make_pass_sra (gcc::context *ctxt)
3699 return new pass_sra (ctxt);
3703 /* Return true iff PARM (which must be a parm_decl) is an unused scalar
3707 is_unused_scalar_param (tree parm)
3710 return (is_gimple_reg (parm)
3711 && (!(name = ssa_default_def (cfun, parm))
3712 || has_zero_uses (name)));
3715 /* Scan immediate uses of a default definition SSA name of a parameter PARM and
3716 examine whether there are any direct or otherwise infeasible ones. If so,
3717 return true, otherwise return false. PARM must be a gimple register with a
3718 non-NULL default definition. */
3721 ptr_parm_has_direct_uses (tree parm)
3723 imm_use_iterator ui;
3725 tree name = ssa_default_def (cfun, parm);
3728 FOR_EACH_IMM_USE_STMT (stmt, ui, name)
3731 use_operand_p use_p;
3733 if (is_gimple_debug (stmt))
3736 /* Valid uses include dereferences on the lhs and the rhs. */
3737 if (gimple_has_lhs (stmt))
3739 tree lhs = gimple_get_lhs (stmt);
3740 while (handled_component_p (lhs))
3741 lhs = TREE_OPERAND (lhs, 0);
3742 if (TREE_CODE (lhs) == MEM_REF
3743 && TREE_OPERAND (lhs, 0) == name
3744 && integer_zerop (TREE_OPERAND (lhs, 1))
3745 && types_compatible_p (TREE_TYPE (lhs),
3746 TREE_TYPE (TREE_TYPE (name)))
3747 && !TREE_THIS_VOLATILE (lhs))
3750 if (gimple_assign_single_p (stmt))
3752 tree rhs = gimple_assign_rhs1 (stmt);
3753 while (handled_component_p (rhs))
3754 rhs = TREE_OPERAND (rhs, 0);
3755 if (TREE_CODE (rhs) == MEM_REF
3756 && TREE_OPERAND (rhs, 0) == name
3757 && integer_zerop (TREE_OPERAND (rhs, 1))
3758 && types_compatible_p (TREE_TYPE (rhs),
3759 TREE_TYPE (TREE_TYPE (name)))
3760 && !TREE_THIS_VOLATILE (rhs))
3763 else if (is_gimple_call (stmt))
3766 for (i = 0; i < gimple_call_num_args (stmt); ++i)
3768 tree arg = gimple_call_arg (stmt, i);
3769 while (handled_component_p (arg))
3770 arg = TREE_OPERAND (arg, 0);
3771 if (TREE_CODE (arg) == MEM_REF
3772 && TREE_OPERAND (arg, 0) == name
3773 && integer_zerop (TREE_OPERAND (arg, 1))
3774 && types_compatible_p (TREE_TYPE (arg),
3775 TREE_TYPE (TREE_TYPE (name)))
3776 && !TREE_THIS_VOLATILE (arg))
3781 /* If the number of valid uses does not match the number of
3782 uses in this stmt there is an unhandled use. */
3783 FOR_EACH_IMM_USE_ON_STMT (use_p, ui)
3790 BREAK_FROM_IMM_USE_STMT (ui);
3796 /* Identify candidates for reduction for IPA-SRA based on their type and mark
3797 them in candidate_bitmap. Note that these do not necessarily include
3798 parameter which are unused and thus can be removed. Return true iff any
3799 such candidate has been found. */
3802 find_param_candidates (void)
3809 for (parm = DECL_ARGUMENTS (current_function_decl);
3811 parm = DECL_CHAIN (parm))
3813 tree type = TREE_TYPE (parm);
3818 if (TREE_THIS_VOLATILE (parm)
3819 || TREE_ADDRESSABLE (parm)
3820 || (!is_gimple_reg_type (type) && is_va_list_type (type)))
3823 if (is_unused_scalar_param (parm))
3829 if (POINTER_TYPE_P (type))
3831 type = TREE_TYPE (type);
3833 if (TREE_CODE (type) == FUNCTION_TYPE
3834 || TYPE_VOLATILE (type)
3835 || (TREE_CODE (type) == ARRAY_TYPE
3836 && TYPE_NONALIASED_COMPONENT (type))
3837 || !is_gimple_reg (parm)
3838 || is_va_list_type (type)
3839 || ptr_parm_has_direct_uses (parm))
3842 else if (!AGGREGATE_TYPE_P (type))
3845 if (!COMPLETE_TYPE_P (type)
3846 || !tree_fits_uhwi_p (TYPE_SIZE (type))
3847 || tree_to_uhwi (TYPE_SIZE (type)) == 0
3848 || (AGGREGATE_TYPE_P (type)
3849 && type_internals_preclude_sra_p (type, &msg)))
3852 bitmap_set_bit (candidate_bitmap, DECL_UID (parm));
3853 slot = candidates->find_slot_with_hash (parm, DECL_UID (parm), INSERT);
3857 if (dump_file && (dump_flags & TDF_DETAILS))
3859 fprintf (dump_file, "Candidate (%d): ", DECL_UID (parm));
3860 print_generic_expr (dump_file, parm, 0);
3861 fprintf (dump_file, "\n");
3865 func_param_count = count;
3869 /* Callback of walk_aliased_vdefs, marks the access passed as DATA as
3873 mark_maybe_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef ATTRIBUTE_UNUSED,
3876 struct access *repr = (struct access *) data;
3878 repr->grp_maybe_modified = 1;
3882 /* Analyze what representatives (in linked lists accessible from
3883 REPRESENTATIVES) can be modified by side effects of statements in the
3884 current function. */
3887 analyze_modified_params (vec<access_p> representatives)
3891 for (i = 0; i < func_param_count; i++)
3893 struct access *repr;
3895 for (repr = representatives[i];
3897 repr = repr->next_grp)
3899 struct access *access;
3903 if (no_accesses_p (repr))
3905 if (!POINTER_TYPE_P (TREE_TYPE (repr->base))
3906 || repr->grp_maybe_modified)
3909 ao_ref_init (&ar, repr->expr);
3910 visited = BITMAP_ALLOC (NULL);
3911 for (access = repr; access; access = access->next_sibling)
3913 /* All accesses are read ones, otherwise grp_maybe_modified would
3914 be trivially set. */
3915 walk_aliased_vdefs (&ar, gimple_vuse (access->stmt),
3916 mark_maybe_modified, repr, &visited);
3917 if (repr->grp_maybe_modified)
3920 BITMAP_FREE (visited);
3925 /* Propagate distances in bb_dereferences in the opposite direction than the
3926 control flow edges, in each step storing the maximum of the current value
3927 and the minimum of all successors. These steps are repeated until the table
3928 stabilizes. Note that BBs which might terminate the functions (according to
3929 final_bbs bitmap) never updated in this way. */
3932 propagate_dereference_distances (void)
3936 auto_vec<basic_block> queue (last_basic_block_for_fn (cfun));
3937 queue.quick_push (ENTRY_BLOCK_PTR_FOR_FN (cfun));
3938 FOR_EACH_BB_FN (bb, cfun)
3940 queue.quick_push (bb);
3944 while (!queue.is_empty ())
3948 bool change = false;
3954 if (bitmap_bit_p (final_bbs, bb->index))
3957 for (i = 0; i < func_param_count; i++)
3959 int idx = bb->index * func_param_count + i;
3961 HOST_WIDE_INT inh = 0;
3963 FOR_EACH_EDGE (e, ei, bb->succs)
3965 int succ_idx = e->dest->index * func_param_count + i;
3967 if (e->src == EXIT_BLOCK_PTR_FOR_FN (cfun))
3973 inh = bb_dereferences [succ_idx];
3975 else if (bb_dereferences [succ_idx] < inh)
3976 inh = bb_dereferences [succ_idx];
3979 if (!first && bb_dereferences[idx] < inh)
3981 bb_dereferences[idx] = inh;
3986 if (change && !bitmap_bit_p (final_bbs, bb->index))
3987 FOR_EACH_EDGE (e, ei, bb->preds)
3992 e->src->aux = e->src;
3993 queue.quick_push (e->src);
3998 /* Dump a dereferences TABLE with heading STR to file F. */
4001 dump_dereferences_table (FILE *f, const char *str, HOST_WIDE_INT *table)
4005 fprintf (dump_file, "%s", str);
4006 FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun),
4007 EXIT_BLOCK_PTR_FOR_FN (cfun), next_bb)
4009 fprintf (f, "%4i %i ", bb->index, bitmap_bit_p (final_bbs, bb->index));
4010 if (bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
4013 for (i = 0; i < func_param_count; i++)
4015 int idx = bb->index * func_param_count + i;
4016 fprintf (f, " %4" HOST_WIDE_INT_PRINT "d", table[idx]);
4021 fprintf (dump_file, "\n");
4024 /* Determine what (parts of) parameters passed by reference that are not
4025 assigned to are not certainly dereferenced in this function and thus the
4026 dereferencing cannot be safely moved to the caller without potentially
4027 introducing a segfault. Mark such REPRESENTATIVES as
4028 grp_not_necessarilly_dereferenced.
4030 The dereferenced maximum "distance," i.e. the offset + size of the accessed
4031 part is calculated rather than simple booleans are calculated for each
4032 pointer parameter to handle cases when only a fraction of the whole
4033 aggregate is allocated (see testsuite/gcc.c-torture/execute/ipa-sra-2.c for
4036 The maximum dereference distances for each pointer parameter and BB are
4037 already stored in bb_dereference. This routine simply propagates these
4038 values upwards by propagate_dereference_distances and then compares the
4039 distances of individual parameters in the ENTRY BB to the equivalent
4040 distances of each representative of a (fraction of a) parameter. */
4043 analyze_caller_dereference_legality (vec<access_p> representatives)
4047 if (dump_file && (dump_flags & TDF_DETAILS))
4048 dump_dereferences_table (dump_file,
4049 "Dereference table before propagation:\n",
4052 propagate_dereference_distances ();
4054 if (dump_file && (dump_flags & TDF_DETAILS))
4055 dump_dereferences_table (dump_file,
4056 "Dereference table after propagation:\n",
4059 for (i = 0; i < func_param_count; i++)
4061 struct access *repr = representatives[i];
4062 int idx = ENTRY_BLOCK_PTR_FOR_FN (cfun)->index * func_param_count + i;
4064 if (!repr || no_accesses_p (repr))
4069 if ((repr->offset + repr->size) > bb_dereferences[idx])
4070 repr->grp_not_necessarilly_dereferenced = 1;
4071 repr = repr->next_grp;
4077 /* Return the representative access for the parameter declaration PARM if it is
4078 a scalar passed by reference which is not written to and the pointer value
4079 is not used directly. Thus, if it is legal to dereference it in the caller
4080 and we can rule out modifications through aliases, such parameter should be
4081 turned into one passed by value. Return NULL otherwise. */
4083 static struct access *
4084 unmodified_by_ref_scalar_representative (tree parm)
4086 int i, access_count;
4087 struct access *repr;
4088 vec<access_p> *access_vec;
4090 access_vec = get_base_access_vector (parm);
4091 gcc_assert (access_vec);
4092 repr = (*access_vec)[0];
4095 repr->group_representative = repr;
4097 access_count = access_vec->length ();
4098 for (i = 1; i < access_count; i++)
4100 struct access *access = (*access_vec)[i];
4103 access->group_representative = repr;
4104 access->next_sibling = repr->next_sibling;
4105 repr->next_sibling = access;
4109 repr->grp_scalar_ptr = 1;
4113 /* Return true iff this ACCESS precludes IPA-SRA of the parameter it is
4114 associated with. REQ_ALIGN is the minimum required alignment. */
4117 access_precludes_ipa_sra_p (struct access *access, unsigned int req_align)
4119 unsigned int exp_align;
4120 /* Avoid issues such as the second simple testcase in PR 42025. The problem
4121 is incompatible assign in a call statement (and possibly even in asm
4122 statements). This can be relaxed by using a new temporary but only for
4123 non-TREE_ADDRESSABLE types and is probably not worth the complexity. (In
4124 intraprocedural SRA we deal with this by keeping the old aggregate around,
4125 something we cannot do in IPA-SRA.) */
4127 && (is_gimple_call (access->stmt)
4128 || gimple_code (access->stmt) == GIMPLE_ASM))
4131 exp_align = get_object_alignment (access->expr);
4132 if (exp_align < req_align)
4139 /* Sort collected accesses for parameter PARM, identify representatives for
4140 each accessed region and link them together. Return NULL if there are
4141 different but overlapping accesses, return the special ptr value meaning
4142 there are no accesses for this parameter if that is the case and return the
4143 first representative otherwise. Set *RO_GRP if there is a group of accesses
4144 with only read (i.e. no write) accesses. */
4146 static struct access *
4147 splice_param_accesses (tree parm, bool *ro_grp)
4149 int i, j, access_count, group_count;
4150 int agg_size, total_size = 0;
4151 struct access *access, *res, **prev_acc_ptr = &res;
4152 vec<access_p> *access_vec;
4154 access_vec = get_base_access_vector (parm);
4156 return &no_accesses_representant;
4157 access_count = access_vec->length ();
4159 access_vec->qsort (compare_access_positions);
4164 while (i < access_count)
4168 access = (*access_vec)[i];
4169 modification = access->write;
4170 if (access_precludes_ipa_sra_p (access, TYPE_ALIGN (access->type)))
4172 a1_alias_type = reference_alias_ptr_type (access->expr);
4174 /* Access is about to become group representative unless we find some
4175 nasty overlap which would preclude us from breaking this parameter
4179 while (j < access_count)
4181 struct access *ac2 = (*access_vec)[j];
4182 if (ac2->offset != access->offset)
4184 /* All or nothing law for parameters. */
4185 if (access->offset + access->size > ac2->offset)
4190 else if (ac2->size != access->size)
4193 if (access_precludes_ipa_sra_p (ac2, TYPE_ALIGN (access->type))
4194 || (ac2->type != access->type
4195 && (TREE_ADDRESSABLE (ac2->type)
4196 || TREE_ADDRESSABLE (access->type)))
4197 || (reference_alias_ptr_type (ac2->expr) != a1_alias_type))
4200 modification |= ac2->write;
4201 ac2->group_representative = access;
4202 ac2->next_sibling = access->next_sibling;
4203 access->next_sibling = ac2;
4208 access->grp_maybe_modified = modification;
4211 *prev_acc_ptr = access;
4212 prev_acc_ptr = &access->next_grp;
4213 total_size += access->size;
4217 if (POINTER_TYPE_P (TREE_TYPE (parm)))
4218 agg_size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (parm))));
4220 agg_size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm)));
4221 if (total_size >= agg_size)
4224 gcc_assert (group_count > 0);
4228 /* Decide whether parameters with representative accesses given by REPR should
4229 be reduced into components. */
4232 decide_one_param_reduction (struct access *repr)
4234 int total_size, cur_parm_size, agg_size, new_param_count, parm_size_limit;
4239 cur_parm_size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm)));
4240 gcc_assert (cur_parm_size > 0);
4242 if (POINTER_TYPE_P (TREE_TYPE (parm)))
4245 agg_size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (parm))));
4250 agg_size = cur_parm_size;
4256 fprintf (dump_file, "Evaluating PARAM group sizes for ");
4257 print_generic_expr (dump_file, parm, 0);
4258 fprintf (dump_file, " (UID: %u): \n", DECL_UID (parm));
4259 for (acc = repr; acc; acc = acc->next_grp)
4260 dump_access (dump_file, acc, true);
4264 new_param_count = 0;
4266 for (; repr; repr = repr->next_grp)
4268 gcc_assert (parm == repr->base);
4270 /* Taking the address of a non-addressable field is verboten. */
4271 if (by_ref && repr->non_addressable)
4274 /* Do not decompose a non-BLKmode param in a way that would
4275 create BLKmode params. Especially for by-reference passing
4276 (thus, pointer-type param) this is hardly worthwhile. */
4277 if (DECL_MODE (parm) != BLKmode
4278 && TYPE_MODE (repr->type) == BLKmode)
4281 if (!by_ref || (!repr->grp_maybe_modified
4282 && !repr->grp_not_necessarilly_dereferenced))
4283 total_size += repr->size;
4285 total_size += cur_parm_size;
4290 gcc_assert (new_param_count > 0);
4292 if (optimize_function_for_size_p (cfun))
4293 parm_size_limit = cur_parm_size;
4295 parm_size_limit = (PARAM_VALUE (PARAM_IPA_SRA_PTR_GROWTH_FACTOR)
4298 if (total_size < agg_size
4299 && total_size <= parm_size_limit)
4302 fprintf (dump_file, " ....will be split into %i components\n",
4304 return new_param_count;
4310 /* The order of the following enums is important, we need to do extra work for
4311 UNUSED_PARAMS, BY_VAL_ACCESSES and UNMODIF_BY_REF_ACCESSES. */
4312 enum ipa_splicing_result { NO_GOOD_ACCESS, UNUSED_PARAMS, BY_VAL_ACCESSES,
4313 MODIF_BY_REF_ACCESSES, UNMODIF_BY_REF_ACCESSES };
4315 /* Identify representatives of all accesses to all candidate parameters for
4316 IPA-SRA. Return result based on what representatives have been found. */
4318 static enum ipa_splicing_result
4319 splice_all_param_accesses (vec<access_p> &representatives)
4321 enum ipa_splicing_result result = NO_GOOD_ACCESS;
4323 struct access *repr;
4325 representatives.create (func_param_count);
4327 for (parm = DECL_ARGUMENTS (current_function_decl);
4329 parm = DECL_CHAIN (parm))
4331 if (is_unused_scalar_param (parm))
4333 representatives.quick_push (&no_accesses_representant);
4334 if (result == NO_GOOD_ACCESS)
4335 result = UNUSED_PARAMS;
4337 else if (POINTER_TYPE_P (TREE_TYPE (parm))
4338 && is_gimple_reg_type (TREE_TYPE (TREE_TYPE (parm)))
4339 && bitmap_bit_p (candidate_bitmap, DECL_UID (parm)))
4341 repr = unmodified_by_ref_scalar_representative (parm);
4342 representatives.quick_push (repr);
4344 result = UNMODIF_BY_REF_ACCESSES;
4346 else if (bitmap_bit_p (candidate_bitmap, DECL_UID (parm)))
4348 bool ro_grp = false;
4349 repr = splice_param_accesses (parm, &ro_grp);
4350 representatives.quick_push (repr);
4352 if (repr && !no_accesses_p (repr))
4354 if (POINTER_TYPE_P (TREE_TYPE (parm)))
4357 result = UNMODIF_BY_REF_ACCESSES;
4358 else if (result < MODIF_BY_REF_ACCESSES)
4359 result = MODIF_BY_REF_ACCESSES;
4361 else if (result < BY_VAL_ACCESSES)
4362 result = BY_VAL_ACCESSES;
4364 else if (no_accesses_p (repr) && (result == NO_GOOD_ACCESS))
4365 result = UNUSED_PARAMS;
4368 representatives.quick_push (NULL);
4371 if (result == NO_GOOD_ACCESS)
4373 representatives.release ();
4374 return NO_GOOD_ACCESS;
4380 /* Return the index of BASE in PARMS. Abort if it is not found. */
4383 get_param_index (tree base, vec<tree> parms)
4387 len = parms.length ();
4388 for (i = 0; i < len; i++)
4389 if (parms[i] == base)
4394 /* Convert the decisions made at the representative level into compact
4395 parameter adjustments. REPRESENTATIVES are pointers to first
4396 representatives of each param accesses, ADJUSTMENTS_COUNT is the expected
4397 final number of adjustments. */
4399 static ipa_parm_adjustment_vec
4400 turn_representatives_into_adjustments (vec<access_p> representatives,
4401 int adjustments_count)
4404 ipa_parm_adjustment_vec adjustments;
4408 gcc_assert (adjustments_count > 0);
4409 parms = ipa_get_vector_of_formal_parms (current_function_decl);
4410 adjustments.create (adjustments_count);
4411 parm = DECL_ARGUMENTS (current_function_decl);
4412 for (i = 0; i < func_param_count; i++, parm = DECL_CHAIN (parm))
4414 struct access *repr = representatives[i];
4416 if (!repr || no_accesses_p (repr))
4418 struct ipa_parm_adjustment adj;
4420 memset (&adj, 0, sizeof (adj));
4421 adj.base_index = get_param_index (parm, parms);
4424 adj.op = IPA_PARM_OP_COPY;
4426 adj.op = IPA_PARM_OP_REMOVE;
4427 adj.arg_prefix = "ISRA";
4428 adjustments.quick_push (adj);
4432 struct ipa_parm_adjustment adj;
4433 int index = get_param_index (parm, parms);
4435 for (; repr; repr = repr->next_grp)
4437 memset (&adj, 0, sizeof (adj));
4438 gcc_assert (repr->base == parm);
4439 adj.base_index = index;
4440 adj.base = repr->base;
4441 adj.type = repr->type;
4442 adj.alias_ptr_type = reference_alias_ptr_type (repr->expr);
4443 adj.offset = repr->offset;
4444 adj.by_ref = (POINTER_TYPE_P (TREE_TYPE (repr->base))
4445 && (repr->grp_maybe_modified
4446 || repr->grp_not_necessarilly_dereferenced));
4447 adj.arg_prefix = "ISRA";
4448 adjustments.quick_push (adj);
4456 /* Analyze the collected accesses and produce a plan what to do with the
4457 parameters in the form of adjustments, NULL meaning nothing. */
4459 static ipa_parm_adjustment_vec
4460 analyze_all_param_acesses (void)
4462 enum ipa_splicing_result repr_state;
4463 bool proceed = false;
4464 int i, adjustments_count = 0;
4465 vec<access_p> representatives;
4466 ipa_parm_adjustment_vec adjustments;
4468 repr_state = splice_all_param_accesses (representatives);
4469 if (repr_state == NO_GOOD_ACCESS)
4470 return ipa_parm_adjustment_vec ();
4472 /* If there are any parameters passed by reference which are not modified
4473 directly, we need to check whether they can be modified indirectly. */
4474 if (repr_state == UNMODIF_BY_REF_ACCESSES)
4476 analyze_caller_dereference_legality (representatives);
4477 analyze_modified_params (representatives);
4480 for (i = 0; i < func_param_count; i++)
4482 struct access *repr = representatives[i];
4484 if (repr && !no_accesses_p (repr))
4486 if (repr->grp_scalar_ptr)
4488 adjustments_count++;
4489 if (repr->grp_not_necessarilly_dereferenced
4490 || repr->grp_maybe_modified)
4491 representatives[i] = NULL;
4495 sra_stats.scalar_by_ref_to_by_val++;
4500 int new_components = decide_one_param_reduction (repr);
4502 if (new_components == 0)
4504 representatives[i] = NULL;
4505 adjustments_count++;
4509 adjustments_count += new_components;
4510 sra_stats.aggregate_params_reduced++;
4511 sra_stats.param_reductions_created += new_components;
4518 if (no_accesses_p (repr))
4521 sra_stats.deleted_unused_parameters++;
4523 adjustments_count++;
4527 if (!proceed && dump_file)
4528 fprintf (dump_file, "NOT proceeding to change params.\n");
4531 adjustments = turn_representatives_into_adjustments (representatives,
4534 adjustments = ipa_parm_adjustment_vec ();
4536 representatives.release ();
4540 /* If a parameter replacement identified by ADJ does not yet exist in the form
4541 of declaration, create it and record it, otherwise return the previously
4545 get_replaced_param_substitute (struct ipa_parm_adjustment *adj)
4548 if (!adj->new_ssa_base)
4550 char *pretty_name = make_fancy_name (adj->base);
4552 repl = create_tmp_reg (TREE_TYPE (adj->base), "ISR");
4553 DECL_NAME (repl) = get_identifier (pretty_name);
4554 obstack_free (&name_obstack, pretty_name);
4556 adj->new_ssa_base = repl;
4559 repl = adj->new_ssa_base;
4563 /* Find the first adjustment for a particular parameter BASE in a vector of
4564 ADJUSTMENTS which is not a copy_param. Return NULL if there is no such
4567 static struct ipa_parm_adjustment *
4568 get_adjustment_for_base (ipa_parm_adjustment_vec adjustments, tree base)
4572 len = adjustments.length ();
4573 for (i = 0; i < len; i++)
4575 struct ipa_parm_adjustment *adj;
4577 adj = &adjustments[i];
4578 if (adj->op != IPA_PARM_OP_COPY && adj->base == base)
4585 /* If OLD_NAME, which is being defined by statement STMT, is an SSA_NAME of a
4586 parameter which is to be removed because its value is not used, create a new
4587 SSA_NAME relating to a replacement VAR_DECL, replace all uses of the
4588 original with it and return it. If there is no need to re-map, return NULL.
4589 ADJUSTMENTS is a pointer to a vector of IPA-SRA adjustments. */
4592 replace_removed_params_ssa_names (tree old_name, gimple stmt,
4593 ipa_parm_adjustment_vec adjustments)
4595 struct ipa_parm_adjustment *adj;
4596 tree decl, repl, new_name;
4598 if (TREE_CODE (old_name) != SSA_NAME)
4601 decl = SSA_NAME_VAR (old_name);
4602 if (decl == NULL_TREE
4603 || TREE_CODE (decl) != PARM_DECL)
4606 adj = get_adjustment_for_base (adjustments, decl);
4610 repl = get_replaced_param_substitute (adj);
4611 new_name = make_ssa_name (repl, stmt);
4612 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_name)
4613 = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (old_name);
4617 fprintf (dump_file, "replacing an SSA name of a removed param ");
4618 print_generic_expr (dump_file, old_name, 0);
4619 fprintf (dump_file, " with ");
4620 print_generic_expr (dump_file, new_name, 0);
4621 fprintf (dump_file, "\n");
4624 replace_uses_by (old_name, new_name);
4628 /* If the statement STMT contains any expressions that need to replaced with a
4629 different one as noted by ADJUSTMENTS, do so. Handle any potential type
4630 incompatibilities (GSI is used to accommodate conversion statements and must
4631 point to the statement). Return true iff the statement was modified. */
4634 sra_ipa_modify_assign (gimple stmt, gimple_stmt_iterator *gsi,
4635 ipa_parm_adjustment_vec adjustments)
4637 tree *lhs_p, *rhs_p;
4640 if (!gimple_assign_single_p (stmt))
4643 rhs_p = gimple_assign_rhs1_ptr (stmt);
4644 lhs_p = gimple_assign_lhs_ptr (stmt);
4646 any = ipa_modify_expr (rhs_p, false, adjustments);
4647 any |= ipa_modify_expr (lhs_p, false, adjustments);
4650 tree new_rhs = NULL_TREE;
4652 if (!useless_type_conversion_p (TREE_TYPE (*lhs_p), TREE_TYPE (*rhs_p)))
4654 if (TREE_CODE (*rhs_p) == CONSTRUCTOR)
4656 /* V_C_Es of constructors can cause trouble (PR 42714). */
4657 if (is_gimple_reg_type (TREE_TYPE (*lhs_p)))
4658 *rhs_p = build_zero_cst (TREE_TYPE (*lhs_p));
4660 *rhs_p = build_constructor (TREE_TYPE (*lhs_p),
4664 new_rhs = fold_build1_loc (gimple_location (stmt),
4665 VIEW_CONVERT_EXPR, TREE_TYPE (*lhs_p),
4668 else if (REFERENCE_CLASS_P (*rhs_p)
4669 && is_gimple_reg_type (TREE_TYPE (*lhs_p))
4670 && !is_gimple_reg (*lhs_p))
4671 /* This can happen when an assignment in between two single field
4672 structures is turned into an assignment in between two pointers to
4673 scalars (PR 42237). */
4678 tree tmp = force_gimple_operand_gsi (gsi, new_rhs, true, NULL_TREE,
4679 true, GSI_SAME_STMT);
4681 gimple_assign_set_rhs_from_tree (gsi, tmp);
4690 /* Traverse the function body and all modifications as described in
4691 ADJUSTMENTS. Return true iff the CFG has been changed. */
4694 ipa_sra_modify_function_body (ipa_parm_adjustment_vec adjustments)
4696 bool cfg_changed = false;
4699 FOR_EACH_BB_FN (bb, cfun)
4701 gimple_stmt_iterator gsi;
4703 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
4705 gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
4706 tree new_lhs, old_lhs = gimple_phi_result (phi);
4707 new_lhs = replace_removed_params_ssa_names (old_lhs, phi, adjustments);
4710 gimple_phi_set_result (phi, new_lhs);
4711 release_ssa_name (old_lhs);
4715 gsi = gsi_start_bb (bb);
4716 while (!gsi_end_p (gsi))
4718 gimple stmt = gsi_stmt (gsi);
4719 bool modified = false;
4723 switch (gimple_code (stmt))
4726 t = gimple_return_retval_ptr (as_a <greturn *> (stmt));
4727 if (*t != NULL_TREE)
4728 modified |= ipa_modify_expr (t, true, adjustments);
4732 modified |= sra_ipa_modify_assign (stmt, &gsi, adjustments);
4736 /* Operands must be processed before the lhs. */
4737 for (i = 0; i < gimple_call_num_args (stmt); i++)
4739 t = gimple_call_arg_ptr (stmt, i);
4740 modified |= ipa_modify_expr (t, true, adjustments);
4743 if (gimple_call_lhs (stmt))
4745 t = gimple_call_lhs_ptr (stmt);
4746 modified |= ipa_modify_expr (t, false, adjustments);
4752 gasm *asm_stmt = as_a <gasm *> (stmt);
4753 for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
4755 t = &TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
4756 modified |= ipa_modify_expr (t, true, adjustments);
4758 for (i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
4760 t = &TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
4761 modified |= ipa_modify_expr (t, false, adjustments);
4772 FOR_EACH_SSA_DEF_OPERAND (defp, stmt, iter, SSA_OP_DEF)
4774 tree old_def = DEF_FROM_PTR (defp);
4775 if (tree new_def = replace_removed_params_ssa_names (old_def, stmt,
4778 SET_DEF (defp, new_def);
4779 release_ssa_name (old_def);
4787 if (maybe_clean_eh_stmt (stmt)
4788 && gimple_purge_dead_eh_edges (gimple_bb (stmt)))
4798 /* Call gimple_debug_bind_reset_value on all debug statements describing
4799 gimple register parameters that are being removed or replaced. */
4802 sra_ipa_reset_debug_stmts (ipa_parm_adjustment_vec adjustments)
4805 gimple_stmt_iterator *gsip = NULL, gsi;
4807 if (MAY_HAVE_DEBUG_STMTS && single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
4809 gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
4812 len = adjustments.length ();
4813 for (i = 0; i < len; i++)
4815 struct ipa_parm_adjustment *adj;
4816 imm_use_iterator ui;
4819 tree name, vexpr, copy = NULL_TREE;
4820 use_operand_p use_p;
4822 adj = &adjustments[i];
4823 if (adj->op == IPA_PARM_OP_COPY || !is_gimple_reg (adj->base))
4825 name = ssa_default_def (cfun, adj->base);
4828 FOR_EACH_IMM_USE_STMT (stmt, ui, name)
4830 if (gimple_clobber_p (stmt))
4832 gimple_stmt_iterator cgsi = gsi_for_stmt (stmt);
4833 unlink_stmt_vdef (stmt);
4834 gsi_remove (&cgsi, true);
4835 release_defs (stmt);
4838 /* All other users must have been removed by
4839 ipa_sra_modify_function_body. */
4840 gcc_assert (is_gimple_debug (stmt));
4841 if (vexpr == NULL && gsip != NULL)
4843 gcc_assert (TREE_CODE (adj->base) == PARM_DECL);
4844 vexpr = make_node (DEBUG_EXPR_DECL);
4845 def_temp = gimple_build_debug_source_bind (vexpr, adj->base,
4847 DECL_ARTIFICIAL (vexpr) = 1;
4848 TREE_TYPE (vexpr) = TREE_TYPE (name);
4849 DECL_MODE (vexpr) = DECL_MODE (adj->base);
4850 gsi_insert_before (gsip, def_temp, GSI_SAME_STMT);
4854 FOR_EACH_IMM_USE_ON_STMT (use_p, ui)
4855 SET_USE (use_p, vexpr);
4858 gimple_debug_bind_reset_value (stmt);
4861 /* Create a VAR_DECL for debug info purposes. */
4862 if (!DECL_IGNORED_P (adj->base))
4864 copy = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
4865 VAR_DECL, DECL_NAME (adj->base),
4866 TREE_TYPE (adj->base));
4867 if (DECL_PT_UID_SET_P (adj->base))
4868 SET_DECL_PT_UID (copy, DECL_PT_UID (adj->base));
4869 TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (adj->base);
4870 TREE_READONLY (copy) = TREE_READONLY (adj->base);
4871 TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (adj->base);
4872 DECL_GIMPLE_REG_P (copy) = DECL_GIMPLE_REG_P (adj->base);
4873 DECL_ARTIFICIAL (copy) = DECL_ARTIFICIAL (adj->base);
4874 DECL_IGNORED_P (copy) = DECL_IGNORED_P (adj->base);
4875 DECL_ABSTRACT_ORIGIN (copy) = DECL_ORIGIN (adj->base);
4876 DECL_SEEN_IN_BIND_EXPR_P (copy) = 1;
4877 SET_DECL_RTL (copy, 0);
4878 TREE_USED (copy) = 1;
4879 DECL_CONTEXT (copy) = current_function_decl;
4880 add_local_decl (cfun, copy);
4882 BLOCK_VARS (DECL_INITIAL (current_function_decl));
4883 BLOCK_VARS (DECL_INITIAL (current_function_decl)) = copy;
4885 if (gsip != NULL && copy && target_for_debug_bind (adj->base))
4887 gcc_assert (TREE_CODE (adj->base) == PARM_DECL);
4889 def_temp = gimple_build_debug_bind (copy, vexpr, NULL);
4891 def_temp = gimple_build_debug_source_bind (copy, adj->base,
4893 gsi_insert_before (gsip, def_temp, GSI_SAME_STMT);
4898 /* Return false if all callers have at least as many actual arguments as there
4899 are formal parameters in the current function and that their types
4903 some_callers_have_mismatched_arguments_p (struct cgraph_node *node,
4904 void *data ATTRIBUTE_UNUSED)
4906 struct cgraph_edge *cs;
4907 for (cs = node->callers; cs; cs = cs->next_caller)
4908 if (!cs->call_stmt || !callsite_arguments_match_p (cs->call_stmt))
4914 /* Return false if all callers have vuse attached to a call statement. */
4917 some_callers_have_no_vuse_p (struct cgraph_node *node,
4918 void *data ATTRIBUTE_UNUSED)
4920 struct cgraph_edge *cs;
4921 for (cs = node->callers; cs; cs = cs->next_caller)
4922 if (!cs->call_stmt || !gimple_vuse (cs->call_stmt))
4928 /* Convert all callers of NODE. */
4931 convert_callers_for_node (struct cgraph_node *node,
4934 ipa_parm_adjustment_vec *adjustments = (ipa_parm_adjustment_vec *) data;
4935 bitmap recomputed_callers = BITMAP_ALLOC (NULL);
4936 struct cgraph_edge *cs;
4938 for (cs = node->callers; cs; cs = cs->next_caller)
4940 push_cfun (DECL_STRUCT_FUNCTION (cs->caller->decl));
4943 fprintf (dump_file, "Adjusting call %s/%i -> %s/%i\n",
4944 xstrdup (cs->caller->name ()),
4946 xstrdup (cs->callee->name ()),
4949 ipa_modify_call_arguments (cs, cs->call_stmt, *adjustments);
4954 for (cs = node->callers; cs; cs = cs->next_caller)
4955 if (bitmap_set_bit (recomputed_callers, cs->caller->uid)
4956 && gimple_in_ssa_p (DECL_STRUCT_FUNCTION (cs->caller->decl)))
4957 compute_inline_parameters (cs->caller, true);
4958 BITMAP_FREE (recomputed_callers);
4963 /* Convert all callers of NODE to pass parameters as given in ADJUSTMENTS. */
4966 convert_callers (struct cgraph_node *node, tree old_decl,
4967 ipa_parm_adjustment_vec adjustments)
4969 basic_block this_block;
4971 node->call_for_symbol_and_aliases (convert_callers_for_node,
4972 &adjustments, false);
4974 if (!encountered_recursive_call)
4977 FOR_EACH_BB_FN (this_block, cfun)
4979 gimple_stmt_iterator gsi;
4981 for (gsi = gsi_start_bb (this_block); !gsi_end_p (gsi); gsi_next (&gsi))
4985 stmt = dyn_cast <gcall *> (gsi_stmt (gsi));
4988 call_fndecl = gimple_call_fndecl (stmt);
4989 if (call_fndecl == old_decl)
4992 fprintf (dump_file, "Adjusting recursive call");
4993 gimple_call_set_fndecl (stmt, node->decl);
4994 ipa_modify_call_arguments (NULL, stmt, adjustments);
5002 /* Perform all the modification required in IPA-SRA for NODE to have parameters
5003 as given in ADJUSTMENTS. Return true iff the CFG has been changed. */
5006 modify_function (struct cgraph_node *node, ipa_parm_adjustment_vec adjustments)
5008 struct cgraph_node *new_node;
5011 cgraph_edge::rebuild_edges ();
5012 free_dominance_info (CDI_DOMINATORS);
5015 /* This must be done after rebuilding cgraph edges for node above.
5016 Otherwise any recursive calls to node that are recorded in
5017 redirect_callers will be corrupted. */
5018 vec<cgraph_edge *> redirect_callers = node->collect_callers ();
5019 new_node = node->create_version_clone_with_body (redirect_callers, NULL,
5020 NULL, false, NULL, NULL,
5022 redirect_callers.release ();
5024 push_cfun (DECL_STRUCT_FUNCTION (new_node->decl));
5025 ipa_modify_formal_parameters (current_function_decl, adjustments);
5026 cfg_changed = ipa_sra_modify_function_body (adjustments);
5027 sra_ipa_reset_debug_stmts (adjustments);
5028 convert_callers (new_node, node->decl, adjustments);
5029 new_node->make_local ();
5033 /* Means of communication between ipa_sra_check_caller and
5034 ipa_sra_preliminary_function_checks. */
5036 struct ipa_sra_check_caller_data
5039 bool bad_arg_alignment;
5043 /* If NODE has a caller, mark that fact in DATA which is pointer to
5044 ipa_sra_check_caller_data. Also check all aggregate arguments in all known
5045 calls if they are unit aligned and if not, set the appropriate flag in DATA
5049 ipa_sra_check_caller (struct cgraph_node *node, void *data)
5054 struct ipa_sra_check_caller_data *iscc;
5055 iscc = (struct ipa_sra_check_caller_data *) data;
5056 iscc->has_callers = true;
5058 for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
5060 if (cs->caller->thunk.thunk_p)
5062 iscc->has_thunk = true;
5065 gimple call_stmt = cs->call_stmt;
5066 unsigned count = gimple_call_num_args (call_stmt);
5067 for (unsigned i = 0; i < count; i++)
5069 tree arg = gimple_call_arg (call_stmt, i);
5070 if (is_gimple_reg (arg))
5074 HOST_WIDE_INT bitsize, bitpos;
5076 int unsignedp, volatilep = 0;
5077 get_inner_reference (arg, &bitsize, &bitpos, &offset, &mode,
5078 &unsignedp, &volatilep, false);
5079 if (bitpos % BITS_PER_UNIT)
5081 iscc->bad_arg_alignment = true;
5090 /* Return false the function is apparently unsuitable for IPA-SRA based on it's
5091 attributes, return true otherwise. NODE is the cgraph node of the current
5095 ipa_sra_preliminary_function_checks (struct cgraph_node *node)
5097 if (!node->can_be_local_p ())
5100 fprintf (dump_file, "Function not local to this compilation unit.\n");
5104 if (!node->local.can_change_signature)
5107 fprintf (dump_file, "Function can not change signature.\n");
5111 if (!tree_versionable_function_p (node->decl))
5114 fprintf (dump_file, "Function is not versionable.\n");
5118 if (!opt_for_fn (node->decl, optimize)
5119 || !opt_for_fn (node->decl, flag_ipa_sra))
5122 fprintf (dump_file, "Function not optimized.\n");
5126 if (DECL_VIRTUAL_P (current_function_decl))
5129 fprintf (dump_file, "Function is a virtual method.\n");
5133 if ((DECL_ONE_ONLY (node->decl) || DECL_EXTERNAL (node->decl))
5134 && inline_summaries->get (node)->size >= MAX_INLINE_INSNS_AUTO)
5137 fprintf (dump_file, "Function too big to be made truly local.\n");
5144 fprintf (dump_file, "Function uses stdarg. \n");
5148 if (TYPE_ATTRIBUTES (TREE_TYPE (node->decl)))
5151 if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
5154 fprintf (dump_file, "Always inline function will be inlined "
5159 struct ipa_sra_check_caller_data iscc;
5160 memset (&iscc, 0, sizeof(iscc));
5161 node->call_for_symbol_and_aliases (ipa_sra_check_caller, &iscc, true);
5162 if (!iscc.has_callers)
5166 "Function has no callers in this compilation unit.\n");
5170 if (iscc.bad_arg_alignment)
5174 "A function call has an argument with non-unit alignment.\n");
5189 /* Perform early interprocedural SRA. */
5192 ipa_early_sra (void)
5194 struct cgraph_node *node = cgraph_node::get (current_function_decl);
5195 ipa_parm_adjustment_vec adjustments;
5198 if (!ipa_sra_preliminary_function_checks (node))
5202 sra_mode = SRA_MODE_EARLY_IPA;
5204 if (!find_param_candidates ())
5207 fprintf (dump_file, "Function has no IPA-SRA candidates.\n");
5211 if (node->call_for_symbol_and_aliases
5212 (some_callers_have_mismatched_arguments_p, NULL, true))
5215 fprintf (dump_file, "There are callers with insufficient number of "
5216 "arguments or arguments with type mismatches.\n");
5220 if (node->call_for_symbol_and_aliases
5221 (some_callers_have_no_vuse_p, NULL, true))
5224 fprintf (dump_file, "There are callers with no VUSE attached "
5225 "to a call stmt.\n");
5229 bb_dereferences = XCNEWVEC (HOST_WIDE_INT,
5231 * last_basic_block_for_fn (cfun));
5232 final_bbs = BITMAP_ALLOC (NULL);
5235 if (encountered_apply_args)
5238 fprintf (dump_file, "Function calls __builtin_apply_args().\n");
5242 if (encountered_unchangable_recursive_call)
5245 fprintf (dump_file, "Function calls itself with insufficient "
5246 "number of arguments.\n");
5250 adjustments = analyze_all_param_acesses ();
5251 if (!adjustments.exists ())
5254 ipa_dump_param_adjustments (dump_file, adjustments, current_function_decl);
5256 if (modify_function (node, adjustments))
5257 ret = TODO_update_ssa | TODO_cleanup_cfg;
5259 ret = TODO_update_ssa;
5260 adjustments.release ();
5262 statistics_counter_event (cfun, "Unused parameters deleted",
5263 sra_stats.deleted_unused_parameters);
5264 statistics_counter_event (cfun, "Scalar parameters converted to by-value",
5265 sra_stats.scalar_by_ref_to_by_val);
5266 statistics_counter_event (cfun, "Aggregate parameters broken up",
5267 sra_stats.aggregate_params_reduced);
5268 statistics_counter_event (cfun, "Aggregate parameter components created",
5269 sra_stats.param_reductions_created);
5272 BITMAP_FREE (final_bbs);
5273 free (bb_dereferences);
5275 sra_deinitialize ();
5281 const pass_data pass_data_early_ipa_sra =
5283 GIMPLE_PASS, /* type */
5284 "eipa_sra", /* name */
5285 OPTGROUP_NONE, /* optinfo_flags */
5286 TV_IPA_SRA, /* tv_id */
5287 0, /* properties_required */
5288 0, /* properties_provided */
5289 0, /* properties_destroyed */
5290 0, /* todo_flags_start */
5291 TODO_dump_symtab, /* todo_flags_finish */
5294 class pass_early_ipa_sra : public gimple_opt_pass
5297 pass_early_ipa_sra (gcc::context *ctxt)
5298 : gimple_opt_pass (pass_data_early_ipa_sra, ctxt)
5301 /* opt_pass methods: */
5302 virtual bool gate (function *) { return flag_ipa_sra && dbg_cnt (eipa_sra); }
5303 virtual unsigned int execute (function *) { return ipa_early_sra (); }
5305 }; // class pass_early_ipa_sra
5310 make_pass_early_ipa_sra (gcc::context *ctxt)
5312 return new pass_early_ipa_sra (ctxt);