1 /* Perform various loop optimizations, including strength reduction.
2 Copyright (C) 1987, 1988, 1989, 1991, 1992, 1993, 1994, 1995, 1996, 1997,
3 1998, 1999, 2000, 2001, 2002, 2003, 2004 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING. If not, write to the Free
19 Software Foundation, 59 Temple Place - Suite 330, Boston, MA
22 /* This is the loop optimization pass of the compiler.
23 It finds invariant computations within loops and moves them
24 to the beginning of the loop. Then it identifies basic and
25 general induction variables.
27 Basic induction variables (BIVs) are a pseudo registers which are set within
28 a loop only by incrementing or decrementing its value. General induction
29 variables (GIVs) are pseudo registers with a value which is a linear function
30 of a basic induction variable. BIVs are recognized by `basic_induction_var';
31 GIVs by `general_induction_var'.
33 Once induction variables are identified, strength reduction is applied to the
34 general induction variables, and induction variable elimination is applied to
35 the basic induction variables.
37 It also finds cases where
38 a register is set within the loop by zero-extending a narrower value
39 and changes these to zero the entire register once before the loop
40 and merely copy the low part within the loop.
42 Most of the complexity is in heuristics to decide when it is worth
43 while to do these things. */
47 #include "coretypes.h"
53 #include "hard-reg-set.h"
54 #include "basic-block.h"
55 #include "insn-config.h"
65 #include "insn-flags.h"
70 /* Not really meaningful values, but at least something. */
71 #ifndef SIMULTANEOUS_PREFETCHES
72 #define SIMULTANEOUS_PREFETCHES 3
74 #ifndef PREFETCH_BLOCK
75 #define PREFETCH_BLOCK 32
78 #define HAVE_prefetch 0
79 #define CODE_FOR_prefetch 0
80 #define gen_prefetch(a,b,c) (abort(), NULL_RTX)
83 /* Give up the prefetch optimizations once we exceed a given threshold.
84 It is unlikely that we would be able to optimize something in a loop
85 with so many detected prefetches. */
86 #define MAX_PREFETCHES 100
87 /* The number of prefetch blocks that are beneficial to fetch at once before
88 a loop with a known (and low) iteration count. */
89 #define PREFETCH_BLOCKS_BEFORE_LOOP_MAX 6
90 /* For very tiny loops it is not worthwhile to prefetch even before the loop,
91 since it is likely that the data are already in the cache. */
92 #define PREFETCH_BLOCKS_BEFORE_LOOP_MIN 2
94 /* Parameterize some prefetch heuristics so they can be turned on and off
95 easily for performance testing on new architectures. These can be
96 defined in target-dependent files. */
98 /* Prefetch is worthwhile only when loads/stores are dense. */
99 #ifndef PREFETCH_ONLY_DENSE_MEM
100 #define PREFETCH_ONLY_DENSE_MEM 1
103 /* Define what we mean by "dense" loads and stores; This value divided by 256
104 is the minimum percentage of memory references that worth prefetching. */
105 #ifndef PREFETCH_DENSE_MEM
106 #define PREFETCH_DENSE_MEM 220
109 /* Do not prefetch for a loop whose iteration count is known to be low. */
110 #ifndef PREFETCH_NO_LOW_LOOPCNT
111 #define PREFETCH_NO_LOW_LOOPCNT 1
114 /* Define what we mean by a "low" iteration count. */
115 #ifndef PREFETCH_LOW_LOOPCNT
116 #define PREFETCH_LOW_LOOPCNT 32
119 /* Do not prefetch for a loop that contains a function call; such a loop is
120 probably not an internal loop. */
121 #ifndef PREFETCH_NO_CALL
122 #define PREFETCH_NO_CALL 1
125 /* Do not prefetch accesses with an extreme stride. */
126 #ifndef PREFETCH_NO_EXTREME_STRIDE
127 #define PREFETCH_NO_EXTREME_STRIDE 1
130 /* Define what we mean by an "extreme" stride. */
131 #ifndef PREFETCH_EXTREME_STRIDE
132 #define PREFETCH_EXTREME_STRIDE 4096
135 /* Define a limit to how far apart indices can be and still be merged
136 into a single prefetch. */
137 #ifndef PREFETCH_EXTREME_DIFFERENCE
138 #define PREFETCH_EXTREME_DIFFERENCE 4096
141 /* Issue prefetch instructions before the loop to fetch data to be used
142 in the first few loop iterations. */
143 #ifndef PREFETCH_BEFORE_LOOP
144 #define PREFETCH_BEFORE_LOOP 1
147 /* Do not handle reversed order prefetches (negative stride). */
148 #ifndef PREFETCH_NO_REVERSE_ORDER
149 #define PREFETCH_NO_REVERSE_ORDER 1
152 /* Prefetch even if the GIV is in conditional code. */
153 #ifndef PREFETCH_CONDITIONAL
154 #define PREFETCH_CONDITIONAL 1
157 #define LOOP_REG_LIFETIME(LOOP, REGNO) \
158 ((REGNO_LAST_LUID (REGNO) - REGNO_FIRST_LUID (REGNO)))
160 #define LOOP_REG_GLOBAL_P(LOOP, REGNO) \
161 ((REGNO_LAST_LUID (REGNO) > INSN_LUID ((LOOP)->end) \
162 || REGNO_FIRST_LUID (REGNO) < INSN_LUID ((LOOP)->start)))
164 #define LOOP_REGNO_NREGS(REGNO, SET_DEST) \
165 ((REGNO) < FIRST_PSEUDO_REGISTER \
166 ? (int) HARD_REGNO_NREGS ((REGNO), GET_MODE (SET_DEST)) : 1)
169 /* Vector mapping INSN_UIDs to luids.
170 The luids are like uids but increase monotonically always.
171 We use them to see whether a jump comes from outside a given loop. */
175 /* Indexed by INSN_UID, contains the ordinal giving the (innermost) loop
176 number the insn is contained in. */
178 struct loop **uid_loop;
180 /* 1 + largest uid of any insn. */
182 int max_uid_for_loop;
184 /* Number of loops detected in current function. Used as index to the
187 static int max_loop_num;
189 /* Bound on pseudo register number before loop optimization.
190 A pseudo has valid regscan info if its number is < max_reg_before_loop. */
191 unsigned int max_reg_before_loop;
193 /* The value to pass to the next call of reg_scan_update. */
194 static int loop_max_reg;
196 /* During the analysis of a loop, a chain of `struct movable's
197 is made to record all the movable insns found.
198 Then the entire chain can be scanned to decide which to move. */
202 rtx insn; /* A movable insn */
203 rtx set_src; /* The expression this reg is set from. */
204 rtx set_dest; /* The destination of this SET. */
205 rtx dependencies; /* When INSN is libcall, this is an EXPR_LIST
206 of any registers used within the LIBCALL. */
207 int consec; /* Number of consecutive following insns
208 that must be moved with this one. */
209 unsigned int regno; /* The register it sets */
210 short lifetime; /* lifetime of that register;
211 may be adjusted when matching movables
212 that load the same value are found. */
213 short savings; /* Number of insns we can move for this reg,
214 including other movables that force this
215 or match this one. */
216 ENUM_BITFIELD(machine_mode) savemode : 8; /* Nonzero means it is a mode for
217 a low part that we should avoid changing when
218 clearing the rest of the reg. */
219 unsigned int cond : 1; /* 1 if only conditionally movable */
220 unsigned int force : 1; /* 1 means MUST move this insn */
221 unsigned int global : 1; /* 1 means reg is live outside this loop */
222 /* If PARTIAL is 1, GLOBAL means something different:
223 that the reg is live outside the range from where it is set
224 to the following label. */
225 unsigned int done : 1; /* 1 inhibits further processing of this */
227 unsigned int partial : 1; /* 1 means this reg is used for zero-extending.
228 In particular, moving it does not make it
230 unsigned int move_insn : 1; /* 1 means that we call emit_move_insn to
231 load SRC, rather than copying INSN. */
232 unsigned int move_insn_first:1;/* Same as above, if this is necessary for the
233 first insn of a consecutive sets group. */
234 unsigned int is_equiv : 1; /* 1 means a REG_EQUIV is present on INSN. */
235 unsigned int insert_temp : 1; /* 1 means we copy to a new pseudo and replace
236 the original insn with a copy from that
237 pseudo, rather than deleting it. */
238 struct movable *match; /* First entry for same value */
239 struct movable *forces; /* An insn that must be moved if this is */
240 struct movable *next;
244 FILE *loop_dump_stream;
246 /* Forward declarations. */
248 static void invalidate_loops_containing_label (rtx);
249 static void find_and_verify_loops (rtx, struct loops *);
250 static void mark_loop_jump (rtx, struct loop *);
251 static void prescan_loop (struct loop *);
252 static int reg_in_basic_block_p (rtx, rtx);
253 static int consec_sets_invariant_p (const struct loop *, rtx, int, rtx);
254 static int labels_in_range_p (rtx, int);
255 static void count_one_set (struct loop_regs *, rtx, rtx, rtx *);
256 static void note_addr_stored (rtx, rtx, void *);
257 static void note_set_pseudo_multiple_uses (rtx, rtx, void *);
258 static int loop_reg_used_before_p (const struct loop *, rtx, rtx);
259 static rtx find_regs_nested (rtx, rtx);
260 static void scan_loop (struct loop*, int);
262 static void replace_call_address (rtx, rtx, rtx);
264 static rtx skip_consec_insns (rtx, int);
265 static int libcall_benefit (rtx);
266 static void ignore_some_movables (struct loop_movables *);
267 static void force_movables (struct loop_movables *);
268 static void combine_movables (struct loop_movables *, struct loop_regs *);
269 static int num_unmoved_movables (const struct loop *);
270 static int regs_match_p (rtx, rtx, struct loop_movables *);
271 static int rtx_equal_for_loop_p (rtx, rtx, struct loop_movables *,
273 static void add_label_notes (rtx, rtx);
274 static void move_movables (struct loop *loop, struct loop_movables *, int,
276 static void loop_movables_add (struct loop_movables *, struct movable *);
277 static void loop_movables_free (struct loop_movables *);
278 static int count_nonfixed_reads (const struct loop *, rtx);
279 static void loop_bivs_find (struct loop *);
280 static void loop_bivs_init_find (struct loop *);
281 static void loop_bivs_check (struct loop *);
282 static void loop_givs_find (struct loop *);
283 static void loop_givs_check (struct loop *);
284 static int loop_biv_eliminable_p (struct loop *, struct iv_class *, int, int);
285 static int loop_giv_reduce_benefit (struct loop *, struct iv_class *,
286 struct induction *, rtx);
287 static void loop_givs_dead_check (struct loop *, struct iv_class *);
288 static void loop_givs_reduce (struct loop *, struct iv_class *);
289 static void loop_givs_rescan (struct loop *, struct iv_class *, rtx *);
290 static void loop_ivs_free (struct loop *);
291 static void strength_reduce (struct loop *, int);
292 static void find_single_use_in_loop (struct loop_regs *, rtx, rtx);
293 static int valid_initial_value_p (rtx, rtx, int, rtx);
294 static void find_mem_givs (const struct loop *, rtx, rtx, int, int);
295 static void record_biv (struct loop *, struct induction *, rtx, rtx, rtx,
296 rtx, rtx *, int, int);
297 static void check_final_value (const struct loop *, struct induction *);
298 static void loop_ivs_dump (const struct loop *, FILE *, int);
299 static void loop_iv_class_dump (const struct iv_class *, FILE *, int);
300 static void loop_biv_dump (const struct induction *, FILE *, int);
301 static void loop_giv_dump (const struct induction *, FILE *, int);
302 static void record_giv (const struct loop *, struct induction *, rtx, rtx,
303 rtx, rtx, rtx, rtx, int, enum g_types, int, int,
305 static void update_giv_derive (const struct loop *, rtx);
306 static void check_ext_dependent_givs (const struct loop *, struct iv_class *);
307 static int basic_induction_var (const struct loop *, rtx, enum machine_mode,
308 rtx, rtx, rtx *, rtx *, rtx **);
309 static rtx simplify_giv_expr (const struct loop *, rtx, rtx *, int *);
310 static int general_induction_var (const struct loop *loop, rtx, rtx *, rtx *,
311 rtx *, rtx *, int, int *, enum machine_mode);
312 static int consec_sets_giv (const struct loop *, int, rtx, rtx, rtx, rtx *,
313 rtx *, rtx *, rtx *);
314 static int check_dbra_loop (struct loop *, int);
315 static rtx express_from_1 (rtx, rtx, rtx);
316 static rtx combine_givs_p (struct induction *, struct induction *);
317 static int cmp_combine_givs_stats (const void *, const void *);
318 static void combine_givs (struct loop_regs *, struct iv_class *);
319 static int product_cheap_p (rtx, rtx);
320 static int maybe_eliminate_biv (const struct loop *, struct iv_class *, int,
322 static int maybe_eliminate_biv_1 (const struct loop *, rtx, rtx,
323 struct iv_class *, int, basic_block, rtx);
324 static int last_use_this_basic_block (rtx, rtx);
325 static void record_initial (rtx, rtx, void *);
326 static void update_reg_last_use (rtx, rtx);
327 static rtx next_insn_in_loop (const struct loop *, rtx);
328 static void loop_regs_scan (const struct loop *, int);
329 static int count_insns_in_loop (const struct loop *);
330 static int find_mem_in_note_1 (rtx *, void *);
331 static rtx find_mem_in_note (rtx);
332 static void load_mems (const struct loop *);
333 static int insert_loop_mem (rtx *, void *);
334 static int replace_loop_mem (rtx *, void *);
335 static void replace_loop_mems (rtx, rtx, rtx, int);
336 static int replace_loop_reg (rtx *, void *);
337 static void replace_loop_regs (rtx insn, rtx, rtx);
338 static void note_reg_stored (rtx, rtx, void *);
339 static void try_copy_prop (const struct loop *, rtx, unsigned int);
340 static void try_swap_copy_prop (const struct loop *, rtx, unsigned int);
341 static rtx check_insn_for_givs (struct loop *, rtx, int, int);
342 static rtx check_insn_for_bivs (struct loop *, rtx, int, int);
343 static rtx gen_add_mult (rtx, rtx, rtx, rtx);
344 static void loop_regs_update (const struct loop *, rtx);
345 static int iv_add_mult_cost (rtx, rtx, rtx, rtx);
347 static rtx loop_insn_emit_after (const struct loop *, basic_block, rtx, rtx);
348 static rtx loop_call_insn_emit_before (const struct loop *, basic_block,
350 static rtx loop_call_insn_hoist (const struct loop *, rtx);
351 static rtx loop_insn_sink_or_swim (const struct loop *, rtx);
353 static void loop_dump_aux (const struct loop *, FILE *, int);
354 static void loop_delete_insns (rtx, rtx);
355 static HOST_WIDE_INT remove_constant_addition (rtx *);
356 static rtx gen_load_of_final_value (rtx, rtx);
357 void debug_ivs (const struct loop *);
358 void debug_iv_class (const struct iv_class *);
359 void debug_biv (const struct induction *);
360 void debug_giv (const struct induction *);
361 void debug_loop (const struct loop *);
362 void debug_loops (const struct loops *);
364 typedef struct loop_replace_args
371 /* Nonzero iff INSN is between START and END, inclusive. */
372 #define INSN_IN_RANGE_P(INSN, START, END) \
373 (INSN_UID (INSN) < max_uid_for_loop \
374 && INSN_LUID (INSN) >= INSN_LUID (START) \
375 && INSN_LUID (INSN) <= INSN_LUID (END))
377 /* Indirect_jump_in_function is computed once per function. */
378 static int indirect_jump_in_function;
379 static int indirect_jump_in_function_p (rtx);
381 static int compute_luids (rtx, rtx, int);
383 static int biv_elimination_giv_has_0_offset (struct induction *,
384 struct induction *, rtx);
386 /* Benefit penalty, if a giv is not replaceable, i.e. must emit an insn to
387 copy the value of the strength reduced giv to its original register. */
388 static int copy_cost;
390 /* Cost of using a register, to normalize the benefits of a giv. */
391 static int reg_address_cost;
396 rtx reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
398 reg_address_cost = address_cost (reg, SImode);
400 copy_cost = COSTS_N_INSNS (1);
403 /* Compute the mapping from uids to luids.
404 LUIDs are numbers assigned to insns, like uids,
405 except that luids increase monotonically through the code.
406 Start at insn START and stop just before END. Assign LUIDs
407 starting with PREV_LUID + 1. Return the last assigned LUID + 1. */
409 compute_luids (rtx start, rtx end, int prev_luid)
414 for (insn = start, i = prev_luid; insn != end; insn = NEXT_INSN (insn))
416 if (INSN_UID (insn) >= max_uid_for_loop)
418 /* Don't assign luids to line-number NOTEs, so that the distance in
419 luids between two insns is not affected by -g. */
420 if (GET_CODE (insn) != NOTE
421 || NOTE_LINE_NUMBER (insn) <= 0)
422 uid_luid[INSN_UID (insn)] = ++i;
424 /* Give a line number note the same luid as preceding insn. */
425 uid_luid[INSN_UID (insn)] = i;
430 /* Entry point of this file. Perform loop optimization
431 on the current function. F is the first insn of the function
432 and DUMPFILE is a stream for output of a trace of actions taken
433 (or 0 if none should be output). */
436 loop_optimize (rtx f, FILE *dumpfile, int flags)
440 struct loops loops_data;
441 struct loops *loops = &loops_data;
442 struct loop_info *loops_info;
444 loop_dump_stream = dumpfile;
446 init_recog_no_volatile ();
448 max_reg_before_loop = max_reg_num ();
449 loop_max_reg = max_reg_before_loop;
453 /* Count the number of loops. */
456 for (insn = f; insn; insn = NEXT_INSN (insn))
458 if (GET_CODE (insn) == NOTE
459 && NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_BEG)
463 /* Don't waste time if no loops. */
464 if (max_loop_num == 0)
467 loops->num = max_loop_num;
469 /* Get size to use for tables indexed by uids.
470 Leave some space for labels allocated by find_and_verify_loops. */
471 max_uid_for_loop = get_max_uid () + 1 + max_loop_num * 32;
473 uid_luid = xcalloc (max_uid_for_loop, sizeof (int));
474 uid_loop = xcalloc (max_uid_for_loop, sizeof (struct loop *));
476 /* Allocate storage for array of loops. */
477 loops->array = xcalloc (loops->num, sizeof (struct loop));
479 /* Find and process each loop.
480 First, find them, and record them in order of their beginnings. */
481 find_and_verify_loops (f, loops);
483 /* Allocate and initialize auxiliary loop information. */
484 loops_info = xcalloc (loops->num, sizeof (struct loop_info));
485 for (i = 0; i < (int) loops->num; i++)
486 loops->array[i].aux = loops_info + i;
488 /* Now find all register lifetimes. This must be done after
489 find_and_verify_loops, because it might reorder the insns in the
491 reg_scan (f, max_reg_before_loop, 1);
493 /* This must occur after reg_scan so that registers created by gcse
494 will have entries in the register tables.
496 We could have added a call to reg_scan after gcse_main in toplev.c,
497 but moving this call to init_alias_analysis is more efficient. */
498 init_alias_analysis ();
500 /* See if we went too far. Note that get_max_uid already returns
501 one more that the maximum uid of all insn. */
502 if (get_max_uid () > max_uid_for_loop)
504 /* Now reset it to the actual size we need. See above. */
505 max_uid_for_loop = get_max_uid ();
507 /* find_and_verify_loops has already called compute_luids, but it
508 might have rearranged code afterwards, so we need to recompute
510 compute_luids (f, NULL_RTX, 0);
512 /* Don't leave gaps in uid_luid for insns that have been
513 deleted. It is possible that the first or last insn
514 using some register has been deleted by cross-jumping.
515 Make sure that uid_luid for that former insn's uid
516 points to the general area where that insn used to be. */
517 for (i = 0; i < max_uid_for_loop; i++)
519 uid_luid[0] = uid_luid[i];
520 if (uid_luid[0] != 0)
523 for (i = 0; i < max_uid_for_loop; i++)
524 if (uid_luid[i] == 0)
525 uid_luid[i] = uid_luid[i - 1];
527 /* Determine if the function has indirect jump. On some systems
528 this prevents low overhead loop instructions from being used. */
529 indirect_jump_in_function = indirect_jump_in_function_p (f);
531 /* Now scan the loops, last ones first, since this means inner ones are done
532 before outer ones. */
533 for (i = max_loop_num - 1; i >= 0; i--)
535 struct loop *loop = &loops->array[i];
537 if (! loop->invalid && loop->end)
539 scan_loop (loop, flags);
544 end_alias_analysis ();
547 for (i = 0; i < (int) loops->num; i++)
548 free (loops_info[i].mems);
556 /* Returns the next insn, in execution order, after INSN. START and
557 END are the NOTE_INSN_LOOP_BEG and NOTE_INSN_LOOP_END for the loop,
558 respectively. LOOP->TOP, if non-NULL, is the top of the loop in the
559 insn-stream; it is used with loops that are entered near the
563 next_insn_in_loop (const struct loop *loop, rtx insn)
565 insn = NEXT_INSN (insn);
567 if (insn == loop->end)
570 /* Go to the top of the loop, and continue there. */
577 if (insn == loop->scan_start)
584 /* Find any register references hidden inside X and add them to
585 the dependency list DEPS. This is used to look inside CLOBBER (MEM
586 when checking whether a PARALLEL can be pulled out of a loop. */
589 find_regs_nested (rtx deps, rtx x)
591 enum rtx_code code = GET_CODE (x);
593 deps = gen_rtx_EXPR_LIST (VOIDmode, x, deps);
596 const char *fmt = GET_RTX_FORMAT (code);
598 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
601 deps = find_regs_nested (deps, XEXP (x, i));
602 else if (fmt[i] == 'E')
603 for (j = 0; j < XVECLEN (x, i); j++)
604 deps = find_regs_nested (deps, XVECEXP (x, i, j));
610 /* Optimize one loop described by LOOP. */
612 /* ??? Could also move memory writes out of loops if the destination address
613 is invariant, the source is invariant, the memory write is not volatile,
614 and if we can prove that no read inside the loop can read this address
615 before the write occurs. If there is a read of this address after the
616 write, then we can also mark the memory read as invariant. */
619 scan_loop (struct loop *loop, int flags)
621 struct loop_info *loop_info = LOOP_INFO (loop);
622 struct loop_regs *regs = LOOP_REGS (loop);
624 rtx loop_start = loop->start;
625 rtx loop_end = loop->end;
627 /* 1 if we are scanning insns that could be executed zero times. */
629 /* 1 if we are scanning insns that might never be executed
630 due to a subroutine call which might exit before they are reached. */
632 /* Number of insns in the loop. */
635 rtx temp, update_start, update_end;
636 /* The SET from an insn, if it is the only SET in the insn. */
638 /* Chain describing insns movable in current loop. */
639 struct loop_movables *movables = LOOP_MOVABLES (loop);
640 /* Ratio of extra register life span we can justify
641 for saving an instruction. More if loop doesn't call subroutines
642 since in that case saving an insn makes more difference
643 and more registers are available. */
645 /* Nonzero if we are scanning instructions in a sub-loop. */
654 /* Determine whether this loop starts with a jump down to a test at
655 the end. This will occur for a small number of loops with a test
656 that is too complex to duplicate in front of the loop.
658 We search for the first insn or label in the loop, skipping NOTEs.
659 However, we must be careful not to skip past a NOTE_INSN_LOOP_BEG
660 (because we might have a loop executed only once that contains a
661 loop which starts with a jump to its exit test) or a NOTE_INSN_LOOP_END
662 (in case we have a degenerate loop).
664 Note that if we mistakenly think that a loop is entered at the top
665 when, in fact, it is entered at the exit test, the only effect will be
666 slightly poorer optimization. Making the opposite error can generate
667 incorrect code. Since very few loops now start with a jump to the
668 exit test, the code here to detect that case is very conservative. */
670 for (p = NEXT_INSN (loop_start);
672 && GET_CODE (p) != CODE_LABEL && ! INSN_P (p)
673 && (GET_CODE (p) != NOTE
674 || (NOTE_LINE_NUMBER (p) != NOTE_INSN_LOOP_BEG
675 && NOTE_LINE_NUMBER (p) != NOTE_INSN_LOOP_END));
679 loop->scan_start = p;
681 /* If loop end is the end of the current function, then emit a
682 NOTE_INSN_DELETED after loop_end and set loop->sink to the dummy
683 note insn. This is the position we use when sinking insns out of
685 if (NEXT_INSN (loop->end) != 0)
686 loop->sink = NEXT_INSN (loop->end);
688 loop->sink = emit_note_after (NOTE_INSN_DELETED, loop->end);
690 /* Set up variables describing this loop. */
692 threshold = (loop_info->has_call ? 1 : 2) * (1 + n_non_fixed_regs);
694 /* If loop has a jump before the first label,
695 the true entry is the target of that jump.
696 Start scan from there.
697 But record in LOOP->TOP the place where the end-test jumps
698 back to so we can scan that after the end of the loop. */
699 if (GET_CODE (p) == JUMP_INSN
700 /* Loop entry must be unconditional jump (and not a RETURN) */
701 && any_uncondjump_p (p)
702 && JUMP_LABEL (p) != 0
703 /* Check to see whether the jump actually
704 jumps out of the loop (meaning it's no loop).
705 This case can happen for things like
706 do {..} while (0). If this label was generated previously
707 by loop, we can't tell anything about it and have to reject
709 && INSN_IN_RANGE_P (JUMP_LABEL (p), loop_start, loop_end))
711 loop->top = next_label (loop->scan_start);
712 loop->scan_start = JUMP_LABEL (p);
715 /* If LOOP->SCAN_START was an insn created by loop, we don't know its luid
716 as required by loop_reg_used_before_p. So skip such loops. (This
717 test may never be true, but it's best to play it safe.)
719 Also, skip loops where we do not start scanning at a label. This
720 test also rejects loops starting with a JUMP_INSN that failed the
723 if (INSN_UID (loop->scan_start) >= max_uid_for_loop
724 || GET_CODE (loop->scan_start) != CODE_LABEL)
726 if (loop_dump_stream)
727 fprintf (loop_dump_stream, "\nLoop from %d to %d is phony.\n\n",
728 INSN_UID (loop_start), INSN_UID (loop_end));
732 /* Allocate extra space for REGs that might be created by load_mems.
733 We allocate a little extra slop as well, in the hopes that we
734 won't have to reallocate the regs array. */
735 loop_regs_scan (loop, loop_info->mems_idx + 16);
736 insn_count = count_insns_in_loop (loop);
738 if (loop_dump_stream)
740 fprintf (loop_dump_stream, "\nLoop from %d to %d: %d real insns.\n",
741 INSN_UID (loop_start), INSN_UID (loop_end), insn_count);
743 fprintf (loop_dump_stream, "Continue at insn %d.\n",
744 INSN_UID (loop->cont));
747 /* Scan through the loop finding insns that are safe to move.
748 Set REGS->ARRAY[I].SET_IN_LOOP negative for the reg I being set, so that
749 this reg will be considered invariant for subsequent insns.
750 We consider whether subsequent insns use the reg
751 in deciding whether it is worth actually moving.
753 MAYBE_NEVER is nonzero if we have passed a conditional jump insn
754 and therefore it is possible that the insns we are scanning
755 would never be executed. At such times, we must make sure
756 that it is safe to execute the insn once instead of zero times.
757 When MAYBE_NEVER is 0, all insns will be executed at least once
758 so that is not a problem. */
760 for (in_libcall = 0, p = next_insn_in_loop (loop, loop->scan_start);
762 p = next_insn_in_loop (loop, p))
764 if (in_libcall && INSN_P (p) && find_reg_note (p, REG_RETVAL, NULL_RTX))
766 if (GET_CODE (p) == INSN)
768 temp = find_reg_note (p, REG_LIBCALL, NULL_RTX);
772 && (set = single_set (p))
773 && GET_CODE (SET_DEST (set)) == REG
774 #ifdef PIC_OFFSET_TABLE_REG_CALL_CLOBBERED
775 && SET_DEST (set) != pic_offset_table_rtx
777 && ! regs->array[REGNO (SET_DEST (set))].may_not_optimize)
783 rtx src = SET_SRC (set);
784 rtx dependencies = 0;
786 /* Figure out what to use as a source of this insn. If a
787 REG_EQUIV note is given or if a REG_EQUAL note with a
788 constant operand is specified, use it as the source and
789 mark that we should move this insn by calling
790 emit_move_insn rather that duplicating the insn.
792 Otherwise, only use the REG_EQUAL contents if a REG_RETVAL
794 temp = find_reg_note (p, REG_EQUIV, NULL_RTX);
796 src = XEXP (temp, 0), move_insn = 1;
799 temp = find_reg_note (p, REG_EQUAL, NULL_RTX);
800 if (temp && CONSTANT_P (XEXP (temp, 0)))
801 src = XEXP (temp, 0), move_insn = 1;
802 if (temp && find_reg_note (p, REG_RETVAL, NULL_RTX))
804 src = XEXP (temp, 0);
805 /* A libcall block can use regs that don't appear in
806 the equivalent expression. To move the libcall,
807 we must move those regs too. */
808 dependencies = libcall_other_reg (p, src);
812 /* For parallels, add any possible uses to the dependencies, as
813 we can't move the insn without resolving them first.
814 MEMs inside CLOBBERs may also reference registers; these
815 count as implicit uses. */
816 if (GET_CODE (PATTERN (p)) == PARALLEL)
818 for (i = 0; i < XVECLEN (PATTERN (p), 0); i++)
820 rtx x = XVECEXP (PATTERN (p), 0, i);
821 if (GET_CODE (x) == USE)
823 = gen_rtx_EXPR_LIST (VOIDmode, XEXP (x, 0),
825 else if (GET_CODE (x) == CLOBBER
826 && GET_CODE (XEXP (x, 0)) == MEM)
827 dependencies = find_regs_nested (dependencies,
828 XEXP (XEXP (x, 0), 0));
832 if (/* The register is used in basic blocks other
833 than the one where it is set (meaning that
834 something after this point in the loop might
835 depend on its value before the set). */
836 ! reg_in_basic_block_p (p, SET_DEST (set))
837 /* And the set is not guaranteed to be executed once
838 the loop starts, or the value before the set is
839 needed before the set occurs...
841 ??? Note we have quadratic behavior here, mitigated
842 by the fact that the previous test will often fail for
843 large loops. Rather than re-scanning the entire loop
844 each time for register usage, we should build tables
845 of the register usage and use them here instead. */
847 || loop_reg_used_before_p (loop, set, p)))
848 /* It is unsafe to move the set. However, it may be OK to
849 move the source into a new pseudo, and substitute a
850 reg-to-reg copy for the original insn.
852 This code used to consider it OK to move a set of a variable
853 which was not created by the user and not used in an exit
855 That behavior is incorrect and was removed. */
858 /* Don't try to optimize a MODE_CC set with a constant
859 source. It probably will be combined with a conditional
861 if (GET_MODE_CLASS (GET_MODE (SET_DEST (set))) == MODE_CC
864 /* Don't try to optimize a register that was made
865 by loop-optimization for an inner loop.
866 We don't know its life-span, so we can't compute
868 else if (REGNO (SET_DEST (set)) >= max_reg_before_loop)
870 /* Don't move the source and add a reg-to-reg copy:
871 - with -Os (this certainly increases size),
872 - if the mode doesn't support copy operations (obviously),
873 - if the source is already a reg (the motion will gain nothing),
874 - if the source is a legitimate constant (likewise). */
877 || ! can_copy_p (GET_MODE (SET_SRC (set)))
878 || GET_CODE (SET_SRC (set)) == REG
879 || (CONSTANT_P (SET_SRC (set))
880 && LEGITIMATE_CONSTANT_P (SET_SRC (set)))))
882 else if ((tem = loop_invariant_p (loop, src))
883 && (dependencies == 0
885 = loop_invariant_p (loop, dependencies)) != 0)
886 && (regs->array[REGNO (SET_DEST (set))].set_in_loop == 1
888 = consec_sets_invariant_p
889 (loop, SET_DEST (set),
890 regs->array[REGNO (SET_DEST (set))].set_in_loop,
892 /* If the insn can cause a trap (such as divide by zero),
893 can't move it unless it's guaranteed to be executed
894 once loop is entered. Even a function call might
895 prevent the trap insn from being reached
896 (since it might exit!) */
897 && ! ((maybe_never || call_passed)
898 && may_trap_p (src)))
901 int regno = REGNO (SET_DEST (set));
903 /* A potential lossage is where we have a case where two insns
904 can be combined as long as they are both in the loop, but
905 we move one of them outside the loop. For large loops,
906 this can lose. The most common case of this is the address
907 of a function being called.
909 Therefore, if this register is marked as being used
910 exactly once if we are in a loop with calls
911 (a "large loop"), see if we can replace the usage of
912 this register with the source of this SET. If we can,
915 Don't do this if P has a REG_RETVAL note or if we have
916 SMALL_REGISTER_CLASSES and SET_SRC is a hard register. */
918 if (loop_info->has_call
919 && regs->array[regno].single_usage != 0
920 && regs->array[regno].single_usage != const0_rtx
921 && REGNO_FIRST_UID (regno) == INSN_UID (p)
922 && (REGNO_LAST_UID (regno)
923 == INSN_UID (regs->array[regno].single_usage))
924 && regs->array[regno].set_in_loop == 1
925 && GET_CODE (SET_SRC (set)) != ASM_OPERANDS
926 && ! side_effects_p (SET_SRC (set))
927 && ! find_reg_note (p, REG_RETVAL, NULL_RTX)
928 && (! SMALL_REGISTER_CLASSES
929 || (! (GET_CODE (SET_SRC (set)) == REG
930 && (REGNO (SET_SRC (set))
931 < FIRST_PSEUDO_REGISTER))))
932 && regno >= FIRST_PSEUDO_REGISTER
933 /* This test is not redundant; SET_SRC (set) might be
934 a call-clobbered register and the life of REGNO
935 might span a call. */
936 && ! modified_between_p (SET_SRC (set), p,
937 regs->array[regno].single_usage)
938 && no_labels_between_p (p,
939 regs->array[regno].single_usage)
940 && validate_replace_rtx (SET_DEST (set), SET_SRC (set),
941 regs->array[regno].single_usage))
943 /* Replace any usage in a REG_EQUAL note. Must copy
944 the new source, so that we don't get rtx sharing
945 between the SET_SOURCE and REG_NOTES of insn p. */
946 REG_NOTES (regs->array[regno].single_usage)
948 (REG_NOTES (regs->array[regno].single_usage),
949 SET_DEST (set), copy_rtx (SET_SRC (set))));
952 for (i = 0; i < LOOP_REGNO_NREGS (regno, SET_DEST (set));
954 regs->array[regno+i].set_in_loop = 0;
958 m = xmalloc (sizeof (struct movable));
962 m->dependencies = dependencies;
963 m->set_dest = SET_DEST (set);
966 = regs->array[REGNO (SET_DEST (set))].set_in_loop - 1;
970 m->move_insn = move_insn;
971 m->move_insn_first = 0;
972 m->insert_temp = insert_temp;
973 m->is_equiv = (find_reg_note (p, REG_EQUIV, NULL_RTX) != 0);
974 m->savemode = VOIDmode;
976 /* Set M->cond if either loop_invariant_p
977 or consec_sets_invariant_p returned 2
978 (only conditionally invariant). */
979 m->cond = ((tem | tem1 | tem2) > 1);
980 m->global = LOOP_REG_GLOBAL_P (loop, regno);
982 m->lifetime = LOOP_REG_LIFETIME (loop, regno);
983 m->savings = regs->array[regno].n_times_set;
984 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
985 m->savings += libcall_benefit (p);
986 for (i = 0; i < LOOP_REGNO_NREGS (regno, SET_DEST (set)); i++)
987 regs->array[regno+i].set_in_loop = move_insn ? -2 : -1;
988 /* Add M to the end of the chain MOVABLES. */
989 loop_movables_add (movables, m);
993 /* It is possible for the first instruction to have a
994 REG_EQUAL note but a non-invariant SET_SRC, so we must
995 remember the status of the first instruction in case
996 the last instruction doesn't have a REG_EQUAL note. */
997 m->move_insn_first = m->move_insn;
999 /* Skip this insn, not checking REG_LIBCALL notes. */
1000 p = next_nonnote_insn (p);
1001 /* Skip the consecutive insns, if there are any. */
1002 p = skip_consec_insns (p, m->consec);
1003 /* Back up to the last insn of the consecutive group. */
1004 p = prev_nonnote_insn (p);
1006 /* We must now reset m->move_insn, m->is_equiv, and
1007 possibly m->set_src to correspond to the effects of
1009 temp = find_reg_note (p, REG_EQUIV, NULL_RTX);
1011 m->set_src = XEXP (temp, 0), m->move_insn = 1;
1014 temp = find_reg_note (p, REG_EQUAL, NULL_RTX);
1015 if (temp && CONSTANT_P (XEXP (temp, 0)))
1016 m->set_src = XEXP (temp, 0), m->move_insn = 1;
1022 = (find_reg_note (p, REG_EQUIV, NULL_RTX) != 0);
1025 /* If this register is always set within a STRICT_LOW_PART
1026 or set to zero, then its high bytes are constant.
1027 So clear them outside the loop and within the loop
1028 just load the low bytes.
1029 We must check that the machine has an instruction to do so.
1030 Also, if the value loaded into the register
1031 depends on the same register, this cannot be done. */
1032 else if (SET_SRC (set) == const0_rtx
1033 && GET_CODE (NEXT_INSN (p)) == INSN
1034 && (set1 = single_set (NEXT_INSN (p)))
1035 && GET_CODE (set1) == SET
1036 && (GET_CODE (SET_DEST (set1)) == STRICT_LOW_PART)
1037 && (GET_CODE (XEXP (SET_DEST (set1), 0)) == SUBREG)
1038 && (SUBREG_REG (XEXP (SET_DEST (set1), 0))
1040 && !reg_mentioned_p (SET_DEST (set), SET_SRC (set1)))
1042 int regno = REGNO (SET_DEST (set));
1043 if (regs->array[regno].set_in_loop == 2)
1046 m = xmalloc (sizeof (struct movable));
1049 m->set_dest = SET_DEST (set);
1050 m->dependencies = 0;
1056 m->move_insn_first = 0;
1057 m->insert_temp = insert_temp;
1059 /* If the insn may not be executed on some cycles,
1060 we can't clear the whole reg; clear just high part.
1061 Not even if the reg is used only within this loop.
1068 Clearing x before the inner loop could clobber a value
1069 being saved from the last time around the outer loop.
1070 However, if the reg is not used outside this loop
1071 and all uses of the register are in the same
1072 basic block as the store, there is no problem.
1074 If this insn was made by loop, we don't know its
1075 INSN_LUID and hence must make a conservative
1077 m->global = (INSN_UID (p) >= max_uid_for_loop
1078 || LOOP_REG_GLOBAL_P (loop, regno)
1079 || (labels_in_range_p
1080 (p, REGNO_FIRST_LUID (regno))));
1081 if (maybe_never && m->global)
1082 m->savemode = GET_MODE (SET_SRC (set1));
1084 m->savemode = VOIDmode;
1088 m->lifetime = LOOP_REG_LIFETIME (loop, regno);
1091 i < LOOP_REGNO_NREGS (regno, SET_DEST (set));
1093 regs->array[regno+i].set_in_loop = -1;
1094 /* Add M to the end of the chain MOVABLES. */
1095 loop_movables_add (movables, m);
1100 /* Past a call insn, we get to insns which might not be executed
1101 because the call might exit. This matters for insns that trap.
1102 Constant and pure call insns always return, so they don't count. */
1103 else if (GET_CODE (p) == CALL_INSN && ! CONST_OR_PURE_CALL_P (p))
1105 /* Past a label or a jump, we get to insns for which we
1106 can't count on whether or how many times they will be
1107 executed during each iteration. Therefore, we can
1108 only move out sets of trivial variables
1109 (those not used after the loop). */
1110 /* Similar code appears twice in strength_reduce. */
1111 else if ((GET_CODE (p) == CODE_LABEL || GET_CODE (p) == JUMP_INSN)
1112 /* If we enter the loop in the middle, and scan around to the
1113 beginning, don't set maybe_never for that. This must be an
1114 unconditional jump, otherwise the code at the top of the
1115 loop might never be executed. Unconditional jumps are
1116 followed by a barrier then the loop_end. */
1117 && ! (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p) == loop->top
1118 && NEXT_INSN (NEXT_INSN (p)) == loop_end
1119 && any_uncondjump_p (p)))
1121 else if (GET_CODE (p) == NOTE)
1123 /* At the virtual top of a converted loop, insns are again known to
1124 be executed: logically, the loop begins here even though the exit
1125 code has been duplicated. */
1126 if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_VTOP && loop_depth == 0)
1127 maybe_never = call_passed = 0;
1128 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_BEG)
1130 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_END)
1135 /* If one movable subsumes another, ignore that other. */
1137 ignore_some_movables (movables);
1139 /* For each movable insn, see if the reg that it loads
1140 leads when it dies right into another conditionally movable insn.
1141 If so, record that the second insn "forces" the first one,
1142 since the second can be moved only if the first is. */
1144 force_movables (movables);
1146 /* See if there are multiple movable insns that load the same value.
1147 If there are, make all but the first point at the first one
1148 through the `match' field, and add the priorities of them
1149 all together as the priority of the first. */
1151 combine_movables (movables, regs);
1153 /* Now consider each movable insn to decide whether it is worth moving.
1154 Store 0 in regs->array[I].set_in_loop for each reg I that is moved.
1156 For machines with few registers this increases code size, so do not
1157 move moveables when optimizing for code size on such machines.
1158 (The 18 below is the value for i386.) */
1161 || (reg_class_size[GENERAL_REGS] > 18 && !loop_info->has_call))
1163 move_movables (loop, movables, threshold, insn_count);
1165 /* Recalculate regs->array if move_movables has created new
1167 if (max_reg_num () > regs->num)
1169 loop_regs_scan (loop, 0);
1170 for (update_start = loop_start;
1171 PREV_INSN (update_start)
1172 && GET_CODE (PREV_INSN (update_start)) != CODE_LABEL;
1173 update_start = PREV_INSN (update_start))
1175 update_end = NEXT_INSN (loop_end);
1177 reg_scan_update (update_start, update_end, loop_max_reg);
1178 loop_max_reg = max_reg_num ();
1182 /* Now candidates that still are negative are those not moved.
1183 Change regs->array[I].set_in_loop to indicate that those are not actually
1185 for (i = 0; i < regs->num; i++)
1186 if (regs->array[i].set_in_loop < 0)
1187 regs->array[i].set_in_loop = regs->array[i].n_times_set;
1189 /* Now that we've moved some things out of the loop, we might be able to
1190 hoist even more memory references. */
1193 /* Recalculate regs->array if load_mems has created new registers. */
1194 if (max_reg_num () > regs->num)
1195 loop_regs_scan (loop, 0);
1197 for (update_start = loop_start;
1198 PREV_INSN (update_start)
1199 && GET_CODE (PREV_INSN (update_start)) != CODE_LABEL;
1200 update_start = PREV_INSN (update_start))
1202 update_end = NEXT_INSN (loop_end);
1204 reg_scan_update (update_start, update_end, loop_max_reg);
1205 loop_max_reg = max_reg_num ();
1207 if (flag_strength_reduce)
1209 if (update_end && GET_CODE (update_end) == CODE_LABEL)
1210 /* Ensure our label doesn't go away. */
1211 LABEL_NUSES (update_end)++;
1213 strength_reduce (loop, flags);
1215 reg_scan_update (update_start, update_end, loop_max_reg);
1216 loop_max_reg = max_reg_num ();
1218 if (update_end && GET_CODE (update_end) == CODE_LABEL
1219 && --LABEL_NUSES (update_end) == 0)
1220 delete_related_insns (update_end);
1224 /* The movable information is required for strength reduction. */
1225 loop_movables_free (movables);
1232 /* Add elements to *OUTPUT to record all the pseudo-regs
1233 mentioned in IN_THIS but not mentioned in NOT_IN_THIS. */
1236 record_excess_regs (rtx in_this, rtx not_in_this, rtx *output)
1242 code = GET_CODE (in_this);
1256 if (REGNO (in_this) >= FIRST_PSEUDO_REGISTER
1257 && ! reg_mentioned_p (in_this, not_in_this))
1258 *output = gen_rtx_EXPR_LIST (VOIDmode, in_this, *output);
1265 fmt = GET_RTX_FORMAT (code);
1266 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1273 for (j = 0; j < XVECLEN (in_this, i); j++)
1274 record_excess_regs (XVECEXP (in_this, i, j), not_in_this, output);
1278 record_excess_regs (XEXP (in_this, i), not_in_this, output);
1284 /* Check what regs are referred to in the libcall block ending with INSN,
1285 aside from those mentioned in the equivalent value.
1286 If there are none, return 0.
1287 If there are one or more, return an EXPR_LIST containing all of them. */
1290 libcall_other_reg (rtx insn, rtx equiv)
1292 rtx note = find_reg_note (insn, REG_RETVAL, NULL_RTX);
1293 rtx p = XEXP (note, 0);
1296 /* First, find all the regs used in the libcall block
1297 that are not mentioned as inputs to the result. */
1301 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
1302 || GET_CODE (p) == CALL_INSN)
1303 record_excess_regs (PATTERN (p), equiv, &output);
1310 /* Return 1 if all uses of REG
1311 are between INSN and the end of the basic block. */
1314 reg_in_basic_block_p (rtx insn, rtx reg)
1316 int regno = REGNO (reg);
1319 if (REGNO_FIRST_UID (regno) != INSN_UID (insn))
1322 /* Search this basic block for the already recorded last use of the reg. */
1323 for (p = insn; p; p = NEXT_INSN (p))
1325 switch (GET_CODE (p))
1332 /* Ordinary insn: if this is the last use, we win. */
1333 if (REGNO_LAST_UID (regno) == INSN_UID (p))
1338 /* Jump insn: if this is the last use, we win. */
1339 if (REGNO_LAST_UID (regno) == INSN_UID (p))
1341 /* Otherwise, it's the end of the basic block, so we lose. */
1346 /* It's the end of the basic block, so we lose. */
1354 /* The "last use" that was recorded can't be found after the first
1355 use. This can happen when the last use was deleted while
1356 processing an inner loop, this inner loop was then completely
1357 unrolled, and the outer loop is always exited after the inner loop,
1358 so that everything after the first use becomes a single basic block. */
1362 /* Compute the benefit of eliminating the insns in the block whose
1363 last insn is LAST. This may be a group of insns used to compute a
1364 value directly or can contain a library call. */
1367 libcall_benefit (rtx last)
1372 for (insn = XEXP (find_reg_note (last, REG_RETVAL, NULL_RTX), 0);
1373 insn != last; insn = NEXT_INSN (insn))
1375 if (GET_CODE (insn) == CALL_INSN)
1376 benefit += 10; /* Assume at least this many insns in a library
1378 else if (GET_CODE (insn) == INSN
1379 && GET_CODE (PATTERN (insn)) != USE
1380 && GET_CODE (PATTERN (insn)) != CLOBBER)
1387 /* Skip COUNT insns from INSN, counting library calls as 1 insn. */
1390 skip_consec_insns (rtx insn, int count)
1392 for (; count > 0; count--)
1396 /* If first insn of libcall sequence, skip to end. */
1397 /* Do this at start of loop, since INSN is guaranteed to
1399 if (GET_CODE (insn) != NOTE
1400 && (temp = find_reg_note (insn, REG_LIBCALL, NULL_RTX)))
1401 insn = XEXP (temp, 0);
1404 insn = NEXT_INSN (insn);
1405 while (GET_CODE (insn) == NOTE);
1411 /* Ignore any movable whose insn falls within a libcall
1412 which is part of another movable.
1413 We make use of the fact that the movable for the libcall value
1414 was made later and so appears later on the chain. */
1417 ignore_some_movables (struct loop_movables *movables)
1419 struct movable *m, *m1;
1421 for (m = movables->head; m; m = m->next)
1423 /* Is this a movable for the value of a libcall? */
1424 rtx note = find_reg_note (m->insn, REG_RETVAL, NULL_RTX);
1428 /* Check for earlier movables inside that range,
1429 and mark them invalid. We cannot use LUIDs here because
1430 insns created by loop.c for prior loops don't have LUIDs.
1431 Rather than reject all such insns from movables, we just
1432 explicitly check each insn in the libcall (since invariant
1433 libcalls aren't that common). */
1434 for (insn = XEXP (note, 0); insn != m->insn; insn = NEXT_INSN (insn))
1435 for (m1 = movables->head; m1 != m; m1 = m1->next)
1436 if (m1->insn == insn)
1442 /* For each movable insn, see if the reg that it loads
1443 leads when it dies right into another conditionally movable insn.
1444 If so, record that the second insn "forces" the first one,
1445 since the second can be moved only if the first is. */
1448 force_movables (struct loop_movables *movables)
1450 struct movable *m, *m1;
1452 for (m1 = movables->head; m1; m1 = m1->next)
1453 /* Omit this if moving just the (SET (REG) 0) of a zero-extend. */
1454 if (!m1->partial && !m1->done)
1456 int regno = m1->regno;
1457 for (m = m1->next; m; m = m->next)
1458 /* ??? Could this be a bug? What if CSE caused the
1459 register of M1 to be used after this insn?
1460 Since CSE does not update regno_last_uid,
1461 this insn M->insn might not be where it dies.
1462 But very likely this doesn't matter; what matters is
1463 that M's reg is computed from M1's reg. */
1464 if (INSN_UID (m->insn) == REGNO_LAST_UID (regno)
1467 if (m != 0 && m->set_src == m1->set_dest
1468 /* If m->consec, m->set_src isn't valid. */
1472 /* Increase the priority of the moving the first insn
1473 since it permits the second to be moved as well.
1474 Likewise for insns already forced by the first insn. */
1480 for (m2 = m1; m2; m2 = m2->forces)
1482 m2->lifetime += m->lifetime;
1483 m2->savings += m->savings;
1489 /* Find invariant expressions that are equal and can be combined into
1493 combine_movables (struct loop_movables *movables, struct loop_regs *regs)
1496 char *matched_regs = xmalloc (regs->num);
1497 enum machine_mode mode;
1499 /* Regs that are set more than once are not allowed to match
1500 or be matched. I'm no longer sure why not. */
1501 /* Only pseudo registers are allowed to match or be matched,
1502 since move_movables does not validate the change. */
1503 /* Perhaps testing m->consec_sets would be more appropriate here? */
1505 for (m = movables->head; m; m = m->next)
1506 if (m->match == 0 && regs->array[m->regno].n_times_set == 1
1507 && m->regno >= FIRST_PSEUDO_REGISTER
1512 int regno = m->regno;
1514 memset (matched_regs, 0, regs->num);
1515 matched_regs[regno] = 1;
1517 /* We want later insns to match the first one. Don't make the first
1518 one match any later ones. So start this loop at m->next. */
1519 for (m1 = m->next; m1; m1 = m1->next)
1520 if (m != m1 && m1->match == 0
1522 && regs->array[m1->regno].n_times_set == 1
1523 && m1->regno >= FIRST_PSEUDO_REGISTER
1524 /* A reg used outside the loop mustn't be eliminated. */
1526 /* A reg used for zero-extending mustn't be eliminated. */
1528 && (matched_regs[m1->regno]
1531 /* Can combine regs with different modes loaded from the
1532 same constant only if the modes are the same or
1533 if both are integer modes with M wider or the same
1534 width as M1. The check for integer is redundant, but
1535 safe, since the only case of differing destination
1536 modes with equal sources is when both sources are
1537 VOIDmode, i.e., CONST_INT. */
1538 (GET_MODE (m->set_dest) == GET_MODE (m1->set_dest)
1539 || (GET_MODE_CLASS (GET_MODE (m->set_dest)) == MODE_INT
1540 && GET_MODE_CLASS (GET_MODE (m1->set_dest)) == MODE_INT
1541 && (GET_MODE_BITSIZE (GET_MODE (m->set_dest))
1542 >= GET_MODE_BITSIZE (GET_MODE (m1->set_dest)))))
1543 /* See if the source of M1 says it matches M. */
1544 && ((GET_CODE (m1->set_src) == REG
1545 && matched_regs[REGNO (m1->set_src)])
1546 || rtx_equal_for_loop_p (m->set_src, m1->set_src,
1548 && ((m->dependencies == m1->dependencies)
1549 || rtx_equal_p (m->dependencies, m1->dependencies)))
1551 m->lifetime += m1->lifetime;
1552 m->savings += m1->savings;
1555 matched_regs[m1->regno] = 1;
1559 /* Now combine the regs used for zero-extension.
1560 This can be done for those not marked `global'
1561 provided their lives don't overlap. */
1563 for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT); mode != VOIDmode;
1564 mode = GET_MODE_WIDER_MODE (mode))
1566 struct movable *m0 = 0;
1568 /* Combine all the registers for extension from mode MODE.
1569 Don't combine any that are used outside this loop. */
1570 for (m = movables->head; m; m = m->next)
1571 if (m->partial && ! m->global
1572 && mode == GET_MODE (SET_SRC (PATTERN (NEXT_INSN (m->insn)))))
1576 int first = REGNO_FIRST_LUID (m->regno);
1577 int last = REGNO_LAST_LUID (m->regno);
1581 /* First one: don't check for overlap, just record it. */
1586 /* Make sure they extend to the same mode.
1587 (Almost always true.) */
1588 if (GET_MODE (m->set_dest) != GET_MODE (m0->set_dest))
1591 /* We already have one: check for overlap with those
1592 already combined together. */
1593 for (m1 = movables->head; m1 != m; m1 = m1->next)
1594 if (m1 == m0 || (m1->partial && m1->match == m0))
1595 if (! (REGNO_FIRST_LUID (m1->regno) > last
1596 || REGNO_LAST_LUID (m1->regno) < first))
1599 /* No overlap: we can combine this with the others. */
1600 m0->lifetime += m->lifetime;
1601 m0->savings += m->savings;
1611 free (matched_regs);
1614 /* Returns the number of movable instructions in LOOP that were not
1615 moved outside the loop. */
1618 num_unmoved_movables (const struct loop *loop)
1623 for (m = LOOP_MOVABLES (loop)->head; m; m = m->next)
1631 /* Return 1 if regs X and Y will become the same if moved. */
1634 regs_match_p (rtx x, rtx y, struct loop_movables *movables)
1636 unsigned int xn = REGNO (x);
1637 unsigned int yn = REGNO (y);
1638 struct movable *mx, *my;
1640 for (mx = movables->head; mx; mx = mx->next)
1641 if (mx->regno == xn)
1644 for (my = movables->head; my; my = my->next)
1645 if (my->regno == yn)
1649 && ((mx->match == my->match && mx->match != 0)
1651 || mx == my->match));
1654 /* Return 1 if X and Y are identical-looking rtx's.
1655 This is the Lisp function EQUAL for rtx arguments.
1657 If two registers are matching movables or a movable register and an
1658 equivalent constant, consider them equal. */
1661 rtx_equal_for_loop_p (rtx x, rtx y, struct loop_movables *movables,
1662 struct loop_regs *regs)
1672 if (x == 0 || y == 0)
1675 code = GET_CODE (x);
1677 /* If we have a register and a constant, they may sometimes be
1679 if (GET_CODE (x) == REG && regs->array[REGNO (x)].set_in_loop == -2
1682 for (m = movables->head; m; m = m->next)
1683 if (m->move_insn && m->regno == REGNO (x)
1684 && rtx_equal_p (m->set_src, y))
1687 else if (GET_CODE (y) == REG && regs->array[REGNO (y)].set_in_loop == -2
1690 for (m = movables->head; m; m = m->next)
1691 if (m->move_insn && m->regno == REGNO (y)
1692 && rtx_equal_p (m->set_src, x))
1696 /* Otherwise, rtx's of different codes cannot be equal. */
1697 if (code != GET_CODE (y))
1700 /* (MULT:SI x y) and (MULT:HI x y) are NOT equivalent.
1701 (REG:SI x) and (REG:HI x) are NOT equivalent. */
1703 if (GET_MODE (x) != GET_MODE (y))
1706 /* These three types of rtx's can be compared nonrecursively. */
1708 return (REGNO (x) == REGNO (y) || regs_match_p (x, y, movables));
1710 if (code == LABEL_REF)
1711 return XEXP (x, 0) == XEXP (y, 0);
1712 if (code == SYMBOL_REF)
1713 return XSTR (x, 0) == XSTR (y, 0);
1715 /* Compare the elements. If any pair of corresponding elements
1716 fail to match, return 0 for the whole things. */
1718 fmt = GET_RTX_FORMAT (code);
1719 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1724 if (XWINT (x, i) != XWINT (y, i))
1729 if (XINT (x, i) != XINT (y, i))
1734 /* Two vectors must have the same length. */
1735 if (XVECLEN (x, i) != XVECLEN (y, i))
1738 /* And the corresponding elements must match. */
1739 for (j = 0; j < XVECLEN (x, i); j++)
1740 if (rtx_equal_for_loop_p (XVECEXP (x, i, j), XVECEXP (y, i, j),
1741 movables, regs) == 0)
1746 if (rtx_equal_for_loop_p (XEXP (x, i), XEXP (y, i), movables, regs)
1752 if (strcmp (XSTR (x, i), XSTR (y, i)))
1757 /* These are just backpointers, so they don't matter. */
1763 /* It is believed that rtx's at this level will never
1764 contain anything but integers and other rtx's,
1765 except for within LABEL_REFs and SYMBOL_REFs. */
1773 /* If X contains any LABEL_REF's, add REG_LABEL notes for them to all
1774 insns in INSNS which use the reference. LABEL_NUSES for CODE_LABEL
1775 references is incremented once for each added note. */
1778 add_label_notes (rtx x, rtx insns)
1780 enum rtx_code code = GET_CODE (x);
1785 if (code == LABEL_REF && !LABEL_REF_NONLOCAL_P (x))
1787 /* This code used to ignore labels that referred to dispatch tables to
1788 avoid flow generating (slightly) worse code.
1790 We no longer ignore such label references (see LABEL_REF handling in
1791 mark_jump_label for additional information). */
1792 for (insn = insns; insn; insn = NEXT_INSN (insn))
1793 if (reg_mentioned_p (XEXP (x, 0), insn))
1795 REG_NOTES (insn) = gen_rtx_INSN_LIST (REG_LABEL, XEXP (x, 0),
1797 if (LABEL_P (XEXP (x, 0)))
1798 LABEL_NUSES (XEXP (x, 0))++;
1802 fmt = GET_RTX_FORMAT (code);
1803 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1806 add_label_notes (XEXP (x, i), insns);
1807 else if (fmt[i] == 'E')
1808 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
1809 add_label_notes (XVECEXP (x, i, j), insns);
1813 /* Scan MOVABLES, and move the insns that deserve to be moved.
1814 If two matching movables are combined, replace one reg with the
1815 other throughout. */
1818 move_movables (struct loop *loop, struct loop_movables *movables,
1819 int threshold, int insn_count)
1821 struct loop_regs *regs = LOOP_REGS (loop);
1822 int nregs = regs->num;
1826 rtx loop_start = loop->start;
1827 rtx loop_end = loop->end;
1828 /* Map of pseudo-register replacements to handle combining
1829 when we move several insns that load the same value
1830 into different pseudo-registers. */
1831 rtx *reg_map = xcalloc (nregs, sizeof (rtx));
1832 char *already_moved = xcalloc (nregs, sizeof (char));
1834 for (m = movables->head; m; m = m->next)
1836 /* Describe this movable insn. */
1838 if (loop_dump_stream)
1840 fprintf (loop_dump_stream, "Insn %d: regno %d (life %d), ",
1841 INSN_UID (m->insn), m->regno, m->lifetime);
1843 fprintf (loop_dump_stream, "consec %d, ", m->consec);
1845 fprintf (loop_dump_stream, "cond ");
1847 fprintf (loop_dump_stream, "force ");
1849 fprintf (loop_dump_stream, "global ");
1851 fprintf (loop_dump_stream, "done ");
1853 fprintf (loop_dump_stream, "move-insn ");
1855 fprintf (loop_dump_stream, "matches %d ",
1856 INSN_UID (m->match->insn));
1858 fprintf (loop_dump_stream, "forces %d ",
1859 INSN_UID (m->forces->insn));
1862 /* Ignore the insn if it's already done (it matched something else).
1863 Otherwise, see if it is now safe to move. */
1867 || (1 == loop_invariant_p (loop, m->set_src)
1868 && (m->dependencies == 0
1869 || 1 == loop_invariant_p (loop, m->dependencies))
1871 || 1 == consec_sets_invariant_p (loop, m->set_dest,
1874 && (! m->forces || m->forces->done))
1878 int savings = m->savings;
1880 /* We have an insn that is safe to move.
1881 Compute its desirability. */
1886 if (loop_dump_stream)
1887 fprintf (loop_dump_stream, "savings %d ", savings);
1889 if (regs->array[regno].moved_once && loop_dump_stream)
1890 fprintf (loop_dump_stream, "halved since already moved ");
1892 /* An insn MUST be moved if we already moved something else
1893 which is safe only if this one is moved too: that is,
1894 if already_moved[REGNO] is nonzero. */
1896 /* An insn is desirable to move if the new lifetime of the
1897 register is no more than THRESHOLD times the old lifetime.
1898 If it's not desirable, it means the loop is so big
1899 that moving won't speed things up much,
1900 and it is liable to make register usage worse. */
1902 /* It is also desirable to move if it can be moved at no
1903 extra cost because something else was already moved. */
1905 if (already_moved[regno]
1906 || flag_move_all_movables
1907 || (threshold * savings * m->lifetime) >=
1908 (regs->array[regno].moved_once ? insn_count * 2 : insn_count)
1909 || (m->forces && m->forces->done
1910 && regs->array[m->forces->regno].n_times_set == 1))
1914 rtx first = NULL_RTX;
1915 rtx newreg = NULL_RTX;
1918 newreg = gen_reg_rtx (GET_MODE (m->set_dest));
1920 /* Now move the insns that set the reg. */
1922 if (m->partial && m->match)
1926 /* Find the end of this chain of matching regs.
1927 Thus, we load each reg in the chain from that one reg.
1928 And that reg is loaded with 0 directly,
1929 since it has ->match == 0. */
1930 for (m1 = m; m1->match; m1 = m1->match);
1931 newpat = gen_move_insn (SET_DEST (PATTERN (m->insn)),
1932 SET_DEST (PATTERN (m1->insn)));
1933 i1 = loop_insn_hoist (loop, newpat);
1935 /* Mark the moved, invariant reg as being allowed to
1936 share a hard reg with the other matching invariant. */
1937 REG_NOTES (i1) = REG_NOTES (m->insn);
1938 r1 = SET_DEST (PATTERN (m->insn));
1939 r2 = SET_DEST (PATTERN (m1->insn));
1941 = gen_rtx_EXPR_LIST (VOIDmode, r1,
1942 gen_rtx_EXPR_LIST (VOIDmode, r2,
1944 delete_insn (m->insn);
1949 if (loop_dump_stream)
1950 fprintf (loop_dump_stream, " moved to %d", INSN_UID (i1));
1952 /* If we are to re-generate the item being moved with a
1953 new move insn, first delete what we have and then emit
1954 the move insn before the loop. */
1955 else if (m->move_insn)
1959 for (count = m->consec; count >= 0; count--)
1961 /* If this is the first insn of a library call sequence,
1962 something is very wrong. */
1963 if (GET_CODE (p) != NOTE
1964 && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
1967 /* If this is the last insn of a libcall sequence, then
1968 delete every insn in the sequence except the last.
1969 The last insn is handled in the normal manner. */
1970 if (GET_CODE (p) != NOTE
1971 && (temp = find_reg_note (p, REG_RETVAL, NULL_RTX)))
1973 temp = XEXP (temp, 0);
1975 temp = delete_insn (temp);
1979 p = delete_insn (p);
1981 /* simplify_giv_expr expects that it can walk the insns
1982 at m->insn forwards and see this old sequence we are
1983 tossing here. delete_insn does preserve the next
1984 pointers, but when we skip over a NOTE we must fix
1985 it up. Otherwise that code walks into the non-deleted
1987 while (p && GET_CODE (p) == NOTE)
1988 p = NEXT_INSN (temp) = NEXT_INSN (p);
1992 /* Replace the original insn with a move from
1993 our newly created temp. */
1995 emit_move_insn (m->set_dest, newreg);
1998 emit_insn_before (seq, p);
2003 emit_move_insn (m->insert_temp ? newreg : m->set_dest,
2008 add_label_notes (m->set_src, seq);
2010 i1 = loop_insn_hoist (loop, seq);
2011 if (! find_reg_note (i1, REG_EQUAL, NULL_RTX))
2012 set_unique_reg_note (i1,
2013 m->is_equiv ? REG_EQUIV : REG_EQUAL,
2016 if (loop_dump_stream)
2017 fprintf (loop_dump_stream, " moved to %d", INSN_UID (i1));
2019 /* The more regs we move, the less we like moving them. */
2024 for (count = m->consec; count >= 0; count--)
2028 /* If first insn of libcall sequence, skip to end. */
2029 /* Do this at start of loop, since p is guaranteed to
2031 if (GET_CODE (p) != NOTE
2032 && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
2035 /* If last insn of libcall sequence, move all
2036 insns except the last before the loop. The last
2037 insn is handled in the normal manner. */
2038 if (GET_CODE (p) != NOTE
2039 && (temp = find_reg_note (p, REG_RETVAL, NULL_RTX)))
2043 rtx fn_address_insn = 0;
2046 for (temp = XEXP (temp, 0); temp != p;
2047 temp = NEXT_INSN (temp))
2053 if (GET_CODE (temp) == NOTE)
2056 body = PATTERN (temp);
2058 /* Find the next insn after TEMP,
2059 not counting USE or NOTE insns. */
2060 for (next = NEXT_INSN (temp); next != p;
2061 next = NEXT_INSN (next))
2062 if (! (GET_CODE (next) == INSN
2063 && GET_CODE (PATTERN (next)) == USE)
2064 && GET_CODE (next) != NOTE)
2067 /* If that is the call, this may be the insn
2068 that loads the function address.
2070 Extract the function address from the insn
2071 that loads it into a register.
2072 If this insn was cse'd, we get incorrect code.
2074 So emit a new move insn that copies the
2075 function address into the register that the
2076 call insn will use. flow.c will delete any
2077 redundant stores that we have created. */
2078 if (GET_CODE (next) == CALL_INSN
2079 && GET_CODE (body) == SET
2080 && GET_CODE (SET_DEST (body)) == REG
2081 && (n = find_reg_note (temp, REG_EQUAL,
2084 fn_reg = SET_SRC (body);
2085 if (GET_CODE (fn_reg) != REG)
2086 fn_reg = SET_DEST (body);
2087 fn_address = XEXP (n, 0);
2088 fn_address_insn = temp;
2090 /* We have the call insn.
2091 If it uses the register we suspect it might,
2092 load it with the correct address directly. */
2093 if (GET_CODE (temp) == CALL_INSN
2095 && reg_referenced_p (fn_reg, body))
2096 loop_insn_emit_after (loop, 0, fn_address_insn,
2098 (fn_reg, fn_address));
2100 if (GET_CODE (temp) == CALL_INSN)
2102 i1 = loop_call_insn_hoist (loop, body);
2103 /* Because the USAGE information potentially
2104 contains objects other than hard registers
2105 we need to copy it. */
2106 if (CALL_INSN_FUNCTION_USAGE (temp))
2107 CALL_INSN_FUNCTION_USAGE (i1)
2108 = copy_rtx (CALL_INSN_FUNCTION_USAGE (temp));
2111 i1 = loop_insn_hoist (loop, body);
2114 if (temp == fn_address_insn)
2115 fn_address_insn = i1;
2116 REG_NOTES (i1) = REG_NOTES (temp);
2117 REG_NOTES (temp) = NULL;
2123 if (m->savemode != VOIDmode)
2125 /* P sets REG to zero; but we should clear only
2126 the bits that are not covered by the mode
2128 rtx reg = m->set_dest;
2133 tem = expand_simple_binop
2134 (GET_MODE (reg), AND, reg,
2135 GEN_INT ((((HOST_WIDE_INT) 1
2136 << GET_MODE_BITSIZE (m->savemode)))
2138 reg, 1, OPTAB_LIB_WIDEN);
2142 emit_move_insn (reg, tem);
2143 sequence = get_insns ();
2145 i1 = loop_insn_hoist (loop, sequence);
2147 else if (GET_CODE (p) == CALL_INSN)
2149 i1 = loop_call_insn_hoist (loop, PATTERN (p));
2150 /* Because the USAGE information potentially
2151 contains objects other than hard registers
2152 we need to copy it. */
2153 if (CALL_INSN_FUNCTION_USAGE (p))
2154 CALL_INSN_FUNCTION_USAGE (i1)
2155 = copy_rtx (CALL_INSN_FUNCTION_USAGE (p));
2157 else if (count == m->consec && m->move_insn_first)
2160 /* The SET_SRC might not be invariant, so we must
2161 use the REG_EQUAL note. */
2163 emit_move_insn (m->insert_temp ? newreg : m->set_dest,
2168 add_label_notes (m->set_src, seq);
2170 i1 = loop_insn_hoist (loop, seq);
2171 if (! find_reg_note (i1, REG_EQUAL, NULL_RTX))
2172 set_unique_reg_note (i1, m->is_equiv ? REG_EQUIV
2173 : REG_EQUAL, m->set_src);
2175 else if (m->insert_temp)
2177 rtx *reg_map2 = xcalloc (REGNO (newreg),
2179 reg_map2 [m->regno] = newreg;
2181 i1 = loop_insn_hoist (loop, copy_rtx (PATTERN (p)));
2182 replace_regs (i1, reg_map2, REGNO (newreg), 1);
2186 i1 = loop_insn_hoist (loop, PATTERN (p));
2188 if (REG_NOTES (i1) == 0)
2190 REG_NOTES (i1) = REG_NOTES (p);
2191 REG_NOTES (p) = NULL;
2193 /* If there is a REG_EQUAL note present whose value
2194 is not loop invariant, then delete it, since it
2195 may cause problems with later optimization passes.
2196 It is possible for cse to create such notes
2197 like this as a result of record_jump_cond. */
2199 if ((temp = find_reg_note (i1, REG_EQUAL, NULL_RTX))
2200 && ! loop_invariant_p (loop, XEXP (temp, 0)))
2201 remove_note (i1, temp);
2207 if (loop_dump_stream)
2208 fprintf (loop_dump_stream, " moved to %d",
2211 /* If library call, now fix the REG_NOTES that contain
2212 insn pointers, namely REG_LIBCALL on FIRST
2213 and REG_RETVAL on I1. */
2214 if ((temp = find_reg_note (i1, REG_RETVAL, NULL_RTX)))
2216 XEXP (temp, 0) = first;
2217 temp = find_reg_note (first, REG_LIBCALL, NULL_RTX);
2218 XEXP (temp, 0) = i1;
2225 /* simplify_giv_expr expects that it can walk the insns
2226 at m->insn forwards and see this old sequence we are
2227 tossing here. delete_insn does preserve the next
2228 pointers, but when we skip over a NOTE we must fix
2229 it up. Otherwise that code walks into the non-deleted
2231 while (p && GET_CODE (p) == NOTE)
2232 p = NEXT_INSN (temp) = NEXT_INSN (p);
2237 /* Replace the original insn with a move from
2238 our newly created temp. */
2240 emit_move_insn (m->set_dest, newreg);
2243 emit_insn_before (seq, p);
2247 /* The more regs we move, the less we like moving them. */
2253 if (!m->insert_temp)
2255 /* Any other movable that loads the same register
2257 already_moved[regno] = 1;
2259 /* This reg has been moved out of one loop. */
2260 regs->array[regno].moved_once = 1;
2262 /* The reg set here is now invariant. */
2266 for (i = 0; i < LOOP_REGNO_NREGS (regno, m->set_dest); i++)
2267 regs->array[regno+i].set_in_loop = 0;
2270 /* Change the length-of-life info for the register
2271 to say it lives at least the full length of this loop.
2272 This will help guide optimizations in outer loops. */
2274 if (REGNO_FIRST_LUID (regno) > INSN_LUID (loop_start))
2275 /* This is the old insn before all the moved insns.
2276 We can't use the moved insn because it is out of range
2277 in uid_luid. Only the old insns have luids. */
2278 REGNO_FIRST_UID (regno) = INSN_UID (loop_start);
2279 if (REGNO_LAST_LUID (regno) < INSN_LUID (loop_end))
2280 REGNO_LAST_UID (regno) = INSN_UID (loop_end);
2283 /* Combine with this moved insn any other matching movables. */
2286 for (m1 = movables->head; m1; m1 = m1->next)
2291 /* Schedule the reg loaded by M1
2292 for replacement so that shares the reg of M.
2293 If the modes differ (only possible in restricted
2294 circumstances, make a SUBREG.
2296 Note this assumes that the target dependent files
2297 treat REG and SUBREG equally, including within
2298 GO_IF_LEGITIMATE_ADDRESS and in all the
2299 predicates since we never verify that replacing the
2300 original register with a SUBREG results in a
2301 recognizable insn. */
2302 if (GET_MODE (m->set_dest) == GET_MODE (m1->set_dest))
2303 reg_map[m1->regno] = m->set_dest;
2306 = gen_lowpart_common (GET_MODE (m1->set_dest),
2309 /* Get rid of the matching insn
2310 and prevent further processing of it. */
2313 /* If library call, delete all insns. */
2314 if ((temp = find_reg_note (m1->insn, REG_RETVAL,
2316 delete_insn_chain (XEXP (temp, 0), m1->insn);
2318 delete_insn (m1->insn);
2320 /* Any other movable that loads the same register
2322 already_moved[m1->regno] = 1;
2324 /* The reg merged here is now invariant,
2325 if the reg it matches is invariant. */
2330 i < LOOP_REGNO_NREGS (regno, m1->set_dest);
2332 regs->array[m1->regno+i].set_in_loop = 0;
2336 else if (loop_dump_stream)
2337 fprintf (loop_dump_stream, "not desirable");
2339 else if (loop_dump_stream && !m->match)
2340 fprintf (loop_dump_stream, "not safe");
2342 if (loop_dump_stream)
2343 fprintf (loop_dump_stream, "\n");
2347 new_start = loop_start;
2349 /* Go through all the instructions in the loop, making
2350 all the register substitutions scheduled in REG_MAP. */
2351 for (p = new_start; p != loop_end; p = NEXT_INSN (p))
2352 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
2353 || GET_CODE (p) == CALL_INSN)
2355 replace_regs (PATTERN (p), reg_map, nregs, 0);
2356 replace_regs (REG_NOTES (p), reg_map, nregs, 0);
2362 free (already_moved);
2367 loop_movables_add (struct loop_movables *movables, struct movable *m)
2369 if (movables->head == 0)
2372 movables->last->next = m;
2378 loop_movables_free (struct loop_movables *movables)
2381 struct movable *m_next;
2383 for (m = movables->head; m; m = m_next)
2391 /* Scan X and replace the address of any MEM in it with ADDR.
2392 REG is the address that MEM should have before the replacement. */
2395 replace_call_address (rtx x, rtx reg, rtx addr)
2403 code = GET_CODE (x);
2417 /* Short cut for very common case. */
2418 replace_call_address (XEXP (x, 1), reg, addr);
2422 /* Short cut for very common case. */
2423 replace_call_address (XEXP (x, 0), reg, addr);
2427 /* If this MEM uses a reg other than the one we expected,
2428 something is wrong. */
2429 if (XEXP (x, 0) != reg)
2438 fmt = GET_RTX_FORMAT (code);
2439 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2442 replace_call_address (XEXP (x, i), reg, addr);
2443 else if (fmt[i] == 'E')
2446 for (j = 0; j < XVECLEN (x, i); j++)
2447 replace_call_address (XVECEXP (x, i, j), reg, addr);
2453 /* Return the number of memory refs to addresses that vary
2457 count_nonfixed_reads (const struct loop *loop, rtx x)
2467 code = GET_CODE (x);
2481 return ((loop_invariant_p (loop, XEXP (x, 0)) != 1)
2482 + count_nonfixed_reads (loop, XEXP (x, 0)));
2489 fmt = GET_RTX_FORMAT (code);
2490 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2493 value += count_nonfixed_reads (loop, XEXP (x, i));
2497 for (j = 0; j < XVECLEN (x, i); j++)
2498 value += count_nonfixed_reads (loop, XVECEXP (x, i, j));
2504 /* Scan a loop setting the elements `cont', `vtop', `loops_enclosed',
2505 `has_call', `has_nonconst_call', `has_volatile', `has_tablejump',
2506 `unknown_address_altered', `unknown_constant_address_altered', and
2507 `num_mem_sets' in LOOP. Also, fill in the array `mems' and the
2508 list `store_mems' in LOOP. */
2511 prescan_loop (struct loop *loop)
2515 struct loop_info *loop_info = LOOP_INFO (loop);
2516 rtx start = loop->start;
2517 rtx end = loop->end;
2518 /* The label after END. Jumping here is just like falling off the
2519 end of the loop. We use next_nonnote_insn instead of next_label
2520 as a hedge against the (pathological) case where some actual insn
2521 might end up between the two. */
2522 rtx exit_target = next_nonnote_insn (end);
2524 loop_info->has_indirect_jump = indirect_jump_in_function;
2525 loop_info->pre_header_has_call = 0;
2526 loop_info->has_call = 0;
2527 loop_info->has_nonconst_call = 0;
2528 loop_info->has_prefetch = 0;
2529 loop_info->has_volatile = 0;
2530 loop_info->has_tablejump = 0;
2531 loop_info->has_multiple_exit_targets = 0;
2534 loop_info->unknown_address_altered = 0;
2535 loop_info->unknown_constant_address_altered = 0;
2536 loop_info->store_mems = NULL_RTX;
2537 loop_info->first_loop_store_insn = NULL_RTX;
2538 loop_info->mems_idx = 0;
2539 loop_info->num_mem_sets = 0;
2540 /* If loop opts run twice, this was set on 1st pass for 2nd. */
2541 loop_info->preconditioned = NOTE_PRECONDITIONED (end);
2543 for (insn = start; insn && GET_CODE (insn) != CODE_LABEL;
2544 insn = PREV_INSN (insn))
2546 if (GET_CODE (insn) == CALL_INSN)
2548 loop_info->pre_header_has_call = 1;
2553 for (insn = NEXT_INSN (start); insn != NEXT_INSN (end);
2554 insn = NEXT_INSN (insn))
2556 switch (GET_CODE (insn))
2559 if (NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_BEG)
2562 /* Count number of loops contained in this one. */
2565 else if (NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_END)
2570 if (! CONST_OR_PURE_CALL_P (insn))
2572 loop_info->unknown_address_altered = 1;
2573 loop_info->has_nonconst_call = 1;
2575 else if (pure_call_p (insn))
2576 loop_info->has_nonconst_call = 1;
2577 loop_info->has_call = 1;
2578 if (can_throw_internal (insn))
2579 loop_info->has_multiple_exit_targets = 1;
2581 /* Calls initializing constant objects have CLOBBER of MEM /u in the
2582 attached FUNCTION_USAGE expression list, not accounted for by the
2583 code above. We should note these to avoid missing dependencies in
2584 later references. */
2588 for (fusage_entry = CALL_INSN_FUNCTION_USAGE (insn);
2589 fusage_entry; fusage_entry = XEXP (fusage_entry, 1))
2591 rtx fusage = XEXP (fusage_entry, 0);
2593 if (GET_CODE (fusage) == CLOBBER
2594 && GET_CODE (XEXP (fusage, 0)) == MEM
2595 && RTX_UNCHANGING_P (XEXP (fusage, 0)))
2597 note_stores (fusage, note_addr_stored, loop_info);
2598 if (! loop_info->first_loop_store_insn
2599 && loop_info->store_mems)
2600 loop_info->first_loop_store_insn = insn;
2607 if (! loop_info->has_multiple_exit_targets)
2609 rtx set = pc_set (insn);
2613 rtx src = SET_SRC (set);
2616 if (GET_CODE (src) == IF_THEN_ELSE)
2618 label1 = XEXP (src, 1);
2619 label2 = XEXP (src, 2);
2629 if (label1 && label1 != pc_rtx)
2631 if (GET_CODE (label1) != LABEL_REF)
2633 /* Something tricky. */
2634 loop_info->has_multiple_exit_targets = 1;
2637 else if (XEXP (label1, 0) != exit_target
2638 && LABEL_OUTSIDE_LOOP_P (label1))
2640 /* A jump outside the current loop. */
2641 loop_info->has_multiple_exit_targets = 1;
2653 /* A return, or something tricky. */
2654 loop_info->has_multiple_exit_targets = 1;
2660 if (volatile_refs_p (PATTERN (insn)))
2661 loop_info->has_volatile = 1;
2663 if (GET_CODE (insn) == JUMP_INSN
2664 && (GET_CODE (PATTERN (insn)) == ADDR_DIFF_VEC
2665 || GET_CODE (PATTERN (insn)) == ADDR_VEC))
2666 loop_info->has_tablejump = 1;
2668 note_stores (PATTERN (insn), note_addr_stored, loop_info);
2669 if (! loop_info->first_loop_store_insn && loop_info->store_mems)
2670 loop_info->first_loop_store_insn = insn;
2672 if (flag_non_call_exceptions && can_throw_internal (insn))
2673 loop_info->has_multiple_exit_targets = 1;
2681 /* Now, rescan the loop, setting up the LOOP_MEMS array. */
2682 if (/* An exception thrown by a called function might land us
2684 ! loop_info->has_nonconst_call
2685 /* We don't want loads for MEMs moved to a location before the
2686 one at which their stack memory becomes allocated. (Note
2687 that this is not a problem for malloc, etc., since those
2688 require actual function calls. */
2689 && ! current_function_calls_alloca
2690 /* There are ways to leave the loop other than falling off the
2692 && ! loop_info->has_multiple_exit_targets)
2693 for (insn = NEXT_INSN (start); insn != NEXT_INSN (end);
2694 insn = NEXT_INSN (insn))
2695 for_each_rtx (&insn, insert_loop_mem, loop_info);
2697 /* BLKmode MEMs are added to LOOP_STORE_MEM as necessary so
2698 that loop_invariant_p and load_mems can use true_dependence
2699 to determine what is really clobbered. */
2700 if (loop_info->unknown_address_altered)
2702 rtx mem = gen_rtx_MEM (BLKmode, const0_rtx);
2704 loop_info->store_mems
2705 = gen_rtx_EXPR_LIST (VOIDmode, mem, loop_info->store_mems);
2707 if (loop_info->unknown_constant_address_altered)
2709 rtx mem = gen_rtx_MEM (BLKmode, const0_rtx);
2711 RTX_UNCHANGING_P (mem) = 1;
2712 loop_info->store_mems
2713 = gen_rtx_EXPR_LIST (VOIDmode, mem, loop_info->store_mems);
2717 /* Invalidate all loops containing LABEL. */
2720 invalidate_loops_containing_label (rtx label)
2723 for (loop = uid_loop[INSN_UID (label)]; loop; loop = loop->outer)
2727 /* Scan the function looking for loops. Record the start and end of each loop.
2728 Also mark as invalid loops any loops that contain a setjmp or are branched
2729 to from outside the loop. */
2732 find_and_verify_loops (rtx f, struct loops *loops)
2737 struct loop *current_loop;
2738 struct loop *next_loop;
2741 num_loops = loops->num;
2743 compute_luids (f, NULL_RTX, 0);
2745 /* If there are jumps to undefined labels,
2746 treat them as jumps out of any/all loops.
2747 This also avoids writing past end of tables when there are no loops. */
2750 /* Find boundaries of loops, mark which loops are contained within
2751 loops, and invalidate loops that have setjmp. */
2754 current_loop = NULL;
2755 for (insn = f; insn; insn = NEXT_INSN (insn))
2757 if (GET_CODE (insn) == NOTE)
2758 switch (NOTE_LINE_NUMBER (insn))
2760 case NOTE_INSN_LOOP_BEG:
2761 next_loop = loops->array + num_loops;
2762 next_loop->num = num_loops;
2764 next_loop->start = insn;
2765 next_loop->outer = current_loop;
2766 current_loop = next_loop;
2769 case NOTE_INSN_LOOP_CONT:
2770 current_loop->cont = insn;
2773 case NOTE_INSN_LOOP_VTOP:
2774 current_loop->vtop = insn;
2777 case NOTE_INSN_LOOP_END:
2781 current_loop->end = insn;
2782 current_loop = current_loop->outer;
2789 if (GET_CODE (insn) == CALL_INSN
2790 && find_reg_note (insn, REG_SETJMP, NULL))
2792 /* In this case, we must invalidate our current loop and any
2794 for (loop = current_loop; loop; loop = loop->outer)
2797 if (loop_dump_stream)
2798 fprintf (loop_dump_stream,
2799 "\nLoop at %d ignored due to setjmp.\n",
2800 INSN_UID (loop->start));
2804 /* Note that this will mark the NOTE_INSN_LOOP_END note as being in the
2805 enclosing loop, but this doesn't matter. */
2806 uid_loop[INSN_UID (insn)] = current_loop;
2809 /* Any loop containing a label used in an initializer must be invalidated,
2810 because it can be jumped into from anywhere. */
2811 for (label = forced_labels; label; label = XEXP (label, 1))
2812 invalidate_loops_containing_label (XEXP (label, 0));
2814 /* Any loop containing a label used for an exception handler must be
2815 invalidated, because it can be jumped into from anywhere. */
2816 for_each_eh_label (invalidate_loops_containing_label);
2818 /* Now scan all insn's in the function. If any JUMP_INSN branches into a
2819 loop that it is not contained within, that loop is marked invalid.
2820 If any INSN or CALL_INSN uses a label's address, then the loop containing
2821 that label is marked invalid, because it could be jumped into from
2824 Also look for blocks of code ending in an unconditional branch that
2825 exits the loop. If such a block is surrounded by a conditional
2826 branch around the block, move the block elsewhere (see below) and
2827 invert the jump to point to the code block. This may eliminate a
2828 label in our loop and will simplify processing by both us and a
2829 possible second cse pass. */
2831 for (insn = f; insn; insn = NEXT_INSN (insn))
2834 struct loop *this_loop = uid_loop[INSN_UID (insn)];
2836 if (GET_CODE (insn) == INSN || GET_CODE (insn) == CALL_INSN)
2838 rtx note = find_reg_note (insn, REG_LABEL, NULL_RTX);
2840 invalidate_loops_containing_label (XEXP (note, 0));
2843 if (GET_CODE (insn) != JUMP_INSN)
2846 mark_loop_jump (PATTERN (insn), this_loop);
2848 /* See if this is an unconditional branch outside the loop. */
2850 && (GET_CODE (PATTERN (insn)) == RETURN
2851 || (any_uncondjump_p (insn)
2852 && onlyjump_p (insn)
2853 && (uid_loop[INSN_UID (JUMP_LABEL (insn))]
2855 && get_max_uid () < max_uid_for_loop)
2858 rtx our_next = next_real_insn (insn);
2859 rtx last_insn_to_move = NEXT_INSN (insn);
2860 struct loop *dest_loop;
2861 struct loop *outer_loop = NULL;
2863 /* Go backwards until we reach the start of the loop, a label,
2865 for (p = PREV_INSN (insn);
2866 GET_CODE (p) != CODE_LABEL
2867 && ! (GET_CODE (p) == NOTE
2868 && NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_BEG)
2869 && GET_CODE (p) != JUMP_INSN;
2873 /* Check for the case where we have a jump to an inner nested
2874 loop, and do not perform the optimization in that case. */
2876 if (JUMP_LABEL (insn))
2878 dest_loop = uid_loop[INSN_UID (JUMP_LABEL (insn))];
2881 for (outer_loop = dest_loop; outer_loop;
2882 outer_loop = outer_loop->outer)
2883 if (outer_loop == this_loop)
2888 /* Make sure that the target of P is within the current loop. */
2890 if (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p)
2891 && uid_loop[INSN_UID (JUMP_LABEL (p))] != this_loop)
2892 outer_loop = this_loop;
2894 /* If we stopped on a JUMP_INSN to the next insn after INSN,
2895 we have a block of code to try to move.
2897 We look backward and then forward from the target of INSN
2898 to find a BARRIER at the same loop depth as the target.
2899 If we find such a BARRIER, we make a new label for the start
2900 of the block, invert the jump in P and point it to that label,
2901 and move the block of code to the spot we found. */
2904 && GET_CODE (p) == JUMP_INSN
2905 && JUMP_LABEL (p) != 0
2906 /* Just ignore jumps to labels that were never emitted.
2907 These always indicate compilation errors. */
2908 && INSN_UID (JUMP_LABEL (p)) != 0
2909 && any_condjump_p (p) && onlyjump_p (p)
2910 && next_real_insn (JUMP_LABEL (p)) == our_next
2911 /* If it's not safe to move the sequence, then we
2913 && insns_safe_to_move_p (p, NEXT_INSN (insn),
2914 &last_insn_to_move))
2917 = JUMP_LABEL (insn) ? JUMP_LABEL (insn) : get_last_insn ();
2918 struct loop *target_loop = uid_loop[INSN_UID (target)];
2922 /* Search for possible garbage past the conditional jumps
2923 and look for the last barrier. */
2924 for (tmp = last_insn_to_move;
2925 tmp && GET_CODE (tmp) != CODE_LABEL; tmp = NEXT_INSN (tmp))
2926 if (GET_CODE (tmp) == BARRIER)
2927 last_insn_to_move = tmp;
2929 for (loc = target; loc; loc = PREV_INSN (loc))
2930 if (GET_CODE (loc) == BARRIER
2931 /* Don't move things inside a tablejump. */
2932 && ((loc2 = next_nonnote_insn (loc)) == 0
2933 || GET_CODE (loc2) != CODE_LABEL
2934 || (loc2 = next_nonnote_insn (loc2)) == 0
2935 || GET_CODE (loc2) != JUMP_INSN
2936 || (GET_CODE (PATTERN (loc2)) != ADDR_VEC
2937 && GET_CODE (PATTERN (loc2)) != ADDR_DIFF_VEC))
2938 && uid_loop[INSN_UID (loc)] == target_loop)
2942 for (loc = target; loc; loc = NEXT_INSN (loc))
2943 if (GET_CODE (loc) == BARRIER
2944 /* Don't move things inside a tablejump. */
2945 && ((loc2 = next_nonnote_insn (loc)) == 0
2946 || GET_CODE (loc2) != CODE_LABEL
2947 || (loc2 = next_nonnote_insn (loc2)) == 0
2948 || GET_CODE (loc2) != JUMP_INSN
2949 || (GET_CODE (PATTERN (loc2)) != ADDR_VEC
2950 && GET_CODE (PATTERN (loc2)) != ADDR_DIFF_VEC))
2951 && uid_loop[INSN_UID (loc)] == target_loop)
2956 rtx cond_label = JUMP_LABEL (p);
2957 rtx new_label = get_label_after (p);
2959 /* Ensure our label doesn't go away. */
2960 LABEL_NUSES (cond_label)++;
2962 /* Verify that uid_loop is large enough and that
2964 if (invert_jump (p, new_label, 1))
2968 /* If no suitable BARRIER was found, create a suitable
2969 one before TARGET. Since TARGET is a fall through
2970 path, we'll need to insert a jump around our block
2971 and add a BARRIER before TARGET.
2973 This creates an extra unconditional jump outside
2974 the loop. However, the benefits of removing rarely
2975 executed instructions from inside the loop usually
2976 outweighs the cost of the extra unconditional jump
2977 outside the loop. */
2982 temp = gen_jump (JUMP_LABEL (insn));
2983 temp = emit_jump_insn_before (temp, target);
2984 JUMP_LABEL (temp) = JUMP_LABEL (insn);
2985 LABEL_NUSES (JUMP_LABEL (insn))++;
2986 loc = emit_barrier_before (target);
2989 /* Include the BARRIER after INSN and copy the
2991 if (squeeze_notes (&new_label, &last_insn_to_move))
2993 reorder_insns (new_label, last_insn_to_move, loc);
2995 /* All those insns are now in TARGET_LOOP. */
2997 q != NEXT_INSN (last_insn_to_move);
2999 uid_loop[INSN_UID (q)] = target_loop;
3001 /* The label jumped to by INSN is no longer a loop
3002 exit. Unless INSN does not have a label (e.g.,
3003 it is a RETURN insn), search loop->exit_labels
3004 to find its label_ref, and remove it. Also turn
3005 off LABEL_OUTSIDE_LOOP_P bit. */
3006 if (JUMP_LABEL (insn))
3008 for (q = 0, r = this_loop->exit_labels;
3010 q = r, r = LABEL_NEXTREF (r))
3011 if (XEXP (r, 0) == JUMP_LABEL (insn))
3013 LABEL_OUTSIDE_LOOP_P (r) = 0;
3015 LABEL_NEXTREF (q) = LABEL_NEXTREF (r);
3017 this_loop->exit_labels = LABEL_NEXTREF (r);
3021 for (loop = this_loop; loop && loop != target_loop;
3025 /* If we didn't find it, then something is
3031 /* P is now a jump outside the loop, so it must be put
3032 in loop->exit_labels, and marked as such.
3033 The easiest way to do this is to just call
3034 mark_loop_jump again for P. */
3035 mark_loop_jump (PATTERN (p), this_loop);
3037 /* If INSN now jumps to the insn after it,
3039 if (JUMP_LABEL (insn) != 0
3040 && (next_real_insn (JUMP_LABEL (insn))
3041 == next_real_insn (insn)))
3042 delete_related_insns (insn);
3045 /* Continue the loop after where the conditional
3046 branch used to jump, since the only branch insn
3047 in the block (if it still remains) is an inter-loop
3048 branch and hence needs no processing. */
3049 insn = NEXT_INSN (cond_label);
3051 if (--LABEL_NUSES (cond_label) == 0)
3052 delete_related_insns (cond_label);
3054 /* This loop will be continued with NEXT_INSN (insn). */
3055 insn = PREV_INSN (insn);
3062 /* If any label in X jumps to a loop different from LOOP_NUM and any of the
3063 loops it is contained in, mark the target loop invalid.
3065 For speed, we assume that X is part of a pattern of a JUMP_INSN. */
3068 mark_loop_jump (rtx x, struct loop *loop)
3070 struct loop *dest_loop;
3071 struct loop *outer_loop;
3074 switch (GET_CODE (x))
3087 /* There could be a label reference in here. */
3088 mark_loop_jump (XEXP (x, 0), loop);
3094 mark_loop_jump (XEXP (x, 0), loop);
3095 mark_loop_jump (XEXP (x, 1), loop);
3099 /* This may refer to a LABEL_REF or SYMBOL_REF. */
3100 mark_loop_jump (XEXP (x, 1), loop);
3105 mark_loop_jump (XEXP (x, 0), loop);
3109 dest_loop = uid_loop[INSN_UID (XEXP (x, 0))];
3111 /* Link together all labels that branch outside the loop. This
3112 is used by final_[bg]iv_value and the loop unrolling code. Also
3113 mark this LABEL_REF so we know that this branch should predict
3116 /* A check to make sure the label is not in an inner nested loop,
3117 since this does not count as a loop exit. */
3120 for (outer_loop = dest_loop; outer_loop;
3121 outer_loop = outer_loop->outer)
3122 if (outer_loop == loop)
3128 if (loop && ! outer_loop)
3130 LABEL_OUTSIDE_LOOP_P (x) = 1;
3131 LABEL_NEXTREF (x) = loop->exit_labels;
3132 loop->exit_labels = x;
3134 for (outer_loop = loop;
3135 outer_loop && outer_loop != dest_loop;
3136 outer_loop = outer_loop->outer)
3137 outer_loop->exit_count++;
3140 /* If this is inside a loop, but not in the current loop or one enclosed
3141 by it, it invalidates at least one loop. */
3146 /* We must invalidate every nested loop containing the target of this
3147 label, except those that also contain the jump insn. */
3149 for (; dest_loop; dest_loop = dest_loop->outer)
3151 /* Stop when we reach a loop that also contains the jump insn. */
3152 for (outer_loop = loop; outer_loop; outer_loop = outer_loop->outer)
3153 if (dest_loop == outer_loop)
3156 /* If we get here, we know we need to invalidate a loop. */
3157 if (loop_dump_stream && ! dest_loop->invalid)
3158 fprintf (loop_dump_stream,
3159 "\nLoop at %d ignored due to multiple entry points.\n",
3160 INSN_UID (dest_loop->start));
3162 dest_loop->invalid = 1;
3167 /* If this is not setting pc, ignore. */
3168 if (SET_DEST (x) == pc_rtx)
3169 mark_loop_jump (SET_SRC (x), loop);
3173 mark_loop_jump (XEXP (x, 1), loop);
3174 mark_loop_jump (XEXP (x, 2), loop);
3179 for (i = 0; i < XVECLEN (x, 0); i++)
3180 mark_loop_jump (XVECEXP (x, 0, i), loop);
3184 for (i = 0; i < XVECLEN (x, 1); i++)
3185 mark_loop_jump (XVECEXP (x, 1, i), loop);
3189 /* Strictly speaking this is not a jump into the loop, only a possible
3190 jump out of the loop. However, we have no way to link the destination
3191 of this jump onto the list of exit labels. To be safe we mark this
3192 loop and any containing loops as invalid. */
3195 for (outer_loop = loop; outer_loop; outer_loop = outer_loop->outer)
3197 if (loop_dump_stream && ! outer_loop->invalid)
3198 fprintf (loop_dump_stream,
3199 "\nLoop at %d ignored due to unknown exit jump.\n",
3200 INSN_UID (outer_loop->start));
3201 outer_loop->invalid = 1;
3208 /* Return nonzero if there is a label in the range from
3209 insn INSN to and including the insn whose luid is END
3210 INSN must have an assigned luid (i.e., it must not have
3211 been previously created by loop.c). */
3214 labels_in_range_p (rtx insn, int end)
3216 while (insn && INSN_LUID (insn) <= end)
3218 if (GET_CODE (insn) == CODE_LABEL)
3220 insn = NEXT_INSN (insn);
3226 /* Record that a memory reference X is being set. */
3229 note_addr_stored (rtx x, rtx y ATTRIBUTE_UNUSED,
3230 void *data ATTRIBUTE_UNUSED)
3232 struct loop_info *loop_info = data;
3234 if (x == 0 || GET_CODE (x) != MEM)
3237 /* Count number of memory writes.
3238 This affects heuristics in strength_reduce. */
3239 loop_info->num_mem_sets++;
3241 /* BLKmode MEM means all memory is clobbered. */
3242 if (GET_MODE (x) == BLKmode)
3244 if (RTX_UNCHANGING_P (x))
3245 loop_info->unknown_constant_address_altered = 1;
3247 loop_info->unknown_address_altered = 1;
3252 loop_info->store_mems = gen_rtx_EXPR_LIST (VOIDmode, x,
3253 loop_info->store_mems);
3256 /* X is a value modified by an INSN that references a biv inside a loop
3257 exit test (ie, X is somehow related to the value of the biv). If X
3258 is a pseudo that is used more than once, then the biv is (effectively)
3259 used more than once. DATA is a pointer to a loop_regs structure. */
3262 note_set_pseudo_multiple_uses (rtx x, rtx y ATTRIBUTE_UNUSED, void *data)
3264 struct loop_regs *regs = (struct loop_regs *) data;
3269 while (GET_CODE (x) == STRICT_LOW_PART
3270 || GET_CODE (x) == SIGN_EXTRACT
3271 || GET_CODE (x) == ZERO_EXTRACT
3272 || GET_CODE (x) == SUBREG)
3275 if (GET_CODE (x) != REG || REGNO (x) < FIRST_PSEUDO_REGISTER)
3278 /* If we do not have usage information, or if we know the register
3279 is used more than once, note that fact for check_dbra_loop. */
3280 if (REGNO (x) >= max_reg_before_loop
3281 || ! regs->array[REGNO (x)].single_usage
3282 || regs->array[REGNO (x)].single_usage == const0_rtx)
3283 regs->multiple_uses = 1;
3286 /* Return nonzero if the rtx X is invariant over the current loop.
3288 The value is 2 if we refer to something only conditionally invariant.
3290 A memory ref is invariant if it is not volatile and does not conflict
3291 with anything stored in `loop_info->store_mems'. */
3294 loop_invariant_p (const struct loop *loop, rtx x)
3296 struct loop_info *loop_info = LOOP_INFO (loop);
3297 struct loop_regs *regs = LOOP_REGS (loop);
3301 int conditional = 0;
3306 code = GET_CODE (x);
3316 /* A LABEL_REF is normally invariant, however, if we are unrolling
3317 loops, and this label is inside the loop, then it isn't invariant.
3318 This is because each unrolled copy of the loop body will have
3319 a copy of this label. If this was invariant, then an insn loading
3320 the address of this label into a register might get moved outside
3321 the loop, and then each loop body would end up using the same label.
3323 We don't know the loop bounds here though, so just fail for all
3325 if (flag_old_unroll_loops)
3332 case UNSPEC_VOLATILE:
3336 /* We used to check RTX_UNCHANGING_P (x) here, but that is invalid
3337 since the reg might be set by initialization within the loop. */
3339 if ((x == frame_pointer_rtx || x == hard_frame_pointer_rtx
3340 || x == arg_pointer_rtx || x == pic_offset_table_rtx)
3341 && ! current_function_has_nonlocal_goto)
3344 if (LOOP_INFO (loop)->has_call
3345 && REGNO (x) < FIRST_PSEUDO_REGISTER && call_used_regs[REGNO (x)])
3348 /* Out-of-range regs can occur when we are called from unrolling.
3349 These registers created by the unroller are set in the loop,
3350 hence are never invariant.
3351 Other out-of-range regs can be generated by load_mems; those that
3352 are written to in the loop are not invariant, while those that are
3353 not written to are invariant. It would be easy for load_mems
3354 to set n_times_set correctly for these registers, however, there
3355 is no easy way to distinguish them from registers created by the
3358 if (REGNO (x) >= (unsigned) regs->num)
3361 if (regs->array[REGNO (x)].set_in_loop < 0)
3364 return regs->array[REGNO (x)].set_in_loop == 0;
3367 /* Volatile memory references must be rejected. Do this before
3368 checking for read-only items, so that volatile read-only items
3369 will be rejected also. */
3370 if (MEM_VOLATILE_P (x))
3373 /* See if there is any dependence between a store and this load. */
3374 mem_list_entry = loop_info->store_mems;
3375 while (mem_list_entry)
3377 if (true_dependence (XEXP (mem_list_entry, 0), VOIDmode,
3381 mem_list_entry = XEXP (mem_list_entry, 1);
3384 /* It's not invalidated by a store in memory
3385 but we must still verify the address is invariant. */
3389 /* Don't mess with insns declared volatile. */
3390 if (MEM_VOLATILE_P (x))
3398 fmt = GET_RTX_FORMAT (code);
3399 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3403 int tem = loop_invariant_p (loop, XEXP (x, i));
3409 else if (fmt[i] == 'E')
3412 for (j = 0; j < XVECLEN (x, i); j++)
3414 int tem = loop_invariant_p (loop, XVECEXP (x, i, j));
3424 return 1 + conditional;
3427 /* Return nonzero if all the insns in the loop that set REG
3428 are INSN and the immediately following insns,
3429 and if each of those insns sets REG in an invariant way
3430 (not counting uses of REG in them).
3432 The value is 2 if some of these insns are only conditionally invariant.
3434 We assume that INSN itself is the first set of REG
3435 and that its source is invariant. */
3438 consec_sets_invariant_p (const struct loop *loop, rtx reg, int n_sets,
3441 struct loop_regs *regs = LOOP_REGS (loop);
3443 unsigned int regno = REGNO (reg);
3445 /* Number of sets we have to insist on finding after INSN. */
3446 int count = n_sets - 1;
3447 int old = regs->array[regno].set_in_loop;
3451 /* If N_SETS hit the limit, we can't rely on its value. */
3455 regs->array[regno].set_in_loop = 0;
3463 code = GET_CODE (p);
3465 /* If library call, skip to end of it. */
3466 if (code == INSN && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
3471 && (set = single_set (p))
3472 && GET_CODE (SET_DEST (set)) == REG
3473 && REGNO (SET_DEST (set)) == regno)
3475 this = loop_invariant_p (loop, SET_SRC (set));
3478 else if ((temp = find_reg_note (p, REG_EQUAL, NULL_RTX)))
3480 /* If this is a libcall, then any invariant REG_EQUAL note is OK.
3481 If this is an ordinary insn, then only CONSTANT_P REG_EQUAL
3483 this = (CONSTANT_P (XEXP (temp, 0))
3484 || (find_reg_note (p, REG_RETVAL, NULL_RTX)
3485 && loop_invariant_p (loop, XEXP (temp, 0))));
3492 else if (code != NOTE)
3494 regs->array[regno].set_in_loop = old;
3499 regs->array[regno].set_in_loop = old;
3500 /* If loop_invariant_p ever returned 2, we return 2. */
3501 return 1 + (value & 2);
3505 /* I don't think this condition is sufficient to allow INSN
3506 to be moved, so we no longer test it. */
3508 /* Return 1 if all insns in the basic block of INSN and following INSN
3509 that set REG are invariant according to TABLE. */
3512 all_sets_invariant_p (rtx reg, rtx insn, short *table)
3515 int regno = REGNO (reg);
3521 code = GET_CODE (p);
3522 if (code == CODE_LABEL || code == JUMP_INSN)
3524 if (code == INSN && GET_CODE (PATTERN (p)) == SET
3525 && GET_CODE (SET_DEST (PATTERN (p))) == REG
3526 && REGNO (SET_DEST (PATTERN (p))) == regno)
3528 if (! loop_invariant_p (loop, SET_SRC (PATTERN (p)), table))
3535 /* Look at all uses (not sets) of registers in X. For each, if it is
3536 the single use, set USAGE[REGNO] to INSN; if there was a previous use in
3537 a different insn, set USAGE[REGNO] to const0_rtx. */
3540 find_single_use_in_loop (struct loop_regs *regs, rtx insn, rtx x)
3542 enum rtx_code code = GET_CODE (x);
3543 const char *fmt = GET_RTX_FORMAT (code);
3547 regs->array[REGNO (x)].single_usage
3548 = (regs->array[REGNO (x)].single_usage != 0
3549 && regs->array[REGNO (x)].single_usage != insn)
3550 ? const0_rtx : insn;
3552 else if (code == SET)
3554 /* Don't count SET_DEST if it is a REG; otherwise count things
3555 in SET_DEST because if a register is partially modified, it won't
3556 show up as a potential movable so we don't care how USAGE is set
3558 if (GET_CODE (SET_DEST (x)) != REG)
3559 find_single_use_in_loop (regs, insn, SET_DEST (x));
3560 find_single_use_in_loop (regs, insn, SET_SRC (x));
3563 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3565 if (fmt[i] == 'e' && XEXP (x, i) != 0)
3566 find_single_use_in_loop (regs, insn, XEXP (x, i));
3567 else if (fmt[i] == 'E')
3568 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
3569 find_single_use_in_loop (regs, insn, XVECEXP (x, i, j));
3573 /* Count and record any set in X which is contained in INSN. Update
3574 REGS->array[I].MAY_NOT_OPTIMIZE and LAST_SET for any register I set
3578 count_one_set (struct loop_regs *regs, rtx insn, rtx x, rtx *last_set)
3580 if (GET_CODE (x) == CLOBBER && GET_CODE (XEXP (x, 0)) == REG)
3581 /* Don't move a reg that has an explicit clobber.
3582 It's not worth the pain to try to do it correctly. */
3583 regs->array[REGNO (XEXP (x, 0))].may_not_optimize = 1;
3585 if (GET_CODE (x) == SET || GET_CODE (x) == CLOBBER)
3587 rtx dest = SET_DEST (x);
3588 while (GET_CODE (dest) == SUBREG
3589 || GET_CODE (dest) == ZERO_EXTRACT
3590 || GET_CODE (dest) == SIGN_EXTRACT
3591 || GET_CODE (dest) == STRICT_LOW_PART)
3592 dest = XEXP (dest, 0);
3593 if (GET_CODE (dest) == REG)
3596 int regno = REGNO (dest);
3597 for (i = 0; i < LOOP_REGNO_NREGS (regno, dest); i++)
3599 /* If this is the first setting of this reg
3600 in current basic block, and it was set before,
3601 it must be set in two basic blocks, so it cannot
3602 be moved out of the loop. */
3603 if (regs->array[regno].set_in_loop > 0
3604 && last_set[regno] == 0)
3605 regs->array[regno+i].may_not_optimize = 1;
3606 /* If this is not first setting in current basic block,
3607 see if reg was used in between previous one and this.
3608 If so, neither one can be moved. */
3609 if (last_set[regno] != 0
3610 && reg_used_between_p (dest, last_set[regno], insn))
3611 regs->array[regno+i].may_not_optimize = 1;
3612 if (regs->array[regno+i].set_in_loop < 127)
3613 ++regs->array[regno+i].set_in_loop;
3614 last_set[regno+i] = insn;
3620 /* Given a loop that is bounded by LOOP->START and LOOP->END and that
3621 is entered at LOOP->SCAN_START, return 1 if the register set in SET
3622 contained in insn INSN is used by any insn that precedes INSN in
3623 cyclic order starting from the loop entry point.
3625 We don't want to use INSN_LUID here because if we restrict INSN to those
3626 that have a valid INSN_LUID, it means we cannot move an invariant out
3627 from an inner loop past two loops. */
3630 loop_reg_used_before_p (const struct loop *loop, rtx set, rtx insn)
3632 rtx reg = SET_DEST (set);
3635 /* Scan forward checking for register usage. If we hit INSN, we
3636 are done. Otherwise, if we hit LOOP->END, wrap around to LOOP->START. */
3637 for (p = loop->scan_start; p != insn; p = NEXT_INSN (p))
3639 if (INSN_P (p) && reg_overlap_mentioned_p (reg, PATTERN (p)))
3650 /* Information we collect about arrays that we might want to prefetch. */
3651 struct prefetch_info
3653 struct iv_class *class; /* Class this prefetch is based on. */
3654 struct induction *giv; /* GIV this prefetch is based on. */
3655 rtx base_address; /* Start prefetching from this address plus
3657 HOST_WIDE_INT index;
3658 HOST_WIDE_INT stride; /* Prefetch stride in bytes in each
3660 unsigned int bytes_accessed; /* Sum of sizes of all accesses to this
3661 prefetch area in one iteration. */
3662 unsigned int total_bytes; /* Total bytes loop will access in this block.
3663 This is set only for loops with known
3664 iteration counts and is 0xffffffff
3666 int prefetch_in_loop; /* Number of prefetch insns in loop. */
3667 int prefetch_before_loop; /* Number of prefetch insns before loop. */
3668 unsigned int write : 1; /* 1 for read/write prefetches. */
3671 /* Data used by check_store function. */
3672 struct check_store_data
3678 static void check_store (rtx, rtx, void *);
3679 static void emit_prefetch_instructions (struct loop *);
3680 static int rtx_equal_for_prefetch_p (rtx, rtx);
3682 /* Set mem_write when mem_address is found. Used as callback to
3685 check_store (rtx x, rtx pat ATTRIBUTE_UNUSED, void *data)
3687 struct check_store_data *d = (struct check_store_data *) data;
3689 if ((GET_CODE (x) == MEM) && rtx_equal_p (d->mem_address, XEXP (x, 0)))
3693 /* Like rtx_equal_p, but attempts to swap commutative operands. This is
3694 important to get some addresses combined. Later more sophisticated
3695 transformations can be added when necessary.
3697 ??? Same trick with swapping operand is done at several other places.
3698 It can be nice to develop some common way to handle this. */
3701 rtx_equal_for_prefetch_p (rtx x, rtx y)
3705 enum rtx_code code = GET_CODE (x);
3710 if (code != GET_CODE (y))
3713 code = GET_CODE (x);
3715 if (GET_RTX_CLASS (code) == 'c')
3717 return ((rtx_equal_for_prefetch_p (XEXP (x, 0), XEXP (y, 0))
3718 && rtx_equal_for_prefetch_p (XEXP (x, 1), XEXP (y, 1)))
3719 || (rtx_equal_for_prefetch_p (XEXP (x, 0), XEXP (y, 1))
3720 && rtx_equal_for_prefetch_p (XEXP (x, 1), XEXP (y, 0))));
3722 /* Compare the elements. If any pair of corresponding elements fails to
3723 match, return 0 for the whole thing. */
3725 fmt = GET_RTX_FORMAT (code);
3726 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3731 if (XWINT (x, i) != XWINT (y, i))
3736 if (XINT (x, i) != XINT (y, i))
3741 /* Two vectors must have the same length. */
3742 if (XVECLEN (x, i) != XVECLEN (y, i))
3745 /* And the corresponding elements must match. */
3746 for (j = 0; j < XVECLEN (x, i); j++)
3747 if (rtx_equal_for_prefetch_p (XVECEXP (x, i, j),
3748 XVECEXP (y, i, j)) == 0)
3753 if (rtx_equal_for_prefetch_p (XEXP (x, i), XEXP (y, i)) == 0)
3758 if (strcmp (XSTR (x, i), XSTR (y, i)))
3763 /* These are just backpointers, so they don't matter. */
3769 /* It is believed that rtx's at this level will never
3770 contain anything but integers and other rtx's,
3771 except for within LABEL_REFs and SYMBOL_REFs. */
3779 /* Remove constant addition value from the expression X (when present)
3782 static HOST_WIDE_INT
3783 remove_constant_addition (rtx *x)
3785 HOST_WIDE_INT addval = 0;
3788 /* Avoid clobbering a shared CONST expression. */
3789 if (GET_CODE (exp) == CONST)
3791 if (GET_CODE (XEXP (exp, 0)) == PLUS
3792 && GET_CODE (XEXP (XEXP (exp, 0), 0)) == SYMBOL_REF
3793 && GET_CODE (XEXP (XEXP (exp, 0), 1)) == CONST_INT)
3795 *x = XEXP (XEXP (exp, 0), 0);
3796 return INTVAL (XEXP (XEXP (exp, 0), 1));
3801 if (GET_CODE (exp) == CONST_INT)
3803 addval = INTVAL (exp);
3807 /* For plus expression recurse on ourself. */
3808 else if (GET_CODE (exp) == PLUS)
3810 addval += remove_constant_addition (&XEXP (exp, 0));
3811 addval += remove_constant_addition (&XEXP (exp, 1));
3813 /* In case our parameter was constant, remove extra zero from the
3815 if (XEXP (exp, 0) == const0_rtx)
3817 else if (XEXP (exp, 1) == const0_rtx)
3824 /* Attempt to identify accesses to arrays that are most likely to cause cache
3825 misses, and emit prefetch instructions a few prefetch blocks forward.
3827 To detect the arrays we use the GIV information that was collected by the
3828 strength reduction pass.
3830 The prefetch instructions are generated after the GIV information is done
3831 and before the strength reduction process. The new GIVs are injected into
3832 the strength reduction tables, so the prefetch addresses are optimized as
3835 GIVs are split into base address, stride, and constant addition values.
3836 GIVs with the same address, stride and close addition values are combined
3837 into a single prefetch. Also writes to GIVs are detected, so that prefetch
3838 for write instructions can be used for the block we write to, on machines
3839 that support write prefetches.
3841 Several heuristics are used to determine when to prefetch. They are
3842 controlled by defined symbols that can be overridden for each target. */
3845 emit_prefetch_instructions (struct loop *loop)
3847 int num_prefetches = 0;
3848 int num_real_prefetches = 0;
3849 int num_real_write_prefetches = 0;
3850 int num_prefetches_before = 0;
3851 int num_write_prefetches_before = 0;
3854 struct iv_class *bl;
3855 struct induction *iv;
3856 struct prefetch_info info[MAX_PREFETCHES];
3857 struct loop_ivs *ivs = LOOP_IVS (loop);
3862 /* Consider only loops w/o calls. When a call is done, the loop is probably
3863 slow enough to read the memory. */
3864 if (PREFETCH_NO_CALL && LOOP_INFO (loop)->has_call)
3866 if (loop_dump_stream)
3867 fprintf (loop_dump_stream, "Prefetch: ignoring loop: has call.\n");
3872 /* Don't prefetch in loops known to have few iterations. */
3873 if (PREFETCH_NO_LOW_LOOPCNT
3874 && LOOP_INFO (loop)->n_iterations
3875 && LOOP_INFO (loop)->n_iterations <= PREFETCH_LOW_LOOPCNT)
3877 if (loop_dump_stream)
3878 fprintf (loop_dump_stream,
3879 "Prefetch: ignoring loop: not enough iterations.\n");
3883 /* Search all induction variables and pick those interesting for the prefetch
3885 for (bl = ivs->list; bl; bl = bl->next)
3887 struct induction *biv = bl->biv, *biv1;
3892 /* Expect all BIVs to be executed in each iteration. This makes our
3893 analysis more conservative. */
3896 /* Discard non-constant additions that we can't handle well yet, and
3897 BIVs that are executed multiple times; such BIVs ought to be
3898 handled in the nested loop. We accept not_every_iteration BIVs,
3899 since these only result in larger strides and make our
3900 heuristics more conservative. */
3901 if (GET_CODE (biv->add_val) != CONST_INT)
3903 if (loop_dump_stream)
3905 fprintf (loop_dump_stream,
3906 "Prefetch: ignoring biv %d: non-constant addition at insn %d:",
3907 REGNO (biv->src_reg), INSN_UID (biv->insn));
3908 print_rtl (loop_dump_stream, biv->add_val);
3909 fprintf (loop_dump_stream, "\n");
3914 if (biv->maybe_multiple)
3916 if (loop_dump_stream)
3918 fprintf (loop_dump_stream,
3919 "Prefetch: ignoring biv %d: maybe_multiple at insn %i:",
3920 REGNO (biv->src_reg), INSN_UID (biv->insn));
3921 print_rtl (loop_dump_stream, biv->add_val);
3922 fprintf (loop_dump_stream, "\n");
3927 basestride += INTVAL (biv1->add_val);
3928 biv1 = biv1->next_iv;
3931 if (biv1 || !basestride)
3934 for (iv = bl->giv; iv; iv = iv->next_iv)
3938 HOST_WIDE_INT index = 0;
3940 HOST_WIDE_INT stride = 0;
3941 int stride_sign = 1;
3942 struct check_store_data d;
3943 const char *ignore_reason = NULL;
3944 int size = GET_MODE_SIZE (GET_MODE (iv));
3946 /* See whether an induction variable is interesting to us and if
3947 not, report the reason. */
3948 if (iv->giv_type != DEST_ADDR)
3949 ignore_reason = "giv is not a destination address";
3951 /* We are interested only in constant stride memory references
3952 in order to be able to compute density easily. */
3953 else if (GET_CODE (iv->mult_val) != CONST_INT)
3954 ignore_reason = "stride is not constant";
3958 stride = INTVAL (iv->mult_val) * basestride;
3965 /* On some targets, reversed order prefetches are not
3967 if (PREFETCH_NO_REVERSE_ORDER && stride_sign < 0)
3968 ignore_reason = "reversed order stride";
3970 /* Prefetch of accesses with an extreme stride might not be
3971 worthwhile, either. */
3972 else if (PREFETCH_NO_EXTREME_STRIDE
3973 && stride > PREFETCH_EXTREME_STRIDE)
3974 ignore_reason = "extreme stride";
3976 /* Ignore GIVs with varying add values; we can't predict the
3977 value for the next iteration. */
3978 else if (!loop_invariant_p (loop, iv->add_val))
3979 ignore_reason = "giv has varying add value";
3981 /* Ignore GIVs in the nested loops; they ought to have been
3983 else if (iv->maybe_multiple)
3984 ignore_reason = "giv is in nested loop";
3987 if (ignore_reason != NULL)
3989 if (loop_dump_stream)
3990 fprintf (loop_dump_stream,
3991 "Prefetch: ignoring giv at %d: %s.\n",
3992 INSN_UID (iv->insn), ignore_reason);
3996 /* Determine the pointer to the basic array we are examining. It is
3997 the sum of the BIV's initial value and the GIV's add_val. */
3998 address = copy_rtx (iv->add_val);
3999 temp = copy_rtx (bl->initial_value);
4001 address = simplify_gen_binary (PLUS, Pmode, temp, address);
4002 index = remove_constant_addition (&address);
4005 d.mem_address = *iv->location;
4007 /* When the GIV is not always executed, we might be better off by
4008 not dirtying the cache pages. */
4009 if (PREFETCH_CONDITIONAL || iv->always_executed)
4010 note_stores (PATTERN (iv->insn), check_store, &d);
4013 if (loop_dump_stream)
4014 fprintf (loop_dump_stream, "Prefetch: Ignoring giv at %d: %s\n",
4015 INSN_UID (iv->insn), "in conditional code.");
4019 /* Attempt to find another prefetch to the same array and see if we
4020 can merge this one. */
4021 for (i = 0; i < num_prefetches; i++)
4022 if (rtx_equal_for_prefetch_p (address, info[i].base_address)
4023 && stride == info[i].stride)
4025 /* In case both access same array (same location
4026 just with small difference in constant indexes), merge
4027 the prefetches. Just do the later and the earlier will
4028 get prefetched from previous iteration.
4029 The artificial threshold should not be too small,
4030 but also not bigger than small portion of memory usually
4031 traversed by single loop. */
4032 if (index >= info[i].index
4033 && index - info[i].index < PREFETCH_EXTREME_DIFFERENCE)
4035 info[i].write |= d.mem_write;
4036 info[i].bytes_accessed += size;
4037 info[i].index = index;
4040 info[num_prefetches].base_address = address;
4045 if (index < info[i].index
4046 && info[i].index - index < PREFETCH_EXTREME_DIFFERENCE)
4048 info[i].write |= d.mem_write;
4049 info[i].bytes_accessed += size;
4055 /* Merging failed. */
4058 info[num_prefetches].giv = iv;
4059 info[num_prefetches].class = bl;
4060 info[num_prefetches].index = index;
4061 info[num_prefetches].stride = stride;
4062 info[num_prefetches].base_address = address;
4063 info[num_prefetches].write = d.mem_write;
4064 info[num_prefetches].bytes_accessed = size;
4066 if (num_prefetches >= MAX_PREFETCHES)
4068 if (loop_dump_stream)
4069 fprintf (loop_dump_stream,
4070 "Maximal number of prefetches exceeded.\n");
4077 for (i = 0; i < num_prefetches; i++)
4081 /* Attempt to calculate the total number of bytes fetched by all
4082 iterations of the loop. Avoid overflow. */
4083 if (LOOP_INFO (loop)->n_iterations
4084 && ((unsigned HOST_WIDE_INT) (0xffffffff / info[i].stride)
4085 >= LOOP_INFO (loop)->n_iterations))
4086 info[i].total_bytes = info[i].stride * LOOP_INFO (loop)->n_iterations;
4088 info[i].total_bytes = 0xffffffff;
4090 density = info[i].bytes_accessed * 100 / info[i].stride;
4092 /* Prefetch might be worthwhile only when the loads/stores are dense. */
4093 if (PREFETCH_ONLY_DENSE_MEM)
4094 if (density * 256 > PREFETCH_DENSE_MEM * 100
4095 && (info[i].total_bytes / PREFETCH_BLOCK
4096 >= PREFETCH_BLOCKS_BEFORE_LOOP_MIN))
4098 info[i].prefetch_before_loop = 1;
4099 info[i].prefetch_in_loop
4100 = (info[i].total_bytes / PREFETCH_BLOCK
4101 > PREFETCH_BLOCKS_BEFORE_LOOP_MAX);
4105 info[i].prefetch_in_loop = 0, info[i].prefetch_before_loop = 0;
4106 if (loop_dump_stream)
4107 fprintf (loop_dump_stream,
4108 "Prefetch: ignoring giv at %d: %d%% density is too low.\n",
4109 INSN_UID (info[i].giv->insn), density);
4112 info[i].prefetch_in_loop = 1, info[i].prefetch_before_loop = 1;
4114 /* Find how many prefetch instructions we'll use within the loop. */
4115 if (info[i].prefetch_in_loop != 0)
4117 info[i].prefetch_in_loop = ((info[i].stride + PREFETCH_BLOCK - 1)
4119 num_real_prefetches += info[i].prefetch_in_loop;
4121 num_real_write_prefetches += info[i].prefetch_in_loop;
4125 /* Determine how many iterations ahead to prefetch within the loop, based
4126 on how many prefetches we currently expect to do within the loop. */
4127 if (num_real_prefetches != 0)
4129 if ((ahead = SIMULTANEOUS_PREFETCHES / num_real_prefetches) == 0)
4131 if (loop_dump_stream)
4132 fprintf (loop_dump_stream,
4133 "Prefetch: ignoring prefetches within loop: ahead is zero; %d < %d\n",
4134 SIMULTANEOUS_PREFETCHES, num_real_prefetches);
4135 num_real_prefetches = 0, num_real_write_prefetches = 0;
4138 /* We'll also use AHEAD to determine how many prefetch instructions to
4139 emit before a loop, so don't leave it zero. */
4141 ahead = PREFETCH_BLOCKS_BEFORE_LOOP_MAX;
4143 for (i = 0; i < num_prefetches; i++)
4145 /* Update if we've decided not to prefetch anything within the loop. */
4146 if (num_real_prefetches == 0)
4147 info[i].prefetch_in_loop = 0;
4149 /* Find how many prefetch instructions we'll use before the loop. */
4150 if (info[i].prefetch_before_loop != 0)
4152 int n = info[i].total_bytes / PREFETCH_BLOCK;
4155 info[i].prefetch_before_loop = n;
4156 num_prefetches_before += n;
4158 num_write_prefetches_before += n;
4161 if (loop_dump_stream)
4163 if (info[i].prefetch_in_loop == 0
4164 && info[i].prefetch_before_loop == 0)
4166 fprintf (loop_dump_stream, "Prefetch insn: %d",
4167 INSN_UID (info[i].giv->insn));
4168 fprintf (loop_dump_stream,
4169 "; in loop: %d; before: %d; %s\n",
4170 info[i].prefetch_in_loop,
4171 info[i].prefetch_before_loop,
4172 info[i].write ? "read/write" : "read only");
4173 fprintf (loop_dump_stream,
4174 " density: %d%%; bytes_accessed: %u; total_bytes: %u\n",
4175 (int) (info[i].bytes_accessed * 100 / info[i].stride),
4176 info[i].bytes_accessed, info[i].total_bytes);
4177 fprintf (loop_dump_stream, " index: " HOST_WIDE_INT_PRINT_DEC
4178 "; stride: " HOST_WIDE_INT_PRINT_DEC "; address: ",
4179 info[i].index, info[i].stride);
4180 print_rtl (loop_dump_stream, info[i].base_address);
4181 fprintf (loop_dump_stream, "\n");
4185 if (num_real_prefetches + num_prefetches_before > 0)
4187 /* Record that this loop uses prefetch instructions. */
4188 LOOP_INFO (loop)->has_prefetch = 1;
4190 if (loop_dump_stream)
4192 fprintf (loop_dump_stream, "Real prefetches needed within loop: %d (write: %d)\n",
4193 num_real_prefetches, num_real_write_prefetches);
4194 fprintf (loop_dump_stream, "Real prefetches needed before loop: %d (write: %d)\n",
4195 num_prefetches_before, num_write_prefetches_before);
4199 for (i = 0; i < num_prefetches; i++)
4203 for (y = 0; y < info[i].prefetch_in_loop; y++)
4205 rtx loc = copy_rtx (*info[i].giv->location);
4207 int bytes_ahead = PREFETCH_BLOCK * (ahead + y);
4208 rtx before_insn = info[i].giv->insn;
4209 rtx prev_insn = PREV_INSN (info[i].giv->insn);
4212 /* We can save some effort by offsetting the address on
4213 architectures with offsettable memory references. */
4214 if (offsettable_address_p (0, VOIDmode, loc))
4215 loc = plus_constant (loc, bytes_ahead);
4218 rtx reg = gen_reg_rtx (Pmode);
4219 loop_iv_add_mult_emit_before (loop, loc, const1_rtx,
4220 GEN_INT (bytes_ahead), reg,
4226 /* Make sure the address operand is valid for prefetch. */
4227 if (! (*insn_data[(int)CODE_FOR_prefetch].operand[0].predicate)
4228 (loc, insn_data[(int)CODE_FOR_prefetch].operand[0].mode))
4229 loc = force_reg (Pmode, loc);
4230 emit_insn (gen_prefetch (loc, GEN_INT (info[i].write),
4234 emit_insn_before (seq, before_insn);
4236 /* Check all insns emitted and record the new GIV
4238 insn = NEXT_INSN (prev_insn);
4239 while (insn != before_insn)
4241 insn = check_insn_for_givs (loop, insn,
4242 info[i].giv->always_executed,
4243 info[i].giv->maybe_multiple);
4244 insn = NEXT_INSN (insn);
4248 if (PREFETCH_BEFORE_LOOP)
4250 /* Emit insns before the loop to fetch the first cache lines or,
4251 if we're not prefetching within the loop, everything we expect
4253 for (y = 0; y < info[i].prefetch_before_loop; y++)
4255 rtx reg = gen_reg_rtx (Pmode);
4256 rtx loop_start = loop->start;
4257 rtx init_val = info[i].class->initial_value;
4258 rtx add_val = simplify_gen_binary (PLUS, Pmode,
4259 info[i].giv->add_val,
4260 GEN_INT (y * PREFETCH_BLOCK));
4262 /* Functions called by LOOP_IV_ADD_EMIT_BEFORE expect a
4263 non-constant INIT_VAL to have the same mode as REG, which
4264 in this case we know to be Pmode. */
4265 if (GET_MODE (init_val) != Pmode && !CONSTANT_P (init_val))
4270 init_val = convert_to_mode (Pmode, init_val, 0);
4273 loop_insn_emit_before (loop, 0, loop_start, seq);
4275 loop_iv_add_mult_emit_before (loop, init_val,
4276 info[i].giv->mult_val,
4277 add_val, reg, 0, loop_start);
4278 emit_insn_before (gen_prefetch (reg, GEN_INT (info[i].write),
4288 /* Communication with routines called via `note_stores'. */
4290 static rtx note_insn;
4292 /* Dummy register to have nonzero DEST_REG for DEST_ADDR type givs. */
4294 static rtx addr_placeholder;
4296 /* ??? Unfinished optimizations, and possible future optimizations,
4297 for the strength reduction code. */
4299 /* ??? The interaction of biv elimination, and recognition of 'constant'
4300 bivs, may cause problems. */
4302 /* ??? Add heuristics so that DEST_ADDR strength reduction does not cause
4303 performance problems.
4305 Perhaps don't eliminate things that can be combined with an addressing
4306 mode. Find all givs that have the same biv, mult_val, and add_val;
4307 then for each giv, check to see if its only use dies in a following
4308 memory address. If so, generate a new memory address and check to see
4309 if it is valid. If it is valid, then store the modified memory address,
4310 otherwise, mark the giv as not done so that it will get its own iv. */
4312 /* ??? Could try to optimize branches when it is known that a biv is always
4315 /* ??? When replace a biv in a compare insn, we should replace with closest
4316 giv so that an optimized branch can still be recognized by the combiner,
4317 e.g. the VAX acb insn. */
4319 /* ??? Many of the checks involving uid_luid could be simplified if regscan
4320 was rerun in loop_optimize whenever a register was added or moved.
4321 Also, some of the optimizations could be a little less conservative. */
4323 /* Scan the loop body and call FNCALL for each insn. In the addition to the
4324 LOOP and INSN parameters pass MAYBE_MULTIPLE and NOT_EVERY_ITERATION to the
4327 NOT_EVERY_ITERATION is 1 if current insn is not known to be executed at
4328 least once for every loop iteration except for the last one.
4330 MAYBE_MULTIPLE is 1 if current insn may be executed more than once for every
4334 for_each_insn_in_loop (struct loop *loop, loop_insn_callback fncall)
4336 int not_every_iteration = 0;
4337 int maybe_multiple = 0;
4338 int past_loop_latch = 0;
4342 /* If loop_scan_start points to the loop exit test, we have to be wary of
4343 subversive use of gotos inside expression statements. */
4344 if (prev_nonnote_insn (loop->scan_start) != prev_nonnote_insn (loop->start))
4345 maybe_multiple = back_branch_in_range_p (loop, loop->scan_start);
4347 /* Scan through loop and update NOT_EVERY_ITERATION and MAYBE_MULTIPLE. */
4348 for (p = next_insn_in_loop (loop, loop->scan_start);
4350 p = next_insn_in_loop (loop, p))
4352 p = fncall (loop, p, not_every_iteration, maybe_multiple);
4354 /* Past CODE_LABEL, we get to insns that may be executed multiple
4355 times. The only way we can be sure that they can't is if every
4356 jump insn between here and the end of the loop either
4357 returns, exits the loop, is a jump to a location that is still
4358 behind the label, or is a jump to the loop start. */
4360 if (GET_CODE (p) == CODE_LABEL)
4368 insn = NEXT_INSN (insn);
4369 if (insn == loop->scan_start)
4371 if (insn == loop->end)
4377 if (insn == loop->scan_start)
4381 if (GET_CODE (insn) == JUMP_INSN
4382 && GET_CODE (PATTERN (insn)) != RETURN
4383 && (!any_condjump_p (insn)
4384 || (JUMP_LABEL (insn) != 0
4385 && JUMP_LABEL (insn) != loop->scan_start
4386 && !loop_insn_first_p (p, JUMP_LABEL (insn)))))
4394 /* Past a jump, we get to insns for which we can't count
4395 on whether they will be executed during each iteration. */
4396 /* This code appears twice in strength_reduce. There is also similar
4397 code in scan_loop. */
4398 if (GET_CODE (p) == JUMP_INSN
4399 /* If we enter the loop in the middle, and scan around to the
4400 beginning, don't set not_every_iteration for that.
4401 This can be any kind of jump, since we want to know if insns
4402 will be executed if the loop is executed. */
4403 && !(JUMP_LABEL (p) == loop->top
4404 && ((NEXT_INSN (NEXT_INSN (p)) == loop->end
4405 && any_uncondjump_p (p))
4406 || (NEXT_INSN (p) == loop->end && any_condjump_p (p)))))
4410 /* If this is a jump outside the loop, then it also doesn't
4411 matter. Check to see if the target of this branch is on the
4412 loop->exits_labels list. */
4414 for (label = loop->exit_labels; label; label = LABEL_NEXTREF (label))
4415 if (XEXP (label, 0) == JUMP_LABEL (p))
4419 not_every_iteration = 1;
4422 else if (GET_CODE (p) == NOTE)
4424 /* At the virtual top of a converted loop, insns are again known to
4425 be executed each iteration: logically, the loop begins here
4426 even though the exit code has been duplicated.
4428 Insns are also again known to be executed each iteration at
4429 the LOOP_CONT note. */
4430 if ((NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_VTOP
4431 || NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_CONT)
4433 not_every_iteration = 0;
4434 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_BEG)
4436 else if (NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_END)
4440 /* Note if we pass a loop latch. If we do, then we can not clear
4441 NOT_EVERY_ITERATION below when we pass the last CODE_LABEL in
4442 a loop since a jump before the last CODE_LABEL may have started
4443 a new loop iteration.
4445 Note that LOOP_TOP is only set for rotated loops and we need
4446 this check for all loops, so compare against the CODE_LABEL
4447 which immediately follows LOOP_START. */
4448 if (GET_CODE (p) == JUMP_INSN
4449 && JUMP_LABEL (p) == NEXT_INSN (loop->start))
4450 past_loop_latch = 1;
4452 /* Unlike in the code motion pass where MAYBE_NEVER indicates that
4453 an insn may never be executed, NOT_EVERY_ITERATION indicates whether
4454 or not an insn is known to be executed each iteration of the
4455 loop, whether or not any iterations are known to occur.
4457 Therefore, if we have just passed a label and have no more labels
4458 between here and the test insn of the loop, and we have not passed
4459 a jump to the top of the loop, then we know these insns will be
4460 executed each iteration. */
4462 if (not_every_iteration
4464 && GET_CODE (p) == CODE_LABEL
4465 && no_labels_between_p (p, loop->end)
4466 && loop_insn_first_p (p, loop->cont))
4467 not_every_iteration = 0;
4472 loop_bivs_find (struct loop *loop)
4474 struct loop_regs *regs = LOOP_REGS (loop);
4475 struct loop_ivs *ivs = LOOP_IVS (loop);
4476 /* Temporary list pointers for traversing ivs->list. */
4477 struct iv_class *bl, **backbl;
4481 for_each_insn_in_loop (loop, check_insn_for_bivs);
4483 /* Scan ivs->list to remove all regs that proved not to be bivs.
4484 Make a sanity check against regs->n_times_set. */
4485 for (backbl = &ivs->list, bl = *backbl; bl; bl = bl->next)
4487 if (REG_IV_TYPE (ivs, bl->regno) != BASIC_INDUCT
4488 /* Above happens if register modified by subreg, etc. */
4489 /* Make sure it is not recognized as a basic induction var: */
4490 || regs->array[bl->regno].n_times_set != bl->biv_count
4491 /* If never incremented, it is invariant that we decided not to
4492 move. So leave it alone. */
4493 || ! bl->incremented)
4495 if (loop_dump_stream)
4496 fprintf (loop_dump_stream, "Biv %d: discarded, %s\n",
4498 (REG_IV_TYPE (ivs, bl->regno) != BASIC_INDUCT
4499 ? "not induction variable"
4500 : (! bl->incremented ? "never incremented"
4503 REG_IV_TYPE (ivs, bl->regno) = NOT_BASIC_INDUCT;
4510 if (loop_dump_stream)
4511 fprintf (loop_dump_stream, "Biv %d: verified\n", bl->regno);
4517 /* Determine how BIVS are initialized by looking through pre-header
4518 extended basic block. */
4520 loop_bivs_init_find (struct loop *loop)
4522 struct loop_ivs *ivs = LOOP_IVS (loop);
4523 /* Temporary list pointers for traversing ivs->list. */
4524 struct iv_class *bl;
4528 /* Find initial value for each biv by searching backwards from loop_start,
4529 halting at first label. Also record any test condition. */
4532 for (p = loop->start; p && GET_CODE (p) != CODE_LABEL; p = PREV_INSN (p))
4538 if (GET_CODE (p) == CALL_INSN)
4542 note_stores (PATTERN (p), record_initial, ivs);
4544 /* Record any test of a biv that branches around the loop if no store
4545 between it and the start of loop. We only care about tests with
4546 constants and registers and only certain of those. */
4547 if (GET_CODE (p) == JUMP_INSN
4548 && JUMP_LABEL (p) != 0
4549 && next_real_insn (JUMP_LABEL (p)) == next_real_insn (loop->end)
4550 && (test = get_condition_for_loop (loop, p)) != 0
4551 && GET_CODE (XEXP (test, 0)) == REG
4552 && REGNO (XEXP (test, 0)) < max_reg_before_loop
4553 && (bl = REG_IV_CLASS (ivs, REGNO (XEXP (test, 0)))) != 0
4554 && valid_initial_value_p (XEXP (test, 1), p, call_seen, loop->start)
4555 && bl->init_insn == 0)
4557 /* If an NE test, we have an initial value! */
4558 if (GET_CODE (test) == NE)
4561 bl->init_set = gen_rtx_SET (VOIDmode,
4562 XEXP (test, 0), XEXP (test, 1));
4565 bl->initial_test = test;
4571 /* Look at the each biv and see if we can say anything better about its
4572 initial value from any initializing insns set up above. (This is done
4573 in two passes to avoid missing SETs in a PARALLEL.) */
4575 loop_bivs_check (struct loop *loop)
4577 struct loop_ivs *ivs = LOOP_IVS (loop);
4578 /* Temporary list pointers for traversing ivs->list. */
4579 struct iv_class *bl;
4580 struct iv_class **backbl;
4582 for (backbl = &ivs->list; (bl = *backbl); backbl = &bl->next)
4587 if (! bl->init_insn)
4590 /* IF INIT_INSN has a REG_EQUAL or REG_EQUIV note and the value
4591 is a constant, use the value of that. */
4592 if (((note = find_reg_note (bl->init_insn, REG_EQUAL, 0)) != NULL
4593 && CONSTANT_P (XEXP (note, 0)))
4594 || ((note = find_reg_note (bl->init_insn, REG_EQUIV, 0)) != NULL
4595 && CONSTANT_P (XEXP (note, 0))))
4596 src = XEXP (note, 0);
4598 src = SET_SRC (bl->init_set);
4600 if (loop_dump_stream)
4601 fprintf (loop_dump_stream,
4602 "Biv %d: initialized at insn %d: initial value ",
4603 bl->regno, INSN_UID (bl->init_insn));
4605 if ((GET_MODE (src) == GET_MODE (regno_reg_rtx[bl->regno])
4606 || GET_MODE (src) == VOIDmode)
4607 && valid_initial_value_p (src, bl->init_insn,
4608 LOOP_INFO (loop)->pre_header_has_call,
4611 bl->initial_value = src;
4613 if (loop_dump_stream)
4615 print_simple_rtl (loop_dump_stream, src);
4616 fputc ('\n', loop_dump_stream);
4619 /* If we can't make it a giv,
4620 let biv keep initial value of "itself". */
4621 else if (loop_dump_stream)
4622 fprintf (loop_dump_stream, "is complex\n");
4627 /* Search the loop for general induction variables. */
4630 loop_givs_find (struct loop* loop)
4632 for_each_insn_in_loop (loop, check_insn_for_givs);
4636 /* For each giv for which we still don't know whether or not it is
4637 replaceable, check to see if it is replaceable because its final value
4638 can be calculated. */
4641 loop_givs_check (struct loop *loop)
4643 struct loop_ivs *ivs = LOOP_IVS (loop);
4644 struct iv_class *bl;
4646 for (bl = ivs->list; bl; bl = bl->next)
4648 struct induction *v;
4650 for (v = bl->giv; v; v = v->next_iv)
4651 if (! v->replaceable && ! v->not_replaceable)
4652 check_final_value (loop, v);
4657 /* Return nonzero if it is possible to eliminate the biv BL provided
4658 all givs are reduced. This is possible if either the reg is not
4659 used outside the loop, or we can compute what its final value will
4663 loop_biv_eliminable_p (struct loop *loop, struct iv_class *bl,
4664 int threshold, int insn_count)
4666 /* For architectures with a decrement_and_branch_until_zero insn,
4667 don't do this if we put a REG_NONNEG note on the endtest for this
4670 #ifdef HAVE_decrement_and_branch_until_zero
4673 if (loop_dump_stream)
4674 fprintf (loop_dump_stream,
4675 "Cannot eliminate nonneg biv %d.\n", bl->regno);
4680 /* Check that biv is used outside loop or if it has a final value.
4681 Compare against bl->init_insn rather than loop->start. We aren't
4682 concerned with any uses of the biv between init_insn and
4683 loop->start since these won't be affected by the value of the biv
4684 elsewhere in the function, so long as init_insn doesn't use the
4687 if ((REGNO_LAST_LUID (bl->regno) < INSN_LUID (loop->end)
4689 && INSN_UID (bl->init_insn) < max_uid_for_loop
4690 && REGNO_FIRST_LUID (bl->regno) >= INSN_LUID (bl->init_insn)
4691 && ! reg_mentioned_p (bl->biv->dest_reg, SET_SRC (bl->init_set)))
4692 || (bl->final_value = final_biv_value (loop, bl)))
4693 return maybe_eliminate_biv (loop, bl, 0, threshold, insn_count);
4695 if (loop_dump_stream)
4697 fprintf (loop_dump_stream,
4698 "Cannot eliminate biv %d.\n",
4700 fprintf (loop_dump_stream,
4701 "First use: insn %d, last use: insn %d.\n",
4702 REGNO_FIRST_UID (bl->regno),
4703 REGNO_LAST_UID (bl->regno));
4709 /* Reduce each giv of BL that we have decided to reduce. */
4712 loop_givs_reduce (struct loop *loop, struct iv_class *bl)
4714 struct induction *v;
4716 for (v = bl->giv; v; v = v->next_iv)
4718 struct induction *tv;
4719 if (! v->ignore && v->same == 0)
4721 int auto_inc_opt = 0;
4723 /* If the code for derived givs immediately below has already
4724 allocated a new_reg, we must keep it. */
4726 v->new_reg = gen_reg_rtx (v->mode);
4729 /* If the target has auto-increment addressing modes, and
4730 this is an address giv, then try to put the increment
4731 immediately after its use, so that flow can create an
4732 auto-increment addressing mode. */
4733 if (v->giv_type == DEST_ADDR && bl->biv_count == 1
4734 && bl->biv->always_executed && ! bl->biv->maybe_multiple
4735 /* We don't handle reversed biv's because bl->biv->insn
4736 does not have a valid INSN_LUID. */
4738 && v->always_executed && ! v->maybe_multiple
4739 && INSN_UID (v->insn) < max_uid_for_loop)
4741 /* If other giv's have been combined with this one, then
4742 this will work only if all uses of the other giv's occur
4743 before this giv's insn. This is difficult to check.
4745 We simplify this by looking for the common case where
4746 there is one DEST_REG giv, and this giv's insn is the
4747 last use of the dest_reg of that DEST_REG giv. If the
4748 increment occurs after the address giv, then we can
4749 perform the optimization. (Otherwise, the increment
4750 would have to go before other_giv, and we would not be
4751 able to combine it with the address giv to get an
4752 auto-inc address.) */
4753 if (v->combined_with)
4755 struct induction *other_giv = 0;
4757 for (tv = bl->giv; tv; tv = tv->next_iv)
4765 if (! tv && other_giv
4766 && REGNO (other_giv->dest_reg) < max_reg_before_loop
4767 && (REGNO_LAST_UID (REGNO (other_giv->dest_reg))
4768 == INSN_UID (v->insn))
4769 && INSN_LUID (v->insn) < INSN_LUID (bl->biv->insn))
4772 /* Check for case where increment is before the address
4773 giv. Do this test in "loop order". */
4774 else if ((INSN_LUID (v->insn) > INSN_LUID (bl->biv->insn)
4775 && (INSN_LUID (v->insn) < INSN_LUID (loop->scan_start)
4776 || (INSN_LUID (bl->biv->insn)
4777 > INSN_LUID (loop->scan_start))))
4778 || (INSN_LUID (v->insn) < INSN_LUID (loop->scan_start)
4779 && (INSN_LUID (loop->scan_start)
4780 < INSN_LUID (bl->biv->insn))))
4789 /* We can't put an insn immediately after one setting
4790 cc0, or immediately before one using cc0. */
4791 if ((auto_inc_opt == 1 && sets_cc0_p (PATTERN (v->insn)))
4792 || (auto_inc_opt == -1
4793 && (prev = prev_nonnote_insn (v->insn)) != 0
4795 && sets_cc0_p (PATTERN (prev))))
4801 v->auto_inc_opt = 1;
4805 /* For each place where the biv is incremented, add an insn
4806 to increment the new, reduced reg for the giv. */
4807 for (tv = bl->biv; tv; tv = tv->next_iv)
4811 /* Skip if location is the same as a previous one. */
4815 insert_before = NEXT_INSN (tv->insn);
4816 else if (auto_inc_opt == 1)
4817 insert_before = NEXT_INSN (v->insn);
4819 insert_before = v->insn;
4821 if (tv->mult_val == const1_rtx)
4822 loop_iv_add_mult_emit_before (loop, tv->add_val, v->mult_val,
4823 v->new_reg, v->new_reg,
4825 else /* tv->mult_val == const0_rtx */
4826 /* A multiply is acceptable here
4827 since this is presumed to be seldom executed. */
4828 loop_iv_add_mult_emit_before (loop, tv->add_val, v->mult_val,
4829 v->add_val, v->new_reg,
4833 /* Add code at loop start to initialize giv's reduced reg. */
4835 loop_iv_add_mult_hoist (loop,
4836 extend_value_for_giv (v, bl->initial_value),
4837 v->mult_val, v->add_val, v->new_reg);
4843 /* Check for givs whose first use is their definition and whose
4844 last use is the definition of another giv. If so, it is likely
4845 dead and should not be used to derive another giv nor to
4849 loop_givs_dead_check (struct loop *loop ATTRIBUTE_UNUSED, struct iv_class *bl)
4851 struct induction *v;
4853 for (v = bl->giv; v; v = v->next_iv)
4856 || (v->same && v->same->ignore))
4859 if (v->giv_type == DEST_REG
4860 && REGNO_FIRST_UID (REGNO (v->dest_reg)) == INSN_UID (v->insn))
4862 struct induction *v1;
4864 for (v1 = bl->giv; v1; v1 = v1->next_iv)
4865 if (REGNO_LAST_UID (REGNO (v->dest_reg)) == INSN_UID (v1->insn))
4873 loop_givs_rescan (struct loop *loop, struct iv_class *bl, rtx *reg_map)
4875 struct induction *v;
4877 for (v = bl->giv; v; v = v->next_iv)
4879 if (v->same && v->same->ignore)
4885 /* Update expression if this was combined, in case other giv was
4888 v->new_reg = replace_rtx (v->new_reg,
4889 v->same->dest_reg, v->same->new_reg);
4891 /* See if this register is known to be a pointer to something. If
4892 so, see if we can find the alignment. First see if there is a
4893 destination register that is a pointer. If so, this shares the
4894 alignment too. Next see if we can deduce anything from the
4895 computational information. If not, and this is a DEST_ADDR
4896 giv, at least we know that it's a pointer, though we don't know
4898 if (GET_CODE (v->new_reg) == REG
4899 && v->giv_type == DEST_REG
4900 && REG_POINTER (v->dest_reg))
4901 mark_reg_pointer (v->new_reg,
4902 REGNO_POINTER_ALIGN (REGNO (v->dest_reg)));
4903 else if (GET_CODE (v->new_reg) == REG
4904 && REG_POINTER (v->src_reg))
4906 unsigned int align = REGNO_POINTER_ALIGN (REGNO (v->src_reg));
4909 || GET_CODE (v->add_val) != CONST_INT
4910 || INTVAL (v->add_val) % (align / BITS_PER_UNIT) != 0)
4913 mark_reg_pointer (v->new_reg, align);
4915 else if (GET_CODE (v->new_reg) == REG
4916 && GET_CODE (v->add_val) == REG
4917 && REG_POINTER (v->add_val))
4919 unsigned int align = REGNO_POINTER_ALIGN (REGNO (v->add_val));
4921 if (align == 0 || GET_CODE (v->mult_val) != CONST_INT
4922 || INTVAL (v->mult_val) % (align / BITS_PER_UNIT) != 0)
4925 mark_reg_pointer (v->new_reg, align);
4927 else if (GET_CODE (v->new_reg) == REG && v->giv_type == DEST_ADDR)
4928 mark_reg_pointer (v->new_reg, 0);
4930 if (v->giv_type == DEST_ADDR)
4931 /* Store reduced reg as the address in the memref where we found
4933 validate_change (v->insn, v->location, v->new_reg, 0);
4934 else if (v->replaceable)
4936 reg_map[REGNO (v->dest_reg)] = v->new_reg;
4940 rtx original_insn = v->insn;
4943 /* Not replaceable; emit an insn to set the original giv reg from
4944 the reduced giv, same as above. */
4945 v->insn = loop_insn_emit_after (loop, 0, original_insn,
4946 gen_move_insn (v->dest_reg,
4949 /* The original insn may have a REG_EQUAL note. This note is
4950 now incorrect and may result in invalid substitutions later.
4951 The original insn is dead, but may be part of a libcall
4952 sequence, which doesn't seem worth the bother of handling. */
4953 note = find_reg_note (original_insn, REG_EQUAL, NULL_RTX);
4955 remove_note (original_insn, note);
4958 /* When a loop is reversed, givs which depend on the reversed
4959 biv, and which are live outside the loop, must be set to their
4960 correct final value. This insn is only needed if the giv is
4961 not replaceable. The correct final value is the same as the
4962 value that the giv starts the reversed loop with. */
4963 if (bl->reversed && ! v->replaceable)
4964 loop_iv_add_mult_sink (loop,
4965 extend_value_for_giv (v, bl->initial_value),
4966 v->mult_val, v->add_val, v->dest_reg);
4967 else if (v->final_value)
4968 loop_insn_sink_or_swim (loop,
4969 gen_load_of_final_value (v->dest_reg,
4972 if (loop_dump_stream)
4974 fprintf (loop_dump_stream, "giv at %d reduced to ",
4975 INSN_UID (v->insn));
4976 print_simple_rtl (loop_dump_stream, v->new_reg);
4977 fprintf (loop_dump_stream, "\n");
4984 loop_giv_reduce_benefit (struct loop *loop ATTRIBUTE_UNUSED,
4985 struct iv_class *bl, struct induction *v,
4991 benefit = v->benefit;
4992 PUT_MODE (test_reg, v->mode);
4993 add_cost = iv_add_mult_cost (bl->biv->add_val, v->mult_val,
4994 test_reg, test_reg);
4996 /* Reduce benefit if not replaceable, since we will insert a
4997 move-insn to replace the insn that calculates this giv. Don't do
4998 this unless the giv is a user variable, since it will often be
4999 marked non-replaceable because of the duplication of the exit
5000 code outside the loop. In such a case, the copies we insert are
5001 dead and will be deleted. So they don't have a cost. Similar
5002 situations exist. */
5003 /* ??? The new final_[bg]iv_value code does a much better job of
5004 finding replaceable giv's, and hence this code may no longer be
5006 if (! v->replaceable && ! bl->eliminable
5007 && REG_USERVAR_P (v->dest_reg))
5008 benefit -= copy_cost;
5010 /* Decrease the benefit to count the add-insns that we will insert
5011 to increment the reduced reg for the giv. ??? This can
5012 overestimate the run-time cost of the additional insns, e.g. if
5013 there are multiple basic blocks that increment the biv, but only
5014 one of these blocks is executed during each iteration. There is
5015 no good way to detect cases like this with the current structure
5016 of the loop optimizer. This code is more accurate for
5017 determining code size than run-time benefits. */
5018 benefit -= add_cost * bl->biv_count;
5020 /* Decide whether to strength-reduce this giv or to leave the code
5021 unchanged (recompute it from the biv each time it is used). This
5022 decision can be made independently for each giv. */
5025 /* Attempt to guess whether autoincrement will handle some of the
5026 new add insns; if so, increase BENEFIT (undo the subtraction of
5027 add_cost that was done above). */
5028 if (v->giv_type == DEST_ADDR
5029 /* Increasing the benefit is risky, since this is only a guess.
5030 Avoid increasing register pressure in cases where there would
5031 be no other benefit from reducing this giv. */
5033 && GET_CODE (v->mult_val) == CONST_INT)
5035 int size = GET_MODE_SIZE (GET_MODE (v->mem));
5037 if (HAVE_POST_INCREMENT
5038 && INTVAL (v->mult_val) == size)
5039 benefit += add_cost * bl->biv_count;
5040 else if (HAVE_PRE_INCREMENT
5041 && INTVAL (v->mult_val) == size)
5042 benefit += add_cost * bl->biv_count;
5043 else if (HAVE_POST_DECREMENT
5044 && -INTVAL (v->mult_val) == size)
5045 benefit += add_cost * bl->biv_count;
5046 else if (HAVE_PRE_DECREMENT
5047 && -INTVAL (v->mult_val) == size)
5048 benefit += add_cost * bl->biv_count;
5056 /* Free IV structures for LOOP. */
5059 loop_ivs_free (struct loop *loop)
5061 struct loop_ivs *ivs = LOOP_IVS (loop);
5062 struct iv_class *iv = ivs->list;
5068 struct iv_class *next = iv->next;
5069 struct induction *induction;
5070 struct induction *next_induction;
5072 for (induction = iv->biv; induction; induction = next_induction)
5074 next_induction = induction->next_iv;
5077 for (induction = iv->giv; induction; induction = next_induction)
5079 next_induction = induction->next_iv;
5089 /* Perform strength reduction and induction variable elimination.
5091 Pseudo registers created during this function will be beyond the
5092 last valid index in several tables including
5093 REGS->ARRAY[I].N_TIMES_SET and REGNO_LAST_UID. This does not cause a
5094 problem here, because the added registers cannot be givs outside of
5095 their loop, and hence will never be reconsidered. But scan_loop
5096 must check regnos to make sure they are in bounds. */
5099 strength_reduce (struct loop *loop, int flags)
5101 struct loop_info *loop_info = LOOP_INFO (loop);
5102 struct loop_regs *regs = LOOP_REGS (loop);
5103 struct loop_ivs *ivs = LOOP_IVS (loop);
5105 /* Temporary list pointer for traversing ivs->list. */
5106 struct iv_class *bl;
5107 /* Ratio of extra register life span we can justify
5108 for saving an instruction. More if loop doesn't call subroutines
5109 since in that case saving an insn makes more difference
5110 and more registers are available. */
5111 /* ??? could set this to last value of threshold in move_movables */
5112 int threshold = (loop_info->has_call ? 1 : 2) * (3 + n_non_fixed_regs);
5113 /* Map of pseudo-register replacements. */
5114 rtx *reg_map = NULL;
5116 int unrolled_insn_copies = 0;
5117 rtx test_reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
5118 int insn_count = count_insns_in_loop (loop);
5120 addr_placeholder = gen_reg_rtx (Pmode);
5122 ivs->n_regs = max_reg_before_loop;
5123 ivs->regs = xcalloc (ivs->n_regs, sizeof (struct iv));
5125 /* Find all BIVs in loop. */
5126 loop_bivs_find (loop);
5128 /* Exit if there are no bivs. */
5131 /* Can still unroll the loop anyways, but indicate that there is no
5132 strength reduction info available. */
5133 if (flags & LOOP_UNROLL)
5134 unroll_loop (loop, insn_count, 0);
5136 loop_ivs_free (loop);
5140 /* Determine how BIVS are initialized by looking through pre-header
5141 extended basic block. */
5142 loop_bivs_init_find (loop);
5144 /* Look at the each biv and see if we can say anything better about its
5145 initial value from any initializing insns set up above. */
5146 loop_bivs_check (loop);
5148 /* Search the loop for general induction variables. */
5149 loop_givs_find (loop);
5151 /* Try to calculate and save the number of loop iterations. This is
5152 set to zero if the actual number can not be calculated. This must
5153 be called after all giv's have been identified, since otherwise it may
5154 fail if the iteration variable is a giv. */
5155 loop_iterations (loop);
5157 #ifdef HAVE_prefetch
5158 if (flags & LOOP_PREFETCH)
5159 emit_prefetch_instructions (loop);
5162 /* Now for each giv for which we still don't know whether or not it is
5163 replaceable, check to see if it is replaceable because its final value
5164 can be calculated. This must be done after loop_iterations is called,
5165 so that final_giv_value will work correctly. */
5166 loop_givs_check (loop);
5168 /* Try to prove that the loop counter variable (if any) is always
5169 nonnegative; if so, record that fact with a REG_NONNEG note
5170 so that "decrement and branch until zero" insn can be used. */
5171 check_dbra_loop (loop, insn_count);
5173 /* Create reg_map to hold substitutions for replaceable giv regs.
5174 Some givs might have been made from biv increments, so look at
5175 ivs->reg_iv_type for a suitable size. */
5176 reg_map_size = ivs->n_regs;
5177 reg_map = xcalloc (reg_map_size, sizeof (rtx));
5179 /* Examine each iv class for feasibility of strength reduction/induction
5180 variable elimination. */
5182 for (bl = ivs->list; bl; bl = bl->next)
5184 struct induction *v;
5187 /* Test whether it will be possible to eliminate this biv
5188 provided all givs are reduced. */
5189 bl->eliminable = loop_biv_eliminable_p (loop, bl, threshold, insn_count);
5191 /* This will be true at the end, if all givs which depend on this
5192 biv have been strength reduced.
5193 We can't (currently) eliminate the biv unless this is so. */
5194 bl->all_reduced = 1;
5196 /* Check each extension dependent giv in this class to see if its
5197 root biv is safe from wrapping in the interior mode. */
5198 check_ext_dependent_givs (loop, bl);
5200 /* Combine all giv's for this iv_class. */
5201 combine_givs (regs, bl);
5203 for (v = bl->giv; v; v = v->next_iv)
5205 struct induction *tv;
5207 if (v->ignore || v->same)
5210 benefit = loop_giv_reduce_benefit (loop, bl, v, test_reg);
5212 /* If an insn is not to be strength reduced, then set its ignore
5213 flag, and clear bl->all_reduced. */
5215 /* A giv that depends on a reversed biv must be reduced if it is
5216 used after the loop exit, otherwise, it would have the wrong
5217 value after the loop exit. To make it simple, just reduce all
5218 of such giv's whether or not we know they are used after the loop
5221 if (! flag_reduce_all_givs
5222 && v->lifetime * threshold * benefit < insn_count
5225 if (loop_dump_stream)
5226 fprintf (loop_dump_stream,
5227 "giv of insn %d not worth while, %d vs %d.\n",
5229 v->lifetime * threshold * benefit, insn_count);
5231 bl->all_reduced = 0;
5235 /* Check that we can increment the reduced giv without a
5236 multiply insn. If not, reject it. */
5238 for (tv = bl->biv; tv; tv = tv->next_iv)
5239 if (tv->mult_val == const1_rtx
5240 && ! product_cheap_p (tv->add_val, v->mult_val))
5242 if (loop_dump_stream)
5243 fprintf (loop_dump_stream,
5244 "giv of insn %d: would need a multiply.\n",
5245 INSN_UID (v->insn));
5247 bl->all_reduced = 0;
5253 /* Check for givs whose first use is their definition and whose
5254 last use is the definition of another giv. If so, it is likely
5255 dead and should not be used to derive another giv nor to
5257 loop_givs_dead_check (loop, bl);
5259 /* Reduce each giv that we decided to reduce. */
5260 loop_givs_reduce (loop, bl);
5262 /* Rescan all givs. If a giv is the same as a giv not reduced, mark it
5265 For each giv register that can be reduced now: if replaceable,
5266 substitute reduced reg wherever the old giv occurs;
5267 else add new move insn "giv_reg = reduced_reg". */
5268 loop_givs_rescan (loop, bl, reg_map);
5270 /* All the givs based on the biv bl have been reduced if they
5273 /* For each giv not marked as maybe dead that has been combined with a
5274 second giv, clear any "maybe dead" mark on that second giv.
5275 v->new_reg will either be or refer to the register of the giv it
5278 Doing this clearing avoids problems in biv elimination where
5279 a giv's new_reg is a complex value that can't be put in the
5280 insn but the giv combined with (with a reg as new_reg) is
5281 marked maybe_dead. Since the register will be used in either
5282 case, we'd prefer it be used from the simpler giv. */
5284 for (v = bl->giv; v; v = v->next_iv)
5285 if (! v->maybe_dead && v->same)
5286 v->same->maybe_dead = 0;
5288 /* Try to eliminate the biv, if it is a candidate.
5289 This won't work if ! bl->all_reduced,
5290 since the givs we planned to use might not have been reduced.
5292 We have to be careful that we didn't initially think we could
5293 eliminate this biv because of a giv that we now think may be
5294 dead and shouldn't be used as a biv replacement.
5296 Also, there is the possibility that we may have a giv that looks
5297 like it can be used to eliminate a biv, but the resulting insn
5298 isn't valid. This can happen, for example, on the 88k, where a
5299 JUMP_INSN can compare a register only with zero. Attempts to
5300 replace it with a compare with a constant will fail.
5302 Note that in cases where this call fails, we may have replaced some
5303 of the occurrences of the biv with a giv, but no harm was done in
5304 doing so in the rare cases where it can occur. */
5306 if (bl->all_reduced == 1 && bl->eliminable
5307 && maybe_eliminate_biv (loop, bl, 1, threshold, insn_count))
5309 /* ?? If we created a new test to bypass the loop entirely,
5310 or otherwise drop straight in, based on this test, then
5311 we might want to rewrite it also. This way some later
5312 pass has more hope of removing the initialization of this
5315 /* If final_value != 0, then the biv may be used after loop end
5316 and we must emit an insn to set it just in case.
5318 Reversed bivs already have an insn after the loop setting their
5319 value, so we don't need another one. We can't calculate the
5320 proper final value for such a biv here anyways. */
5321 if (bl->final_value && ! bl->reversed)
5322 loop_insn_sink_or_swim (loop,
5323 gen_load_of_final_value (bl->biv->dest_reg,
5326 if (loop_dump_stream)
5327 fprintf (loop_dump_stream, "Reg %d: biv eliminated\n",
5330 /* See above note wrt final_value. But since we couldn't eliminate
5331 the biv, we must set the value after the loop instead of before. */
5332 else if (bl->final_value && ! bl->reversed)
5333 loop_insn_sink (loop, gen_load_of_final_value (bl->biv->dest_reg,
5337 /* Go through all the instructions in the loop, making all the
5338 register substitutions scheduled in REG_MAP. */
5340 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
5341 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
5342 || GET_CODE (p) == CALL_INSN)
5344 replace_regs (PATTERN (p), reg_map, reg_map_size, 0);
5345 replace_regs (REG_NOTES (p), reg_map, reg_map_size, 0);
5349 if (loop_info->n_iterations > 0)
5351 /* When we completely unroll a loop we will likely not need the increment
5352 of the loop BIV and we will not need the conditional branch at the
5354 unrolled_insn_copies = insn_count - 2;
5357 /* When we completely unroll a loop on a HAVE_cc0 machine we will not
5358 need the comparison before the conditional branch at the end of the
5360 unrolled_insn_copies -= 1;
5363 /* We'll need one copy for each loop iteration. */
5364 unrolled_insn_copies *= loop_info->n_iterations;
5366 /* A little slop to account for the ability to remove initialization
5367 code, better CSE, and other secondary benefits of completely
5368 unrolling some loops. */
5369 unrolled_insn_copies -= 1;
5371 /* Clamp the value. */
5372 if (unrolled_insn_copies < 0)
5373 unrolled_insn_copies = 0;
5376 /* Unroll loops from within strength reduction so that we can use the
5377 induction variable information that strength_reduce has already
5378 collected. Always unroll loops that would be as small or smaller
5379 unrolled than when rolled. */
5380 if ((flags & LOOP_UNROLL)
5381 || ((flags & LOOP_AUTO_UNROLL)
5382 && loop_info->n_iterations > 0
5383 && unrolled_insn_copies <= insn_count))
5384 unroll_loop (loop, insn_count, 1);
5386 #ifdef HAVE_doloop_end
5387 if (HAVE_doloop_end && (flags & LOOP_BCT) && flag_branch_on_count_reg)
5388 doloop_optimize (loop);
5389 #endif /* HAVE_doloop_end */
5391 /* In case number of iterations is known, drop branch prediction note
5392 in the branch. Do that only in second loop pass, as loop unrolling
5393 may change the number of iterations performed. */
5394 if (flags & LOOP_BCT)
5396 unsigned HOST_WIDE_INT n
5397 = loop_info->n_iterations / loop_info->unroll_number;
5399 predict_insn (prev_nonnote_insn (loop->end), PRED_LOOP_ITERATIONS,
5400 REG_BR_PROB_BASE - REG_BR_PROB_BASE / n);
5403 if (loop_dump_stream)
5404 fprintf (loop_dump_stream, "\n");
5406 loop_ivs_free (loop);
5411 /*Record all basic induction variables calculated in the insn. */
5413 check_insn_for_bivs (struct loop *loop, rtx p, int not_every_iteration,
5416 struct loop_ivs *ivs = LOOP_IVS (loop);
5423 if (GET_CODE (p) == INSN
5424 && (set = single_set (p))
5425 && GET_CODE (SET_DEST (set)) == REG)
5427 dest_reg = SET_DEST (set);
5428 if (REGNO (dest_reg) < max_reg_before_loop
5429 && REGNO (dest_reg) >= FIRST_PSEUDO_REGISTER
5430 && REG_IV_TYPE (ivs, REGNO (dest_reg)) != NOT_BASIC_INDUCT)
5432 if (basic_induction_var (loop, SET_SRC (set),
5433 GET_MODE (SET_SRC (set)),
5434 dest_reg, p, &inc_val, &mult_val,
5437 /* It is a possible basic induction variable.
5438 Create and initialize an induction structure for it. */
5440 struct induction *v = xmalloc (sizeof (struct induction));
5442 record_biv (loop, v, p, dest_reg, inc_val, mult_val, location,
5443 not_every_iteration, maybe_multiple);
5444 REG_IV_TYPE (ivs, REGNO (dest_reg)) = BASIC_INDUCT;
5446 else if (REGNO (dest_reg) < ivs->n_regs)
5447 REG_IV_TYPE (ivs, REGNO (dest_reg)) = NOT_BASIC_INDUCT;
5453 /* Record all givs calculated in the insn.
5454 A register is a giv if: it is only set once, it is a function of a
5455 biv and a constant (or invariant), and it is not a biv. */
5457 check_insn_for_givs (struct loop *loop, rtx p, int not_every_iteration,
5460 struct loop_regs *regs = LOOP_REGS (loop);
5463 /* Look for a general induction variable in a register. */
5464 if (GET_CODE (p) == INSN
5465 && (set = single_set (p))
5466 && GET_CODE (SET_DEST (set)) == REG
5467 && ! regs->array[REGNO (SET_DEST (set))].may_not_optimize)
5476 rtx last_consec_insn;
5478 dest_reg = SET_DEST (set);
5479 if (REGNO (dest_reg) < FIRST_PSEUDO_REGISTER)
5482 if (/* SET_SRC is a giv. */
5483 (general_induction_var (loop, SET_SRC (set), &src_reg, &add_val,
5484 &mult_val, &ext_val, 0, &benefit, VOIDmode)
5485 /* Equivalent expression is a giv. */
5486 || ((regnote = find_reg_note (p, REG_EQUAL, NULL_RTX))
5487 && general_induction_var (loop, XEXP (regnote, 0), &src_reg,
5488 &add_val, &mult_val, &ext_val, 0,
5489 &benefit, VOIDmode)))
5490 /* Don't try to handle any regs made by loop optimization.
5491 We have nothing on them in regno_first_uid, etc. */
5492 && REGNO (dest_reg) < max_reg_before_loop
5493 /* Don't recognize a BASIC_INDUCT_VAR here. */
5494 && dest_reg != src_reg
5495 /* This must be the only place where the register is set. */
5496 && (regs->array[REGNO (dest_reg)].n_times_set == 1
5497 /* or all sets must be consecutive and make a giv. */
5498 || (benefit = consec_sets_giv (loop, benefit, p,
5500 &add_val, &mult_val, &ext_val,
5501 &last_consec_insn))))
5503 struct induction *v = xmalloc (sizeof (struct induction));
5505 /* If this is a library call, increase benefit. */
5506 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
5507 benefit += libcall_benefit (p);
5509 /* Skip the consecutive insns, if there are any. */
5510 if (regs->array[REGNO (dest_reg)].n_times_set != 1)
5511 p = last_consec_insn;
5513 record_giv (loop, v, p, src_reg, dest_reg, mult_val, add_val,
5514 ext_val, benefit, DEST_REG, not_every_iteration,
5515 maybe_multiple, (rtx*) 0);
5520 /* Look for givs which are memory addresses. */
5521 if (GET_CODE (p) == INSN)
5522 find_mem_givs (loop, PATTERN (p), p, not_every_iteration,
5525 /* Update the status of whether giv can derive other givs. This can
5526 change when we pass a label or an insn that updates a biv. */
5527 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
5528 || GET_CODE (p) == CODE_LABEL)
5529 update_giv_derive (loop, p);
5533 /* Return 1 if X is a valid source for an initial value (or as value being
5534 compared against in an initial test).
5536 X must be either a register or constant and must not be clobbered between
5537 the current insn and the start of the loop.
5539 INSN is the insn containing X. */
5542 valid_initial_value_p (rtx x, rtx insn, int call_seen, rtx loop_start)
5547 /* Only consider pseudos we know about initialized in insns whose luids
5549 if (GET_CODE (x) != REG
5550 || REGNO (x) >= max_reg_before_loop)
5553 /* Don't use call-clobbered registers across a call which clobbers it. On
5554 some machines, don't use any hard registers at all. */
5555 if (REGNO (x) < FIRST_PSEUDO_REGISTER
5556 && (SMALL_REGISTER_CLASSES
5557 || (call_used_regs[REGNO (x)] && call_seen)))
5560 /* Don't use registers that have been clobbered before the start of the
5562 if (reg_set_between_p (x, insn, loop_start))
5568 /* Scan X for memory refs and check each memory address
5569 as a possible giv. INSN is the insn whose pattern X comes from.
5570 NOT_EVERY_ITERATION is 1 if the insn might not be executed during
5571 every loop iteration. MAYBE_MULTIPLE is 1 if the insn might be executed
5572 more than once in each loop iteration. */
5575 find_mem_givs (const struct loop *loop, rtx x, rtx insn,
5576 int not_every_iteration, int maybe_multiple)
5585 code = GET_CODE (x);
5610 /* This code used to disable creating GIVs with mult_val == 1 and
5611 add_val == 0. However, this leads to lost optimizations when
5612 it comes time to combine a set of related DEST_ADDR GIVs, since
5613 this one would not be seen. */
5615 if (general_induction_var (loop, XEXP (x, 0), &src_reg, &add_val,
5616 &mult_val, &ext_val, 1, &benefit,
5619 /* Found one; record it. */
5620 struct induction *v = xmalloc (sizeof (struct induction));
5622 record_giv (loop, v, insn, src_reg, addr_placeholder, mult_val,
5623 add_val, ext_val, benefit, DEST_ADDR,
5624 not_every_iteration, maybe_multiple, &XEXP (x, 0));
5635 /* Recursively scan the subexpressions for other mem refs. */
5637 fmt = GET_RTX_FORMAT (code);
5638 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
5640 find_mem_givs (loop, XEXP (x, i), insn, not_every_iteration,
5642 else if (fmt[i] == 'E')
5643 for (j = 0; j < XVECLEN (x, i); j++)
5644 find_mem_givs (loop, XVECEXP (x, i, j), insn, not_every_iteration,
5648 /* Fill in the data about one biv update.
5649 V is the `struct induction' in which we record the biv. (It is
5650 allocated by the caller, with alloca.)
5651 INSN is the insn that sets it.
5652 DEST_REG is the biv's reg.
5654 MULT_VAL is const1_rtx if the biv is being incremented here, in which case
5655 INC_VAL is the increment. Otherwise, MULT_VAL is const0_rtx and the biv is
5656 being set to INC_VAL.
5658 NOT_EVERY_ITERATION is nonzero if this biv update is not know to be
5659 executed every iteration; MAYBE_MULTIPLE is nonzero if this biv update
5660 can be executed more than once per iteration. If MAYBE_MULTIPLE
5661 and NOT_EVERY_ITERATION are both zero, we know that the biv update is
5662 executed exactly once per iteration. */
5665 record_biv (struct loop *loop, struct induction *v, rtx insn, rtx dest_reg,
5666 rtx inc_val, rtx mult_val, rtx *location,
5667 int not_every_iteration, int maybe_multiple)
5669 struct loop_ivs *ivs = LOOP_IVS (loop);
5670 struct iv_class *bl;
5673 v->src_reg = dest_reg;
5674 v->dest_reg = dest_reg;
5675 v->mult_val = mult_val;
5676 v->add_val = inc_val;
5677 v->ext_dependent = NULL_RTX;
5678 v->location = location;
5679 v->mode = GET_MODE (dest_reg);
5680 v->always_computable = ! not_every_iteration;
5681 v->always_executed = ! not_every_iteration;
5682 v->maybe_multiple = maybe_multiple;
5685 /* Add this to the reg's iv_class, creating a class
5686 if this is the first incrementation of the reg. */
5688 bl = REG_IV_CLASS (ivs, REGNO (dest_reg));
5691 /* Create and initialize new iv_class. */
5693 bl = xmalloc (sizeof (struct iv_class));
5695 bl->regno = REGNO (dest_reg);
5701 /* Set initial value to the reg itself. */
5702 bl->initial_value = dest_reg;
5703 bl->final_value = 0;
5704 /* We haven't seen the initializing insn yet. */
5707 bl->initial_test = 0;
5708 bl->incremented = 0;
5712 bl->total_benefit = 0;
5714 /* Add this class to ivs->list. */
5715 bl->next = ivs->list;
5718 /* Put it in the array of biv register classes. */
5719 REG_IV_CLASS (ivs, REGNO (dest_reg)) = bl;
5723 /* Check if location is the same as a previous one. */
5724 struct induction *induction;
5725 for (induction = bl->biv; induction; induction = induction->next_iv)
5726 if (location == induction->location)
5728 v->same = induction;
5733 /* Update IV_CLASS entry for this biv. */
5734 v->next_iv = bl->biv;
5737 if (mult_val == const1_rtx)
5738 bl->incremented = 1;
5740 if (loop_dump_stream)
5741 loop_biv_dump (v, loop_dump_stream, 0);
5744 /* Fill in the data about one giv.
5745 V is the `struct induction' in which we record the giv. (It is
5746 allocated by the caller, with alloca.)
5747 INSN is the insn that sets it.
5748 BENEFIT estimates the savings from deleting this insn.
5749 TYPE is DEST_REG or DEST_ADDR; it says whether the giv is computed
5750 into a register or is used as a memory address.
5752 SRC_REG is the biv reg which the giv is computed from.
5753 DEST_REG is the giv's reg (if the giv is stored in a reg).
5754 MULT_VAL and ADD_VAL are the coefficients used to compute the giv.
5755 LOCATION points to the place where this giv's value appears in INSN. */
5758 record_giv (const struct loop *loop, struct induction *v, rtx insn,
5759 rtx src_reg, rtx dest_reg, rtx mult_val, rtx add_val,
5760 rtx ext_val, int benefit, enum g_types type,
5761 int not_every_iteration, int maybe_multiple, rtx *location)
5763 struct loop_ivs *ivs = LOOP_IVS (loop);
5764 struct induction *b;
5765 struct iv_class *bl;
5766 rtx set = single_set (insn);
5769 /* Attempt to prove constantness of the values. Don't let simplify_rtx
5770 undo the MULT canonicalization that we performed earlier. */
5771 temp = simplify_rtx (add_val);
5773 && ! (GET_CODE (add_val) == MULT
5774 && GET_CODE (temp) == ASHIFT))
5778 v->src_reg = src_reg;
5780 v->dest_reg = dest_reg;
5781 v->mult_val = mult_val;
5782 v->add_val = add_val;
5783 v->ext_dependent = ext_val;
5784 v->benefit = benefit;
5785 v->location = location;
5787 v->combined_with = 0;
5788 v->maybe_multiple = maybe_multiple;
5790 v->derive_adjustment = 0;
5796 v->auto_inc_opt = 0;
5800 /* The v->always_computable field is used in update_giv_derive, to
5801 determine whether a giv can be used to derive another giv. For a
5802 DEST_REG giv, INSN computes a new value for the giv, so its value
5803 isn't computable if INSN insn't executed every iteration.
5804 However, for a DEST_ADDR giv, INSN merely uses the value of the giv;
5805 it does not compute a new value. Hence the value is always computable
5806 regardless of whether INSN is executed each iteration. */
5808 if (type == DEST_ADDR)
5809 v->always_computable = 1;
5811 v->always_computable = ! not_every_iteration;
5813 v->always_executed = ! not_every_iteration;
5815 if (type == DEST_ADDR)
5817 v->mode = GET_MODE (*location);
5820 else /* type == DEST_REG */
5822 v->mode = GET_MODE (SET_DEST (set));
5824 v->lifetime = LOOP_REG_LIFETIME (loop, REGNO (dest_reg));
5826 /* If the lifetime is zero, it means that this register is
5827 really a dead store. So mark this as a giv that can be
5828 ignored. This will not prevent the biv from being eliminated. */
5829 if (v->lifetime == 0)
5832 REG_IV_TYPE (ivs, REGNO (dest_reg)) = GENERAL_INDUCT;
5833 REG_IV_INFO (ivs, REGNO (dest_reg)) = v;
5836 /* Add the giv to the class of givs computed from one biv. */
5838 bl = REG_IV_CLASS (ivs, REGNO (src_reg));
5841 v->next_iv = bl->giv;
5843 /* Don't count DEST_ADDR. This is supposed to count the number of
5844 insns that calculate givs. */
5845 if (type == DEST_REG)
5847 bl->total_benefit += benefit;
5850 /* Fatal error, biv missing for this giv? */
5853 if (type == DEST_ADDR)
5856 v->not_replaceable = 0;
5860 /* The giv can be replaced outright by the reduced register only if all
5861 of the following conditions are true:
5862 - the insn that sets the giv is always executed on any iteration
5863 on which the giv is used at all
5864 (there are two ways to deduce this:
5865 either the insn is executed on every iteration,
5866 or all uses follow that insn in the same basic block),
5867 - the giv is not used outside the loop
5868 - no assignments to the biv occur during the giv's lifetime. */
5870 if (REGNO_FIRST_UID (REGNO (dest_reg)) == INSN_UID (insn)
5871 /* Previous line always fails if INSN was moved by loop opt. */
5872 && REGNO_LAST_LUID (REGNO (dest_reg))
5873 < INSN_LUID (loop->end)
5874 && (! not_every_iteration
5875 || last_use_this_basic_block (dest_reg, insn)))
5877 /* Now check that there are no assignments to the biv within the
5878 giv's lifetime. This requires two separate checks. */
5880 /* Check each biv update, and fail if any are between the first
5881 and last use of the giv.
5883 If this loop contains an inner loop that was unrolled, then
5884 the insn modifying the biv may have been emitted by the loop
5885 unrolling code, and hence does not have a valid luid. Just
5886 mark the biv as not replaceable in this case. It is not very
5887 useful as a biv, because it is used in two different loops.
5888 It is very unlikely that we would be able to optimize the giv
5889 using this biv anyways. */
5892 v->not_replaceable = 0;
5893 for (b = bl->biv; b; b = b->next_iv)
5895 if (INSN_UID (b->insn) >= max_uid_for_loop
5896 || ((INSN_LUID (b->insn)
5897 >= REGNO_FIRST_LUID (REGNO (dest_reg)))
5898 && (INSN_LUID (b->insn)
5899 <= REGNO_LAST_LUID (REGNO (dest_reg)))))
5902 v->not_replaceable = 1;
5907 /* If there are any backwards branches that go from after the
5908 biv update to before it, then this giv is not replaceable. */
5910 for (b = bl->biv; b; b = b->next_iv)
5911 if (back_branch_in_range_p (loop, b->insn))
5914 v->not_replaceable = 1;
5920 /* May still be replaceable, we don't have enough info here to
5923 v->not_replaceable = 0;
5927 /* Record whether the add_val contains a const_int, for later use by
5932 v->no_const_addval = 1;
5933 if (tem == const0_rtx)
5935 else if (CONSTANT_P (add_val))
5936 v->no_const_addval = 0;
5937 if (GET_CODE (tem) == PLUS)
5941 if (GET_CODE (XEXP (tem, 0)) == PLUS)
5942 tem = XEXP (tem, 0);
5943 else if (GET_CODE (XEXP (tem, 1)) == PLUS)
5944 tem = XEXP (tem, 1);
5948 if (CONSTANT_P (XEXP (tem, 1)))
5949 v->no_const_addval = 0;
5953 if (loop_dump_stream)
5954 loop_giv_dump (v, loop_dump_stream, 0);
5957 /* All this does is determine whether a giv can be made replaceable because
5958 its final value can be calculated. This code can not be part of record_giv
5959 above, because final_giv_value requires that the number of loop iterations
5960 be known, and that can not be accurately calculated until after all givs
5961 have been identified. */
5964 check_final_value (const struct loop *loop, struct induction *v)
5966 rtx final_value = 0;
5968 /* DEST_ADDR givs will never reach here, because they are always marked
5969 replaceable above in record_giv. */
5971 /* The giv can be replaced outright by the reduced register only if all
5972 of the following conditions are true:
5973 - the insn that sets the giv is always executed on any iteration
5974 on which the giv is used at all
5975 (there are two ways to deduce this:
5976 either the insn is executed on every iteration,
5977 or all uses follow that insn in the same basic block),
5978 - its final value can be calculated (this condition is different
5979 than the one above in record_giv)
5980 - it's not used before the it's set
5981 - no assignments to the biv occur during the giv's lifetime. */
5984 /* This is only called now when replaceable is known to be false. */
5985 /* Clear replaceable, so that it won't confuse final_giv_value. */
5989 if ((final_value = final_giv_value (loop, v))
5990 && (v->always_executed
5991 || last_use_this_basic_block (v->dest_reg, v->insn)))
5993 int biv_increment_seen = 0, before_giv_insn = 0;
5998 v->not_replaceable = 0;
6000 /* When trying to determine whether or not a biv increment occurs
6001 during the lifetime of the giv, we can ignore uses of the variable
6002 outside the loop because final_value is true. Hence we can not
6003 use regno_last_uid and regno_first_uid as above in record_giv. */
6005 /* Search the loop to determine whether any assignments to the
6006 biv occur during the giv's lifetime. Start with the insn
6007 that sets the giv, and search around the loop until we come
6008 back to that insn again.
6010 Also fail if there is a jump within the giv's lifetime that jumps
6011 to somewhere outside the lifetime but still within the loop. This
6012 catches spaghetti code where the execution order is not linear, and
6013 hence the above test fails. Here we assume that the giv lifetime
6014 does not extend from one iteration of the loop to the next, so as
6015 to make the test easier. Since the lifetime isn't known yet,
6016 this requires two loops. See also record_giv above. */
6018 last_giv_use = v->insn;
6025 before_giv_insn = 1;
6026 p = NEXT_INSN (loop->start);
6031 if (GET_CODE (p) == INSN || GET_CODE (p) == JUMP_INSN
6032 || GET_CODE (p) == CALL_INSN)
6034 /* It is possible for the BIV increment to use the GIV if we
6035 have a cycle. Thus we must be sure to check each insn for
6036 both BIV and GIV uses, and we must check for BIV uses
6039 if (! biv_increment_seen
6040 && reg_set_p (v->src_reg, PATTERN (p)))
6041 biv_increment_seen = 1;
6043 if (reg_mentioned_p (v->dest_reg, PATTERN (p)))
6045 if (biv_increment_seen || before_giv_insn)
6048 v->not_replaceable = 1;
6056 /* Now that the lifetime of the giv is known, check for branches
6057 from within the lifetime to outside the lifetime if it is still
6067 p = NEXT_INSN (loop->start);
6068 if (p == last_giv_use)
6071 if (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p)
6072 && LABEL_NAME (JUMP_LABEL (p))
6073 && ((loop_insn_first_p (JUMP_LABEL (p), v->insn)
6074 && loop_insn_first_p (loop->start, JUMP_LABEL (p)))
6075 || (loop_insn_first_p (last_giv_use, JUMP_LABEL (p))
6076 && loop_insn_first_p (JUMP_LABEL (p), loop->end))))
6079 v->not_replaceable = 1;
6081 if (loop_dump_stream)
6082 fprintf (loop_dump_stream,
6083 "Found branch outside giv lifetime.\n");
6090 /* If it is replaceable, then save the final value. */
6092 v->final_value = final_value;
6095 if (loop_dump_stream && v->replaceable)
6096 fprintf (loop_dump_stream, "Insn %d: giv reg %d final_value replaceable\n",
6097 INSN_UID (v->insn), REGNO (v->dest_reg));
6100 /* Update the status of whether a giv can derive other givs.
6102 We need to do something special if there is or may be an update to the biv
6103 between the time the giv is defined and the time it is used to derive
6106 In addition, a giv that is only conditionally set is not allowed to
6107 derive another giv once a label has been passed.
6109 The cases we look at are when a label or an update to a biv is passed. */
6112 update_giv_derive (const struct loop *loop, rtx p)
6114 struct loop_ivs *ivs = LOOP_IVS (loop);
6115 struct iv_class *bl;
6116 struct induction *biv, *giv;
6120 /* Search all IV classes, then all bivs, and finally all givs.
6122 There are three cases we are concerned with. First we have the situation
6123 of a giv that is only updated conditionally. In that case, it may not
6124 derive any givs after a label is passed.
6126 The second case is when a biv update occurs, or may occur, after the
6127 definition of a giv. For certain biv updates (see below) that are
6128 known to occur between the giv definition and use, we can adjust the
6129 giv definition. For others, or when the biv update is conditional,
6130 we must prevent the giv from deriving any other givs. There are two
6131 sub-cases within this case.
6133 If this is a label, we are concerned with any biv update that is done
6134 conditionally, since it may be done after the giv is defined followed by
6135 a branch here (actually, we need to pass both a jump and a label, but
6136 this extra tracking doesn't seem worth it).
6138 If this is a jump, we are concerned about any biv update that may be
6139 executed multiple times. We are actually only concerned about
6140 backward jumps, but it is probably not worth performing the test
6141 on the jump again here.
6143 If this is a biv update, we must adjust the giv status to show that a
6144 subsequent biv update was performed. If this adjustment cannot be done,
6145 the giv cannot derive further givs. */
6147 for (bl = ivs->list; bl; bl = bl->next)
6148 for (biv = bl->biv; biv; biv = biv->next_iv)
6149 if (GET_CODE (p) == CODE_LABEL || GET_CODE (p) == JUMP_INSN
6152 /* Skip if location is the same as a previous one. */
6156 for (giv = bl->giv; giv; giv = giv->next_iv)
6158 /* If cant_derive is already true, there is no point in
6159 checking all of these conditions again. */
6160 if (giv->cant_derive)
6163 /* If this giv is conditionally set and we have passed a label,
6164 it cannot derive anything. */
6165 if (GET_CODE (p) == CODE_LABEL && ! giv->always_computable)
6166 giv->cant_derive = 1;
6168 /* Skip givs that have mult_val == 0, since
6169 they are really invariants. Also skip those that are
6170 replaceable, since we know their lifetime doesn't contain
6172 else if (giv->mult_val == const0_rtx || giv->replaceable)
6175 /* The only way we can allow this giv to derive another
6176 is if this is a biv increment and we can form the product
6177 of biv->add_val and giv->mult_val. In this case, we will
6178 be able to compute a compensation. */
6179 else if (biv->insn == p)
6184 if (biv->mult_val == const1_rtx)
6185 tem = simplify_giv_expr (loop,
6186 gen_rtx_MULT (giv->mode,
6189 &ext_val_dummy, &dummy);
6191 if (tem && giv->derive_adjustment)
6192 tem = simplify_giv_expr
6194 gen_rtx_PLUS (giv->mode, tem, giv->derive_adjustment),
6195 &ext_val_dummy, &dummy);
6198 giv->derive_adjustment = tem;
6200 giv->cant_derive = 1;
6202 else if ((GET_CODE (p) == CODE_LABEL && ! biv->always_computable)
6203 || (GET_CODE (p) == JUMP_INSN && biv->maybe_multiple))
6204 giv->cant_derive = 1;
6209 /* Check whether an insn is an increment legitimate for a basic induction var.
6210 X is the source of insn P, or a part of it.
6211 MODE is the mode in which X should be interpreted.
6213 DEST_REG is the putative biv, also the destination of the insn.
6214 We accept patterns of these forms:
6215 REG = REG + INVARIANT (includes REG = REG - CONSTANT)
6216 REG = INVARIANT + REG
6218 If X is suitable, we return 1, set *MULT_VAL to CONST1_RTX,
6219 store the additive term into *INC_VAL, and store the place where
6220 we found the additive term into *LOCATION.
6222 If X is an assignment of an invariant into DEST_REG, we set
6223 *MULT_VAL to CONST0_RTX, and store the invariant into *INC_VAL.
6225 We also want to detect a BIV when it corresponds to a variable
6226 whose mode was promoted. In that case, an increment
6227 of the variable may be a PLUS that adds a SUBREG of that variable to
6228 an invariant and then sign- or zero-extends the result of the PLUS
6231 Most GIVs in such cases will be in the promoted mode, since that is the
6232 probably the natural computation mode (and almost certainly the mode
6233 used for addresses) on the machine. So we view the pseudo-reg containing
6234 the variable as the BIV, as if it were simply incremented.
6236 Note that treating the entire pseudo as a BIV will result in making
6237 simple increments to any GIVs based on it. However, if the variable
6238 overflows in its declared mode but not its promoted mode, the result will
6239 be incorrect. This is acceptable if the variable is signed, since
6240 overflows in such cases are undefined, but not if it is unsigned, since
6241 those overflows are defined. So we only check for SIGN_EXTEND and
6244 If we cannot find a biv, we return 0. */
6247 basic_induction_var (const struct loop *loop, rtx x, enum machine_mode mode,
6248 rtx dest_reg, rtx p, rtx *inc_val, rtx *mult_val,
6253 rtx insn, set = 0, last, inc;
6255 code = GET_CODE (x);
6260 if (rtx_equal_p (XEXP (x, 0), dest_reg)
6261 || (GET_CODE (XEXP (x, 0)) == SUBREG
6262 && SUBREG_PROMOTED_VAR_P (XEXP (x, 0))
6263 && SUBREG_REG (XEXP (x, 0)) == dest_reg))
6265 argp = &XEXP (x, 1);
6267 else if (rtx_equal_p (XEXP (x, 1), dest_reg)
6268 || (GET_CODE (XEXP (x, 1)) == SUBREG
6269 && SUBREG_PROMOTED_VAR_P (XEXP (x, 1))
6270 && SUBREG_REG (XEXP (x, 1)) == dest_reg))
6272 argp = &XEXP (x, 0);
6278 if (loop_invariant_p (loop, arg) != 1)
6281 /* convert_modes can emit new instructions, e.g. when arg is a loop
6282 invariant MEM and dest_reg has a different mode.
6283 These instructions would be emitted after the end of the function
6284 and then *inc_val would be an uninitialized pseudo.
6285 Detect this and bail in this case.
6286 Other alternatives to solve this can be introducing a convert_modes
6287 variant which is allowed to fail but not allowed to emit new
6288 instructions, emit these instructions before loop start and let
6289 it be garbage collected if *inc_val is never used or saving the
6290 *inc_val initialization sequence generated here and when *inc_val
6291 is going to be actually used, emit it at some suitable place. */
6292 last = get_last_insn ();
6293 inc = convert_modes (GET_MODE (dest_reg), GET_MODE (x), arg, 0);
6294 if (get_last_insn () != last)
6296 delete_insns_since (last);
6301 *mult_val = const1_rtx;
6306 /* If what's inside the SUBREG is a BIV, then the SUBREG. This will
6307 handle addition of promoted variables.
6308 ??? The comment at the start of this function is wrong: promoted
6309 variable increments don't look like it says they do. */
6310 return basic_induction_var (loop, SUBREG_REG (x),
6311 GET_MODE (SUBREG_REG (x)),
6312 dest_reg, p, inc_val, mult_val, location);
6315 /* If this register is assigned in a previous insn, look at its
6316 source, but don't go outside the loop or past a label. */
6318 /* If this sets a register to itself, we would repeat any previous
6319 biv increment if we applied this strategy blindly. */
6320 if (rtx_equal_p (dest_reg, x))
6329 insn = PREV_INSN (insn);
6331 while (insn && GET_CODE (insn) == NOTE
6332 && NOTE_LINE_NUMBER (insn) != NOTE_INSN_LOOP_BEG);
6336 set = single_set (insn);
6339 dest = SET_DEST (set);
6341 || (GET_CODE (dest) == SUBREG
6342 && (GET_MODE_SIZE (GET_MODE (dest)) <= UNITS_PER_WORD)
6343 && (GET_MODE_CLASS (GET_MODE (dest)) == MODE_INT)
6344 && SUBREG_REG (dest) == x))
6345 return basic_induction_var (loop, SET_SRC (set),
6346 (GET_MODE (SET_SRC (set)) == VOIDmode
6348 : GET_MODE (SET_SRC (set))),
6350 inc_val, mult_val, location);
6352 while (GET_CODE (dest) == SIGN_EXTRACT
6353 || GET_CODE (dest) == ZERO_EXTRACT
6354 || GET_CODE (dest) == SUBREG
6355 || GET_CODE (dest) == STRICT_LOW_PART)
6356 dest = XEXP (dest, 0);
6362 /* Can accept constant setting of biv only when inside inner most loop.
6363 Otherwise, a biv of an inner loop may be incorrectly recognized
6364 as a biv of the outer loop,
6365 causing code to be moved INTO the inner loop. */
6367 if (loop_invariant_p (loop, x) != 1)
6372 /* convert_modes aborts if we try to convert to or from CCmode, so just
6373 exclude that case. It is very unlikely that a condition code value
6374 would be a useful iterator anyways. convert_modes aborts if we try to
6375 convert a float mode to non-float or vice versa too. */
6376 if (loop->level == 1
6377 && GET_MODE_CLASS (mode) == GET_MODE_CLASS (GET_MODE (dest_reg))
6378 && GET_MODE_CLASS (mode) != MODE_CC)
6380 /* Possible bug here? Perhaps we don't know the mode of X. */
6381 last = get_last_insn ();
6382 inc = convert_modes (GET_MODE (dest_reg), mode, x, 0);
6383 if (get_last_insn () != last)
6385 delete_insns_since (last);
6390 *mult_val = const0_rtx;
6397 /* Ignore this BIV if signed arithmetic overflow is defined. */
6400 return basic_induction_var (loop, XEXP (x, 0), GET_MODE (XEXP (x, 0)),
6401 dest_reg, p, inc_val, mult_val, location);
6404 /* Similar, since this can be a sign extension. */
6405 for (insn = PREV_INSN (p);
6406 (insn && GET_CODE (insn) == NOTE
6407 && NOTE_LINE_NUMBER (insn) != NOTE_INSN_LOOP_BEG);
6408 insn = PREV_INSN (insn))
6412 set = single_set (insn);
6414 if (! rtx_equal_p (dest_reg, XEXP (x, 0))
6415 && set && SET_DEST (set) == XEXP (x, 0)
6416 && GET_CODE (XEXP (x, 1)) == CONST_INT
6417 && INTVAL (XEXP (x, 1)) >= 0
6418 && GET_CODE (SET_SRC (set)) == ASHIFT
6419 && XEXP (x, 1) == XEXP (SET_SRC (set), 1))
6420 return basic_induction_var (loop, XEXP (SET_SRC (set), 0),
6421 GET_MODE (XEXP (x, 0)),
6422 dest_reg, insn, inc_val, mult_val,
6431 /* A general induction variable (giv) is any quantity that is a linear
6432 function of a basic induction variable,
6433 i.e. giv = biv * mult_val + add_val.
6434 The coefficients can be any loop invariant quantity.
6435 A giv need not be computed directly from the biv;
6436 it can be computed by way of other givs. */
6438 /* Determine whether X computes a giv.
6439 If it does, return a nonzero value
6440 which is the benefit from eliminating the computation of X;
6441 set *SRC_REG to the register of the biv that it is computed from;
6442 set *ADD_VAL and *MULT_VAL to the coefficients,
6443 such that the value of X is biv * mult + add; */
6446 general_induction_var (const struct loop *loop, rtx x, rtx *src_reg,
6447 rtx *add_val, rtx *mult_val, rtx *ext_val,
6448 int is_addr, int *pbenefit,
6449 enum machine_mode addr_mode)
6451 struct loop_ivs *ivs = LOOP_IVS (loop);
6454 /* If this is an invariant, forget it, it isn't a giv. */
6455 if (loop_invariant_p (loop, x) == 1)
6459 *ext_val = NULL_RTX;
6460 x = simplify_giv_expr (loop, x, ext_val, pbenefit);
6464 switch (GET_CODE (x))
6468 /* Since this is now an invariant and wasn't before, it must be a giv
6469 with MULT_VAL == 0. It doesn't matter which BIV we associate this
6471 *src_reg = ivs->list->biv->dest_reg;
6472 *mult_val = const0_rtx;
6477 /* This is equivalent to a BIV. */
6479 *mult_val = const1_rtx;
6480 *add_val = const0_rtx;
6484 /* Either (plus (biv) (invar)) or
6485 (plus (mult (biv) (invar_1)) (invar_2)). */
6486 if (GET_CODE (XEXP (x, 0)) == MULT)
6488 *src_reg = XEXP (XEXP (x, 0), 0);
6489 *mult_val = XEXP (XEXP (x, 0), 1);
6493 *src_reg = XEXP (x, 0);
6494 *mult_val = const1_rtx;
6496 *add_val = XEXP (x, 1);
6500 /* ADD_VAL is zero. */
6501 *src_reg = XEXP (x, 0);
6502 *mult_val = XEXP (x, 1);
6503 *add_val = const0_rtx;
6510 /* Remove any enclosing USE from ADD_VAL and MULT_VAL (there will be
6511 unless they are CONST_INT). */
6512 if (GET_CODE (*add_val) == USE)
6513 *add_val = XEXP (*add_val, 0);
6514 if (GET_CODE (*mult_val) == USE)
6515 *mult_val = XEXP (*mult_val, 0);
6518 *pbenefit += address_cost (orig_x, addr_mode) - reg_address_cost;
6520 *pbenefit += rtx_cost (orig_x, SET);
6522 /* Always return true if this is a giv so it will be detected as such,
6523 even if the benefit is zero or negative. This allows elimination
6524 of bivs that might otherwise not be eliminated. */
6528 /* Given an expression, X, try to form it as a linear function of a biv.
6529 We will canonicalize it to be of the form
6530 (plus (mult (BIV) (invar_1))
6532 with possible degeneracies.
6534 The invariant expressions must each be of a form that can be used as a
6535 machine operand. We surround then with a USE rtx (a hack, but localized
6536 and certainly unambiguous!) if not a CONST_INT for simplicity in this
6537 routine; it is the caller's responsibility to strip them.
6539 If no such canonicalization is possible (i.e., two biv's are used or an
6540 expression that is neither invariant nor a biv or giv), this routine
6543 For a nonzero return, the result will have a code of CONST_INT, USE,
6544 REG (for a BIV), PLUS, or MULT. No other codes will occur.
6546 *BENEFIT will be incremented by the benefit of any sub-giv encountered. */
6548 static rtx sge_plus (enum machine_mode, rtx, rtx);
6549 static rtx sge_plus_constant (rtx, rtx);
6552 simplify_giv_expr (const struct loop *loop, rtx x, rtx *ext_val, int *benefit)
6554 struct loop_ivs *ivs = LOOP_IVS (loop);
6555 struct loop_regs *regs = LOOP_REGS (loop);
6556 enum machine_mode mode = GET_MODE (x);
6560 /* If this is not an integer mode, or if we cannot do arithmetic in this
6561 mode, this can't be a giv. */
6562 if (mode != VOIDmode
6563 && (GET_MODE_CLASS (mode) != MODE_INT
6564 || GET_MODE_BITSIZE (mode) > HOST_BITS_PER_WIDE_INT))
6567 switch (GET_CODE (x))
6570 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
6571 arg1 = simplify_giv_expr (loop, XEXP (x, 1), ext_val, benefit);
6572 if (arg0 == 0 || arg1 == 0)
6575 /* Put constant last, CONST_INT last if both constant. */
6576 if ((GET_CODE (arg0) == USE
6577 || GET_CODE (arg0) == CONST_INT)
6578 && ! ((GET_CODE (arg0) == USE
6579 && GET_CODE (arg1) == USE)
6580 || GET_CODE (arg1) == CONST_INT))
6581 tem = arg0, arg0 = arg1, arg1 = tem;
6583 /* Handle addition of zero, then addition of an invariant. */
6584 if (arg1 == const0_rtx)
6586 else if (GET_CODE (arg1) == CONST_INT || GET_CODE (arg1) == USE)
6587 switch (GET_CODE (arg0))
6591 /* Adding two invariants must result in an invariant, so enclose
6592 addition operation inside a USE and return it. */
6593 if (GET_CODE (arg0) == USE)
6594 arg0 = XEXP (arg0, 0);
6595 if (GET_CODE (arg1) == USE)
6596 arg1 = XEXP (arg1, 0);
6598 if (GET_CODE (arg0) == CONST_INT)
6599 tem = arg0, arg0 = arg1, arg1 = tem;
6600 if (GET_CODE (arg1) == CONST_INT)
6601 tem = sge_plus_constant (arg0, arg1);
6603 tem = sge_plus (mode, arg0, arg1);
6605 if (GET_CODE (tem) != CONST_INT)
6606 tem = gen_rtx_USE (mode, tem);
6611 /* biv + invar or mult + invar. Return sum. */
6612 return gen_rtx_PLUS (mode, arg0, arg1);
6615 /* (a + invar_1) + invar_2. Associate. */
6617 simplify_giv_expr (loop,
6629 /* Each argument must be either REG, PLUS, or MULT. Convert REG to
6630 MULT to reduce cases. */
6631 if (GET_CODE (arg0) == REG)
6632 arg0 = gen_rtx_MULT (mode, arg0, const1_rtx);
6633 if (GET_CODE (arg1) == REG)
6634 arg1 = gen_rtx_MULT (mode, arg1, const1_rtx);
6636 /* Now have PLUS + PLUS, PLUS + MULT, MULT + PLUS, or MULT + MULT.
6637 Put a MULT first, leaving PLUS + PLUS, MULT + PLUS, or MULT + MULT.
6638 Recurse to associate the second PLUS. */
6639 if (GET_CODE (arg1) == MULT)
6640 tem = arg0, arg0 = arg1, arg1 = tem;
6642 if (GET_CODE (arg1) == PLUS)
6644 simplify_giv_expr (loop,
6646 gen_rtx_PLUS (mode, arg0,
6651 /* Now must have MULT + MULT. Distribute if same biv, else not giv. */
6652 if (GET_CODE (arg0) != MULT || GET_CODE (arg1) != MULT)
6655 if (!rtx_equal_p (arg0, arg1))
6658 return simplify_giv_expr (loop,
6667 /* Handle "a - b" as "a + b * (-1)". */
6668 return simplify_giv_expr (loop,
6677 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
6678 arg1 = simplify_giv_expr (loop, XEXP (x, 1), ext_val, benefit);
6679 if (arg0 == 0 || arg1 == 0)
6682 /* Put constant last, CONST_INT last if both constant. */
6683 if ((GET_CODE (arg0) == USE || GET_CODE (arg0) == CONST_INT)
6684 && GET_CODE (arg1) != CONST_INT)
6685 tem = arg0, arg0 = arg1, arg1 = tem;
6687 /* If second argument is not now constant, not giv. */
6688 if (GET_CODE (arg1) != USE && GET_CODE (arg1) != CONST_INT)
6691 /* Handle multiply by 0 or 1. */
6692 if (arg1 == const0_rtx)
6695 else if (arg1 == const1_rtx)
6698 switch (GET_CODE (arg0))
6701 /* biv * invar. Done. */
6702 return gen_rtx_MULT (mode, arg0, arg1);
6705 /* Product of two constants. */
6706 return GEN_INT (INTVAL (arg0) * INTVAL (arg1));
6709 /* invar * invar is a giv, but attempt to simplify it somehow. */
6710 if (GET_CODE (arg1) != CONST_INT)
6713 arg0 = XEXP (arg0, 0);
6714 if (GET_CODE (arg0) == MULT)
6716 /* (invar_0 * invar_1) * invar_2. Associate. */
6717 return simplify_giv_expr (loop,
6726 /* Propagate the MULT expressions to the innermost nodes. */
6727 else if (GET_CODE (arg0) == PLUS)
6729 /* (invar_0 + invar_1) * invar_2. Distribute. */
6730 return simplify_giv_expr (loop,
6742 return gen_rtx_USE (mode, gen_rtx_MULT (mode, arg0, arg1));
6745 /* (a * invar_1) * invar_2. Associate. */
6746 return simplify_giv_expr (loop,
6755 /* (a + invar_1) * invar_2. Distribute. */
6756 return simplify_giv_expr (loop,
6771 /* Shift by constant is multiply by power of two. */
6772 if (GET_CODE (XEXP (x, 1)) != CONST_INT)
6776 simplify_giv_expr (loop,
6779 GEN_INT ((HOST_WIDE_INT) 1
6780 << INTVAL (XEXP (x, 1)))),
6784 /* "-a" is "a * (-1)" */
6785 return simplify_giv_expr (loop,
6786 gen_rtx_MULT (mode, XEXP (x, 0), constm1_rtx),
6790 /* "~a" is "-a - 1". Silly, but easy. */
6791 return simplify_giv_expr (loop,
6792 gen_rtx_MINUS (mode,
6793 gen_rtx_NEG (mode, XEXP (x, 0)),
6798 /* Already in proper form for invariant. */
6804 /* Conditionally recognize extensions of simple IVs. After we've
6805 computed loop traversal counts and verified the range of the
6806 source IV, we'll reevaluate this as a GIV. */
6807 if (*ext_val == NULL_RTX)
6809 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
6810 if (arg0 && *ext_val == NULL_RTX && GET_CODE (arg0) == REG)
6812 *ext_val = gen_rtx_fmt_e (GET_CODE (x), mode, arg0);
6819 /* If this is a new register, we can't deal with it. */
6820 if (REGNO (x) >= max_reg_before_loop)
6823 /* Check for biv or giv. */
6824 switch (REG_IV_TYPE (ivs, REGNO (x)))
6828 case GENERAL_INDUCT:
6830 struct induction *v = REG_IV_INFO (ivs, REGNO (x));
6832 /* Form expression from giv and add benefit. Ensure this giv
6833 can derive another and subtract any needed adjustment if so. */
6835 /* Increasing the benefit here is risky. The only case in which it
6836 is arguably correct is if this is the only use of V. In other
6837 cases, this will artificially inflate the benefit of the current
6838 giv, and lead to suboptimal code. Thus, it is disabled, since
6839 potentially not reducing an only marginally beneficial giv is
6840 less harmful than reducing many givs that are not really
6843 rtx single_use = regs->array[REGNO (x)].single_usage;
6844 if (single_use && single_use != const0_rtx)
6845 *benefit += v->benefit;
6851 tem = gen_rtx_PLUS (mode, gen_rtx_MULT (mode,
6852 v->src_reg, v->mult_val),
6855 if (v->derive_adjustment)
6856 tem = gen_rtx_MINUS (mode, tem, v->derive_adjustment);
6857 arg0 = simplify_giv_expr (loop, tem, ext_val, benefit);
6860 if (!v->ext_dependent)
6865 *ext_val = v->ext_dependent;
6873 /* If it isn't an induction variable, and it is invariant, we
6874 may be able to simplify things further by looking through
6875 the bits we just moved outside the loop. */
6876 if (loop_invariant_p (loop, x) == 1)
6879 struct loop_movables *movables = LOOP_MOVABLES (loop);
6881 for (m = movables->head; m; m = m->next)
6882 if (rtx_equal_p (x, m->set_dest))
6884 /* Ok, we found a match. Substitute and simplify. */
6886 /* If we match another movable, we must use that, as
6887 this one is going away. */
6889 return simplify_giv_expr (loop, m->match->set_dest,
6892 /* If consec is nonzero, this is a member of a group of
6893 instructions that were moved together. We handle this
6894 case only to the point of seeking to the last insn and
6895 looking for a REG_EQUAL. Fail if we don't find one. */
6902 tem = NEXT_INSN (tem);
6906 tem = find_reg_note (tem, REG_EQUAL, NULL_RTX);
6908 tem = XEXP (tem, 0);
6912 tem = single_set (m->insn);
6914 tem = SET_SRC (tem);
6919 /* What we are most interested in is pointer
6920 arithmetic on invariants -- only take
6921 patterns we may be able to do something with. */
6922 if (GET_CODE (tem) == PLUS
6923 || GET_CODE (tem) == MULT
6924 || GET_CODE (tem) == ASHIFT
6925 || GET_CODE (tem) == CONST_INT
6926 || GET_CODE (tem) == SYMBOL_REF)
6928 tem = simplify_giv_expr (loop, tem, ext_val,
6933 else if (GET_CODE (tem) == CONST
6934 && GET_CODE (XEXP (tem, 0)) == PLUS
6935 && GET_CODE (XEXP (XEXP (tem, 0), 0)) == SYMBOL_REF
6936 && GET_CODE (XEXP (XEXP (tem, 0), 1)) == CONST_INT)
6938 tem = simplify_giv_expr (loop, XEXP (tem, 0),
6950 /* Fall through to general case. */
6952 /* If invariant, return as USE (unless CONST_INT).
6953 Otherwise, not giv. */
6954 if (GET_CODE (x) == USE)
6957 if (loop_invariant_p (loop, x) == 1)
6959 if (GET_CODE (x) == CONST_INT)
6961 if (GET_CODE (x) == CONST
6962 && GET_CODE (XEXP (x, 0)) == PLUS
6963 && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
6964 && GET_CODE (XEXP (XEXP (x, 0), 1)) == CONST_INT)
6966 return gen_rtx_USE (mode, x);
6973 /* This routine folds invariants such that there is only ever one
6974 CONST_INT in the summation. It is only used by simplify_giv_expr. */
6977 sge_plus_constant (rtx x, rtx c)
6979 if (GET_CODE (x) == CONST_INT)
6980 return GEN_INT (INTVAL (x) + INTVAL (c));
6981 else if (GET_CODE (x) != PLUS)
6982 return gen_rtx_PLUS (GET_MODE (x), x, c);
6983 else if (GET_CODE (XEXP (x, 1)) == CONST_INT)
6985 return gen_rtx_PLUS (GET_MODE (x), XEXP (x, 0),
6986 GEN_INT (INTVAL (XEXP (x, 1)) + INTVAL (c)));
6988 else if (GET_CODE (XEXP (x, 0)) == PLUS
6989 || GET_CODE (XEXP (x, 1)) != PLUS)
6991 return gen_rtx_PLUS (GET_MODE (x),
6992 sge_plus_constant (XEXP (x, 0), c), XEXP (x, 1));
6996 return gen_rtx_PLUS (GET_MODE (x),
6997 sge_plus_constant (XEXP (x, 1), c), XEXP (x, 0));
7002 sge_plus (enum machine_mode mode, rtx x, rtx y)
7004 while (GET_CODE (y) == PLUS)
7006 rtx a = XEXP (y, 0);
7007 if (GET_CODE (a) == CONST_INT)
7008 x = sge_plus_constant (x, a);
7010 x = gen_rtx_PLUS (mode, x, a);
7013 if (GET_CODE (y) == CONST_INT)
7014 x = sge_plus_constant (x, y);
7016 x = gen_rtx_PLUS (mode, x, y);
7020 /* Help detect a giv that is calculated by several consecutive insns;
7024 The caller has already identified the first insn P as having a giv as dest;
7025 we check that all other insns that set the same register follow
7026 immediately after P, that they alter nothing else,
7027 and that the result of the last is still a giv.
7029 The value is 0 if the reg set in P is not really a giv.
7030 Otherwise, the value is the amount gained by eliminating
7031 all the consecutive insns that compute the value.
7033 FIRST_BENEFIT is the amount gained by eliminating the first insn, P.
7034 SRC_REG is the reg of the biv; DEST_REG is the reg of the giv.
7036 The coefficients of the ultimate giv value are stored in
7037 *MULT_VAL and *ADD_VAL. */
7040 consec_sets_giv (const struct loop *loop, int first_benefit, rtx p,
7041 rtx src_reg, rtx dest_reg, rtx *add_val, rtx *mult_val,
7042 rtx *ext_val, rtx *last_consec_insn)
7044 struct loop_ivs *ivs = LOOP_IVS (loop);
7045 struct loop_regs *regs = LOOP_REGS (loop);
7052 /* Indicate that this is a giv so that we can update the value produced in
7053 each insn of the multi-insn sequence.
7055 This induction structure will be used only by the call to
7056 general_induction_var below, so we can allocate it on our stack.
7057 If this is a giv, our caller will replace the induct var entry with
7058 a new induction structure. */
7059 struct induction *v;
7061 if (REG_IV_TYPE (ivs, REGNO (dest_reg)) != UNKNOWN_INDUCT)
7064 v = alloca (sizeof (struct induction));
7065 v->src_reg = src_reg;
7066 v->mult_val = *mult_val;
7067 v->add_val = *add_val;
7068 v->benefit = first_benefit;
7070 v->derive_adjustment = 0;
7071 v->ext_dependent = NULL_RTX;
7073 REG_IV_TYPE (ivs, REGNO (dest_reg)) = GENERAL_INDUCT;
7074 REG_IV_INFO (ivs, REGNO (dest_reg)) = v;
7076 count = regs->array[REGNO (dest_reg)].n_times_set - 1;
7081 code = GET_CODE (p);
7083 /* If libcall, skip to end of call sequence. */
7084 if (code == INSN && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
7088 && (set = single_set (p))
7089 && GET_CODE (SET_DEST (set)) == REG
7090 && SET_DEST (set) == dest_reg
7091 && (general_induction_var (loop, SET_SRC (set), &src_reg,
7092 add_val, mult_val, ext_val, 0,
7094 /* Giv created by equivalent expression. */
7095 || ((temp = find_reg_note (p, REG_EQUAL, NULL_RTX))
7096 && general_induction_var (loop, XEXP (temp, 0), &src_reg,
7097 add_val, mult_val, ext_val, 0,
7098 &benefit, VOIDmode)))
7099 && src_reg == v->src_reg)
7101 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
7102 benefit += libcall_benefit (p);
7105 v->mult_val = *mult_val;
7106 v->add_val = *add_val;
7107 v->benefit += benefit;
7109 else if (code != NOTE)
7111 /* Allow insns that set something other than this giv to a
7112 constant. Such insns are needed on machines which cannot
7113 include long constants and should not disqualify a giv. */
7115 && (set = single_set (p))
7116 && SET_DEST (set) != dest_reg
7117 && CONSTANT_P (SET_SRC (set)))
7120 REG_IV_TYPE (ivs, REGNO (dest_reg)) = UNKNOWN_INDUCT;
7125 REG_IV_TYPE (ivs, REGNO (dest_reg)) = UNKNOWN_INDUCT;
7126 *last_consec_insn = p;
7130 /* Return an rtx, if any, that expresses giv G2 as a function of the register
7131 represented by G1. If no such expression can be found, or it is clear that
7132 it cannot possibly be a valid address, 0 is returned.
7134 To perform the computation, we note that
7137 where `v' is the biv.
7139 So G2 = (y/b) * G1 + (b - a*y/x).
7141 Note that MULT = y/x.
7143 Update: A and B are now allowed to be additive expressions such that
7144 B contains all variables in A. That is, computing B-A will not require
7145 subtracting variables. */
7148 express_from_1 (rtx a, rtx b, rtx mult)
7150 /* If MULT is zero, then A*MULT is zero, and our expression is B. */
7152 if (mult == const0_rtx)
7155 /* If MULT is not 1, we cannot handle A with non-constants, since we
7156 would then be required to subtract multiples of the registers in A.
7157 This is theoretically possible, and may even apply to some Fortran
7158 constructs, but it is a lot of work and we do not attempt it here. */
7160 if (mult != const1_rtx && GET_CODE (a) != CONST_INT)
7163 /* In general these structures are sorted top to bottom (down the PLUS
7164 chain), but not left to right across the PLUS. If B is a higher
7165 order giv than A, we can strip one level and recurse. If A is higher
7166 order, we'll eventually bail out, but won't know that until the end.
7167 If they are the same, we'll strip one level around this loop. */
7169 while (GET_CODE (a) == PLUS && GET_CODE (b) == PLUS)
7171 rtx ra, rb, oa, ob, tmp;
7173 ra = XEXP (a, 0), oa = XEXP (a, 1);
7174 if (GET_CODE (ra) == PLUS)
7175 tmp = ra, ra = oa, oa = tmp;
7177 rb = XEXP (b, 0), ob = XEXP (b, 1);
7178 if (GET_CODE (rb) == PLUS)
7179 tmp = rb, rb = ob, ob = tmp;
7181 if (rtx_equal_p (ra, rb))
7182 /* We matched: remove one reg completely. */
7184 else if (GET_CODE (ob) != PLUS && rtx_equal_p (ra, ob))
7185 /* An alternate match. */
7187 else if (GET_CODE (oa) != PLUS && rtx_equal_p (oa, rb))
7188 /* An alternate match. */
7192 /* Indicates an extra register in B. Strip one level from B and
7193 recurse, hoping B was the higher order expression. */
7194 ob = express_from_1 (a, ob, mult);
7197 return gen_rtx_PLUS (GET_MODE (b), rb, ob);
7201 /* Here we are at the last level of A, go through the cases hoping to
7202 get rid of everything but a constant. */
7204 if (GET_CODE (a) == PLUS)
7208 ra = XEXP (a, 0), oa = XEXP (a, 1);
7209 if (rtx_equal_p (oa, b))
7211 else if (!rtx_equal_p (ra, b))
7214 if (GET_CODE (oa) != CONST_INT)
7217 return GEN_INT (-INTVAL (oa) * INTVAL (mult));
7219 else if (GET_CODE (a) == CONST_INT)
7221 return plus_constant (b, -INTVAL (a) * INTVAL (mult));
7223 else if (CONSTANT_P (a))
7225 enum machine_mode mode_a = GET_MODE (a);
7226 enum machine_mode mode_b = GET_MODE (b);
7227 enum machine_mode mode = mode_b == VOIDmode ? mode_a : mode_b;
7228 return simplify_gen_binary (MINUS, mode, b, a);
7230 else if (GET_CODE (b) == PLUS)
7232 if (rtx_equal_p (a, XEXP (b, 0)))
7234 else if (rtx_equal_p (a, XEXP (b, 1)))
7239 else if (rtx_equal_p (a, b))
7246 express_from (struct induction *g1, struct induction *g2)
7250 /* The value that G1 will be multiplied by must be a constant integer. Also,
7251 the only chance we have of getting a valid address is if b*c/a (see above
7252 for notation) is also an integer. */
7253 if (GET_CODE (g1->mult_val) == CONST_INT
7254 && GET_CODE (g2->mult_val) == CONST_INT)
7256 if (g1->mult_val == const0_rtx
7257 || (g1->mult_val == constm1_rtx
7258 && INTVAL (g2->mult_val)
7259 == (HOST_WIDE_INT) 1 << (HOST_BITS_PER_WIDE_INT - 1))
7260 || INTVAL (g2->mult_val) % INTVAL (g1->mult_val) != 0)
7262 mult = GEN_INT (INTVAL (g2->mult_val) / INTVAL (g1->mult_val));
7264 else if (rtx_equal_p (g1->mult_val, g2->mult_val))
7268 /* ??? Find out if the one is a multiple of the other? */
7272 add = express_from_1 (g1->add_val, g2->add_val, mult);
7273 if (add == NULL_RTX)
7275 /* Failed. If we've got a multiplication factor between G1 and G2,
7276 scale G1's addend and try again. */
7277 if (INTVAL (mult) > 1)
7279 rtx g1_add_val = g1->add_val;
7280 if (GET_CODE (g1_add_val) == MULT
7281 && GET_CODE (XEXP (g1_add_val, 1)) == CONST_INT)
7284 m = INTVAL (mult) * INTVAL (XEXP (g1_add_val, 1));
7285 g1_add_val = gen_rtx_MULT (GET_MODE (g1_add_val),
7286 XEXP (g1_add_val, 0), GEN_INT (m));
7290 g1_add_val = gen_rtx_MULT (GET_MODE (g1_add_val), g1_add_val,
7294 add = express_from_1 (g1_add_val, g2->add_val, const1_rtx);
7297 if (add == NULL_RTX)
7300 /* Form simplified final result. */
7301 if (mult == const0_rtx)
7303 else if (mult == const1_rtx)
7304 mult = g1->dest_reg;
7306 mult = gen_rtx_MULT (g2->mode, g1->dest_reg, mult);
7308 if (add == const0_rtx)
7312 if (GET_CODE (add) == PLUS
7313 && CONSTANT_P (XEXP (add, 1)))
7315 rtx tem = XEXP (add, 1);
7316 mult = gen_rtx_PLUS (g2->mode, mult, XEXP (add, 0));
7320 return gen_rtx_PLUS (g2->mode, mult, add);
7324 /* Return an rtx, if any, that expresses giv G2 as a function of the register
7325 represented by G1. This indicates that G2 should be combined with G1 and
7326 that G2 can use (either directly or via an address expression) a register
7327 used to represent G1. */
7330 combine_givs_p (struct induction *g1, struct induction *g2)
7334 /* With the introduction of ext dependent givs, we must care for modes.
7335 G2 must not use a wider mode than G1. */
7336 if (GET_MODE_SIZE (g1->mode) < GET_MODE_SIZE (g2->mode))
7339 ret = comb = express_from (g1, g2);
7340 if (comb == NULL_RTX)
7342 if (g1->mode != g2->mode)
7343 ret = gen_lowpart (g2->mode, comb);
7345 /* If these givs are identical, they can be combined. We use the results
7346 of express_from because the addends are not in a canonical form, so
7347 rtx_equal_p is a weaker test. */
7348 /* But don't combine a DEST_REG giv with a DEST_ADDR giv; we want the
7349 combination to be the other way round. */
7350 if (comb == g1->dest_reg
7351 && (g1->giv_type == DEST_REG || g2->giv_type == DEST_ADDR))
7356 /* If G2 can be expressed as a function of G1 and that function is valid
7357 as an address and no more expensive than using a register for G2,
7358 the expression of G2 in terms of G1 can be used. */
7360 && g2->giv_type == DEST_ADDR
7361 && memory_address_p (GET_MODE (g2->mem), ret))
7367 /* Check each extension dependent giv in this class to see if its
7368 root biv is safe from wrapping in the interior mode, which would
7369 make the giv illegal. */
7372 check_ext_dependent_givs (const struct loop *loop, struct iv_class *bl)
7374 struct loop_info *loop_info = LOOP_INFO (loop);
7375 int ze_ok = 0, se_ok = 0, info_ok = 0;
7376 enum machine_mode biv_mode = GET_MODE (bl->biv->src_reg);
7377 HOST_WIDE_INT start_val;
7378 unsigned HOST_WIDE_INT u_end_val = 0;
7379 unsigned HOST_WIDE_INT u_start_val = 0;
7381 struct induction *v;
7383 /* Make sure the iteration data is available. We must have
7384 constants in order to be certain of no overflow. */
7385 if (loop_info->n_iterations > 0
7386 && bl->initial_value
7387 && GET_CODE (bl->initial_value) == CONST_INT
7388 && (incr = biv_total_increment (bl))
7389 && GET_CODE (incr) == CONST_INT
7390 /* Make sure the host can represent the arithmetic. */
7391 && HOST_BITS_PER_WIDE_INT >= GET_MODE_BITSIZE (biv_mode))
7393 unsigned HOST_WIDE_INT abs_incr, total_incr;
7394 HOST_WIDE_INT s_end_val;
7398 start_val = INTVAL (bl->initial_value);
7399 u_start_val = start_val;
7401 neg_incr = 0, abs_incr = INTVAL (incr);
7402 if (INTVAL (incr) < 0)
7403 neg_incr = 1, abs_incr = -abs_incr;
7404 total_incr = abs_incr * loop_info->n_iterations;
7406 /* Check for host arithmetic overflow. */
7407 if (total_incr / loop_info->n_iterations == abs_incr)
7409 unsigned HOST_WIDE_INT u_max;
7410 HOST_WIDE_INT s_max;
7412 u_end_val = start_val + (neg_incr ? -total_incr : total_incr);
7413 s_end_val = u_end_val;
7414 u_max = GET_MODE_MASK (biv_mode);
7417 /* Check zero extension of biv ok. */
7419 /* Check for host arithmetic overflow. */
7421 ? u_end_val < u_start_val
7422 : u_end_val > u_start_val)
7423 /* Check for target arithmetic overflow. */
7425 ? 1 /* taken care of with host overflow */
7426 : u_end_val <= u_max))
7431 /* Check sign extension of biv ok. */
7432 /* ??? While it is true that overflow with signed and pointer
7433 arithmetic is undefined, I fear too many programmers don't
7434 keep this fact in mind -- myself included on occasion.
7435 So leave alone with the signed overflow optimizations. */
7436 if (start_val >= -s_max - 1
7437 /* Check for host arithmetic overflow. */
7439 ? s_end_val < start_val
7440 : s_end_val > start_val)
7441 /* Check for target arithmetic overflow. */
7443 ? s_end_val >= -s_max - 1
7444 : s_end_val <= s_max))
7451 /* If we know the BIV is compared at run-time against an
7452 invariant value, and the increment is +/- 1, we may also
7453 be able to prove that the BIV cannot overflow. */
7454 else if (bl->biv->src_reg == loop_info->iteration_var
7455 && loop_info->comparison_value
7456 && loop_invariant_p (loop, loop_info->comparison_value)
7457 && (incr = biv_total_increment (bl))
7458 && GET_CODE (incr) == CONST_INT)
7460 /* If the increment is +1, and the exit test is a <,
7461 the BIV cannot overflow. (For <=, we have the
7462 problematic case that the comparison value might
7463 be the maximum value of the range.) */
7464 if (INTVAL (incr) == 1)
7466 if (loop_info->comparison_code == LT)
7468 else if (loop_info->comparison_code == LTU)
7472 /* Likewise for increment -1 and exit test >. */
7473 if (INTVAL (incr) == -1)
7475 if (loop_info->comparison_code == GT)
7477 else if (loop_info->comparison_code == GTU)
7482 /* Invalidate givs that fail the tests. */
7483 for (v = bl->giv; v; v = v->next_iv)
7484 if (v->ext_dependent)
7486 enum rtx_code code = GET_CODE (v->ext_dependent);
7499 /* We don't know whether this value is being used as either
7500 signed or unsigned, so to safely truncate we must satisfy
7501 both. The initial check here verifies the BIV itself;
7502 once that is successful we may check its range wrt the
7503 derived GIV. This works only if we were able to determine
7504 constant start and end values above. */
7505 if (se_ok && ze_ok && info_ok)
7507 enum machine_mode outer_mode = GET_MODE (v->ext_dependent);
7508 unsigned HOST_WIDE_INT max = GET_MODE_MASK (outer_mode) >> 1;
7510 /* We know from the above that both endpoints are nonnegative,
7511 and that there is no wrapping. Verify that both endpoints
7512 are within the (signed) range of the outer mode. */
7513 if (u_start_val <= max && u_end_val <= max)
7524 if (loop_dump_stream)
7526 fprintf (loop_dump_stream,
7527 "Verified ext dependent giv at %d of reg %d\n",
7528 INSN_UID (v->insn), bl->regno);
7533 if (loop_dump_stream)
7538 why = "biv iteration values overflowed";
7542 incr = biv_total_increment (bl);
7543 if (incr == const1_rtx)
7544 why = "biv iteration info incomplete; incr by 1";
7546 why = "biv iteration info incomplete";
7549 fprintf (loop_dump_stream,
7550 "Failed ext dependent giv at %d, %s\n",
7551 INSN_UID (v->insn), why);
7554 bl->all_reduced = 0;
7559 /* Generate a version of VALUE in a mode appropriate for initializing V. */
7562 extend_value_for_giv (struct induction *v, rtx value)
7564 rtx ext_dep = v->ext_dependent;
7569 /* Recall that check_ext_dependent_givs verified that the known bounds
7570 of a biv did not overflow or wrap with respect to the extension for
7571 the giv. Therefore, constants need no additional adjustment. */
7572 if (CONSTANT_P (value) && GET_MODE (value) == VOIDmode)
7575 /* Otherwise, we must adjust the value to compensate for the
7576 differing modes of the biv and the giv. */
7577 return gen_rtx_fmt_e (GET_CODE (ext_dep), GET_MODE (ext_dep), value);
7580 struct combine_givs_stats
7587 cmp_combine_givs_stats (const void *xp, const void *yp)
7589 const struct combine_givs_stats * const x =
7590 (const struct combine_givs_stats *) xp;
7591 const struct combine_givs_stats * const y =
7592 (const struct combine_givs_stats *) yp;
7594 d = y->total_benefit - x->total_benefit;
7595 /* Stabilize the sort. */
7597 d = x->giv_number - y->giv_number;
7601 /* Check all pairs of givs for iv_class BL and see if any can be combined with
7602 any other. If so, point SAME to the giv combined with and set NEW_REG to
7603 be an expression (in terms of the other giv's DEST_REG) equivalent to the
7604 giv. Also, update BENEFIT and related fields for cost/benefit analysis. */
7607 combine_givs (struct loop_regs *regs, struct iv_class *bl)
7609 /* Additional benefit to add for being combined multiple times. */
7610 const int extra_benefit = 3;
7612 struct induction *g1, *g2, **giv_array;
7613 int i, j, k, giv_count;
7614 struct combine_givs_stats *stats;
7617 /* Count givs, because bl->giv_count is incorrect here. */
7619 for (g1 = bl->giv; g1; g1 = g1->next_iv)
7623 giv_array = alloca (giv_count * sizeof (struct induction *));
7625 for (g1 = bl->giv; g1; g1 = g1->next_iv)
7627 giv_array[i++] = g1;
7629 stats = xcalloc (giv_count, sizeof (*stats));
7630 can_combine = xcalloc (giv_count, giv_count * sizeof (rtx));
7632 for (i = 0; i < giv_count; i++)
7638 stats[i].giv_number = i;
7640 /* If a DEST_REG GIV is used only once, do not allow it to combine
7641 with anything, for in doing so we will gain nothing that cannot
7642 be had by simply letting the GIV with which we would have combined
7643 to be reduced on its own. The losage shows up in particular with
7644 DEST_ADDR targets on hosts with reg+reg addressing, though it can
7645 be seen elsewhere as well. */
7646 if (g1->giv_type == DEST_REG
7647 && (single_use = regs->array[REGNO (g1->dest_reg)].single_usage)
7648 && single_use != const0_rtx)
7651 this_benefit = g1->benefit;
7652 /* Add an additional weight for zero addends. */
7653 if (g1->no_const_addval)
7656 for (j = 0; j < giv_count; j++)
7662 && (this_combine = combine_givs_p (g1, g2)) != NULL_RTX)
7664 can_combine[i * giv_count + j] = this_combine;
7665 this_benefit += g2->benefit + extra_benefit;
7668 stats[i].total_benefit = this_benefit;
7671 /* Iterate, combining until we can't. */
7673 qsort (stats, giv_count, sizeof (*stats), cmp_combine_givs_stats);
7675 if (loop_dump_stream)
7677 fprintf (loop_dump_stream, "Sorted combine statistics:\n");
7678 for (k = 0; k < giv_count; k++)
7680 g1 = giv_array[stats[k].giv_number];
7681 if (!g1->combined_with && !g1->same)
7682 fprintf (loop_dump_stream, " {%d, %d}",
7683 INSN_UID (giv_array[stats[k].giv_number]->insn),
7684 stats[k].total_benefit);
7686 putc ('\n', loop_dump_stream);
7689 for (k = 0; k < giv_count; k++)
7691 int g1_add_benefit = 0;
7693 i = stats[k].giv_number;
7696 /* If it has already been combined, skip. */
7697 if (g1->combined_with || g1->same)
7700 for (j = 0; j < giv_count; j++)
7703 if (g1 != g2 && can_combine[i * giv_count + j]
7704 /* If it has already been combined, skip. */
7705 && ! g2->same && ! g2->combined_with)
7709 g2->new_reg = can_combine[i * giv_count + j];
7711 /* For destination, we now may replace by mem expression instead
7712 of register. This changes the costs considerably, so add the
7714 if (g2->giv_type == DEST_ADDR)
7715 g2->benefit = (g2->benefit + reg_address_cost
7716 - address_cost (g2->new_reg,
7717 GET_MODE (g2->mem)));
7718 g1->combined_with++;
7719 g1->lifetime += g2->lifetime;
7721 g1_add_benefit += g2->benefit;
7723 /* ??? The new final_[bg]iv_value code does a much better job
7724 of finding replaceable giv's, and hence this code may no
7725 longer be necessary. */
7726 if (! g2->replaceable && REG_USERVAR_P (g2->dest_reg))
7727 g1_add_benefit -= copy_cost;
7729 /* To help optimize the next set of combinations, remove
7730 this giv from the benefits of other potential mates. */
7731 for (l = 0; l < giv_count; ++l)
7733 int m = stats[l].giv_number;
7734 if (can_combine[m * giv_count + j])
7735 stats[l].total_benefit -= g2->benefit + extra_benefit;
7738 if (loop_dump_stream)
7739 fprintf (loop_dump_stream,
7740 "giv at %d combined with giv at %d; new benefit %d + %d, lifetime %d\n",
7741 INSN_UID (g2->insn), INSN_UID (g1->insn),
7742 g1->benefit, g1_add_benefit, g1->lifetime);
7746 /* To help optimize the next set of combinations, remove
7747 this giv from the benefits of other potential mates. */
7748 if (g1->combined_with)
7750 for (j = 0; j < giv_count; ++j)
7752 int m = stats[j].giv_number;
7753 if (can_combine[m * giv_count + i])
7754 stats[j].total_benefit -= g1->benefit + extra_benefit;
7757 g1->benefit += g1_add_benefit;
7759 /* We've finished with this giv, and everything it touched.
7760 Restart the combination so that proper weights for the
7761 rest of the givs are properly taken into account. */
7762 /* ??? Ideally we would compact the arrays at this point, so
7763 as to not cover old ground. But sanely compacting
7764 can_combine is tricky. */
7774 /* Generate sequence for REG = B * M + A. B is the initial value of
7775 the basic induction variable, M a multiplicative constant, A an
7776 additive constant and REG the destination register. */
7779 gen_add_mult (rtx b, rtx m, rtx a, rtx reg)
7785 /* Use unsigned arithmetic. */
7786 result = expand_mult_add (b, reg, m, a, GET_MODE (reg), 1);
7788 emit_move_insn (reg, result);
7796 /* Update registers created in insn sequence SEQ. */
7799 loop_regs_update (const struct loop *loop ATTRIBUTE_UNUSED, rtx seq)
7803 /* Update register info for alias analysis. */
7806 while (insn != NULL_RTX)
7808 rtx set = single_set (insn);
7810 if (set && GET_CODE (SET_DEST (set)) == REG)
7811 record_base_value (REGNO (SET_DEST (set)), SET_SRC (set), 0);
7813 insn = NEXT_INSN (insn);
7818 /* EMIT code before BEFORE_BB/BEFORE_INSN to set REG = B * M + A. B
7819 is the initial value of the basic induction variable, M a
7820 multiplicative constant, A an additive constant and REG the
7821 destination register. */
7824 loop_iv_add_mult_emit_before (const struct loop *loop, rtx b, rtx m, rtx a,
7825 rtx reg, basic_block before_bb, rtx before_insn)
7831 loop_iv_add_mult_hoist (loop, b, m, a, reg);
7835 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
7836 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
7838 /* Increase the lifetime of any invariants moved further in code. */
7839 update_reg_last_use (a, before_insn);
7840 update_reg_last_use (b, before_insn);
7841 update_reg_last_use (m, before_insn);
7843 /* It is possible that the expansion created lots of new registers.
7844 Iterate over the sequence we just created and record them all. We
7845 must do this before inserting the sequence. */
7846 loop_regs_update (loop, seq);
7848 loop_insn_emit_before (loop, before_bb, before_insn, seq);
7852 /* Emit insns in loop pre-header to set REG = B * M + A. B is the
7853 initial value of the basic induction variable, M a multiplicative
7854 constant, A an additive constant and REG the destination
7858 loop_iv_add_mult_sink (const struct loop *loop, rtx b, rtx m, rtx a, rtx reg)
7862 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
7863 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
7865 /* Increase the lifetime of any invariants moved further in code.
7866 ???? Is this really necessary? */
7867 update_reg_last_use (a, loop->sink);
7868 update_reg_last_use (b, loop->sink);
7869 update_reg_last_use (m, loop->sink);
7871 /* It is possible that the expansion created lots of new registers.
7872 Iterate over the sequence we just created and record them all. We
7873 must do this before inserting the sequence. */
7874 loop_regs_update (loop, seq);
7876 loop_insn_sink (loop, seq);
7880 /* Emit insns after loop to set REG = B * M + A. B is the initial
7881 value of the basic induction variable, M a multiplicative constant,
7882 A an additive constant and REG the destination register. */
7885 loop_iv_add_mult_hoist (const struct loop *loop, rtx b, rtx m, rtx a, rtx reg)
7889 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
7890 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
7892 /* It is possible that the expansion created lots of new registers.
7893 Iterate over the sequence we just created and record them all. We
7894 must do this before inserting the sequence. */
7895 loop_regs_update (loop, seq);
7897 loop_insn_hoist (loop, seq);
7902 /* Similar to gen_add_mult, but compute cost rather than generating
7906 iv_add_mult_cost (rtx b, rtx m, rtx a, rtx reg)
7912 result = expand_mult_add (b, reg, m, a, GET_MODE (reg), 1);
7914 emit_move_insn (reg, result);
7915 last = get_last_insn ();
7918 rtx t = single_set (last);
7920 cost += rtx_cost (SET_SRC (t), SET);
7921 last = PREV_INSN (last);
7927 /* Test whether A * B can be computed without
7928 an actual multiply insn. Value is 1 if so.
7930 ??? This function stinks because it generates a ton of wasted RTL
7931 ??? and as a result fragments GC memory to no end. There are other
7932 ??? places in the compiler which are invoked a lot and do the same
7933 ??? thing, generate wasted RTL just to see if something is possible. */
7936 product_cheap_p (rtx a, rtx b)
7941 /* If only one is constant, make it B. */
7942 if (GET_CODE (a) == CONST_INT)
7943 tmp = a, a = b, b = tmp;
7945 /* If first constant, both constant, so don't need multiply. */
7946 if (GET_CODE (a) == CONST_INT)
7949 /* If second not constant, neither is constant, so would need multiply. */
7950 if (GET_CODE (b) != CONST_INT)
7953 /* One operand is constant, so might not need multiply insn. Generate the
7954 code for the multiply and see if a call or multiply, or long sequence
7955 of insns is generated. */
7958 expand_mult (GET_MODE (a), a, b, NULL_RTX, 1);
7966 while (tmp != NULL_RTX)
7968 rtx next = NEXT_INSN (tmp);
7971 || GET_CODE (tmp) != INSN
7972 || (GET_CODE (PATTERN (tmp)) == SET
7973 && GET_CODE (SET_SRC (PATTERN (tmp))) == MULT)
7974 || (GET_CODE (PATTERN (tmp)) == PARALLEL
7975 && GET_CODE (XVECEXP (PATTERN (tmp), 0, 0)) == SET
7976 && GET_CODE (SET_SRC (XVECEXP (PATTERN (tmp), 0, 0))) == MULT))
7985 else if (GET_CODE (tmp) == SET
7986 && GET_CODE (SET_SRC (tmp)) == MULT)
7988 else if (GET_CODE (tmp) == PARALLEL
7989 && GET_CODE (XVECEXP (tmp, 0, 0)) == SET
7990 && GET_CODE (SET_SRC (XVECEXP (tmp, 0, 0))) == MULT)
7996 /* Check to see if loop can be terminated by a "decrement and branch until
7997 zero" instruction. If so, add a REG_NONNEG note to the branch insn if so.
7998 Also try reversing an increment loop to a decrement loop
7999 to see if the optimization can be performed.
8000 Value is nonzero if optimization was performed. */
8002 /* This is useful even if the architecture doesn't have such an insn,
8003 because it might change a loops which increments from 0 to n to a loop
8004 which decrements from n to 0. A loop that decrements to zero is usually
8005 faster than one that increments from zero. */
8007 /* ??? This could be rewritten to use some of the loop unrolling procedures,
8008 such as approx_final_value, biv_total_increment, loop_iterations, and
8009 final_[bg]iv_value. */
8012 check_dbra_loop (struct loop *loop, int insn_count)
8014 struct loop_info *loop_info = LOOP_INFO (loop);
8015 struct loop_regs *regs = LOOP_REGS (loop);
8016 struct loop_ivs *ivs = LOOP_IVS (loop);
8017 struct iv_class *bl;
8019 enum machine_mode mode;
8025 rtx before_comparison;
8029 int compare_and_branch;
8030 rtx loop_start = loop->start;
8031 rtx loop_end = loop->end;
8033 /* If last insn is a conditional branch, and the insn before tests a
8034 register value, try to optimize it. Otherwise, we can't do anything. */
8036 jump = PREV_INSN (loop_end);
8037 comparison = get_condition_for_loop (loop, jump);
8038 if (comparison == 0)
8040 if (!onlyjump_p (jump))
8043 /* Try to compute whether the compare/branch at the loop end is one or
8044 two instructions. */
8045 get_condition (jump, &first_compare, false);
8046 if (first_compare == jump)
8047 compare_and_branch = 1;
8048 else if (first_compare == prev_nonnote_insn (jump))
8049 compare_and_branch = 2;
8054 /* If more than one condition is present to control the loop, then
8055 do not proceed, as this function does not know how to rewrite
8056 loop tests with more than one condition.
8058 Look backwards from the first insn in the last comparison
8059 sequence and see if we've got another comparison sequence. */
8062 if ((jump1 = prev_nonnote_insn (first_compare)) != loop->cont)
8063 if (GET_CODE (jump1) == JUMP_INSN)
8067 /* Check all of the bivs to see if the compare uses one of them.
8068 Skip biv's set more than once because we can't guarantee that
8069 it will be zero on the last iteration. Also skip if the biv is
8070 used between its update and the test insn. */
8072 for (bl = ivs->list; bl; bl = bl->next)
8074 if (bl->biv_count == 1
8075 && ! bl->biv->maybe_multiple
8076 && bl->biv->dest_reg == XEXP (comparison, 0)
8077 && ! reg_used_between_p (regno_reg_rtx[bl->regno], bl->biv->insn,
8082 /* Try swapping the comparison to identify a suitable biv. */
8084 for (bl = ivs->list; bl; bl = bl->next)
8085 if (bl->biv_count == 1
8086 && ! bl->biv->maybe_multiple
8087 && bl->biv->dest_reg == XEXP (comparison, 1)
8088 && ! reg_used_between_p (regno_reg_rtx[bl->regno], bl->biv->insn,
8091 comparison = gen_rtx_fmt_ee (swap_condition (GET_CODE (comparison)),
8093 XEXP (comparison, 1),
8094 XEXP (comparison, 0));
8101 /* Look for the case where the basic induction variable is always
8102 nonnegative, and equals zero on the last iteration.
8103 In this case, add a reg_note REG_NONNEG, which allows the
8104 m68k DBRA instruction to be used. */
8106 if (((GET_CODE (comparison) == GT && XEXP (comparison, 1) == constm1_rtx)
8107 || (GET_CODE (comparison) == NE && XEXP (comparison, 1) == const0_rtx))
8108 && GET_CODE (bl->biv->add_val) == CONST_INT
8109 && INTVAL (bl->biv->add_val) < 0)
8111 /* Initial value must be greater than 0,
8112 init_val % -dec_value == 0 to ensure that it equals zero on
8113 the last iteration */
8115 if (GET_CODE (bl->initial_value) == CONST_INT
8116 && INTVAL (bl->initial_value) > 0
8117 && (INTVAL (bl->initial_value)
8118 % (-INTVAL (bl->biv->add_val))) == 0)
8120 /* Register always nonnegative, add REG_NOTE to branch. */
8121 if (! find_reg_note (jump, REG_NONNEG, NULL_RTX))
8123 = gen_rtx_EXPR_LIST (REG_NONNEG, bl->biv->dest_reg,
8130 /* If the decrement is 1 and the value was tested as >= 0 before
8131 the loop, then we can safely optimize. */
8132 for (p = loop_start; p; p = PREV_INSN (p))
8134 if (GET_CODE (p) == CODE_LABEL)
8136 if (GET_CODE (p) != JUMP_INSN)
8139 before_comparison = get_condition_for_loop (loop, p);
8140 if (before_comparison
8141 && XEXP (before_comparison, 0) == bl->biv->dest_reg
8142 && (GET_CODE (before_comparison) == LT
8143 || GET_CODE (before_comparison) == LTU)
8144 && XEXP (before_comparison, 1) == const0_rtx
8145 && ! reg_set_between_p (bl->biv->dest_reg, p, loop_start)
8146 && INTVAL (bl->biv->add_val) == -1)
8148 if (! find_reg_note (jump, REG_NONNEG, NULL_RTX))
8150 = gen_rtx_EXPR_LIST (REG_NONNEG, bl->biv->dest_reg,
8158 else if (GET_CODE (bl->biv->add_val) == CONST_INT
8159 && INTVAL (bl->biv->add_val) > 0)
8161 /* Try to change inc to dec, so can apply above optimization. */
8163 all registers modified are induction variables or invariant,
8164 all memory references have non-overlapping addresses
8165 (obviously true if only one write)
8166 allow 2 insns for the compare/jump at the end of the loop. */
8167 /* Also, we must avoid any instructions which use both the reversed
8168 biv and another biv. Such instructions will fail if the loop is
8169 reversed. We meet this condition by requiring that either
8170 no_use_except_counting is true, or else that there is only
8172 int num_nonfixed_reads = 0;
8173 /* 1 if the iteration var is used only to count iterations. */
8174 int no_use_except_counting = 0;
8175 /* 1 if the loop has no memory store, or it has a single memory store
8176 which is reversible. */
8177 int reversible_mem_store = 1;
8179 if (bl->giv_count == 0
8180 && !loop->exit_count
8181 && !loop_info->has_multiple_exit_targets)
8183 rtx bivreg = regno_reg_rtx[bl->regno];
8184 struct iv_class *blt;
8186 /* If there are no givs for this biv, and the only exit is the
8187 fall through at the end of the loop, then
8188 see if perhaps there are no uses except to count. */
8189 no_use_except_counting = 1;
8190 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
8193 rtx set = single_set (p);
8195 if (set && GET_CODE (SET_DEST (set)) == REG
8196 && REGNO (SET_DEST (set)) == bl->regno)
8197 /* An insn that sets the biv is okay. */
8199 else if (!reg_mentioned_p (bivreg, PATTERN (p)))
8200 /* An insn that doesn't mention the biv is okay. */
8202 else if (p == prev_nonnote_insn (prev_nonnote_insn (loop_end))
8203 || p == prev_nonnote_insn (loop_end))
8205 /* If either of these insns uses the biv and sets a pseudo
8206 that has more than one usage, then the biv has uses
8207 other than counting since it's used to derive a value
8208 that is used more than one time. */
8209 note_stores (PATTERN (p), note_set_pseudo_multiple_uses,
8211 if (regs->multiple_uses)
8213 no_use_except_counting = 0;
8219 no_use_except_counting = 0;
8224 /* A biv has uses besides counting if it is used to set
8226 for (blt = ivs->list; blt; blt = blt->next)
8228 && reg_mentioned_p (bivreg, SET_SRC (blt->init_set)))
8230 no_use_except_counting = 0;
8235 if (no_use_except_counting)
8236 /* No need to worry about MEMs. */
8238 else if (loop_info->num_mem_sets <= 1)
8240 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
8242 num_nonfixed_reads += count_nonfixed_reads (loop, PATTERN (p));
8244 /* If the loop has a single store, and the destination address is
8245 invariant, then we can't reverse the loop, because this address
8246 might then have the wrong value at loop exit.
8247 This would work if the source was invariant also, however, in that
8248 case, the insn should have been moved out of the loop. */
8250 if (loop_info->num_mem_sets == 1)
8252 struct induction *v;
8254 /* If we could prove that each of the memory locations
8255 written to was different, then we could reverse the
8256 store -- but we don't presently have any way of
8258 reversible_mem_store = 0;
8260 /* If the store depends on a register that is set after the
8261 store, it depends on the initial value, and is thus not
8263 for (v = bl->giv; reversible_mem_store && v; v = v->next_iv)
8265 if (v->giv_type == DEST_REG
8266 && reg_mentioned_p (v->dest_reg,
8267 PATTERN (loop_info->first_loop_store_insn))
8268 && loop_insn_first_p (loop_info->first_loop_store_insn,
8270 reversible_mem_store = 0;
8277 /* This code only acts for innermost loops. Also it simplifies
8278 the memory address check by only reversing loops with
8279 zero or one memory access.
8280 Two memory accesses could involve parts of the same array,
8281 and that can't be reversed.
8282 If the biv is used only for counting, than we don't need to worry
8283 about all these things. */
8285 if ((num_nonfixed_reads <= 1
8286 && ! loop_info->has_nonconst_call
8287 && ! loop_info->has_prefetch
8288 && ! loop_info->has_volatile
8289 && reversible_mem_store
8290 && (bl->giv_count + bl->biv_count + loop_info->num_mem_sets
8291 + num_unmoved_movables (loop) + compare_and_branch == insn_count)
8292 && (bl == ivs->list && bl->next == 0))
8293 || (no_use_except_counting && ! loop_info->has_prefetch))
8297 /* Loop can be reversed. */
8298 if (loop_dump_stream)
8299 fprintf (loop_dump_stream, "Can reverse loop\n");
8301 /* Now check other conditions:
8303 The increment must be a constant, as must the initial value,
8304 and the comparison code must be LT.
8306 This test can probably be improved since +/- 1 in the constant
8307 can be obtained by changing LT to LE and vice versa; this is
8311 /* for constants, LE gets turned into LT */
8312 && (GET_CODE (comparison) == LT
8313 || (GET_CODE (comparison) == LE
8314 && no_use_except_counting)
8315 || GET_CODE (comparison) == LTU))
8317 HOST_WIDE_INT add_val, add_adjust, comparison_val = 0;
8318 rtx initial_value, comparison_value;
8320 enum rtx_code cmp_code;
8321 int comparison_const_width;
8322 unsigned HOST_WIDE_INT comparison_sign_mask;
8323 bool keep_first_compare;
8325 add_val = INTVAL (bl->biv->add_val);
8326 comparison_value = XEXP (comparison, 1);
8327 if (GET_MODE (comparison_value) == VOIDmode)
8328 comparison_const_width
8329 = GET_MODE_BITSIZE (GET_MODE (XEXP (comparison, 0)));
8331 comparison_const_width
8332 = GET_MODE_BITSIZE (GET_MODE (comparison_value));
8333 if (comparison_const_width > HOST_BITS_PER_WIDE_INT)
8334 comparison_const_width = HOST_BITS_PER_WIDE_INT;
8335 comparison_sign_mask
8336 = (unsigned HOST_WIDE_INT) 1 << (comparison_const_width - 1);
8338 /* If the comparison value is not a loop invariant, then we
8339 can not reverse this loop.
8341 ??? If the insns which initialize the comparison value as
8342 a whole compute an invariant result, then we could move
8343 them out of the loop and proceed with loop reversal. */
8344 if (! loop_invariant_p (loop, comparison_value))
8347 if (GET_CODE (comparison_value) == CONST_INT)
8348 comparison_val = INTVAL (comparison_value);
8349 initial_value = bl->initial_value;
8351 /* Normalize the initial value if it is an integer and
8352 has no other use except as a counter. This will allow
8353 a few more loops to be reversed. */
8354 if (no_use_except_counting
8355 && GET_CODE (comparison_value) == CONST_INT
8356 && GET_CODE (initial_value) == CONST_INT)
8358 comparison_val = comparison_val - INTVAL (bl->initial_value);
8359 /* The code below requires comparison_val to be a multiple
8360 of add_val in order to do the loop reversal, so
8361 round up comparison_val to a multiple of add_val.
8362 Since comparison_value is constant, we know that the
8363 current comparison code is LT. */
8364 comparison_val = comparison_val + add_val - 1;
8366 -= (unsigned HOST_WIDE_INT) comparison_val % add_val;
8367 /* We postpone overflow checks for COMPARISON_VAL here;
8368 even if there is an overflow, we might still be able to
8369 reverse the loop, if converting the loop exit test to
8371 initial_value = const0_rtx;
8374 /* First check if we can do a vanilla loop reversal. */
8375 if (initial_value == const0_rtx
8376 /* If we have a decrement_and_branch_on_count,
8377 prefer the NE test, since this will allow that
8378 instruction to be generated. Note that we must
8379 use a vanilla loop reversal if the biv is used to
8380 calculate a giv or has a non-counting use. */
8381 #if ! defined (HAVE_decrement_and_branch_until_zero) \
8382 && defined (HAVE_decrement_and_branch_on_count)
8383 && (! (add_val == 1 && loop->vtop
8384 && (bl->biv_count == 0
8385 || no_use_except_counting)))
8387 && GET_CODE (comparison_value) == CONST_INT
8388 /* Now do postponed overflow checks on COMPARISON_VAL. */
8389 && ! (((comparison_val - add_val) ^ INTVAL (comparison_value))
8390 & comparison_sign_mask))
8392 /* Register will always be nonnegative, with value
8393 0 on last iteration */
8394 add_adjust = add_val;
8398 else if (add_val == 1 && loop->vtop
8399 && (bl->biv_count == 0
8400 || no_use_except_counting))
8408 if (GET_CODE (comparison) == LE)
8409 add_adjust -= add_val;
8411 /* If the initial value is not zero, or if the comparison
8412 value is not an exact multiple of the increment, then we
8413 can not reverse this loop. */
8414 if (initial_value == const0_rtx
8415 && GET_CODE (comparison_value) == CONST_INT)
8417 if (((unsigned HOST_WIDE_INT) comparison_val % add_val) != 0)
8422 if (! no_use_except_counting || add_val != 1)
8426 final_value = comparison_value;
8428 /* Reset these in case we normalized the initial value
8429 and comparison value above. */
8430 if (GET_CODE (comparison_value) == CONST_INT
8431 && GET_CODE (initial_value) == CONST_INT)
8433 comparison_value = GEN_INT (comparison_val);
8435 = GEN_INT (comparison_val + INTVAL (bl->initial_value));
8437 bl->initial_value = initial_value;
8439 /* Save some info needed to produce the new insns. */
8440 reg = bl->biv->dest_reg;
8441 mode = GET_MODE (reg);
8442 jump_label = condjump_label (PREV_INSN (loop_end));
8443 new_add_val = GEN_INT (-INTVAL (bl->biv->add_val));
8445 /* Set start_value; if this is not a CONST_INT, we need
8447 Initialize biv to start_value before loop start.
8448 The old initializing insn will be deleted as a
8449 dead store by flow.c. */
8450 if (initial_value == const0_rtx
8451 && GET_CODE (comparison_value) == CONST_INT)
8454 = gen_int_mode (comparison_val - add_adjust, mode);
8455 loop_insn_hoist (loop, gen_move_insn (reg, start_value));
8457 else if (GET_CODE (initial_value) == CONST_INT)
8459 rtx offset = GEN_INT (-INTVAL (initial_value) - add_adjust);
8460 rtx add_insn = gen_add3_insn (reg, comparison_value, offset);
8466 = gen_rtx_PLUS (mode, comparison_value, offset);
8467 loop_insn_hoist (loop, add_insn);
8468 if (GET_CODE (comparison) == LE)
8469 final_value = gen_rtx_PLUS (mode, comparison_value,
8472 else if (! add_adjust)
8474 rtx sub_insn = gen_sub3_insn (reg, comparison_value,
8480 = gen_rtx_MINUS (mode, comparison_value, initial_value);
8481 loop_insn_hoist (loop, sub_insn);
8484 /* We could handle the other cases too, but it'll be
8485 better to have a testcase first. */
8488 /* We may not have a single insn which can increment a reg, so
8489 create a sequence to hold all the insns from expand_inc. */
8491 expand_inc (reg, new_add_val);
8495 p = loop_insn_emit_before (loop, 0, bl->biv->insn, tem);
8496 delete_insn (bl->biv->insn);
8498 /* Update biv info to reflect its new status. */
8500 bl->initial_value = start_value;
8501 bl->biv->add_val = new_add_val;
8503 /* Update loop info. */
8504 loop_info->initial_value = reg;
8505 loop_info->initial_equiv_value = reg;
8506 loop_info->final_value = const0_rtx;
8507 loop_info->final_equiv_value = const0_rtx;
8508 loop_info->comparison_value = const0_rtx;
8509 loop_info->comparison_code = cmp_code;
8510 loop_info->increment = new_add_val;
8512 /* Inc LABEL_NUSES so that delete_insn will
8513 not delete the label. */
8514 LABEL_NUSES (XEXP (jump_label, 0))++;
8516 /* If we have a separate comparison insn that does more
8517 than just set cc0, the result of the comparison might
8518 be used outside the loop. */
8519 keep_first_compare = (compare_and_branch == 2
8521 && sets_cc0_p (first_compare) <= 0
8525 /* Emit an insn after the end of the loop to set the biv's
8526 proper exit value if it is used anywhere outside the loop. */
8527 if (keep_first_compare
8528 || (REGNO_LAST_UID (bl->regno) != INSN_UID (first_compare))
8530 || REGNO_FIRST_UID (bl->regno) != INSN_UID (bl->init_insn))
8531 loop_insn_sink (loop, gen_load_of_final_value (reg, final_value));
8533 if (keep_first_compare)
8534 loop_insn_sink (loop, PATTERN (first_compare));
8536 /* Delete compare/branch at end of loop. */
8537 delete_related_insns (PREV_INSN (loop_end));
8538 if (compare_and_branch == 2)
8539 delete_related_insns (first_compare);
8541 /* Add new compare/branch insn at end of loop. */
8543 emit_cmp_and_jump_insns (reg, const0_rtx, cmp_code, NULL_RTX,
8545 XEXP (jump_label, 0));
8548 emit_jump_insn_before (tem, loop_end);
8550 for (tem = PREV_INSN (loop_end);
8551 tem && GET_CODE (tem) != JUMP_INSN;
8552 tem = PREV_INSN (tem))
8556 JUMP_LABEL (tem) = XEXP (jump_label, 0);
8562 /* Increment of LABEL_NUSES done above. */
8563 /* Register is now always nonnegative,
8564 so add REG_NONNEG note to the branch. */
8565 REG_NOTES (tem) = gen_rtx_EXPR_LIST (REG_NONNEG, reg,
8571 /* No insn may reference both the reversed and another biv or it
8572 will fail (see comment near the top of the loop reversal
8574 Earlier on, we have verified that the biv has no use except
8575 counting, or it is the only biv in this function.
8576 However, the code that computes no_use_except_counting does
8577 not verify reg notes. It's possible to have an insn that
8578 references another biv, and has a REG_EQUAL note with an
8579 expression based on the reversed biv. To avoid this case,
8580 remove all REG_EQUAL notes based on the reversed biv
8582 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
8586 rtx set = single_set (p);
8587 /* If this is a set of a GIV based on the reversed biv, any
8588 REG_EQUAL notes should still be correct. */
8590 || GET_CODE (SET_DEST (set)) != REG
8591 || (size_t) REGNO (SET_DEST (set)) >= ivs->n_regs
8592 || REG_IV_TYPE (ivs, REGNO (SET_DEST (set))) != GENERAL_INDUCT
8593 || REG_IV_INFO (ivs, REGNO (SET_DEST (set)))->src_reg != bl->biv->src_reg)
8594 for (pnote = ®_NOTES (p); *pnote;)
8596 if (REG_NOTE_KIND (*pnote) == REG_EQUAL
8597 && reg_mentioned_p (regno_reg_rtx[bl->regno],
8599 *pnote = XEXP (*pnote, 1);
8601 pnote = &XEXP (*pnote, 1);
8605 /* Mark that this biv has been reversed. Each giv which depends
8606 on this biv, and which is also live past the end of the loop
8607 will have to be fixed up. */
8611 if (loop_dump_stream)
8613 fprintf (loop_dump_stream, "Reversed loop");
8615 fprintf (loop_dump_stream, " and added reg_nonneg\n");
8617 fprintf (loop_dump_stream, "\n");
8628 /* Verify whether the biv BL appears to be eliminable,
8629 based on the insns in the loop that refer to it.
8631 If ELIMINATE_P is nonzero, actually do the elimination.
8633 THRESHOLD and INSN_COUNT are from loop_optimize and are used to
8634 determine whether invariant insns should be placed inside or at the
8635 start of the loop. */
8638 maybe_eliminate_biv (const struct loop *loop, struct iv_class *bl,
8639 int eliminate_p, int threshold, int insn_count)
8641 struct loop_ivs *ivs = LOOP_IVS (loop);
8642 rtx reg = bl->biv->dest_reg;
8645 /* Scan all insns in the loop, stopping if we find one that uses the
8646 biv in a way that we cannot eliminate. */
8648 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
8650 enum rtx_code code = GET_CODE (p);
8651 basic_block where_bb = 0;
8652 rtx where_insn = threshold >= insn_count ? 0 : p;
8655 /* If this is a libcall that sets a giv, skip ahead to its end. */
8656 if (GET_RTX_CLASS (code) == 'i')
8658 note = find_reg_note (p, REG_LIBCALL, NULL_RTX);
8662 rtx last = XEXP (note, 0);
8663 rtx set = single_set (last);
8665 if (set && GET_CODE (SET_DEST (set)) == REG)
8667 unsigned int regno = REGNO (SET_DEST (set));
8669 if (regno < ivs->n_regs
8670 && REG_IV_TYPE (ivs, regno) == GENERAL_INDUCT
8671 && REG_IV_INFO (ivs, regno)->src_reg == bl->biv->src_reg)
8677 /* Closely examine the insn if the biv is mentioned. */
8678 if ((code == INSN || code == JUMP_INSN || code == CALL_INSN)
8679 && reg_mentioned_p (reg, PATTERN (p))
8680 && ! maybe_eliminate_biv_1 (loop, PATTERN (p), p, bl,
8681 eliminate_p, where_bb, where_insn))
8683 if (loop_dump_stream)
8684 fprintf (loop_dump_stream,
8685 "Cannot eliminate biv %d: biv used in insn %d.\n",
8686 bl->regno, INSN_UID (p));
8690 /* If we are eliminating, kill REG_EQUAL notes mentioning the biv. */
8692 && (note = find_reg_note (p, REG_EQUAL, NULL_RTX)) != NULL_RTX
8693 && reg_mentioned_p (reg, XEXP (note, 0)))
8694 remove_note (p, note);
8699 if (loop_dump_stream)
8700 fprintf (loop_dump_stream, "biv %d %s eliminated.\n",
8701 bl->regno, eliminate_p ? "was" : "can be");
8708 /* INSN and REFERENCE are instructions in the same insn chain.
8709 Return nonzero if INSN is first. */
8712 loop_insn_first_p (rtx insn, rtx reference)
8716 for (p = insn, q = reference;;)
8718 /* Start with test for not first so that INSN == REFERENCE yields not
8720 if (q == insn || ! p)
8722 if (p == reference || ! q)
8725 /* Either of P or Q might be a NOTE. Notes have the same LUID as the
8726 previous insn, hence the <= comparison below does not work if
8728 if (INSN_UID (p) < max_uid_for_loop
8729 && INSN_UID (q) < max_uid_for_loop
8730 && GET_CODE (p) != NOTE)
8731 return INSN_LUID (p) <= INSN_LUID (q);
8733 if (INSN_UID (p) >= max_uid_for_loop
8734 || GET_CODE (p) == NOTE)
8736 if (INSN_UID (q) >= max_uid_for_loop)
8741 /* We are trying to eliminate BIV in INSN using GIV. Return nonzero if
8742 the offset that we have to take into account due to auto-increment /
8743 div derivation is zero. */
8745 biv_elimination_giv_has_0_offset (struct induction *biv,
8746 struct induction *giv, rtx insn)
8748 /* If the giv V had the auto-inc address optimization applied
8749 to it, and INSN occurs between the giv insn and the biv
8750 insn, then we'd have to adjust the value used here.
8751 This is rare, so we don't bother to make this possible. */
8752 if (giv->auto_inc_opt
8753 && ((loop_insn_first_p (giv->insn, insn)
8754 && loop_insn_first_p (insn, biv->insn))
8755 || (loop_insn_first_p (biv->insn, insn)
8756 && loop_insn_first_p (insn, giv->insn))))
8762 /* If BL appears in X (part of the pattern of INSN), see if we can
8763 eliminate its use. If so, return 1. If not, return 0.
8765 If BIV does not appear in X, return 1.
8767 If ELIMINATE_P is nonzero, actually do the elimination.
8768 WHERE_INSN/WHERE_BB indicate where extra insns should be added.
8769 Depending on how many items have been moved out of the loop, it
8770 will either be before INSN (when WHERE_INSN is nonzero) or at the
8771 start of the loop (when WHERE_INSN is zero). */
8774 maybe_eliminate_biv_1 (const struct loop *loop, rtx x, rtx insn,
8775 struct iv_class *bl, int eliminate_p,
8776 basic_block where_bb, rtx where_insn)
8778 enum rtx_code code = GET_CODE (x);
8779 rtx reg = bl->biv->dest_reg;
8780 enum machine_mode mode = GET_MODE (reg);
8781 struct induction *v;
8793 /* If we haven't already been able to do something with this BIV,
8794 we can't eliminate it. */
8800 /* If this sets the BIV, it is not a problem. */
8801 if (SET_DEST (x) == reg)
8804 /* If this is an insn that defines a giv, it is also ok because
8805 it will go away when the giv is reduced. */
8806 for (v = bl->giv; v; v = v->next_iv)
8807 if (v->giv_type == DEST_REG && SET_DEST (x) == v->dest_reg)
8811 if (SET_DEST (x) == cc0_rtx && SET_SRC (x) == reg)
8813 /* Can replace with any giv that was reduced and
8814 that has (MULT_VAL != 0) and (ADD_VAL == 0).
8815 Require a constant for MULT_VAL, so we know it's nonzero.
8816 ??? We disable this optimization to avoid potential
8819 for (v = bl->giv; v; v = v->next_iv)
8820 if (GET_CODE (v->mult_val) == CONST_INT && v->mult_val != const0_rtx
8821 && v->add_val == const0_rtx
8822 && ! v->ignore && ! v->maybe_dead && v->always_computable
8826 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8832 /* If the giv has the opposite direction of change,
8833 then reverse the comparison. */
8834 if (INTVAL (v->mult_val) < 0)
8835 new = gen_rtx_COMPARE (GET_MODE (v->new_reg),
8836 const0_rtx, v->new_reg);
8840 /* We can probably test that giv's reduced reg. */
8841 if (validate_change (insn, &SET_SRC (x), new, 0))
8845 /* Look for a giv with (MULT_VAL != 0) and (ADD_VAL != 0);
8846 replace test insn with a compare insn (cmp REDUCED_GIV ADD_VAL).
8847 Require a constant for MULT_VAL, so we know it's nonzero.
8848 ??? Do this only if ADD_VAL is a pointer to avoid a potential
8849 overflow problem. */
8851 for (v = bl->giv; v; v = v->next_iv)
8852 if (GET_CODE (v->mult_val) == CONST_INT
8853 && v->mult_val != const0_rtx
8854 && ! v->ignore && ! v->maybe_dead && v->always_computable
8856 && (GET_CODE (v->add_val) == SYMBOL_REF
8857 || GET_CODE (v->add_val) == LABEL_REF
8858 || GET_CODE (v->add_val) == CONST
8859 || (GET_CODE (v->add_val) == REG
8860 && REG_POINTER (v->add_val))))
8862 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8868 /* If the giv has the opposite direction of change,
8869 then reverse the comparison. */
8870 if (INTVAL (v->mult_val) < 0)
8871 new = gen_rtx_COMPARE (VOIDmode, copy_rtx (v->add_val),
8874 new = gen_rtx_COMPARE (VOIDmode, v->new_reg,
8875 copy_rtx (v->add_val));
8877 /* Replace biv with the giv's reduced register. */
8878 update_reg_last_use (v->add_val, insn);
8879 if (validate_change (insn, &SET_SRC (PATTERN (insn)), new, 0))
8882 /* Insn doesn't support that constant or invariant. Copy it
8883 into a register (it will be a loop invariant.) */
8884 tem = gen_reg_rtx (GET_MODE (v->new_reg));
8886 loop_insn_emit_before (loop, 0, where_insn,
8888 copy_rtx (v->add_val)));
8890 /* Substitute the new register for its invariant value in
8891 the compare expression. */
8892 XEXP (new, (INTVAL (v->mult_val) < 0) ? 0 : 1) = tem;
8893 if (validate_change (insn, &SET_SRC (PATTERN (insn)), new, 0))
8902 case GT: case GE: case GTU: case GEU:
8903 case LT: case LE: case LTU: case LEU:
8904 /* See if either argument is the biv. */
8905 if (XEXP (x, 0) == reg)
8906 arg = XEXP (x, 1), arg_operand = 1;
8907 else if (XEXP (x, 1) == reg)
8908 arg = XEXP (x, 0), arg_operand = 0;
8912 if (CONSTANT_P (arg))
8914 /* First try to replace with any giv that has constant positive
8915 mult_val and constant add_val. We might be able to support
8916 negative mult_val, but it seems complex to do it in general. */
8918 for (v = bl->giv; v; v = v->next_iv)
8919 if (GET_CODE (v->mult_val) == CONST_INT
8920 && INTVAL (v->mult_val) > 0
8921 && (GET_CODE (v->add_val) == SYMBOL_REF
8922 || GET_CODE (v->add_val) == LABEL_REF
8923 || GET_CODE (v->add_val) == CONST
8924 || (GET_CODE (v->add_val) == REG
8925 && REG_POINTER (v->add_val)))
8926 && ! v->ignore && ! v->maybe_dead && v->always_computable
8929 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8932 /* Don't eliminate if the linear combination that makes up
8933 the giv overflows when it is applied to ARG. */
8934 if (GET_CODE (arg) == CONST_INT)
8938 if (GET_CODE (v->add_val) == CONST_INT)
8939 add_val = v->add_val;
8941 add_val = const0_rtx;
8943 if (const_mult_add_overflow_p (arg, v->mult_val,
8951 /* Replace biv with the giv's reduced reg. */
8952 validate_change (insn, &XEXP (x, 1 - arg_operand), v->new_reg, 1);
8954 /* If all constants are actually constant integers and
8955 the derived constant can be directly placed in the COMPARE,
8957 if (GET_CODE (arg) == CONST_INT
8958 && GET_CODE (v->add_val) == CONST_INT)
8960 tem = expand_mult_add (arg, NULL_RTX, v->mult_val,
8961 v->add_val, mode, 1);
8965 /* Otherwise, load it into a register. */
8966 tem = gen_reg_rtx (mode);
8967 loop_iv_add_mult_emit_before (loop, arg,
8968 v->mult_val, v->add_val,
8969 tem, where_bb, where_insn);
8972 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
8974 if (apply_change_group ())
8978 /* Look for giv with positive constant mult_val and nonconst add_val.
8979 Insert insns to calculate new compare value.
8980 ??? Turn this off due to possible overflow. */
8982 for (v = bl->giv; v; v = v->next_iv)
8983 if (GET_CODE (v->mult_val) == CONST_INT
8984 && INTVAL (v->mult_val) > 0
8985 && ! v->ignore && ! v->maybe_dead && v->always_computable
8991 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
8997 tem = gen_reg_rtx (mode);
8999 /* Replace biv with giv's reduced register. */
9000 validate_change (insn, &XEXP (x, 1 - arg_operand),
9003 /* Compute value to compare against. */
9004 loop_iv_add_mult_emit_before (loop, arg,
9005 v->mult_val, v->add_val,
9006 tem, where_bb, where_insn);
9007 /* Use it in this insn. */
9008 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
9009 if (apply_change_group ())
9013 else if (GET_CODE (arg) == REG || GET_CODE (arg) == MEM)
9015 if (loop_invariant_p (loop, arg) == 1)
9017 /* Look for giv with constant positive mult_val and nonconst
9018 add_val. Insert insns to compute new compare value.
9019 ??? Turn this off due to possible overflow. */
9021 for (v = bl->giv; v; v = v->next_iv)
9022 if (GET_CODE (v->mult_val) == CONST_INT && INTVAL (v->mult_val) > 0
9023 && ! v->ignore && ! v->maybe_dead && v->always_computable
9029 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
9035 tem = gen_reg_rtx (mode);
9037 /* Replace biv with giv's reduced register. */
9038 validate_change (insn, &XEXP (x, 1 - arg_operand),
9041 /* Compute value to compare against. */
9042 loop_iv_add_mult_emit_before (loop, arg,
9043 v->mult_val, v->add_val,
9044 tem, where_bb, where_insn);
9045 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
9046 if (apply_change_group ())
9051 /* This code has problems. Basically, you can't know when
9052 seeing if we will eliminate BL, whether a particular giv
9053 of ARG will be reduced. If it isn't going to be reduced,
9054 we can't eliminate BL. We can try forcing it to be reduced,
9055 but that can generate poor code.
9057 The problem is that the benefit of reducing TV, below should
9058 be increased if BL can actually be eliminated, but this means
9059 we might have to do a topological sort of the order in which
9060 we try to process biv. It doesn't seem worthwhile to do
9061 this sort of thing now. */
9064 /* Otherwise the reg compared with had better be a biv. */
9065 if (GET_CODE (arg) != REG
9066 || REG_IV_TYPE (ivs, REGNO (arg)) != BASIC_INDUCT)
9069 /* Look for a pair of givs, one for each biv,
9070 with identical coefficients. */
9071 for (v = bl->giv; v; v = v->next_iv)
9073 struct induction *tv;
9075 if (v->ignore || v->maybe_dead || v->mode != mode)
9078 for (tv = REG_IV_CLASS (ivs, REGNO (arg))->giv; tv;
9080 if (! tv->ignore && ! tv->maybe_dead
9081 && rtx_equal_p (tv->mult_val, v->mult_val)
9082 && rtx_equal_p (tv->add_val, v->add_val)
9083 && tv->mode == mode)
9085 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
9091 /* Replace biv with its giv's reduced reg. */
9092 XEXP (x, 1 - arg_operand) = v->new_reg;
9093 /* Replace other operand with the other giv's
9095 XEXP (x, arg_operand) = tv->new_reg;
9102 /* If we get here, the biv can't be eliminated. */
9106 /* If this address is a DEST_ADDR giv, it doesn't matter if the
9107 biv is used in it, since it will be replaced. */
9108 for (v = bl->giv; v; v = v->next_iv)
9109 if (v->giv_type == DEST_ADDR && v->location == &XEXP (x, 0))
9117 /* See if any subexpression fails elimination. */
9118 fmt = GET_RTX_FORMAT (code);
9119 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
9124 if (! maybe_eliminate_biv_1 (loop, XEXP (x, i), insn, bl,
9125 eliminate_p, where_bb, where_insn))
9130 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
9131 if (! maybe_eliminate_biv_1 (loop, XVECEXP (x, i, j), insn, bl,
9132 eliminate_p, where_bb, where_insn))
9141 /* Return nonzero if the last use of REG
9142 is in an insn following INSN in the same basic block. */
9145 last_use_this_basic_block (rtx reg, rtx insn)
9149 n && GET_CODE (n) != CODE_LABEL && GET_CODE (n) != JUMP_INSN;
9152 if (REGNO_LAST_UID (REGNO (reg)) == INSN_UID (n))
9158 /* Called via `note_stores' to record the initial value of a biv. Here we
9159 just record the location of the set and process it later. */
9162 record_initial (rtx dest, rtx set, void *data ATTRIBUTE_UNUSED)
9164 struct loop_ivs *ivs = (struct loop_ivs *) data;
9165 struct iv_class *bl;
9167 if (GET_CODE (dest) != REG
9168 || REGNO (dest) >= ivs->n_regs
9169 || REG_IV_TYPE (ivs, REGNO (dest)) != BASIC_INDUCT)
9172 bl = REG_IV_CLASS (ivs, REGNO (dest));
9174 /* If this is the first set found, record it. */
9175 if (bl->init_insn == 0)
9177 bl->init_insn = note_insn;
9182 /* If any of the registers in X are "old" and currently have a last use earlier
9183 than INSN, update them to have a last use of INSN. Their actual last use
9184 will be the previous insn but it will not have a valid uid_luid so we can't
9185 use it. X must be a source expression only. */
9188 update_reg_last_use (rtx x, rtx insn)
9190 /* Check for the case where INSN does not have a valid luid. In this case,
9191 there is no need to modify the regno_last_uid, as this can only happen
9192 when code is inserted after the loop_end to set a pseudo's final value,
9193 and hence this insn will never be the last use of x.
9194 ???? This comment is not correct. See for example loop_givs_reduce.
9195 This may insert an insn before another new insn. */
9196 if (GET_CODE (x) == REG && REGNO (x) < max_reg_before_loop
9197 && INSN_UID (insn) < max_uid_for_loop
9198 && REGNO_LAST_LUID (REGNO (x)) < INSN_LUID (insn))
9200 REGNO_LAST_UID (REGNO (x)) = INSN_UID (insn);
9205 const char *fmt = GET_RTX_FORMAT (GET_CODE (x));
9206 for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
9209 update_reg_last_use (XEXP (x, i), insn);
9210 else if (fmt[i] == 'E')
9211 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
9212 update_reg_last_use (XVECEXP (x, i, j), insn);
9217 /* Given an insn INSN and condition COND, return the condition in a
9218 canonical form to simplify testing by callers. Specifically:
9220 (1) The code will always be a comparison operation (EQ, NE, GT, etc.).
9221 (2) Both operands will be machine operands; (cc0) will have been replaced.
9222 (3) If an operand is a constant, it will be the second operand.
9223 (4) (LE x const) will be replaced with (LT x <const+1>) and similarly
9224 for GE, GEU, and LEU.
9226 If the condition cannot be understood, or is an inequality floating-point
9227 comparison which needs to be reversed, 0 will be returned.
9229 If REVERSE is nonzero, then reverse the condition prior to canonizing it.
9231 If EARLIEST is nonzero, it is a pointer to a place where the earliest
9232 insn used in locating the condition was found. If a replacement test
9233 of the condition is desired, it should be placed in front of that
9234 insn and we will be sure that the inputs are still valid.
9236 If WANT_REG is nonzero, we wish the condition to be relative to that
9237 register, if possible. Therefore, do not canonicalize the condition
9238 further. If ALLOW_CC_MODE is nonzero, allow the condition returned
9239 to be a compare to a CC mode register. */
9242 canonicalize_condition (rtx insn, rtx cond, int reverse, rtx *earliest,
9243 rtx want_reg, int allow_cc_mode)
9250 int reverse_code = 0;
9251 enum machine_mode mode;
9253 code = GET_CODE (cond);
9254 mode = GET_MODE (cond);
9255 op0 = XEXP (cond, 0);
9256 op1 = XEXP (cond, 1);
9259 code = reversed_comparison_code (cond, insn);
9260 if (code == UNKNOWN)
9266 /* If we are comparing a register with zero, see if the register is set
9267 in the previous insn to a COMPARE or a comparison operation. Perform
9268 the same tests as a function of STORE_FLAG_VALUE as find_comparison_args
9271 while (GET_RTX_CLASS (code) == '<'
9272 && op1 == CONST0_RTX (GET_MODE (op0))
9275 /* Set nonzero when we find something of interest. */
9279 /* If comparison with cc0, import actual comparison from compare
9283 if ((prev = prev_nonnote_insn (prev)) == 0
9284 || GET_CODE (prev) != INSN
9285 || (set = single_set (prev)) == 0
9286 || SET_DEST (set) != cc0_rtx)
9289 op0 = SET_SRC (set);
9290 op1 = CONST0_RTX (GET_MODE (op0));
9296 /* If this is a COMPARE, pick up the two things being compared. */
9297 if (GET_CODE (op0) == COMPARE)
9299 op1 = XEXP (op0, 1);
9300 op0 = XEXP (op0, 0);
9303 else if (GET_CODE (op0) != REG)
9306 /* Go back to the previous insn. Stop if it is not an INSN. We also
9307 stop if it isn't a single set or if it has a REG_INC note because
9308 we don't want to bother dealing with it. */
9310 if ((prev = prev_nonnote_insn (prev)) == 0
9311 || GET_CODE (prev) != INSN
9312 || FIND_REG_INC_NOTE (prev, NULL_RTX))
9315 set = set_of (op0, prev);
9318 && (GET_CODE (set) != SET
9319 || !rtx_equal_p (SET_DEST (set), op0)))
9322 /* If this is setting OP0, get what it sets it to if it looks
9326 enum machine_mode inner_mode = GET_MODE (SET_DEST (set));
9327 #ifdef FLOAT_STORE_FLAG_VALUE
9328 REAL_VALUE_TYPE fsfv;
9331 /* ??? We may not combine comparisons done in a CCmode with
9332 comparisons not done in a CCmode. This is to aid targets
9333 like Alpha that have an IEEE compliant EQ instruction, and
9334 a non-IEEE compliant BEQ instruction. The use of CCmode is
9335 actually artificial, simply to prevent the combination, but
9336 should not affect other platforms.
9338 However, we must allow VOIDmode comparisons to match either
9339 CCmode or non-CCmode comparison, because some ports have
9340 modeless comparisons inside branch patterns.
9342 ??? This mode check should perhaps look more like the mode check
9343 in simplify_comparison in combine. */
9345 if ((GET_CODE (SET_SRC (set)) == COMPARE
9348 && GET_MODE_CLASS (inner_mode) == MODE_INT
9349 && (GET_MODE_BITSIZE (inner_mode)
9350 <= HOST_BITS_PER_WIDE_INT)
9351 && (STORE_FLAG_VALUE
9352 & ((HOST_WIDE_INT) 1
9353 << (GET_MODE_BITSIZE (inner_mode) - 1))))
9354 #ifdef FLOAT_STORE_FLAG_VALUE
9356 && GET_MODE_CLASS (inner_mode) == MODE_FLOAT
9357 && (fsfv = FLOAT_STORE_FLAG_VALUE (inner_mode),
9358 REAL_VALUE_NEGATIVE (fsfv)))
9361 && GET_RTX_CLASS (GET_CODE (SET_SRC (set))) == '<'))
9362 && (((GET_MODE_CLASS (mode) == MODE_CC)
9363 == (GET_MODE_CLASS (inner_mode) == MODE_CC))
9364 || mode == VOIDmode || inner_mode == VOIDmode))
9366 else if (((code == EQ
9368 && (GET_MODE_BITSIZE (inner_mode)
9369 <= HOST_BITS_PER_WIDE_INT)
9370 && GET_MODE_CLASS (inner_mode) == MODE_INT
9371 && (STORE_FLAG_VALUE
9372 & ((HOST_WIDE_INT) 1
9373 << (GET_MODE_BITSIZE (inner_mode) - 1))))
9374 #ifdef FLOAT_STORE_FLAG_VALUE
9376 && GET_MODE_CLASS (inner_mode) == MODE_FLOAT
9377 && (fsfv = FLOAT_STORE_FLAG_VALUE (inner_mode),
9378 REAL_VALUE_NEGATIVE (fsfv)))
9381 && GET_RTX_CLASS (GET_CODE (SET_SRC (set))) == '<'
9382 && (((GET_MODE_CLASS (mode) == MODE_CC)
9383 == (GET_MODE_CLASS (inner_mode) == MODE_CC))
9384 || mode == VOIDmode || inner_mode == VOIDmode))
9394 else if (reg_set_p (op0, prev))
9395 /* If this sets OP0, but not directly, we have to give up. */
9400 if (GET_RTX_CLASS (GET_CODE (x)) == '<')
9401 code = GET_CODE (x);
9404 code = reversed_comparison_code (x, prev);
9405 if (code == UNKNOWN)
9410 op0 = XEXP (x, 0), op1 = XEXP (x, 1);
9416 /* If constant is first, put it last. */
9417 if (CONSTANT_P (op0))
9418 code = swap_condition (code), tem = op0, op0 = op1, op1 = tem;
9420 /* If OP0 is the result of a comparison, we weren't able to find what
9421 was really being compared, so fail. */
9423 && GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC)
9426 /* Canonicalize any ordered comparison with integers involving equality
9427 if we can do computations in the relevant mode and we do not
9430 if (GET_MODE_CLASS (GET_MODE (op0)) != MODE_CC
9431 && GET_CODE (op1) == CONST_INT
9432 && GET_MODE (op0) != VOIDmode
9433 && GET_MODE_BITSIZE (GET_MODE (op0)) <= HOST_BITS_PER_WIDE_INT)
9435 HOST_WIDE_INT const_val = INTVAL (op1);
9436 unsigned HOST_WIDE_INT uconst_val = const_val;
9437 unsigned HOST_WIDE_INT max_val
9438 = (unsigned HOST_WIDE_INT) GET_MODE_MASK (GET_MODE (op0));
9443 if ((unsigned HOST_WIDE_INT) const_val != max_val >> 1)
9444 code = LT, op1 = gen_int_mode (const_val + 1, GET_MODE (op0));
9447 /* When cross-compiling, const_val might be sign-extended from
9448 BITS_PER_WORD to HOST_BITS_PER_WIDE_INT */
9450 if ((HOST_WIDE_INT) (const_val & max_val)
9451 != (((HOST_WIDE_INT) 1
9452 << (GET_MODE_BITSIZE (GET_MODE (op0)) - 1))))
9453 code = GT, op1 = gen_int_mode (const_val - 1, GET_MODE (op0));
9457 if (uconst_val < max_val)
9458 code = LTU, op1 = gen_int_mode (uconst_val + 1, GET_MODE (op0));
9462 if (uconst_val != 0)
9463 code = GTU, op1 = gen_int_mode (uconst_val - 1, GET_MODE (op0));
9471 /* Never return CC0; return zero instead. */
9475 return gen_rtx_fmt_ee (code, VOIDmode, op0, op1);
9478 /* Given a jump insn JUMP, return the condition that will cause it to branch
9479 to its JUMP_LABEL. If the condition cannot be understood, or is an
9480 inequality floating-point comparison which needs to be reversed, 0 will
9483 If EARLIEST is nonzero, it is a pointer to a place where the earliest
9484 insn used in locating the condition was found. If a replacement test
9485 of the condition is desired, it should be placed in front of that
9486 insn and we will be sure that the inputs are still valid.
9488 If ALLOW_CC_MODE is nonzero, allow the condition returned to be a
9489 compare CC mode register. */
9492 get_condition (rtx jump, rtx *earliest, int allow_cc_mode)
9498 /* If this is not a standard conditional jump, we can't parse it. */
9499 if (GET_CODE (jump) != JUMP_INSN
9500 || ! any_condjump_p (jump))
9502 set = pc_set (jump);
9504 cond = XEXP (SET_SRC (set), 0);
9506 /* If this branches to JUMP_LABEL when the condition is false, reverse
9509 = GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
9510 && XEXP (XEXP (SET_SRC (set), 2), 0) == JUMP_LABEL (jump);
9512 return canonicalize_condition (jump, cond, reverse, earliest, NULL_RTX,
9516 /* Similar to above routine, except that we also put an invariant last
9517 unless both operands are invariants. */
9520 get_condition_for_loop (const struct loop *loop, rtx x)
9522 rtx comparison = get_condition (x, (rtx*) 0, false);
9525 || ! loop_invariant_p (loop, XEXP (comparison, 0))
9526 || loop_invariant_p (loop, XEXP (comparison, 1)))
9529 return gen_rtx_fmt_ee (swap_condition (GET_CODE (comparison)), VOIDmode,
9530 XEXP (comparison, 1), XEXP (comparison, 0));
9533 /* Scan the function and determine whether it has indirect (computed) jumps.
9535 This is taken mostly from flow.c; similar code exists elsewhere
9536 in the compiler. It may be useful to put this into rtlanal.c. */
9538 indirect_jump_in_function_p (rtx start)
9542 for (insn = start; insn; insn = NEXT_INSN (insn))
9543 if (computed_jump_p (insn))
9549 /* Add MEM to the LOOP_MEMS array, if appropriate. See the
9550 documentation for LOOP_MEMS for the definition of `appropriate'.
9551 This function is called from prescan_loop via for_each_rtx. */
9554 insert_loop_mem (rtx *mem, void *data ATTRIBUTE_UNUSED)
9556 struct loop_info *loop_info = data;
9563 switch (GET_CODE (m))
9569 /* We're not interested in MEMs that are only clobbered. */
9573 /* We're not interested in the MEM associated with a
9574 CONST_DOUBLE, so there's no need to traverse into this. */
9578 /* We're not interested in any MEMs that only appear in notes. */
9582 /* This is not a MEM. */
9586 /* See if we've already seen this MEM. */
9587 for (i = 0; i < loop_info->mems_idx; ++i)
9588 if (rtx_equal_p (m, loop_info->mems[i].mem))
9590 if (MEM_VOLATILE_P (m) && !MEM_VOLATILE_P (loop_info->mems[i].mem))
9591 loop_info->mems[i].mem = m;
9592 if (GET_MODE (m) != GET_MODE (loop_info->mems[i].mem))
9593 /* The modes of the two memory accesses are different. If
9594 this happens, something tricky is going on, and we just
9595 don't optimize accesses to this MEM. */
9596 loop_info->mems[i].optimize = 0;
9601 /* Resize the array, if necessary. */
9602 if (loop_info->mems_idx == loop_info->mems_allocated)
9604 if (loop_info->mems_allocated != 0)
9605 loop_info->mems_allocated *= 2;
9607 loop_info->mems_allocated = 32;
9609 loop_info->mems = xrealloc (loop_info->mems,
9610 loop_info->mems_allocated * sizeof (loop_mem_info));
9613 /* Actually insert the MEM. */
9614 loop_info->mems[loop_info->mems_idx].mem = m;
9615 /* We can't hoist this MEM out of the loop if it's a BLKmode MEM
9616 because we can't put it in a register. We still store it in the
9617 table, though, so that if we see the same address later, but in a
9618 non-BLK mode, we'll not think we can optimize it at that point. */
9619 loop_info->mems[loop_info->mems_idx].optimize = (GET_MODE (m) != BLKmode);
9620 loop_info->mems[loop_info->mems_idx].reg = NULL_RTX;
9621 ++loop_info->mems_idx;
9627 /* Allocate REGS->ARRAY or reallocate it if it is too small.
9629 Increment REGS->ARRAY[I].SET_IN_LOOP at the index I of each
9630 register that is modified by an insn between FROM and TO. If the
9631 value of an element of REGS->array[I].SET_IN_LOOP becomes 127 or
9632 more, stop incrementing it, to avoid overflow.
9634 Store in REGS->ARRAY[I].SINGLE_USAGE the single insn in which
9635 register I is used, if it is only used once. Otherwise, it is set
9636 to 0 (for no uses) or const0_rtx for more than one use. This
9637 parameter may be zero, in which case this processing is not done.
9639 Set REGS->ARRAY[I].MAY_NOT_OPTIMIZE nonzero if we should not
9640 optimize register I. */
9643 loop_regs_scan (const struct loop *loop, int extra_size)
9645 struct loop_regs *regs = LOOP_REGS (loop);
9647 /* last_set[n] is nonzero iff reg n has been set in the current
9648 basic block. In that case, it is the insn that last set reg n. */
9653 old_nregs = regs->num;
9654 regs->num = max_reg_num ();
9656 /* Grow the regs array if not allocated or too small. */
9657 if (regs->num >= regs->size)
9659 regs->size = regs->num + extra_size;
9661 regs->array = xrealloc (regs->array, regs->size * sizeof (*regs->array));
9663 /* Zero the new elements. */
9664 memset (regs->array + old_nregs, 0,
9665 (regs->size - old_nregs) * sizeof (*regs->array));
9668 /* Clear previously scanned fields but do not clear n_times_set. */
9669 for (i = 0; i < old_nregs; i++)
9671 regs->array[i].set_in_loop = 0;
9672 regs->array[i].may_not_optimize = 0;
9673 regs->array[i].single_usage = NULL_RTX;
9676 last_set = xcalloc (regs->num, sizeof (rtx));
9678 /* Scan the loop, recording register usage. */
9679 for (insn = loop->top ? loop->top : loop->start; insn != loop->end;
9680 insn = NEXT_INSN (insn))
9684 /* Record registers that have exactly one use. */
9685 find_single_use_in_loop (regs, insn, PATTERN (insn));
9687 /* Include uses in REG_EQUAL notes. */
9688 if (REG_NOTES (insn))
9689 find_single_use_in_loop (regs, insn, REG_NOTES (insn));
9691 if (GET_CODE (PATTERN (insn)) == SET
9692 || GET_CODE (PATTERN (insn)) == CLOBBER)
9693 count_one_set (regs, insn, PATTERN (insn), last_set);
9694 else if (GET_CODE (PATTERN (insn)) == PARALLEL)
9697 for (i = XVECLEN (PATTERN (insn), 0) - 1; i >= 0; i--)
9698 count_one_set (regs, insn, XVECEXP (PATTERN (insn), 0, i),
9703 if (GET_CODE (insn) == CODE_LABEL || GET_CODE (insn) == JUMP_INSN)
9704 memset (last_set, 0, regs->num * sizeof (rtx));
9706 /* Invalidate all registers used for function argument passing.
9707 We check rtx_varies_p for the same reason as below, to allow
9708 optimizing PIC calculations. */
9709 if (GET_CODE (insn) == CALL_INSN)
9712 for (link = CALL_INSN_FUNCTION_USAGE (insn);
9714 link = XEXP (link, 1))
9718 if (GET_CODE (op = XEXP (link, 0)) == USE
9719 && GET_CODE (reg = XEXP (op, 0)) == REG
9720 && rtx_varies_p (reg, 1))
9721 regs->array[REGNO (reg)].may_not_optimize = 1;
9726 /* Invalidate all hard registers clobbered by calls. With one exception:
9727 a call-clobbered PIC register is still function-invariant for our
9728 purposes, since we can hoist any PIC calculations out of the loop.
9729 Thus the call to rtx_varies_p. */
9730 if (LOOP_INFO (loop)->has_call)
9731 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
9732 if (TEST_HARD_REG_BIT (regs_invalidated_by_call, i)
9733 && rtx_varies_p (regno_reg_rtx[i], 1))
9735 regs->array[i].may_not_optimize = 1;
9736 regs->array[i].set_in_loop = 1;
9739 #ifdef AVOID_CCMODE_COPIES
9740 /* Don't try to move insns which set CC registers if we should not
9741 create CCmode register copies. */
9742 for (i = regs->num - 1; i >= FIRST_PSEUDO_REGISTER; i--)
9743 if (GET_MODE_CLASS (GET_MODE (regno_reg_rtx[i])) == MODE_CC)
9744 regs->array[i].may_not_optimize = 1;
9747 /* Set regs->array[I].n_times_set for the new registers. */
9748 for (i = old_nregs; i < regs->num; i++)
9749 regs->array[i].n_times_set = regs->array[i].set_in_loop;
9754 /* Returns the number of real INSNs in the LOOP. */
9757 count_insns_in_loop (const struct loop *loop)
9762 for (insn = loop->top ? loop->top : loop->start; insn != loop->end;
9763 insn = NEXT_INSN (insn))
9770 /* Move MEMs into registers for the duration of the loop. */
9773 load_mems (const struct loop *loop)
9775 struct loop_info *loop_info = LOOP_INFO (loop);
9776 struct loop_regs *regs = LOOP_REGS (loop);
9777 int maybe_never = 0;
9779 rtx p, prev_ebb_head;
9780 rtx label = NULL_RTX;
9782 /* Nonzero if the next instruction may never be executed. */
9783 int next_maybe_never = 0;
9784 unsigned int last_max_reg = max_reg_num ();
9786 if (loop_info->mems_idx == 0)
9789 /* We cannot use next_label here because it skips over normal insns. */
9790 end_label = next_nonnote_insn (loop->end);
9791 if (end_label && GET_CODE (end_label) != CODE_LABEL)
9792 end_label = NULL_RTX;
9794 /* Check to see if it's possible that some instructions in the loop are
9795 never executed. Also check if there is a goto out of the loop other
9796 than right after the end of the loop. */
9797 for (p = next_insn_in_loop (loop, loop->scan_start);
9799 p = next_insn_in_loop (loop, p))
9801 if (GET_CODE (p) == CODE_LABEL)
9803 else if (GET_CODE (p) == JUMP_INSN
9804 /* If we enter the loop in the middle, and scan
9805 around to the beginning, don't set maybe_never
9806 for that. This must be an unconditional jump,
9807 otherwise the code at the top of the loop might
9808 never be executed. Unconditional jumps are
9809 followed a by barrier then loop end. */
9810 && ! (GET_CODE (p) == JUMP_INSN
9811 && JUMP_LABEL (p) == loop->top
9812 && NEXT_INSN (NEXT_INSN (p)) == loop->end
9813 && any_uncondjump_p (p)))
9815 /* If this is a jump outside of the loop but not right
9816 after the end of the loop, we would have to emit new fixup
9817 sequences for each such label. */
9818 if (/* If we can't tell where control might go when this
9819 JUMP_INSN is executed, we must be conservative. */
9821 || (JUMP_LABEL (p) != end_label
9822 && (INSN_UID (JUMP_LABEL (p)) >= max_uid_for_loop
9823 || INSN_LUID (JUMP_LABEL (p)) < INSN_LUID (loop->start)
9824 || INSN_LUID (JUMP_LABEL (p)) > INSN_LUID (loop->end))))
9827 if (!any_condjump_p (p))
9828 /* Something complicated. */
9831 /* If there are any more instructions in the loop, they
9832 might not be reached. */
9833 next_maybe_never = 1;
9835 else if (next_maybe_never)
9839 /* Find start of the extended basic block that enters the loop. */
9840 for (p = loop->start;
9841 PREV_INSN (p) && GET_CODE (p) != CODE_LABEL;
9848 /* Build table of mems that get set to constant values before the
9850 for (; p != loop->start; p = NEXT_INSN (p))
9851 cselib_process_insn (p);
9853 /* Actually move the MEMs. */
9854 for (i = 0; i < loop_info->mems_idx; ++i)
9856 regset_head load_copies;
9857 regset_head store_copies;
9860 rtx mem = loop_info->mems[i].mem;
9863 if (MEM_VOLATILE_P (mem)
9864 || loop_invariant_p (loop, XEXP (mem, 0)) != 1)
9865 /* There's no telling whether or not MEM is modified. */
9866 loop_info->mems[i].optimize = 0;
9868 /* Go through the MEMs written to in the loop to see if this
9869 one is aliased by one of them. */
9870 mem_list_entry = loop_info->store_mems;
9871 while (mem_list_entry)
9873 if (rtx_equal_p (mem, XEXP (mem_list_entry, 0)))
9875 else if (true_dependence (XEXP (mem_list_entry, 0), VOIDmode,
9878 /* MEM is indeed aliased by this store. */
9879 loop_info->mems[i].optimize = 0;
9882 mem_list_entry = XEXP (mem_list_entry, 1);
9885 if (flag_float_store && written
9886 && GET_MODE_CLASS (GET_MODE (mem)) == MODE_FLOAT)
9887 loop_info->mems[i].optimize = 0;
9889 /* If this MEM is written to, we must be sure that there
9890 are no reads from another MEM that aliases this one. */
9891 if (loop_info->mems[i].optimize && written)
9895 for (j = 0; j < loop_info->mems_idx; ++j)
9899 else if (true_dependence (mem,
9901 loop_info->mems[j].mem,
9904 /* It's not safe to hoist loop_info->mems[i] out of
9905 the loop because writes to it might not be
9906 seen by reads from loop_info->mems[j]. */
9907 loop_info->mems[i].optimize = 0;
9913 if (maybe_never && may_trap_p (mem))
9914 /* We can't access the MEM outside the loop; it might
9915 cause a trap that wouldn't have happened otherwise. */
9916 loop_info->mems[i].optimize = 0;
9918 if (!loop_info->mems[i].optimize)
9919 /* We thought we were going to lift this MEM out of the
9920 loop, but later discovered that we could not. */
9923 INIT_REG_SET (&load_copies);
9924 INIT_REG_SET (&store_copies);
9926 /* Allocate a pseudo for this MEM. We set REG_USERVAR_P in
9927 order to keep scan_loop from moving stores to this MEM
9928 out of the loop just because this REG is neither a
9929 user-variable nor used in the loop test. */
9930 reg = gen_reg_rtx (GET_MODE (mem));
9931 REG_USERVAR_P (reg) = 1;
9932 loop_info->mems[i].reg = reg;
9934 /* Now, replace all references to the MEM with the
9935 corresponding pseudos. */
9937 for (p = next_insn_in_loop (loop, loop->scan_start);
9939 p = next_insn_in_loop (loop, p))
9945 set = single_set (p);
9947 /* See if this copies the mem into a register that isn't
9948 modified afterwards. We'll try to do copy propagation
9949 a little further on. */
9951 /* @@@ This test is _way_ too conservative. */
9953 && GET_CODE (SET_DEST (set)) == REG
9954 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER
9955 && REGNO (SET_DEST (set)) < last_max_reg
9956 && regs->array[REGNO (SET_DEST (set))].n_times_set == 1
9957 && rtx_equal_p (SET_SRC (set), mem))
9958 SET_REGNO_REG_SET (&load_copies, REGNO (SET_DEST (set)));
9960 /* See if this copies the mem from a register that isn't
9961 modified afterwards. We'll try to remove the
9962 redundant copy later on by doing a little register
9963 renaming and copy propagation. This will help
9964 to untangle things for the BIV detection code. */
9967 && GET_CODE (SET_SRC (set)) == REG
9968 && REGNO (SET_SRC (set)) >= FIRST_PSEUDO_REGISTER
9969 && REGNO (SET_SRC (set)) < last_max_reg
9970 && regs->array[REGNO (SET_SRC (set))].n_times_set == 1
9971 && rtx_equal_p (SET_DEST (set), mem))
9972 SET_REGNO_REG_SET (&store_copies, REGNO (SET_SRC (set)));
9974 /* If this is a call which uses / clobbers this memory
9975 location, we must not change the interface here. */
9976 if (GET_CODE (p) == CALL_INSN
9977 && reg_mentioned_p (loop_info->mems[i].mem,
9978 CALL_INSN_FUNCTION_USAGE (p)))
9981 loop_info->mems[i].optimize = 0;
9985 /* Replace the memory reference with the shadow register. */
9986 replace_loop_mems (p, loop_info->mems[i].mem,
9987 loop_info->mems[i].reg, written);
9990 if (GET_CODE (p) == CODE_LABEL
9991 || GET_CODE (p) == JUMP_INSN)
9995 if (! loop_info->mems[i].optimize)
9996 ; /* We found we couldn't do the replacement, so do nothing. */
9997 else if (! apply_change_group ())
9998 /* We couldn't replace all occurrences of the MEM. */
9999 loop_info->mems[i].optimize = 0;
10002 /* Load the memory immediately before LOOP->START, which is
10003 the NOTE_LOOP_BEG. */
10004 cselib_val *e = cselib_lookup (mem, VOIDmode, 0);
10008 struct elt_loc_list *const_equiv = 0;
10012 struct elt_loc_list *equiv;
10013 struct elt_loc_list *best_equiv = 0;
10014 for (equiv = e->locs; equiv; equiv = equiv->next)
10016 if (CONSTANT_P (equiv->loc))
10017 const_equiv = equiv;
10018 else if (GET_CODE (equiv->loc) == REG
10019 /* Extending hard register lifetimes causes crash
10020 on SRC targets. Doing so on non-SRC is
10021 probably also not good idea, since we most
10022 probably have pseudoregister equivalence as
10024 && REGNO (equiv->loc) >= FIRST_PSEUDO_REGISTER)
10025 best_equiv = equiv;
10027 /* Use the constant equivalence if that is cheap enough. */
10029 best_equiv = const_equiv;
10030 else if (const_equiv
10031 && (rtx_cost (const_equiv->loc, SET)
10032 <= rtx_cost (best_equiv->loc, SET)))
10034 best_equiv = const_equiv;
10038 /* If best_equiv is nonzero, we know that MEM is set to a
10039 constant or register before the loop. We will use this
10040 knowledge to initialize the shadow register with that
10041 constant or reg rather than by loading from MEM. */
10043 best = copy_rtx (best_equiv->loc);
10046 set = gen_move_insn (reg, best);
10047 set = loop_insn_hoist (loop, set);
10050 for (p = prev_ebb_head; p != loop->start; p = NEXT_INSN (p))
10051 if (REGNO_LAST_UID (REGNO (best)) == INSN_UID (p))
10053 REGNO_LAST_UID (REGNO (best)) = INSN_UID (set);
10059 set_unique_reg_note (set, REG_EQUAL, copy_rtx (const_equiv->loc));
10063 if (label == NULL_RTX)
10065 label = gen_label_rtx ();
10066 emit_label_after (label, loop->end);
10069 /* Store the memory immediately after END, which is
10070 the NOTE_LOOP_END. */
10071 set = gen_move_insn (copy_rtx (mem), reg);
10072 loop_insn_emit_after (loop, 0, label, set);
10075 if (loop_dump_stream)
10077 fprintf (loop_dump_stream, "Hoisted regno %d %s from ",
10078 REGNO (reg), (written ? "r/w" : "r/o"));
10079 print_rtl (loop_dump_stream, mem);
10080 fputc ('\n', loop_dump_stream);
10083 /* Attempt a bit of copy propagation. This helps untangle the
10084 data flow, and enables {basic,general}_induction_var to find
10086 EXECUTE_IF_SET_IN_REG_SET
10087 (&load_copies, FIRST_PSEUDO_REGISTER, j,
10089 try_copy_prop (loop, reg, j);
10091 CLEAR_REG_SET (&load_copies);
10093 EXECUTE_IF_SET_IN_REG_SET
10094 (&store_copies, FIRST_PSEUDO_REGISTER, j,
10096 try_swap_copy_prop (loop, reg, j);
10098 CLEAR_REG_SET (&store_copies);
10102 /* Now, we need to replace all references to the previous exit
10103 label with the new one. */
10104 if (label != NULL_RTX && end_label != NULL_RTX)
10105 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
10106 if (GET_CODE (p) == JUMP_INSN && JUMP_LABEL (p) == end_label)
10107 redirect_jump (p, label, false);
10112 /* For communication between note_reg_stored and its caller. */
10113 struct note_reg_stored_arg
10119 /* Called via note_stores, record in SET_SEEN whether X, which is written,
10120 is equal to ARG. */
10122 note_reg_stored (rtx x, rtx setter ATTRIBUTE_UNUSED, void *arg)
10124 struct note_reg_stored_arg *t = (struct note_reg_stored_arg *) arg;
10129 /* Try to replace every occurrence of pseudo REGNO with REPLACEMENT.
10130 There must be exactly one insn that sets this pseudo; it will be
10131 deleted if all replacements succeed and we can prove that the register
10132 is not used after the loop. */
10135 try_copy_prop (const struct loop *loop, rtx replacement, unsigned int regno)
10137 /* This is the reg that we are copying from. */
10138 rtx reg_rtx = regno_reg_rtx[regno];
10141 /* These help keep track of whether we replaced all uses of the reg. */
10142 int replaced_last = 0;
10143 int store_is_first = 0;
10145 for (insn = next_insn_in_loop (loop, loop->scan_start);
10147 insn = next_insn_in_loop (loop, insn))
10151 /* Only substitute within one extended basic block from the initializing
10153 if (GET_CODE (insn) == CODE_LABEL && init_insn)
10156 if (! INSN_P (insn))
10159 /* Is this the initializing insn? */
10160 set = single_set (insn);
10162 && GET_CODE (SET_DEST (set)) == REG
10163 && REGNO (SET_DEST (set)) == regno)
10169 if (REGNO_FIRST_UID (regno) == INSN_UID (insn))
10170 store_is_first = 1;
10173 /* Only substitute after seeing the initializing insn. */
10174 if (init_insn && insn != init_insn)
10176 struct note_reg_stored_arg arg;
10178 replace_loop_regs (insn, reg_rtx, replacement);
10179 if (REGNO_LAST_UID (regno) == INSN_UID (insn))
10182 /* Stop replacing when REPLACEMENT is modified. */
10183 arg.reg = replacement;
10185 note_stores (PATTERN (insn), note_reg_stored, &arg);
10188 rtx note = find_reg_note (insn, REG_EQUAL, NULL);
10190 /* It is possible that we've turned previously valid REG_EQUAL to
10191 invalid, as we change the REGNO to REPLACEMENT and unlike REGNO,
10192 REPLACEMENT is modified, we get different meaning. */
10193 if (note && reg_mentioned_p (replacement, XEXP (note, 0)))
10194 remove_note (insn, note);
10201 if (apply_change_group ())
10203 if (loop_dump_stream)
10204 fprintf (loop_dump_stream, " Replaced reg %d", regno);
10205 if (store_is_first && replaced_last)
10210 /* Assume we're just deleting INIT_INSN. */
10212 /* Look for REG_RETVAL note. If we're deleting the end of
10213 the libcall sequence, the whole sequence can go. */
10214 retval_note = find_reg_note (init_insn, REG_RETVAL, NULL_RTX);
10215 /* If we found a REG_RETVAL note, find the first instruction
10216 in the sequence. */
10218 first = XEXP (retval_note, 0);
10220 /* Delete the instructions. */
10221 loop_delete_insns (first, init_insn);
10223 if (loop_dump_stream)
10224 fprintf (loop_dump_stream, ".\n");
10228 /* Replace all the instructions from FIRST up to and including LAST
10229 with NOTE_INSN_DELETED notes. */
10232 loop_delete_insns (rtx first, rtx last)
10236 if (loop_dump_stream)
10237 fprintf (loop_dump_stream, ", deleting init_insn (%d)",
10239 delete_insn (first);
10241 /* If this was the LAST instructions we're supposed to delete,
10246 first = NEXT_INSN (first);
10250 /* Try to replace occurrences of pseudo REGNO with REPLACEMENT within
10251 loop LOOP if the order of the sets of these registers can be
10252 swapped. There must be exactly one insn within the loop that sets
10253 this pseudo followed immediately by a move insn that sets
10254 REPLACEMENT with REGNO. */
10256 try_swap_copy_prop (const struct loop *loop, rtx replacement,
10257 unsigned int regno)
10260 rtx set = NULL_RTX;
10261 unsigned int new_regno;
10263 new_regno = REGNO (replacement);
10265 for (insn = next_insn_in_loop (loop, loop->scan_start);
10267 insn = next_insn_in_loop (loop, insn))
10269 /* Search for the insn that copies REGNO to NEW_REGNO? */
10271 && (set = single_set (insn))
10272 && GET_CODE (SET_DEST (set)) == REG
10273 && REGNO (SET_DEST (set)) == new_regno
10274 && GET_CODE (SET_SRC (set)) == REG
10275 && REGNO (SET_SRC (set)) == regno)
10279 if (insn != NULL_RTX)
10284 /* Some DEF-USE info would come in handy here to make this
10285 function more general. For now, just check the previous insn
10286 which is the most likely candidate for setting REGNO. */
10288 prev_insn = PREV_INSN (insn);
10291 && (prev_set = single_set (prev_insn))
10292 && GET_CODE (SET_DEST (prev_set)) == REG
10293 && REGNO (SET_DEST (prev_set)) == regno)
10296 (set (reg regno) (expr))
10297 (set (reg new_regno) (reg regno))
10299 so try converting this to:
10300 (set (reg new_regno) (expr))
10301 (set (reg regno) (reg new_regno))
10303 The former construct is often generated when a global
10304 variable used for an induction variable is shadowed by a
10305 register (NEW_REGNO). The latter construct improves the
10306 chances of GIV replacement and BIV elimination. */
10308 validate_change (prev_insn, &SET_DEST (prev_set),
10310 validate_change (insn, &SET_DEST (set),
10312 validate_change (insn, &SET_SRC (set),
10315 if (apply_change_group ())
10317 if (loop_dump_stream)
10318 fprintf (loop_dump_stream,
10319 " Swapped set of reg %d at %d with reg %d at %d.\n",
10320 regno, INSN_UID (insn),
10321 new_regno, INSN_UID (prev_insn));
10323 /* Update first use of REGNO. */
10324 if (REGNO_FIRST_UID (regno) == INSN_UID (prev_insn))
10325 REGNO_FIRST_UID (regno) = INSN_UID (insn);
10327 /* Now perform copy propagation to hopefully
10328 remove all uses of REGNO within the loop. */
10329 try_copy_prop (loop, replacement, regno);
10335 /* Worker function for find_mem_in_note, called via for_each_rtx. */
10338 find_mem_in_note_1 (rtx *x, void *data)
10340 if (*x != NULL_RTX && GET_CODE (*x) == MEM)
10342 rtx *res = (rtx *) data;
10349 /* Returns the first MEM found in NOTE by depth-first search. */
10352 find_mem_in_note (rtx note)
10354 if (note && for_each_rtx (¬e, find_mem_in_note_1, ¬e))
10359 /* Replace MEM with its associated pseudo register. This function is
10360 called from load_mems via for_each_rtx. DATA is actually a pointer
10361 to a structure describing the instruction currently being scanned
10362 and the MEM we are currently replacing. */
10365 replace_loop_mem (rtx *mem, void *data)
10367 loop_replace_args *args = (loop_replace_args *) data;
10373 switch (GET_CODE (m))
10379 /* We're not interested in the MEM associated with a
10380 CONST_DOUBLE, so there's no need to traverse into one. */
10384 /* This is not a MEM. */
10388 if (!rtx_equal_p (args->match, m))
10389 /* This is not the MEM we are currently replacing. */
10392 /* Actually replace the MEM. */
10393 validate_change (args->insn, mem, args->replacement, 1);
10399 replace_loop_mems (rtx insn, rtx mem, rtx reg, int written)
10401 loop_replace_args args;
10405 args.replacement = reg;
10407 for_each_rtx (&insn, replace_loop_mem, &args);
10409 /* If we hoist a mem write out of the loop, then REG_EQUAL
10410 notes referring to the mem are no longer valid. */
10416 for (link = ®_NOTES (insn); (note = *link); link = &XEXP (note, 1))
10418 if (REG_NOTE_KIND (note) == REG_EQUAL
10419 && (sub = find_mem_in_note (note))
10420 && true_dependence (mem, VOIDmode, sub, rtx_varies_p))
10422 /* Remove the note. */
10423 validate_change (NULL_RTX, link, XEXP (note, 1), 1);
10430 /* Replace one register with another. Called through for_each_rtx; PX points
10431 to the rtx being scanned. DATA is actually a pointer to
10432 a structure of arguments. */
10435 replace_loop_reg (rtx *px, void *data)
10438 loop_replace_args *args = (loop_replace_args *) data;
10443 if (x == args->match)
10444 validate_change (args->insn, px, args->replacement, 1);
10450 replace_loop_regs (rtx insn, rtx reg, rtx replacement)
10452 loop_replace_args args;
10456 args.replacement = replacement;
10458 for_each_rtx (&insn, replace_loop_reg, &args);
10461 /* Emit insn for PATTERN after WHERE_INSN in basic block WHERE_BB
10462 (ignored in the interim). */
10465 loop_insn_emit_after (const struct loop *loop ATTRIBUTE_UNUSED,
10466 basic_block where_bb ATTRIBUTE_UNUSED, rtx where_insn,
10469 return emit_insn_after (pattern, where_insn);
10473 /* If WHERE_INSN is nonzero emit insn for PATTERN before WHERE_INSN
10474 in basic block WHERE_BB (ignored in the interim) within the loop
10475 otherwise hoist PATTERN into the loop pre-header. */
10478 loop_insn_emit_before (const struct loop *loop,
10479 basic_block where_bb ATTRIBUTE_UNUSED,
10480 rtx where_insn, rtx pattern)
10483 return loop_insn_hoist (loop, pattern);
10484 return emit_insn_before (pattern, where_insn);
10488 /* Emit call insn for PATTERN before WHERE_INSN in basic block
10489 WHERE_BB (ignored in the interim) within the loop. */
10492 loop_call_insn_emit_before (const struct loop *loop ATTRIBUTE_UNUSED,
10493 basic_block where_bb ATTRIBUTE_UNUSED,
10494 rtx where_insn, rtx pattern)
10496 return emit_call_insn_before (pattern, where_insn);
10500 /* Hoist insn for PATTERN into the loop pre-header. */
10503 loop_insn_hoist (const struct loop *loop, rtx pattern)
10505 return loop_insn_emit_before (loop, 0, loop->start, pattern);
10509 /* Hoist call insn for PATTERN into the loop pre-header. */
10512 loop_call_insn_hoist (const struct loop *loop, rtx pattern)
10514 return loop_call_insn_emit_before (loop, 0, loop->start, pattern);
10518 /* Sink insn for PATTERN after the loop end. */
10521 loop_insn_sink (const struct loop *loop, rtx pattern)
10523 return loop_insn_emit_before (loop, 0, loop->sink, pattern);
10526 /* bl->final_value can be either general_operand or PLUS of general_operand
10527 and constant. Emit sequence of instructions to load it into REG. */
10529 gen_load_of_final_value (rtx reg, rtx final_value)
10533 final_value = force_operand (final_value, reg);
10534 if (final_value != reg)
10535 emit_move_insn (reg, final_value);
10536 seq = get_insns ();
10541 /* If the loop has multiple exits, emit insn for PATTERN before the
10542 loop to ensure that it will always be executed no matter how the
10543 loop exits. Otherwise, emit the insn for PATTERN after the loop,
10544 since this is slightly more efficient. */
10547 loop_insn_sink_or_swim (const struct loop *loop, rtx pattern)
10549 if (loop->exit_count)
10550 return loop_insn_hoist (loop, pattern);
10552 return loop_insn_sink (loop, pattern);
10556 loop_ivs_dump (const struct loop *loop, FILE *file, int verbose)
10558 struct iv_class *bl;
10561 if (! loop || ! file)
10564 for (bl = LOOP_IVS (loop)->list; bl; bl = bl->next)
10567 fprintf (file, "Loop %d: %d IV classes\n", loop->num, iv_num);
10569 for (bl = LOOP_IVS (loop)->list; bl; bl = bl->next)
10571 loop_iv_class_dump (bl, file, verbose);
10572 fputc ('\n', file);
10578 loop_iv_class_dump (const struct iv_class *bl, FILE *file,
10579 int verbose ATTRIBUTE_UNUSED)
10581 struct induction *v;
10585 if (! bl || ! file)
10588 fprintf (file, "IV class for reg %d, benefit %d\n",
10589 bl->regno, bl->total_benefit);
10591 fprintf (file, " Init insn %d", INSN_UID (bl->init_insn));
10592 if (bl->initial_value)
10594 fprintf (file, ", init val: ");
10595 print_simple_rtl (file, bl->initial_value);
10597 if (bl->initial_test)
10599 fprintf (file, ", init test: ");
10600 print_simple_rtl (file, bl->initial_test);
10602 fputc ('\n', file);
10604 if (bl->final_value)
10606 fprintf (file, " Final val: ");
10607 print_simple_rtl (file, bl->final_value);
10608 fputc ('\n', file);
10611 if ((incr = biv_total_increment (bl)))
10613 fprintf (file, " Total increment: ");
10614 print_simple_rtl (file, incr);
10615 fputc ('\n', file);
10618 /* List the increments. */
10619 for (i = 0, v = bl->biv; v; v = v->next_iv, i++)
10621 fprintf (file, " Inc%d: insn %d, incr: ", i, INSN_UID (v->insn));
10622 print_simple_rtl (file, v->add_val);
10623 fputc ('\n', file);
10626 /* List the givs. */
10627 for (i = 0, v = bl->giv; v; v = v->next_iv, i++)
10629 fprintf (file, " Giv%d: insn %d, benefit %d, ",
10630 i, INSN_UID (v->insn), v->benefit);
10631 if (v->giv_type == DEST_ADDR)
10632 print_simple_rtl (file, v->mem);
10634 print_simple_rtl (file, single_set (v->insn));
10635 fputc ('\n', file);
10641 loop_biv_dump (const struct induction *v, FILE *file, int verbose)
10648 REGNO (v->dest_reg), INSN_UID (v->insn));
10649 fprintf (file, " const ");
10650 print_simple_rtl (file, v->add_val);
10652 if (verbose && v->final_value)
10654 fputc ('\n', file);
10655 fprintf (file, " final ");
10656 print_simple_rtl (file, v->final_value);
10659 fputc ('\n', file);
10664 loop_giv_dump (const struct induction *v, FILE *file, int verbose)
10669 if (v->giv_type == DEST_REG)
10670 fprintf (file, "Giv %d: insn %d",
10671 REGNO (v->dest_reg), INSN_UID (v->insn));
10673 fprintf (file, "Dest address: insn %d",
10674 INSN_UID (v->insn));
10676 fprintf (file, " src reg %d benefit %d",
10677 REGNO (v->src_reg), v->benefit);
10678 fprintf (file, " lifetime %d",
10681 if (v->replaceable)
10682 fprintf (file, " replaceable");
10684 if (v->no_const_addval)
10685 fprintf (file, " ncav");
10687 if (v->ext_dependent)
10689 switch (GET_CODE (v->ext_dependent))
10692 fprintf (file, " ext se");
10695 fprintf (file, " ext ze");
10698 fprintf (file, " ext tr");
10705 fputc ('\n', file);
10706 fprintf (file, " mult ");
10707 print_simple_rtl (file, v->mult_val);
10709 fputc ('\n', file);
10710 fprintf (file, " add ");
10711 print_simple_rtl (file, v->add_val);
10713 if (verbose && v->final_value)
10715 fputc ('\n', file);
10716 fprintf (file, " final ");
10717 print_simple_rtl (file, v->final_value);
10720 fputc ('\n', file);
10725 debug_ivs (const struct loop *loop)
10727 loop_ivs_dump (loop, stderr, 1);
10732 debug_iv_class (const struct iv_class *bl)
10734 loop_iv_class_dump (bl, stderr, 1);
10739 debug_biv (const struct induction *v)
10741 loop_biv_dump (v, stderr, 1);
10746 debug_giv (const struct induction *v)
10748 loop_giv_dump (v, stderr, 1);
10752 #define LOOP_BLOCK_NUM_1(INSN) \
10753 ((INSN) ? (BLOCK_FOR_INSN (INSN) ? BLOCK_NUM (INSN) : - 1) : -1)
10755 /* The notes do not have an assigned block, so look at the next insn. */
10756 #define LOOP_BLOCK_NUM(INSN) \
10757 ((INSN) ? (GET_CODE (INSN) == NOTE \
10758 ? LOOP_BLOCK_NUM_1 (next_nonnote_insn (INSN)) \
10759 : LOOP_BLOCK_NUM_1 (INSN)) \
10762 #define LOOP_INSN_UID(INSN) ((INSN) ? INSN_UID (INSN) : -1)
10765 loop_dump_aux (const struct loop *loop, FILE *file,
10766 int verbose ATTRIBUTE_UNUSED)
10770 if (! loop || ! file)
10773 /* Print diagnostics to compare our concept of a loop with
10774 what the loop notes say. */
10775 if (! PREV_INSN (BB_HEAD (loop->first))
10776 || GET_CODE (PREV_INSN (BB_HEAD (loop->first))) != NOTE
10777 || NOTE_LINE_NUMBER (PREV_INSN (BB_HEAD (loop->first)))
10778 != NOTE_INSN_LOOP_BEG)
10779 fprintf (file, ";; No NOTE_INSN_LOOP_BEG at %d\n",
10780 INSN_UID (PREV_INSN (BB_HEAD (loop->first))));
10781 if (! NEXT_INSN (BB_END (loop->last))
10782 || GET_CODE (NEXT_INSN (BB_END (loop->last))) != NOTE
10783 || NOTE_LINE_NUMBER (NEXT_INSN (BB_END (loop->last)))
10784 != NOTE_INSN_LOOP_END)
10785 fprintf (file, ";; No NOTE_INSN_LOOP_END at %d\n",
10786 INSN_UID (NEXT_INSN (BB_END (loop->last))));
10791 ";; start %d (%d), cont dom %d (%d), cont %d (%d), vtop %d (%d), end %d (%d)\n",
10792 LOOP_BLOCK_NUM (loop->start),
10793 LOOP_INSN_UID (loop->start),
10794 LOOP_BLOCK_NUM (loop->cont),
10795 LOOP_INSN_UID (loop->cont),
10796 LOOP_BLOCK_NUM (loop->cont),
10797 LOOP_INSN_UID (loop->cont),
10798 LOOP_BLOCK_NUM (loop->vtop),
10799 LOOP_INSN_UID (loop->vtop),
10800 LOOP_BLOCK_NUM (loop->end),
10801 LOOP_INSN_UID (loop->end));
10802 fprintf (file, ";; top %d (%d), scan start %d (%d)\n",
10803 LOOP_BLOCK_NUM (loop->top),
10804 LOOP_INSN_UID (loop->top),
10805 LOOP_BLOCK_NUM (loop->scan_start),
10806 LOOP_INSN_UID (loop->scan_start));
10807 fprintf (file, ";; exit_count %d", loop->exit_count);
10808 if (loop->exit_count)
10810 fputs (", labels:", file);
10811 for (label = loop->exit_labels; label; label = LABEL_NEXTREF (label))
10813 fprintf (file, " %d ",
10814 LOOP_INSN_UID (XEXP (label, 0)));
10817 fputs ("\n", file);
10819 /* This can happen when a marked loop appears as two nested loops,
10820 say from while (a || b) {}. The inner loop won't match
10821 the loop markers but the outer one will. */
10822 if (LOOP_BLOCK_NUM (loop->cont) != loop->latch->index)
10823 fprintf (file, ";; NOTE_INSN_LOOP_CONT not in loop latch\n");
10827 /* Call this function from the debugger to dump LOOP. */
10830 debug_loop (const struct loop *loop)
10832 flow_loop_dump (loop, stderr, loop_dump_aux, 1);
10835 /* Call this function from the debugger to dump LOOPS. */
10838 debug_loops (const struct loops *loops)
10840 flow_loops_dump (loops, stderr, loop_dump_aux, 1);