| 1 | /* Lowering pass for OMP directives. Converts OMP directives into explicit |
| 2 | calls to the runtime library (libgomp), data marshalling to implement data |
| 3 | sharing and copying clauses, offloading to accelerators, and more. |
| 4 | |
| 5 | Contributed by Diego Novillo <dnovillo@redhat.com> |
| 6 | |
| 7 | Copyright (C) 2005-2015 Free Software Foundation, Inc. |
| 8 | |
| 9 | This file is part of GCC. |
| 10 | |
| 11 | GCC is free software; you can redistribute it and/or modify it under |
| 12 | the terms of the GNU General Public License as published by the Free |
| 13 | Software Foundation; either version 3, or (at your option) any later |
| 14 | version. |
| 15 | |
| 16 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
| 17 | WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 18 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 19 | for more details. |
| 20 | |
| 21 | You should have received a copy of the GNU General Public License |
| 22 | along with GCC; see the file COPYING3. If not see |
| 23 | <http://www.gnu.org/licenses/>. */ |
| 24 | |
| 25 | #include "config.h" |
| 26 | #include "system.h" |
| 27 | #include "coretypes.h" |
| 28 | #include "tm.h" |
| 29 | #include "hash-set.h" |
| 30 | #include "machmode.h" |
| 31 | #include "vec.h" |
| 32 | #include "double-int.h" |
| 33 | #include "input.h" |
| 34 | #include "alias.h" |
| 35 | #include "symtab.h" |
| 36 | #include "wide-int.h" |
| 37 | #include "inchash.h" |
| 38 | #include "tree.h" |
| 39 | #include "fold-const.h" |
| 40 | #include "stringpool.h" |
| 41 | #include "stor-layout.h" |
| 42 | #include "rtl.h" |
| 43 | #include "predict.h" |
| 44 | #include "hard-reg-set.h" |
| 45 | #include "function.h" |
| 46 | #include "dominance.h" |
| 47 | #include "cfg.h" |
| 48 | #include "cfganal.h" |
| 49 | #include "basic-block.h" |
| 50 | #include "tree-ssa-alias.h" |
| 51 | #include "internal-fn.h" |
| 52 | #include "gimple-fold.h" |
| 53 | #include "gimple-expr.h" |
| 54 | #include "is-a.h" |
| 55 | #include "gimple.h" |
| 56 | #include "gimplify.h" |
| 57 | #include "gimple-iterator.h" |
| 58 | #include "gimplify-me.h" |
| 59 | #include "gimple-walk.h" |
| 60 | #include "tree-iterator.h" |
| 61 | #include "tree-inline.h" |
| 62 | #include "langhooks.h" |
| 63 | #include "diagnostic-core.h" |
| 64 | #include "gimple-ssa.h" |
| 65 | #include "hash-map.h" |
| 66 | #include "plugin-api.h" |
| 67 | #include "ipa-ref.h" |
| 68 | #include "cgraph.h" |
| 69 | #include "tree-cfg.h" |
| 70 | #include "tree-phinodes.h" |
| 71 | #include "ssa-iterators.h" |
| 72 | #include "tree-ssanames.h" |
| 73 | #include "tree-into-ssa.h" |
| 74 | #include "hashtab.h" |
| 75 | #include "flags.h" |
| 76 | #include "statistics.h" |
| 77 | #include "real.h" |
| 78 | #include "fixed-value.h" |
| 79 | #include "insn-config.h" |
| 80 | #include "expmed.h" |
| 81 | #include "dojump.h" |
| 82 | #include "explow.h" |
| 83 | #include "calls.h" |
| 84 | #include "emit-rtl.h" |
| 85 | #include "varasm.h" |
| 86 | #include "stmt.h" |
| 87 | #include "expr.h" |
| 88 | #include "tree-dfa.h" |
| 89 | #include "tree-ssa.h" |
| 90 | #include "tree-pass.h" |
| 91 | #include "except.h" |
| 92 | #include "splay-tree.h" |
| 93 | #include "insn-codes.h" |
| 94 | #include "optabs.h" |
| 95 | #include "cfgloop.h" |
| 96 | #include "target.h" |
| 97 | #include "common/common-target.h" |
| 98 | #include "omp-low.h" |
| 99 | #include "gimple-low.h" |
| 100 | #include "tree-cfgcleanup.h" |
| 101 | #include "pretty-print.h" |
| 102 | #include "alloc-pool.h" |
| 103 | #include "symbol-summary.h" |
| 104 | #include "ipa-prop.h" |
| 105 | #include "tree-nested.h" |
| 106 | #include "tree-eh.h" |
| 107 | #include "cilk.h" |
| 108 | #include "context.h" |
| 109 | #include "lto-section-names.h" |
| 110 | #include "gomp-constants.h" |
| 111 | |
| 112 | |
| 113 | /* Lowering of OMP parallel and workshare constructs proceeds in two |
| 114 | phases. The first phase scans the function looking for OMP statements |
| 115 | and then for variables that must be replaced to satisfy data sharing |
| 116 | clauses. The second phase expands code for the constructs, as well as |
| 117 | re-gimplifying things when variables have been replaced with complex |
| 118 | expressions. |
| 119 | |
| 120 | Final code generation is done by pass_expand_omp. The flowgraph is |
| 121 | scanned for regions which are then moved to a new |
| 122 | function, to be invoked by the thread library, or offloaded. */ |
| 123 | |
| 124 | /* OMP region information. Every parallel and workshare |
| 125 | directive is enclosed between two markers, the OMP_* directive |
| 126 | and a corresponding OMP_RETURN statement. */ |
| 127 | |
| 128 | struct omp_region |
| 129 | { |
| 130 | /* The enclosing region. */ |
| 131 | struct omp_region *outer; |
| 132 | |
| 133 | /* First child region. */ |
| 134 | struct omp_region *inner; |
| 135 | |
| 136 | /* Next peer region. */ |
| 137 | struct omp_region *next; |
| 138 | |
| 139 | /* Block containing the omp directive as its last stmt. */ |
| 140 | basic_block entry; |
| 141 | |
| 142 | /* Block containing the OMP_RETURN as its last stmt. */ |
| 143 | basic_block exit; |
| 144 | |
| 145 | /* Block containing the OMP_CONTINUE as its last stmt. */ |
| 146 | basic_block cont; |
| 147 | |
| 148 | /* If this is a combined parallel+workshare region, this is a list |
| 149 | of additional arguments needed by the combined parallel+workshare |
| 150 | library call. */ |
| 151 | vec<tree, va_gc> *ws_args; |
| 152 | |
| 153 | /* The code for the omp directive of this region. */ |
| 154 | enum gimple_code type; |
| 155 | |
| 156 | /* Schedule kind, only used for OMP_FOR type regions. */ |
| 157 | enum omp_clause_schedule_kind sched_kind; |
| 158 | |
| 159 | /* True if this is a combined parallel+workshare region. */ |
| 160 | bool is_combined_parallel; |
| 161 | }; |
| 162 | |
| 163 | /* Levels of parallelism as defined by OpenACC. Increasing numbers |
| 164 | correspond to deeper loop nesting levels. */ |
| 165 | #define MASK_GANG 1 |
| 166 | #define MASK_WORKER 2 |
| 167 | #define MASK_VECTOR 4 |
| 168 | |
| 169 | /* Context structure. Used to store information about each parallel |
| 170 | directive in the code. */ |
| 171 | |
| 172 | typedef struct omp_context |
| 173 | { |
| 174 | /* This field must be at the beginning, as we do "inheritance": Some |
| 175 | callback functions for tree-inline.c (e.g., omp_copy_decl) |
| 176 | receive a copy_body_data pointer that is up-casted to an |
| 177 | omp_context pointer. */ |
| 178 | copy_body_data cb; |
| 179 | |
| 180 | /* The tree of contexts corresponding to the encountered constructs. */ |
| 181 | struct omp_context *outer; |
| 182 | gimple stmt; |
| 183 | |
| 184 | /* Map variables to fields in a structure that allows communication |
| 185 | between sending and receiving threads. */ |
| 186 | splay_tree field_map; |
| 187 | tree record_type; |
| 188 | tree sender_decl; |
| 189 | tree receiver_decl; |
| 190 | |
| 191 | /* These are used just by task contexts, if task firstprivate fn is |
| 192 | needed. srecord_type is used to communicate from the thread |
| 193 | that encountered the task construct to task firstprivate fn, |
| 194 | record_type is allocated by GOMP_task, initialized by task firstprivate |
| 195 | fn and passed to the task body fn. */ |
| 196 | splay_tree sfield_map; |
| 197 | tree srecord_type; |
| 198 | |
| 199 | /* A chain of variables to add to the top-level block surrounding the |
| 200 | construct. In the case of a parallel, this is in the child function. */ |
| 201 | tree block_vars; |
| 202 | |
| 203 | /* A map of reduction pointer variables. For accelerators, each |
| 204 | reduction variable is replaced with an array. Each thread, in turn, |
| 205 | is assigned to a slot on that array. */ |
| 206 | splay_tree reduction_map; |
| 207 | |
| 208 | /* Label to which GOMP_cancel{,llation_point} and explicit and implicit |
| 209 | barriers should jump to during omplower pass. */ |
| 210 | tree cancel_label; |
| 211 | |
| 212 | /* What to do with variables with implicitly determined sharing |
| 213 | attributes. */ |
| 214 | enum omp_clause_default_kind default_kind; |
| 215 | |
| 216 | /* Nesting depth of this context. Used to beautify error messages re |
| 217 | invalid gotos. The outermost ctx is depth 1, with depth 0 being |
| 218 | reserved for the main body of the function. */ |
| 219 | int depth; |
| 220 | |
| 221 | /* True if this parallel directive is nested within another. */ |
| 222 | bool is_nested; |
| 223 | |
| 224 | /* True if this construct can be cancelled. */ |
| 225 | bool cancellable; |
| 226 | |
| 227 | /* For OpenACC loops, a mask of gang, worker and vector used at |
| 228 | levels below this one. */ |
| 229 | int gwv_below; |
| 230 | /* For OpenACC loops, a mask of gang, worker and vector used at |
| 231 | this level and above. For parallel and kernels clauses, a mask |
| 232 | indicating which of num_gangs/num_workers/num_vectors was used. */ |
| 233 | int gwv_this; |
| 234 | } omp_context; |
| 235 | |
| 236 | /* A structure holding the elements of: |
| 237 | for (V = N1; V cond N2; V += STEP) [...] */ |
| 238 | |
| 239 | struct omp_for_data_loop |
| 240 | { |
| 241 | tree v, n1, n2, step; |
| 242 | enum tree_code cond_code; |
| 243 | }; |
| 244 | |
| 245 | /* A structure describing the main elements of a parallel loop. */ |
| 246 | |
| 247 | struct omp_for_data |
| 248 | { |
| 249 | struct omp_for_data_loop loop; |
| 250 | tree chunk_size; |
| 251 | gomp_for *for_stmt; |
| 252 | tree pre, iter_type; |
| 253 | int collapse; |
| 254 | bool have_nowait, have_ordered; |
| 255 | enum omp_clause_schedule_kind sched_kind; |
| 256 | struct omp_for_data_loop *loops; |
| 257 | }; |
| 258 | |
| 259 | |
| 260 | static splay_tree all_contexts; |
| 261 | static int taskreg_nesting_level; |
| 262 | static int target_nesting_level; |
| 263 | static struct omp_region *root_omp_region; |
| 264 | static bitmap task_shared_vars; |
| 265 | static vec<omp_context *> taskreg_contexts; |
| 266 | |
| 267 | static void scan_omp (gimple_seq *, omp_context *); |
| 268 | static tree scan_omp_1_op (tree *, int *, void *); |
| 269 | |
| 270 | #define WALK_SUBSTMTS \ |
| 271 | case GIMPLE_BIND: \ |
| 272 | case GIMPLE_TRY: \ |
| 273 | case GIMPLE_CATCH: \ |
| 274 | case GIMPLE_EH_FILTER: \ |
| 275 | case GIMPLE_TRANSACTION: \ |
| 276 | /* The sub-statements for these should be walked. */ \ |
| 277 | *handled_ops_p = false; \ |
| 278 | break; |
| 279 | |
| 280 | /* Helper function to get the name of the array containing the partial |
| 281 | reductions for OpenACC reductions. */ |
| 282 | static const char * |
| 283 | oacc_get_reduction_array_id (tree node) |
| 284 | { |
| 285 | const char *id = IDENTIFIER_POINTER (DECL_NAME (node)); |
| 286 | int len = strlen ("OACC") + strlen (id); |
| 287 | char *temp_name = XALLOCAVEC (char, len + 1); |
| 288 | snprintf (temp_name, len + 1, "OACC%s", id); |
| 289 | return IDENTIFIER_POINTER (get_identifier (temp_name)); |
| 290 | } |
| 291 | |
| 292 | /* Determine the number of threads OpenACC threads used to determine the |
| 293 | size of the array of partial reductions. Currently, this is num_gangs |
| 294 | * vector_length. This value may be different than GOACC_GET_NUM_THREADS, |
| 295 | because it is independed of the device used. */ |
| 296 | |
| 297 | static tree |
| 298 | oacc_max_threads (omp_context *ctx) |
| 299 | { |
| 300 | tree nthreads, vector_length, gangs, clauses; |
| 301 | |
| 302 | gangs = fold_convert (sizetype, integer_one_node); |
| 303 | vector_length = gangs; |
| 304 | |
| 305 | /* The reduction clause may be nested inside a loop directive. |
| 306 | Scan for the innermost vector_length clause. */ |
| 307 | for (omp_context *oc = ctx; oc; oc = oc->outer) |
| 308 | { |
| 309 | if (gimple_code (oc->stmt) != GIMPLE_OMP_TARGET |
| 310 | || (gimple_omp_target_kind (oc->stmt) |
| 311 | != GF_OMP_TARGET_KIND_OACC_PARALLEL)) |
| 312 | continue; |
| 313 | |
| 314 | clauses = gimple_omp_target_clauses (oc->stmt); |
| 315 | |
| 316 | vector_length = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH); |
| 317 | if (vector_length) |
| 318 | vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length), |
| 319 | sizetype, |
| 320 | OMP_CLAUSE_VECTOR_LENGTH_EXPR |
| 321 | (vector_length)); |
| 322 | else |
| 323 | vector_length = fold_convert (sizetype, integer_one_node); |
| 324 | |
| 325 | gangs = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS); |
| 326 | if (gangs) |
| 327 | gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (gangs), sizetype, |
| 328 | OMP_CLAUSE_NUM_GANGS_EXPR (gangs)); |
| 329 | else |
| 330 | gangs = fold_convert (sizetype, integer_one_node); |
| 331 | |
| 332 | break; |
| 333 | } |
| 334 | |
| 335 | nthreads = fold_build2 (MULT_EXPR, sizetype, gangs, vector_length); |
| 336 | |
| 337 | return nthreads; |
| 338 | } |
| 339 | |
| 340 | /* Holds offload tables with decls. */ |
| 341 | vec<tree, va_gc> *offload_funcs, *offload_vars; |
| 342 | |
| 343 | /* Convenience function for calling scan_omp_1_op on tree operands. */ |
| 344 | |
| 345 | static inline tree |
| 346 | scan_omp_op (tree *tp, omp_context *ctx) |
| 347 | { |
| 348 | struct walk_stmt_info wi; |
| 349 | |
| 350 | memset (&wi, 0, sizeof (wi)); |
| 351 | wi.info = ctx; |
| 352 | wi.want_locations = true; |
| 353 | |
| 354 | return walk_tree (tp, scan_omp_1_op, &wi, NULL); |
| 355 | } |
| 356 | |
| 357 | static void lower_omp (gimple_seq *, omp_context *); |
| 358 | static tree lookup_decl_in_outer_ctx (tree, omp_context *); |
| 359 | static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *); |
| 360 | |
| 361 | /* Find an OMP clause of type KIND within CLAUSES. */ |
| 362 | |
| 363 | tree |
| 364 | find_omp_clause (tree clauses, enum omp_clause_code kind) |
| 365 | { |
| 366 | for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses)) |
| 367 | if (OMP_CLAUSE_CODE (clauses) == kind) |
| 368 | return clauses; |
| 369 | |
| 370 | return NULL_TREE; |
| 371 | } |
| 372 | |
| 373 | /* Return true if CTX is for an omp parallel. */ |
| 374 | |
| 375 | static inline bool |
| 376 | is_parallel_ctx (omp_context *ctx) |
| 377 | { |
| 378 | return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL; |
| 379 | } |
| 380 | |
| 381 | |
| 382 | /* Return true if CTX is for an omp task. */ |
| 383 | |
| 384 | static inline bool |
| 385 | is_task_ctx (omp_context *ctx) |
| 386 | { |
| 387 | return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK; |
| 388 | } |
| 389 | |
| 390 | |
| 391 | /* Return true if CTX is for an omp parallel or omp task. */ |
| 392 | |
| 393 | static inline bool |
| 394 | is_taskreg_ctx (omp_context *ctx) |
| 395 | { |
| 396 | return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL |
| 397 | || gimple_code (ctx->stmt) == GIMPLE_OMP_TASK; |
| 398 | } |
| 399 | |
| 400 | |
| 401 | /* Return true if REGION is a combined parallel+workshare region. */ |
| 402 | |
| 403 | static inline bool |
| 404 | is_combined_parallel (struct omp_region *region) |
| 405 | { |
| 406 | return region->is_combined_parallel; |
| 407 | } |
| 408 | |
| 409 | |
| 410 | /* Extract the header elements of parallel loop FOR_STMT and store |
| 411 | them into *FD. */ |
| 412 | |
| 413 | static void |
| 414 | extract_omp_for_data (gomp_for *for_stmt, struct omp_for_data *fd, |
| 415 | struct omp_for_data_loop *loops) |
| 416 | { |
| 417 | tree t, var, *collapse_iter, *collapse_count; |
| 418 | tree count = NULL_TREE, iter_type = long_integer_type_node; |
| 419 | struct omp_for_data_loop *loop; |
| 420 | int i; |
| 421 | struct omp_for_data_loop dummy_loop; |
| 422 | location_t loc = gimple_location (for_stmt); |
| 423 | bool simd = gimple_omp_for_kind (for_stmt) & GF_OMP_FOR_SIMD; |
| 424 | bool distribute = gimple_omp_for_kind (for_stmt) |
| 425 | == GF_OMP_FOR_KIND_DISTRIBUTE; |
| 426 | |
| 427 | fd->for_stmt = for_stmt; |
| 428 | fd->pre = NULL; |
| 429 | fd->collapse = gimple_omp_for_collapse (for_stmt); |
| 430 | if (fd->collapse > 1) |
| 431 | fd->loops = loops; |
| 432 | else |
| 433 | fd->loops = &fd->loop; |
| 434 | |
| 435 | fd->have_nowait = distribute || simd; |
| 436 | fd->have_ordered = false; |
| 437 | fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC; |
| 438 | fd->chunk_size = NULL_TREE; |
| 439 | if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_CILKFOR) |
| 440 | fd->sched_kind = OMP_CLAUSE_SCHEDULE_CILKFOR; |
| 441 | collapse_iter = NULL; |
| 442 | collapse_count = NULL; |
| 443 | |
| 444 | for (t = gimple_omp_for_clauses (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t)) |
| 445 | switch (OMP_CLAUSE_CODE (t)) |
| 446 | { |
| 447 | case OMP_CLAUSE_NOWAIT: |
| 448 | fd->have_nowait = true; |
| 449 | break; |
| 450 | case OMP_CLAUSE_ORDERED: |
| 451 | fd->have_ordered = true; |
| 452 | break; |
| 453 | case OMP_CLAUSE_SCHEDULE: |
| 454 | gcc_assert (!distribute); |
| 455 | fd->sched_kind = OMP_CLAUSE_SCHEDULE_KIND (t); |
| 456 | fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t); |
| 457 | break; |
| 458 | case OMP_CLAUSE_DIST_SCHEDULE: |
| 459 | gcc_assert (distribute); |
| 460 | fd->chunk_size = OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t); |
| 461 | break; |
| 462 | case OMP_CLAUSE_COLLAPSE: |
| 463 | if (fd->collapse > 1) |
| 464 | { |
| 465 | collapse_iter = &OMP_CLAUSE_COLLAPSE_ITERVAR (t); |
| 466 | collapse_count = &OMP_CLAUSE_COLLAPSE_COUNT (t); |
| 467 | } |
| 468 | break; |
| 469 | default: |
| 470 | break; |
| 471 | } |
| 472 | |
| 473 | /* FIXME: for now map schedule(auto) to schedule(static). |
| 474 | There should be analysis to determine whether all iterations |
| 475 | are approximately the same amount of work (then schedule(static) |
| 476 | is best) or if it varies (then schedule(dynamic,N) is better). */ |
| 477 | if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_AUTO) |
| 478 | { |
| 479 | fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC; |
| 480 | gcc_assert (fd->chunk_size == NULL); |
| 481 | } |
| 482 | gcc_assert (fd->collapse == 1 || collapse_iter != NULL); |
| 483 | if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME) |
| 484 | gcc_assert (fd->chunk_size == NULL); |
| 485 | else if (fd->chunk_size == NULL) |
| 486 | { |
| 487 | /* We only need to compute a default chunk size for ordered |
| 488 | static loops and dynamic loops. */ |
| 489 | if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC |
| 490 | || fd->have_ordered) |
| 491 | fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC) |
| 492 | ? integer_zero_node : integer_one_node; |
| 493 | } |
| 494 | |
| 495 | for (i = 0; i < fd->collapse; i++) |
| 496 | { |
| 497 | if (fd->collapse == 1) |
| 498 | loop = &fd->loop; |
| 499 | else if (loops != NULL) |
| 500 | loop = loops + i; |
| 501 | else |
| 502 | loop = &dummy_loop; |
| 503 | |
| 504 | loop->v = gimple_omp_for_index (for_stmt, i); |
| 505 | gcc_assert (SSA_VAR_P (loop->v)); |
| 506 | gcc_assert (TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE |
| 507 | || TREE_CODE (TREE_TYPE (loop->v)) == POINTER_TYPE); |
| 508 | var = TREE_CODE (loop->v) == SSA_NAME ? SSA_NAME_VAR (loop->v) : loop->v; |
| 509 | loop->n1 = gimple_omp_for_initial (for_stmt, i); |
| 510 | |
| 511 | loop->cond_code = gimple_omp_for_cond (for_stmt, i); |
| 512 | loop->n2 = gimple_omp_for_final (for_stmt, i); |
| 513 | switch (loop->cond_code) |
| 514 | { |
| 515 | case LT_EXPR: |
| 516 | case GT_EXPR: |
| 517 | break; |
| 518 | case NE_EXPR: |
| 519 | gcc_assert (gimple_omp_for_kind (for_stmt) |
| 520 | == GF_OMP_FOR_KIND_CILKSIMD |
| 521 | || (gimple_omp_for_kind (for_stmt) |
| 522 | == GF_OMP_FOR_KIND_CILKFOR)); |
| 523 | break; |
| 524 | case LE_EXPR: |
| 525 | if (POINTER_TYPE_P (TREE_TYPE (loop->n2))) |
| 526 | loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, 1); |
| 527 | else |
| 528 | loop->n2 = fold_build2_loc (loc, |
| 529 | PLUS_EXPR, TREE_TYPE (loop->n2), loop->n2, |
| 530 | build_int_cst (TREE_TYPE (loop->n2), 1)); |
| 531 | loop->cond_code = LT_EXPR; |
| 532 | break; |
| 533 | case GE_EXPR: |
| 534 | if (POINTER_TYPE_P (TREE_TYPE (loop->n2))) |
| 535 | loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, -1); |
| 536 | else |
| 537 | loop->n2 = fold_build2_loc (loc, |
| 538 | MINUS_EXPR, TREE_TYPE (loop->n2), loop->n2, |
| 539 | build_int_cst (TREE_TYPE (loop->n2), 1)); |
| 540 | loop->cond_code = GT_EXPR; |
| 541 | break; |
| 542 | default: |
| 543 | gcc_unreachable (); |
| 544 | } |
| 545 | |
| 546 | t = gimple_omp_for_incr (for_stmt, i); |
| 547 | gcc_assert (TREE_OPERAND (t, 0) == var); |
| 548 | switch (TREE_CODE (t)) |
| 549 | { |
| 550 | case PLUS_EXPR: |
| 551 | loop->step = TREE_OPERAND (t, 1); |
| 552 | break; |
| 553 | case POINTER_PLUS_EXPR: |
| 554 | loop->step = fold_convert (ssizetype, TREE_OPERAND (t, 1)); |
| 555 | break; |
| 556 | case MINUS_EXPR: |
| 557 | loop->step = TREE_OPERAND (t, 1); |
| 558 | loop->step = fold_build1_loc (loc, |
| 559 | NEGATE_EXPR, TREE_TYPE (loop->step), |
| 560 | loop->step); |
| 561 | break; |
| 562 | default: |
| 563 | gcc_unreachable (); |
| 564 | } |
| 565 | |
| 566 | if (simd |
| 567 | || (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC |
| 568 | && !fd->have_ordered)) |
| 569 | { |
| 570 | if (fd->collapse == 1) |
| 571 | iter_type = TREE_TYPE (loop->v); |
| 572 | else if (i == 0 |
| 573 | || TYPE_PRECISION (iter_type) |
| 574 | < TYPE_PRECISION (TREE_TYPE (loop->v))) |
| 575 | iter_type |
| 576 | = build_nonstandard_integer_type |
| 577 | (TYPE_PRECISION (TREE_TYPE (loop->v)), 1); |
| 578 | } |
| 579 | else if (iter_type != long_long_unsigned_type_node) |
| 580 | { |
| 581 | if (POINTER_TYPE_P (TREE_TYPE (loop->v))) |
| 582 | iter_type = long_long_unsigned_type_node; |
| 583 | else if (TYPE_UNSIGNED (TREE_TYPE (loop->v)) |
| 584 | && TYPE_PRECISION (TREE_TYPE (loop->v)) |
| 585 | >= TYPE_PRECISION (iter_type)) |
| 586 | { |
| 587 | tree n; |
| 588 | |
| 589 | if (loop->cond_code == LT_EXPR) |
| 590 | n = fold_build2_loc (loc, |
| 591 | PLUS_EXPR, TREE_TYPE (loop->v), |
| 592 | loop->n2, loop->step); |
| 593 | else |
| 594 | n = loop->n1; |
| 595 | if (TREE_CODE (n) != INTEGER_CST |
| 596 | || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type), n)) |
| 597 | iter_type = long_long_unsigned_type_node; |
| 598 | } |
| 599 | else if (TYPE_PRECISION (TREE_TYPE (loop->v)) |
| 600 | > TYPE_PRECISION (iter_type)) |
| 601 | { |
| 602 | tree n1, n2; |
| 603 | |
| 604 | if (loop->cond_code == LT_EXPR) |
| 605 | { |
| 606 | n1 = loop->n1; |
| 607 | n2 = fold_build2_loc (loc, |
| 608 | PLUS_EXPR, TREE_TYPE (loop->v), |
| 609 | loop->n2, loop->step); |
| 610 | } |
| 611 | else |
| 612 | { |
| 613 | n1 = fold_build2_loc (loc, |
| 614 | MINUS_EXPR, TREE_TYPE (loop->v), |
| 615 | loop->n2, loop->step); |
| 616 | n2 = loop->n1; |
| 617 | } |
| 618 | if (TREE_CODE (n1) != INTEGER_CST |
| 619 | || TREE_CODE (n2) != INTEGER_CST |
| 620 | || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type), n1) |
| 621 | || !tree_int_cst_lt (n2, TYPE_MAX_VALUE (iter_type))) |
| 622 | iter_type = long_long_unsigned_type_node; |
| 623 | } |
| 624 | } |
| 625 | |
| 626 | if (collapse_count && *collapse_count == NULL) |
| 627 | { |
| 628 | t = fold_binary (loop->cond_code, boolean_type_node, |
| 629 | fold_convert (TREE_TYPE (loop->v), loop->n1), |
| 630 | fold_convert (TREE_TYPE (loop->v), loop->n2)); |
| 631 | if (t && integer_zerop (t)) |
| 632 | count = build_zero_cst (long_long_unsigned_type_node); |
| 633 | else if ((i == 0 || count != NULL_TREE) |
| 634 | && TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE |
| 635 | && TREE_CONSTANT (loop->n1) |
| 636 | && TREE_CONSTANT (loop->n2) |
| 637 | && TREE_CODE (loop->step) == INTEGER_CST) |
| 638 | { |
| 639 | tree itype = TREE_TYPE (loop->v); |
| 640 | |
| 641 | if (POINTER_TYPE_P (itype)) |
| 642 | itype = signed_type_for (itype); |
| 643 | t = build_int_cst (itype, (loop->cond_code == LT_EXPR ? -1 : 1)); |
| 644 | t = fold_build2_loc (loc, |
| 645 | PLUS_EXPR, itype, |
| 646 | fold_convert_loc (loc, itype, loop->step), t); |
| 647 | t = fold_build2_loc (loc, PLUS_EXPR, itype, t, |
| 648 | fold_convert_loc (loc, itype, loop->n2)); |
| 649 | t = fold_build2_loc (loc, MINUS_EXPR, itype, t, |
| 650 | fold_convert_loc (loc, itype, loop->n1)); |
| 651 | if (TYPE_UNSIGNED (itype) && loop->cond_code == GT_EXPR) |
| 652 | t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, |
| 653 | fold_build1_loc (loc, NEGATE_EXPR, itype, t), |
| 654 | fold_build1_loc (loc, NEGATE_EXPR, itype, |
| 655 | fold_convert_loc (loc, itype, |
| 656 | loop->step))); |
| 657 | else |
| 658 | t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, t, |
| 659 | fold_convert_loc (loc, itype, loop->step)); |
| 660 | t = fold_convert_loc (loc, long_long_unsigned_type_node, t); |
| 661 | if (count != NULL_TREE) |
| 662 | count = fold_build2_loc (loc, |
| 663 | MULT_EXPR, long_long_unsigned_type_node, |
| 664 | count, t); |
| 665 | else |
| 666 | count = t; |
| 667 | if (TREE_CODE (count) != INTEGER_CST) |
| 668 | count = NULL_TREE; |
| 669 | } |
| 670 | else if (count && !integer_zerop (count)) |
| 671 | count = NULL_TREE; |
| 672 | } |
| 673 | } |
| 674 | |
| 675 | if (count |
| 676 | && !simd |
| 677 | && (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC |
| 678 | || fd->have_ordered)) |
| 679 | { |
| 680 | if (!tree_int_cst_lt (count, TYPE_MAX_VALUE (long_integer_type_node))) |
| 681 | iter_type = long_long_unsigned_type_node; |
| 682 | else |
| 683 | iter_type = long_integer_type_node; |
| 684 | } |
| 685 | else if (collapse_iter && *collapse_iter != NULL) |
| 686 | iter_type = TREE_TYPE (*collapse_iter); |
| 687 | fd->iter_type = iter_type; |
| 688 | if (collapse_iter && *collapse_iter == NULL) |
| 689 | *collapse_iter = create_tmp_var (iter_type, ".iter"); |
| 690 | if (collapse_count && *collapse_count == NULL) |
| 691 | { |
| 692 | if (count) |
| 693 | *collapse_count = fold_convert_loc (loc, iter_type, count); |
| 694 | else |
| 695 | *collapse_count = create_tmp_var (iter_type, ".count"); |
| 696 | } |
| 697 | |
| 698 | if (fd->collapse > 1) |
| 699 | { |
| 700 | fd->loop.v = *collapse_iter; |
| 701 | fd->loop.n1 = build_int_cst (TREE_TYPE (fd->loop.v), 0); |
| 702 | fd->loop.n2 = *collapse_count; |
| 703 | fd->loop.step = build_int_cst (TREE_TYPE (fd->loop.v), 1); |
| 704 | fd->loop.cond_code = LT_EXPR; |
| 705 | } |
| 706 | |
| 707 | /* For OpenACC loops, force a chunk size of one, as this avoids the default |
| 708 | scheduling where several subsequent iterations are being executed by the |
| 709 | same thread. */ |
| 710 | if (gimple_omp_for_kind (for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP) |
| 711 | { |
| 712 | gcc_assert (fd->chunk_size == NULL_TREE); |
| 713 | fd->chunk_size = build_int_cst (TREE_TYPE (fd->loop.v), 1); |
| 714 | } |
| 715 | } |
| 716 | |
| 717 | |
| 718 | /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB |
| 719 | is the immediate dominator of PAR_ENTRY_BB, return true if there |
| 720 | are no data dependencies that would prevent expanding the parallel |
| 721 | directive at PAR_ENTRY_BB as a combined parallel+workshare region. |
| 722 | |
| 723 | When expanding a combined parallel+workshare region, the call to |
| 724 | the child function may need additional arguments in the case of |
| 725 | GIMPLE_OMP_FOR regions. In some cases, these arguments are |
| 726 | computed out of variables passed in from the parent to the child |
| 727 | via 'struct .omp_data_s'. For instance: |
| 728 | |
| 729 | #pragma omp parallel for schedule (guided, i * 4) |
| 730 | for (j ...) |
| 731 | |
| 732 | Is lowered into: |
| 733 | |
| 734 | # BLOCK 2 (PAR_ENTRY_BB) |
| 735 | .omp_data_o.i = i; |
| 736 | #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598) |
| 737 | |
| 738 | # BLOCK 3 (WS_ENTRY_BB) |
| 739 | .omp_data_i = &.omp_data_o; |
| 740 | D.1667 = .omp_data_i->i; |
| 741 | D.1598 = D.1667 * 4; |
| 742 | #pragma omp for schedule (guided, D.1598) |
| 743 | |
| 744 | When we outline the parallel region, the call to the child function |
| 745 | 'bar.omp_fn.0' will need the value D.1598 in its argument list, but |
| 746 | that value is computed *after* the call site. So, in principle we |
| 747 | cannot do the transformation. |
| 748 | |
| 749 | To see whether the code in WS_ENTRY_BB blocks the combined |
| 750 | parallel+workshare call, we collect all the variables used in the |
| 751 | GIMPLE_OMP_FOR header check whether they appear on the LHS of any |
| 752 | statement in WS_ENTRY_BB. If so, then we cannot emit the combined |
| 753 | call. |
| 754 | |
| 755 | FIXME. If we had the SSA form built at this point, we could merely |
| 756 | hoist the code in block 3 into block 2 and be done with it. But at |
| 757 | this point we don't have dataflow information and though we could |
| 758 | hack something up here, it is really not worth the aggravation. */ |
| 759 | |
| 760 | static bool |
| 761 | workshare_safe_to_combine_p (basic_block ws_entry_bb) |
| 762 | { |
| 763 | struct omp_for_data fd; |
| 764 | gimple ws_stmt = last_stmt (ws_entry_bb); |
| 765 | |
| 766 | if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS) |
| 767 | return true; |
| 768 | |
| 769 | gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR); |
| 770 | |
| 771 | extract_omp_for_data (as_a <gomp_for *> (ws_stmt), &fd, NULL); |
| 772 | |
| 773 | if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST) |
| 774 | return false; |
| 775 | if (fd.iter_type != long_integer_type_node) |
| 776 | return false; |
| 777 | |
| 778 | /* FIXME. We give up too easily here. If any of these arguments |
| 779 | are not constants, they will likely involve variables that have |
| 780 | been mapped into fields of .omp_data_s for sharing with the child |
| 781 | function. With appropriate data flow, it would be possible to |
| 782 | see through this. */ |
| 783 | if (!is_gimple_min_invariant (fd.loop.n1) |
| 784 | || !is_gimple_min_invariant (fd.loop.n2) |
| 785 | || !is_gimple_min_invariant (fd.loop.step) |
| 786 | || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size))) |
| 787 | return false; |
| 788 | |
| 789 | return true; |
| 790 | } |
| 791 | |
| 792 | |
| 793 | /* Collect additional arguments needed to emit a combined |
| 794 | parallel+workshare call. WS_STMT is the workshare directive being |
| 795 | expanded. */ |
| 796 | |
| 797 | static vec<tree, va_gc> * |
| 798 | get_ws_args_for (gimple par_stmt, gimple ws_stmt) |
| 799 | { |
| 800 | tree t; |
| 801 | location_t loc = gimple_location (ws_stmt); |
| 802 | vec<tree, va_gc> *ws_args; |
| 803 | |
| 804 | if (gomp_for *for_stmt = dyn_cast <gomp_for *> (ws_stmt)) |
| 805 | { |
| 806 | struct omp_for_data fd; |
| 807 | tree n1, n2; |
| 808 | |
| 809 | extract_omp_for_data (for_stmt, &fd, NULL); |
| 810 | n1 = fd.loop.n1; |
| 811 | n2 = fd.loop.n2; |
| 812 | |
| 813 | if (gimple_omp_for_combined_into_p (for_stmt)) |
| 814 | { |
| 815 | tree innerc |
| 816 | = find_omp_clause (gimple_omp_parallel_clauses (par_stmt), |
| 817 | OMP_CLAUSE__LOOPTEMP_); |
| 818 | gcc_assert (innerc); |
| 819 | n1 = OMP_CLAUSE_DECL (innerc); |
| 820 | innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), |
| 821 | OMP_CLAUSE__LOOPTEMP_); |
| 822 | gcc_assert (innerc); |
| 823 | n2 = OMP_CLAUSE_DECL (innerc); |
| 824 | } |
| 825 | |
| 826 | vec_alloc (ws_args, 3 + (fd.chunk_size != 0)); |
| 827 | |
| 828 | t = fold_convert_loc (loc, long_integer_type_node, n1); |
| 829 | ws_args->quick_push (t); |
| 830 | |
| 831 | t = fold_convert_loc (loc, long_integer_type_node, n2); |
| 832 | ws_args->quick_push (t); |
| 833 | |
| 834 | t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step); |
| 835 | ws_args->quick_push (t); |
| 836 | |
| 837 | if (fd.chunk_size) |
| 838 | { |
| 839 | t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size); |
| 840 | ws_args->quick_push (t); |
| 841 | } |
| 842 | |
| 843 | return ws_args; |
| 844 | } |
| 845 | else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS) |
| 846 | { |
| 847 | /* Number of sections is equal to the number of edges from the |
| 848 | GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to |
| 849 | the exit of the sections region. */ |
| 850 | basic_block bb = single_succ (gimple_bb (ws_stmt)); |
| 851 | t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1); |
| 852 | vec_alloc (ws_args, 1); |
| 853 | ws_args->quick_push (t); |
| 854 | return ws_args; |
| 855 | } |
| 856 | |
| 857 | gcc_unreachable (); |
| 858 | } |
| 859 | |
| 860 | |
| 861 | /* Discover whether REGION is a combined parallel+workshare region. */ |
| 862 | |
| 863 | static void |
| 864 | determine_parallel_type (struct omp_region *region) |
| 865 | { |
| 866 | basic_block par_entry_bb, par_exit_bb; |
| 867 | basic_block ws_entry_bb, ws_exit_bb; |
| 868 | |
| 869 | if (region == NULL || region->inner == NULL |
| 870 | || region->exit == NULL || region->inner->exit == NULL |
| 871 | || region->inner->cont == NULL) |
| 872 | return; |
| 873 | |
| 874 | /* We only support parallel+for and parallel+sections. */ |
| 875 | if (region->type != GIMPLE_OMP_PARALLEL |
| 876 | || (region->inner->type != GIMPLE_OMP_FOR |
| 877 | && region->inner->type != GIMPLE_OMP_SECTIONS)) |
| 878 | return; |
| 879 | |
| 880 | /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and |
| 881 | WS_EXIT_BB -> PAR_EXIT_BB. */ |
| 882 | par_entry_bb = region->entry; |
| 883 | par_exit_bb = region->exit; |
| 884 | ws_entry_bb = region->inner->entry; |
| 885 | ws_exit_bb = region->inner->exit; |
| 886 | |
| 887 | if (single_succ (par_entry_bb) == ws_entry_bb |
| 888 | && single_succ (ws_exit_bb) == par_exit_bb |
| 889 | && workshare_safe_to_combine_p (ws_entry_bb) |
| 890 | && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb)) |
| 891 | || (last_and_only_stmt (ws_entry_bb) |
| 892 | && last_and_only_stmt (par_exit_bb)))) |
| 893 | { |
| 894 | gimple par_stmt = last_stmt (par_entry_bb); |
| 895 | gimple ws_stmt = last_stmt (ws_entry_bb); |
| 896 | |
| 897 | if (region->inner->type == GIMPLE_OMP_FOR) |
| 898 | { |
| 899 | /* If this is a combined parallel loop, we need to determine |
| 900 | whether or not to use the combined library calls. There |
| 901 | are two cases where we do not apply the transformation: |
| 902 | static loops and any kind of ordered loop. In the first |
| 903 | case, we already open code the loop so there is no need |
| 904 | to do anything else. In the latter case, the combined |
| 905 | parallel loop call would still need extra synchronization |
| 906 | to implement ordered semantics, so there would not be any |
| 907 | gain in using the combined call. */ |
| 908 | tree clauses = gimple_omp_for_clauses (ws_stmt); |
| 909 | tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE); |
| 910 | if (c == NULL |
| 911 | || OMP_CLAUSE_SCHEDULE_KIND (c) == OMP_CLAUSE_SCHEDULE_STATIC |
| 912 | || find_omp_clause (clauses, OMP_CLAUSE_ORDERED)) |
| 913 | { |
| 914 | region->is_combined_parallel = false; |
| 915 | region->inner->is_combined_parallel = false; |
| 916 | return; |
| 917 | } |
| 918 | } |
| 919 | |
| 920 | region->is_combined_parallel = true; |
| 921 | region->inner->is_combined_parallel = true; |
| 922 | region->ws_args = get_ws_args_for (par_stmt, ws_stmt); |
| 923 | } |
| 924 | } |
| 925 | |
| 926 | |
| 927 | /* Return true if EXPR is variable sized. */ |
| 928 | |
| 929 | static inline bool |
| 930 | is_variable_sized (const_tree expr) |
| 931 | { |
| 932 | return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr))); |
| 933 | } |
| 934 | |
| 935 | /* Return true if DECL is a reference type. */ |
| 936 | |
| 937 | static inline bool |
| 938 | is_reference (tree decl) |
| 939 | { |
| 940 | return lang_hooks.decls.omp_privatize_by_reference (decl); |
| 941 | } |
| 942 | |
| 943 | /* Return the type of a decl. If the decl is reference type, |
| 944 | return its base type. */ |
| 945 | static inline tree |
| 946 | get_base_type (tree decl) |
| 947 | { |
| 948 | tree type = TREE_TYPE (decl); |
| 949 | if (is_reference (decl)) |
| 950 | type = TREE_TYPE (type); |
| 951 | return type; |
| 952 | } |
| 953 | |
| 954 | /* Lookup variables. The "maybe" form |
| 955 | allows for the variable form to not have been entered, otherwise we |
| 956 | assert that the variable must have been entered. */ |
| 957 | |
| 958 | static inline tree |
| 959 | lookup_decl (tree var, omp_context *ctx) |
| 960 | { |
| 961 | tree *n = ctx->cb.decl_map->get (var); |
| 962 | return *n; |
| 963 | } |
| 964 | |
| 965 | static inline tree |
| 966 | maybe_lookup_decl (const_tree var, omp_context *ctx) |
| 967 | { |
| 968 | tree *n = ctx->cb.decl_map->get (const_cast<tree> (var)); |
| 969 | return n ? *n : NULL_TREE; |
| 970 | } |
| 971 | |
| 972 | static inline tree |
| 973 | lookup_field (tree var, omp_context *ctx) |
| 974 | { |
| 975 | splay_tree_node n; |
| 976 | n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var); |
| 977 | return (tree) n->value; |
| 978 | } |
| 979 | |
| 980 | static inline tree |
| 981 | lookup_sfield (tree var, omp_context *ctx) |
| 982 | { |
| 983 | splay_tree_node n; |
| 984 | n = splay_tree_lookup (ctx->sfield_map |
| 985 | ? ctx->sfield_map : ctx->field_map, |
| 986 | (splay_tree_key) var); |
| 987 | return (tree) n->value; |
| 988 | } |
| 989 | |
| 990 | static inline tree |
| 991 | maybe_lookup_field (tree var, omp_context *ctx) |
| 992 | { |
| 993 | splay_tree_node n; |
| 994 | n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var); |
| 995 | return n ? (tree) n->value : NULL_TREE; |
| 996 | } |
| 997 | |
| 998 | static inline tree |
| 999 | lookup_oacc_reduction (const char *id, omp_context *ctx) |
| 1000 | { |
| 1001 | splay_tree_node n; |
| 1002 | n = splay_tree_lookup (ctx->reduction_map, (splay_tree_key) id); |
| 1003 | return (tree) n->value; |
| 1004 | } |
| 1005 | |
| 1006 | static inline tree |
| 1007 | maybe_lookup_oacc_reduction (tree var, omp_context *ctx) |
| 1008 | { |
| 1009 | splay_tree_node n = NULL; |
| 1010 | if (ctx->reduction_map) |
| 1011 | n = splay_tree_lookup (ctx->reduction_map, (splay_tree_key) var); |
| 1012 | return n ? (tree) n->value : NULL_TREE; |
| 1013 | } |
| 1014 | |
| 1015 | /* Return true if DECL should be copied by pointer. SHARED_CTX is |
| 1016 | the parallel context if DECL is to be shared. */ |
| 1017 | |
| 1018 | static bool |
| 1019 | use_pointer_for_field (tree decl, omp_context *shared_ctx) |
| 1020 | { |
| 1021 | if (AGGREGATE_TYPE_P (TREE_TYPE (decl))) |
| 1022 | return true; |
| 1023 | |
| 1024 | /* We can only use copy-in/copy-out semantics for shared variables |
| 1025 | when we know the value is not accessible from an outer scope. */ |
| 1026 | if (shared_ctx) |
| 1027 | { |
| 1028 | gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt)); |
| 1029 | |
| 1030 | /* ??? Trivially accessible from anywhere. But why would we even |
| 1031 | be passing an address in this case? Should we simply assert |
| 1032 | this to be false, or should we have a cleanup pass that removes |
| 1033 | these from the list of mappings? */ |
| 1034 | if (TREE_STATIC (decl) || DECL_EXTERNAL (decl)) |
| 1035 | return true; |
| 1036 | |
| 1037 | /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell |
| 1038 | without analyzing the expression whether or not its location |
| 1039 | is accessible to anyone else. In the case of nested parallel |
| 1040 | regions it certainly may be. */ |
| 1041 | if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl)) |
| 1042 | return true; |
| 1043 | |
| 1044 | /* Do not use copy-in/copy-out for variables that have their |
| 1045 | address taken. */ |
| 1046 | if (TREE_ADDRESSABLE (decl)) |
| 1047 | return true; |
| 1048 | |
| 1049 | /* lower_send_shared_vars only uses copy-in, but not copy-out |
| 1050 | for these. */ |
| 1051 | if (TREE_READONLY (decl) |
| 1052 | || ((TREE_CODE (decl) == RESULT_DECL |
| 1053 | || TREE_CODE (decl) == PARM_DECL) |
| 1054 | && DECL_BY_REFERENCE (decl))) |
| 1055 | return false; |
| 1056 | |
| 1057 | /* Disallow copy-in/out in nested parallel if |
| 1058 | decl is shared in outer parallel, otherwise |
| 1059 | each thread could store the shared variable |
| 1060 | in its own copy-in location, making the |
| 1061 | variable no longer really shared. */ |
| 1062 | if (shared_ctx->is_nested) |
| 1063 | { |
| 1064 | omp_context *up; |
| 1065 | |
| 1066 | for (up = shared_ctx->outer; up; up = up->outer) |
| 1067 | if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up)) |
| 1068 | break; |
| 1069 | |
| 1070 | if (up) |
| 1071 | { |
| 1072 | tree c; |
| 1073 | |
| 1074 | for (c = gimple_omp_taskreg_clauses (up->stmt); |
| 1075 | c; c = OMP_CLAUSE_CHAIN (c)) |
| 1076 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED |
| 1077 | && OMP_CLAUSE_DECL (c) == decl) |
| 1078 | break; |
| 1079 | |
| 1080 | if (c) |
| 1081 | goto maybe_mark_addressable_and_ret; |
| 1082 | } |
| 1083 | } |
| 1084 | |
| 1085 | /* For tasks avoid using copy-in/out. As tasks can be |
| 1086 | deferred or executed in different thread, when GOMP_task |
| 1087 | returns, the task hasn't necessarily terminated. */ |
| 1088 | if (is_task_ctx (shared_ctx)) |
| 1089 | { |
| 1090 | tree outer; |
| 1091 | maybe_mark_addressable_and_ret: |
| 1092 | outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx); |
| 1093 | if (is_gimple_reg (outer)) |
| 1094 | { |
| 1095 | /* Taking address of OUTER in lower_send_shared_vars |
| 1096 | might need regimplification of everything that uses the |
| 1097 | variable. */ |
| 1098 | if (!task_shared_vars) |
| 1099 | task_shared_vars = BITMAP_ALLOC (NULL); |
| 1100 | bitmap_set_bit (task_shared_vars, DECL_UID (outer)); |
| 1101 | TREE_ADDRESSABLE (outer) = 1; |
| 1102 | } |
| 1103 | return true; |
| 1104 | } |
| 1105 | } |
| 1106 | |
| 1107 | return false; |
| 1108 | } |
| 1109 | |
| 1110 | /* Construct a new automatic decl similar to VAR. */ |
| 1111 | |
| 1112 | static tree |
| 1113 | omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx) |
| 1114 | { |
| 1115 | tree copy = copy_var_decl (var, name, type); |
| 1116 | |
| 1117 | DECL_CONTEXT (copy) = current_function_decl; |
| 1118 | DECL_CHAIN (copy) = ctx->block_vars; |
| 1119 | ctx->block_vars = copy; |
| 1120 | |
| 1121 | return copy; |
| 1122 | } |
| 1123 | |
| 1124 | static tree |
| 1125 | omp_copy_decl_1 (tree var, omp_context *ctx) |
| 1126 | { |
| 1127 | return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx); |
| 1128 | } |
| 1129 | |
| 1130 | /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it |
| 1131 | as appropriate. */ |
| 1132 | static tree |
| 1133 | omp_build_component_ref (tree obj, tree field) |
| 1134 | { |
| 1135 | tree ret = build3 (COMPONENT_REF, TREE_TYPE (field), obj, field, NULL); |
| 1136 | if (TREE_THIS_VOLATILE (field)) |
| 1137 | TREE_THIS_VOLATILE (ret) |= 1; |
| 1138 | if (TREE_READONLY (field)) |
| 1139 | TREE_READONLY (ret) |= 1; |
| 1140 | return ret; |
| 1141 | } |
| 1142 | |
| 1143 | /* Build tree nodes to access the field for VAR on the receiver side. */ |
| 1144 | |
| 1145 | static tree |
| 1146 | build_receiver_ref (tree var, bool by_ref, omp_context *ctx) |
| 1147 | { |
| 1148 | tree x, field = lookup_field (var, ctx); |
| 1149 | |
| 1150 | /* If the receiver record type was remapped in the child function, |
| 1151 | remap the field into the new record type. */ |
| 1152 | x = maybe_lookup_field (field, ctx); |
| 1153 | if (x != NULL) |
| 1154 | field = x; |
| 1155 | |
| 1156 | x = build_simple_mem_ref (ctx->receiver_decl); |
| 1157 | x = omp_build_component_ref (x, field); |
| 1158 | if (by_ref) |
| 1159 | x = build_simple_mem_ref (x); |
| 1160 | |
| 1161 | return x; |
| 1162 | } |
| 1163 | |
| 1164 | /* Build tree nodes to access VAR in the scope outer to CTX. In the case |
| 1165 | of a parallel, this is a component reference; for workshare constructs |
| 1166 | this is some variable. */ |
| 1167 | |
| 1168 | static tree |
| 1169 | build_outer_var_ref (tree var, omp_context *ctx) |
| 1170 | { |
| 1171 | tree x; |
| 1172 | |
| 1173 | if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx))) |
| 1174 | x = var; |
| 1175 | else if (is_variable_sized (var)) |
| 1176 | { |
| 1177 | x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0); |
| 1178 | x = build_outer_var_ref (x, ctx); |
| 1179 | x = build_simple_mem_ref (x); |
| 1180 | } |
| 1181 | else if (is_taskreg_ctx (ctx)) |
| 1182 | { |
| 1183 | bool by_ref = use_pointer_for_field (var, NULL); |
| 1184 | x = build_receiver_ref (var, by_ref, ctx); |
| 1185 | } |
| 1186 | else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR |
| 1187 | && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) |
| 1188 | { |
| 1189 | /* #pragma omp simd isn't a worksharing construct, and can reference even |
| 1190 | private vars in its linear etc. clauses. */ |
| 1191 | x = NULL_TREE; |
| 1192 | if (ctx->outer && is_taskreg_ctx (ctx)) |
| 1193 | x = lookup_decl (var, ctx->outer); |
| 1194 | else if (ctx->outer) |
| 1195 | x = maybe_lookup_decl_in_outer_ctx (var, ctx); |
| 1196 | if (x == NULL_TREE) |
| 1197 | x = var; |
| 1198 | } |
| 1199 | else if (ctx->outer) |
| 1200 | x = lookup_decl (var, ctx->outer); |
| 1201 | else if (is_reference (var)) |
| 1202 | /* This can happen with orphaned constructs. If var is reference, it is |
| 1203 | possible it is shared and as such valid. */ |
| 1204 | x = var; |
| 1205 | else |
| 1206 | gcc_unreachable (); |
| 1207 | |
| 1208 | if (is_reference (var)) |
| 1209 | x = build_simple_mem_ref (x); |
| 1210 | |
| 1211 | return x; |
| 1212 | } |
| 1213 | |
| 1214 | /* Build tree nodes to access the field for VAR on the sender side. */ |
| 1215 | |
| 1216 | static tree |
| 1217 | build_sender_ref (tree var, omp_context *ctx) |
| 1218 | { |
| 1219 | tree field = lookup_sfield (var, ctx); |
| 1220 | return omp_build_component_ref (ctx->sender_decl, field); |
| 1221 | } |
| 1222 | |
| 1223 | /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */ |
| 1224 | |
| 1225 | static void |
| 1226 | install_var_field (tree var, bool by_ref, int mask, omp_context *ctx) |
| 1227 | { |
| 1228 | tree field, type, sfield = NULL_TREE; |
| 1229 | |
| 1230 | gcc_assert ((mask & 1) == 0 |
| 1231 | || !splay_tree_lookup (ctx->field_map, (splay_tree_key) var)); |
| 1232 | gcc_assert ((mask & 2) == 0 || !ctx->sfield_map |
| 1233 | || !splay_tree_lookup (ctx->sfield_map, (splay_tree_key) var)); |
| 1234 | gcc_assert ((mask & 3) == 3 |
| 1235 | || !is_gimple_omp_oacc (ctx->stmt)); |
| 1236 | |
| 1237 | type = TREE_TYPE (var); |
| 1238 | if (mask & 4) |
| 1239 | { |
| 1240 | gcc_assert (TREE_CODE (type) == ARRAY_TYPE); |
| 1241 | type = build_pointer_type (build_pointer_type (type)); |
| 1242 | } |
| 1243 | else if (by_ref) |
| 1244 | type = build_pointer_type (type); |
| 1245 | else if ((mask & 3) == 1 && is_reference (var)) |
| 1246 | type = TREE_TYPE (type); |
| 1247 | |
| 1248 | field = build_decl (DECL_SOURCE_LOCATION (var), |
| 1249 | FIELD_DECL, DECL_NAME (var), type); |
| 1250 | |
| 1251 | /* Remember what variable this field was created for. This does have a |
| 1252 | side effect of making dwarf2out ignore this member, so for helpful |
| 1253 | debugging we clear it later in delete_omp_context. */ |
| 1254 | DECL_ABSTRACT_ORIGIN (field) = var; |
| 1255 | if (type == TREE_TYPE (var)) |
| 1256 | { |
| 1257 | DECL_ALIGN (field) = DECL_ALIGN (var); |
| 1258 | DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var); |
| 1259 | TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var); |
| 1260 | } |
| 1261 | else |
| 1262 | DECL_ALIGN (field) = TYPE_ALIGN (type); |
| 1263 | |
| 1264 | if ((mask & 3) == 3) |
| 1265 | { |
| 1266 | insert_field_into_struct (ctx->record_type, field); |
| 1267 | if (ctx->srecord_type) |
| 1268 | { |
| 1269 | sfield = build_decl (DECL_SOURCE_LOCATION (var), |
| 1270 | FIELD_DECL, DECL_NAME (var), type); |
| 1271 | DECL_ABSTRACT_ORIGIN (sfield) = var; |
| 1272 | DECL_ALIGN (sfield) = DECL_ALIGN (field); |
| 1273 | DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field); |
| 1274 | TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field); |
| 1275 | insert_field_into_struct (ctx->srecord_type, sfield); |
| 1276 | } |
| 1277 | } |
| 1278 | else |
| 1279 | { |
| 1280 | if (ctx->srecord_type == NULL_TREE) |
| 1281 | { |
| 1282 | tree t; |
| 1283 | |
| 1284 | ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE); |
| 1285 | ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
| 1286 | for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t)) |
| 1287 | { |
| 1288 | sfield = build_decl (DECL_SOURCE_LOCATION (var), |
| 1289 | FIELD_DECL, DECL_NAME (t), TREE_TYPE (t)); |
| 1290 | DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t); |
| 1291 | insert_field_into_struct (ctx->srecord_type, sfield); |
| 1292 | splay_tree_insert (ctx->sfield_map, |
| 1293 | (splay_tree_key) DECL_ABSTRACT_ORIGIN (t), |
| 1294 | (splay_tree_value) sfield); |
| 1295 | } |
| 1296 | } |
| 1297 | sfield = field; |
| 1298 | insert_field_into_struct ((mask & 1) ? ctx->record_type |
| 1299 | : ctx->srecord_type, field); |
| 1300 | } |
| 1301 | |
| 1302 | if (mask & 1) |
| 1303 | splay_tree_insert (ctx->field_map, (splay_tree_key) var, |
| 1304 | (splay_tree_value) field); |
| 1305 | if ((mask & 2) && ctx->sfield_map) |
| 1306 | splay_tree_insert (ctx->sfield_map, (splay_tree_key) var, |
| 1307 | (splay_tree_value) sfield); |
| 1308 | } |
| 1309 | |
| 1310 | static tree |
| 1311 | install_var_local (tree var, omp_context *ctx) |
| 1312 | { |
| 1313 | tree new_var = omp_copy_decl_1 (var, ctx); |
| 1314 | insert_decl_map (&ctx->cb, var, new_var); |
| 1315 | return new_var; |
| 1316 | } |
| 1317 | |
| 1318 | /* Adjust the replacement for DECL in CTX for the new context. This means |
| 1319 | copying the DECL_VALUE_EXPR, and fixing up the type. */ |
| 1320 | |
| 1321 | static void |
| 1322 | fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug) |
| 1323 | { |
| 1324 | tree new_decl, size; |
| 1325 | |
| 1326 | new_decl = lookup_decl (decl, ctx); |
| 1327 | |
| 1328 | TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb); |
| 1329 | |
| 1330 | if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug) |
| 1331 | && DECL_HAS_VALUE_EXPR_P (decl)) |
| 1332 | { |
| 1333 | tree ve = DECL_VALUE_EXPR (decl); |
| 1334 | walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL); |
| 1335 | SET_DECL_VALUE_EXPR (new_decl, ve); |
| 1336 | DECL_HAS_VALUE_EXPR_P (new_decl) = 1; |
| 1337 | } |
| 1338 | |
| 1339 | if (!TREE_CONSTANT (DECL_SIZE (new_decl))) |
| 1340 | { |
| 1341 | size = remap_decl (DECL_SIZE (decl), &ctx->cb); |
| 1342 | if (size == error_mark_node) |
| 1343 | size = TYPE_SIZE (TREE_TYPE (new_decl)); |
| 1344 | DECL_SIZE (new_decl) = size; |
| 1345 | |
| 1346 | size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb); |
| 1347 | if (size == error_mark_node) |
| 1348 | size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl)); |
| 1349 | DECL_SIZE_UNIT (new_decl) = size; |
| 1350 | } |
| 1351 | } |
| 1352 | |
| 1353 | /* The callback for remap_decl. Search all containing contexts for a |
| 1354 | mapping of the variable; this avoids having to duplicate the splay |
| 1355 | tree ahead of time. We know a mapping doesn't already exist in the |
| 1356 | given context. Create new mappings to implement default semantics. */ |
| 1357 | |
| 1358 | static tree |
| 1359 | omp_copy_decl (tree var, copy_body_data *cb) |
| 1360 | { |
| 1361 | omp_context *ctx = (omp_context *) cb; |
| 1362 | tree new_var; |
| 1363 | |
| 1364 | if (TREE_CODE (var) == LABEL_DECL) |
| 1365 | { |
| 1366 | new_var = create_artificial_label (DECL_SOURCE_LOCATION (var)); |
| 1367 | DECL_CONTEXT (new_var) = current_function_decl; |
| 1368 | insert_decl_map (&ctx->cb, var, new_var); |
| 1369 | return new_var; |
| 1370 | } |
| 1371 | |
| 1372 | while (!is_taskreg_ctx (ctx)) |
| 1373 | { |
| 1374 | ctx = ctx->outer; |
| 1375 | if (ctx == NULL) |
| 1376 | return var; |
| 1377 | new_var = maybe_lookup_decl (var, ctx); |
| 1378 | if (new_var) |
| 1379 | return new_var; |
| 1380 | } |
| 1381 | |
| 1382 | if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn) |
| 1383 | return var; |
| 1384 | |
| 1385 | return error_mark_node; |
| 1386 | } |
| 1387 | |
| 1388 | |
| 1389 | /* Debugging dumps for parallel regions. */ |
| 1390 | void dump_omp_region (FILE *, struct omp_region *, int); |
| 1391 | void debug_omp_region (struct omp_region *); |
| 1392 | void debug_all_omp_regions (void); |
| 1393 | |
| 1394 | /* Dump the parallel region tree rooted at REGION. */ |
| 1395 | |
| 1396 | void |
| 1397 | dump_omp_region (FILE *file, struct omp_region *region, int indent) |
| 1398 | { |
| 1399 | fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index, |
| 1400 | gimple_code_name[region->type]); |
| 1401 | |
| 1402 | if (region->inner) |
| 1403 | dump_omp_region (file, region->inner, indent + 4); |
| 1404 | |
| 1405 | if (region->cont) |
| 1406 | { |
| 1407 | fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "", |
| 1408 | region->cont->index); |
| 1409 | } |
| 1410 | |
| 1411 | if (region->exit) |
| 1412 | fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "", |
| 1413 | region->exit->index); |
| 1414 | else |
| 1415 | fprintf (file, "%*s[no exit marker]\n", indent, ""); |
| 1416 | |
| 1417 | if (region->next) |
| 1418 | dump_omp_region (file, region->next, indent); |
| 1419 | } |
| 1420 | |
| 1421 | DEBUG_FUNCTION void |
| 1422 | debug_omp_region (struct omp_region *region) |
| 1423 | { |
| 1424 | dump_omp_region (stderr, region, 0); |
| 1425 | } |
| 1426 | |
| 1427 | DEBUG_FUNCTION void |
| 1428 | debug_all_omp_regions (void) |
| 1429 | { |
| 1430 | dump_omp_region (stderr, root_omp_region, 0); |
| 1431 | } |
| 1432 | |
| 1433 | |
| 1434 | /* Create a new parallel region starting at STMT inside region PARENT. */ |
| 1435 | |
| 1436 | static struct omp_region * |
| 1437 | new_omp_region (basic_block bb, enum gimple_code type, |
| 1438 | struct omp_region *parent) |
| 1439 | { |
| 1440 | struct omp_region *region = XCNEW (struct omp_region); |
| 1441 | |
| 1442 | region->outer = parent; |
| 1443 | region->entry = bb; |
| 1444 | region->type = type; |
| 1445 | |
| 1446 | if (parent) |
| 1447 | { |
| 1448 | /* This is a nested region. Add it to the list of inner |
| 1449 | regions in PARENT. */ |
| 1450 | region->next = parent->inner; |
| 1451 | parent->inner = region; |
| 1452 | } |
| 1453 | else |
| 1454 | { |
| 1455 | /* This is a toplevel region. Add it to the list of toplevel |
| 1456 | regions in ROOT_OMP_REGION. */ |
| 1457 | region->next = root_omp_region; |
| 1458 | root_omp_region = region; |
| 1459 | } |
| 1460 | |
| 1461 | return region; |
| 1462 | } |
| 1463 | |
| 1464 | /* Release the memory associated with the region tree rooted at REGION. */ |
| 1465 | |
| 1466 | static void |
| 1467 | free_omp_region_1 (struct omp_region *region) |
| 1468 | { |
| 1469 | struct omp_region *i, *n; |
| 1470 | |
| 1471 | for (i = region->inner; i ; i = n) |
| 1472 | { |
| 1473 | n = i->next; |
| 1474 | free_omp_region_1 (i); |
| 1475 | } |
| 1476 | |
| 1477 | free (region); |
| 1478 | } |
| 1479 | |
| 1480 | /* Release the memory for the entire omp region tree. */ |
| 1481 | |
| 1482 | void |
| 1483 | free_omp_regions (void) |
| 1484 | { |
| 1485 | struct omp_region *r, *n; |
| 1486 | for (r = root_omp_region; r ; r = n) |
| 1487 | { |
| 1488 | n = r->next; |
| 1489 | free_omp_region_1 (r); |
| 1490 | } |
| 1491 | root_omp_region = NULL; |
| 1492 | } |
| 1493 | |
| 1494 | |
| 1495 | /* Create a new context, with OUTER_CTX being the surrounding context. */ |
| 1496 | |
| 1497 | static omp_context * |
| 1498 | new_omp_context (gimple stmt, omp_context *outer_ctx) |
| 1499 | { |
| 1500 | omp_context *ctx = XCNEW (omp_context); |
| 1501 | |
| 1502 | splay_tree_insert (all_contexts, (splay_tree_key) stmt, |
| 1503 | (splay_tree_value) ctx); |
| 1504 | ctx->stmt = stmt; |
| 1505 | |
| 1506 | if (outer_ctx) |
| 1507 | { |
| 1508 | ctx->outer = outer_ctx; |
| 1509 | ctx->cb = outer_ctx->cb; |
| 1510 | ctx->cb.block = NULL; |
| 1511 | ctx->depth = outer_ctx->depth + 1; |
| 1512 | ctx->reduction_map = outer_ctx->reduction_map; |
| 1513 | } |
| 1514 | else |
| 1515 | { |
| 1516 | ctx->cb.src_fn = current_function_decl; |
| 1517 | ctx->cb.dst_fn = current_function_decl; |
| 1518 | ctx->cb.src_node = cgraph_node::get (current_function_decl); |
| 1519 | gcc_checking_assert (ctx->cb.src_node); |
| 1520 | ctx->cb.dst_node = ctx->cb.src_node; |
| 1521 | ctx->cb.src_cfun = cfun; |
| 1522 | ctx->cb.copy_decl = omp_copy_decl; |
| 1523 | ctx->cb.eh_lp_nr = 0; |
| 1524 | ctx->cb.transform_call_graph_edges = CB_CGE_MOVE; |
| 1525 | ctx->depth = 1; |
| 1526 | } |
| 1527 | |
| 1528 | ctx->cb.decl_map = new hash_map<tree, tree>; |
| 1529 | |
| 1530 | return ctx; |
| 1531 | } |
| 1532 | |
| 1533 | static gimple_seq maybe_catch_exception (gimple_seq); |
| 1534 | |
| 1535 | /* Finalize task copyfn. */ |
| 1536 | |
| 1537 | static void |
| 1538 | finalize_task_copyfn (gomp_task *task_stmt) |
| 1539 | { |
| 1540 | struct function *child_cfun; |
| 1541 | tree child_fn; |
| 1542 | gimple_seq seq = NULL, new_seq; |
| 1543 | gbind *bind; |
| 1544 | |
| 1545 | child_fn = gimple_omp_task_copy_fn (task_stmt); |
| 1546 | if (child_fn == NULL_TREE) |
| 1547 | return; |
| 1548 | |
| 1549 | child_cfun = DECL_STRUCT_FUNCTION (child_fn); |
| 1550 | DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties; |
| 1551 | |
| 1552 | push_cfun (child_cfun); |
| 1553 | bind = gimplify_body (child_fn, false); |
| 1554 | gimple_seq_add_stmt (&seq, bind); |
| 1555 | new_seq = maybe_catch_exception (seq); |
| 1556 | if (new_seq != seq) |
| 1557 | { |
| 1558 | bind = gimple_build_bind (NULL, new_seq, NULL); |
| 1559 | seq = NULL; |
| 1560 | gimple_seq_add_stmt (&seq, bind); |
| 1561 | } |
| 1562 | gimple_set_body (child_fn, seq); |
| 1563 | pop_cfun (); |
| 1564 | |
| 1565 | /* Inform the callgraph about the new function. */ |
| 1566 | cgraph_node::add_new_function (child_fn, false); |
| 1567 | } |
| 1568 | |
| 1569 | /* Destroy a omp_context data structures. Called through the splay tree |
| 1570 | value delete callback. */ |
| 1571 | |
| 1572 | static void |
| 1573 | delete_omp_context (splay_tree_value value) |
| 1574 | { |
| 1575 | omp_context *ctx = (omp_context *) value; |
| 1576 | |
| 1577 | delete ctx->cb.decl_map; |
| 1578 | |
| 1579 | if (ctx->field_map) |
| 1580 | splay_tree_delete (ctx->field_map); |
| 1581 | if (ctx->sfield_map) |
| 1582 | splay_tree_delete (ctx->sfield_map); |
| 1583 | if (ctx->reduction_map |
| 1584 | /* Shared over several omp_contexts. */ |
| 1585 | && (ctx->outer == NULL |
| 1586 | || ctx->reduction_map != ctx->outer->reduction_map)) |
| 1587 | splay_tree_delete (ctx->reduction_map); |
| 1588 | |
| 1589 | /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before |
| 1590 | it produces corrupt debug information. */ |
| 1591 | if (ctx->record_type) |
| 1592 | { |
| 1593 | tree t; |
| 1594 | for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t)) |
| 1595 | DECL_ABSTRACT_ORIGIN (t) = NULL; |
| 1596 | } |
| 1597 | if (ctx->srecord_type) |
| 1598 | { |
| 1599 | tree t; |
| 1600 | for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t)) |
| 1601 | DECL_ABSTRACT_ORIGIN (t) = NULL; |
| 1602 | } |
| 1603 | |
| 1604 | if (is_task_ctx (ctx)) |
| 1605 | finalize_task_copyfn (as_a <gomp_task *> (ctx->stmt)); |
| 1606 | |
| 1607 | XDELETE (ctx); |
| 1608 | } |
| 1609 | |
| 1610 | /* Fix up RECEIVER_DECL with a type that has been remapped to the child |
| 1611 | context. */ |
| 1612 | |
| 1613 | static void |
| 1614 | fixup_child_record_type (omp_context *ctx) |
| 1615 | { |
| 1616 | tree f, type = ctx->record_type; |
| 1617 | |
| 1618 | /* ??? It isn't sufficient to just call remap_type here, because |
| 1619 | variably_modified_type_p doesn't work the way we expect for |
| 1620 | record types. Testing each field for whether it needs remapping |
| 1621 | and creating a new record by hand works, however. */ |
| 1622 | for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f)) |
| 1623 | if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) |
| 1624 | break; |
| 1625 | if (f) |
| 1626 | { |
| 1627 | tree name, new_fields = NULL; |
| 1628 | |
| 1629 | type = lang_hooks.types.make_type (RECORD_TYPE); |
| 1630 | name = DECL_NAME (TYPE_NAME (ctx->record_type)); |
| 1631 | name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl), |
| 1632 | TYPE_DECL, name, type); |
| 1633 | TYPE_NAME (type) = name; |
| 1634 | |
| 1635 | for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f)) |
| 1636 | { |
| 1637 | tree new_f = copy_node (f); |
| 1638 | DECL_CONTEXT (new_f) = type; |
| 1639 | TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb); |
| 1640 | DECL_CHAIN (new_f) = new_fields; |
| 1641 | walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL); |
| 1642 | walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, |
| 1643 | &ctx->cb, NULL); |
| 1644 | walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r, |
| 1645 | &ctx->cb, NULL); |
| 1646 | new_fields = new_f; |
| 1647 | |
| 1648 | /* Arrange to be able to look up the receiver field |
| 1649 | given the sender field. */ |
| 1650 | splay_tree_insert (ctx->field_map, (splay_tree_key) f, |
| 1651 | (splay_tree_value) new_f); |
| 1652 | } |
| 1653 | TYPE_FIELDS (type) = nreverse (new_fields); |
| 1654 | layout_type (type); |
| 1655 | } |
| 1656 | |
| 1657 | TREE_TYPE (ctx->receiver_decl) |
| 1658 | = build_qualified_type (build_reference_type (type), TYPE_QUAL_RESTRICT); |
| 1659 | } |
| 1660 | |
| 1661 | /* Instantiate decls as necessary in CTX to satisfy the data sharing |
| 1662 | specified by CLAUSES. */ |
| 1663 | |
| 1664 | static void |
| 1665 | scan_sharing_clauses (tree clauses, omp_context *ctx) |
| 1666 | { |
| 1667 | tree c, decl; |
| 1668 | bool scan_array_reductions = false; |
| 1669 | |
| 1670 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
| 1671 | { |
| 1672 | bool by_ref; |
| 1673 | |
| 1674 | switch (OMP_CLAUSE_CODE (c)) |
| 1675 | { |
| 1676 | case OMP_CLAUSE_PRIVATE: |
| 1677 | decl = OMP_CLAUSE_DECL (c); |
| 1678 | if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) |
| 1679 | goto do_private; |
| 1680 | else if (!is_variable_sized (decl)) |
| 1681 | install_var_local (decl, ctx); |
| 1682 | break; |
| 1683 | |
| 1684 | case OMP_CLAUSE_SHARED: |
| 1685 | decl = OMP_CLAUSE_DECL (c); |
| 1686 | /* Ignore shared directives in teams construct. */ |
| 1687 | if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) |
| 1688 | { |
| 1689 | /* Global variables don't need to be copied, |
| 1690 | the receiver side will use them directly. */ |
| 1691 | tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx); |
| 1692 | if (is_global_var (odecl)) |
| 1693 | break; |
| 1694 | insert_decl_map (&ctx->cb, decl, odecl); |
| 1695 | break; |
| 1696 | } |
| 1697 | gcc_assert (is_taskreg_ctx (ctx)); |
| 1698 | gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl)) |
| 1699 | || !is_variable_sized (decl)); |
| 1700 | /* Global variables don't need to be copied, |
| 1701 | the receiver side will use them directly. */ |
| 1702 | if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))) |
| 1703 | break; |
| 1704 | by_ref = use_pointer_for_field (decl, ctx); |
| 1705 | if (! TREE_READONLY (decl) |
| 1706 | || TREE_ADDRESSABLE (decl) |
| 1707 | || by_ref |
| 1708 | || is_reference (decl)) |
| 1709 | { |
| 1710 | install_var_field (decl, by_ref, 3, ctx); |
| 1711 | install_var_local (decl, ctx); |
| 1712 | break; |
| 1713 | } |
| 1714 | /* We don't need to copy const scalar vars back. */ |
| 1715 | OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE); |
| 1716 | goto do_private; |
| 1717 | |
| 1718 | case OMP_CLAUSE_LASTPRIVATE: |
| 1719 | /* Let the corresponding firstprivate clause create |
| 1720 | the variable. */ |
| 1721 | if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) |
| 1722 | break; |
| 1723 | /* FALLTHRU */ |
| 1724 | |
| 1725 | case OMP_CLAUSE_FIRSTPRIVATE: |
| 1726 | if (is_gimple_omp_oacc (ctx->stmt)) |
| 1727 | { |
| 1728 | sorry ("clause not supported yet"); |
| 1729 | break; |
| 1730 | } |
| 1731 | /* FALLTHRU */ |
| 1732 | case OMP_CLAUSE_REDUCTION: |
| 1733 | case OMP_CLAUSE_LINEAR: |
| 1734 | decl = OMP_CLAUSE_DECL (c); |
| 1735 | do_private: |
| 1736 | if (is_variable_sized (decl)) |
| 1737 | { |
| 1738 | if (is_task_ctx (ctx)) |
| 1739 | install_var_field (decl, false, 1, ctx); |
| 1740 | break; |
| 1741 | } |
| 1742 | else if (is_taskreg_ctx (ctx)) |
| 1743 | { |
| 1744 | bool global |
| 1745 | = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)); |
| 1746 | by_ref = use_pointer_for_field (decl, NULL); |
| 1747 | |
| 1748 | if (is_task_ctx (ctx) |
| 1749 | && (global || by_ref || is_reference (decl))) |
| 1750 | { |
| 1751 | install_var_field (decl, false, 1, ctx); |
| 1752 | if (!global) |
| 1753 | install_var_field (decl, by_ref, 2, ctx); |
| 1754 | } |
| 1755 | else if (!global) |
| 1756 | install_var_field (decl, by_ref, 3, ctx); |
| 1757 | } |
| 1758 | install_var_local (decl, ctx); |
| 1759 | if (is_gimple_omp_oacc (ctx->stmt) |
| 1760 | && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) |
| 1761 | { |
| 1762 | /* Create a decl for the reduction array. */ |
| 1763 | tree var = OMP_CLAUSE_DECL (c); |
| 1764 | tree type = get_base_type (var); |
| 1765 | tree ptype = build_pointer_type (type); |
| 1766 | tree array = create_tmp_var (ptype, |
| 1767 | oacc_get_reduction_array_id (var)); |
| 1768 | omp_context *c = (ctx->field_map ? ctx : ctx->outer); |
| 1769 | install_var_field (array, true, 3, c); |
| 1770 | install_var_local (array, c); |
| 1771 | |
| 1772 | /* Insert it into the current context. */ |
| 1773 | splay_tree_insert (ctx->reduction_map, (splay_tree_key) |
| 1774 | oacc_get_reduction_array_id (var), |
| 1775 | (splay_tree_value) array); |
| 1776 | splay_tree_insert (ctx->reduction_map, |
| 1777 | (splay_tree_key) array, |
| 1778 | (splay_tree_value) array); |
| 1779 | } |
| 1780 | break; |
| 1781 | |
| 1782 | case OMP_CLAUSE__LOOPTEMP_: |
| 1783 | gcc_assert (is_parallel_ctx (ctx)); |
| 1784 | decl = OMP_CLAUSE_DECL (c); |
| 1785 | install_var_field (decl, false, 3, ctx); |
| 1786 | install_var_local (decl, ctx); |
| 1787 | break; |
| 1788 | |
| 1789 | case OMP_CLAUSE_COPYPRIVATE: |
| 1790 | case OMP_CLAUSE_COPYIN: |
| 1791 | decl = OMP_CLAUSE_DECL (c); |
| 1792 | by_ref = use_pointer_for_field (decl, NULL); |
| 1793 | install_var_field (decl, by_ref, 3, ctx); |
| 1794 | break; |
| 1795 | |
| 1796 | case OMP_CLAUSE_DEFAULT: |
| 1797 | ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c); |
| 1798 | break; |
| 1799 | |
| 1800 | case OMP_CLAUSE_FINAL: |
| 1801 | case OMP_CLAUSE_IF: |
| 1802 | case OMP_CLAUSE_NUM_THREADS: |
| 1803 | case OMP_CLAUSE_NUM_TEAMS: |
| 1804 | case OMP_CLAUSE_THREAD_LIMIT: |
| 1805 | case OMP_CLAUSE_DEVICE: |
| 1806 | case OMP_CLAUSE_SCHEDULE: |
| 1807 | case OMP_CLAUSE_DIST_SCHEDULE: |
| 1808 | case OMP_CLAUSE_DEPEND: |
| 1809 | case OMP_CLAUSE__CILK_FOR_COUNT_: |
| 1810 | case OMP_CLAUSE_NUM_GANGS: |
| 1811 | case OMP_CLAUSE_NUM_WORKERS: |
| 1812 | case OMP_CLAUSE_VECTOR_LENGTH: |
| 1813 | if (ctx->outer) |
| 1814 | scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer); |
| 1815 | break; |
| 1816 | |
| 1817 | case OMP_CLAUSE_TO: |
| 1818 | case OMP_CLAUSE_FROM: |
| 1819 | case OMP_CLAUSE_MAP: |
| 1820 | if (ctx->outer) |
| 1821 | scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer); |
| 1822 | decl = OMP_CLAUSE_DECL (c); |
| 1823 | /* Global variables with "omp declare target" attribute |
| 1824 | don't need to be copied, the receiver side will use them |
| 1825 | directly. */ |
| 1826 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
| 1827 | && DECL_P (decl) |
| 1828 | && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)) |
| 1829 | && varpool_node::get_create (decl)->offloadable) |
| 1830 | break; |
| 1831 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
| 1832 | && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER) |
| 1833 | { |
| 1834 | /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are |
| 1835 | not offloaded; there is nothing to map for those. */ |
| 1836 | if (!is_gimple_omp_offloaded (ctx->stmt) |
| 1837 | && !POINTER_TYPE_P (TREE_TYPE (decl)) |
| 1838 | && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)) |
| 1839 | break; |
| 1840 | } |
| 1841 | if (DECL_P (decl)) |
| 1842 | { |
| 1843 | if (DECL_SIZE (decl) |
| 1844 | && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST) |
| 1845 | { |
| 1846 | tree decl2 = DECL_VALUE_EXPR (decl); |
| 1847 | gcc_assert (TREE_CODE (decl2) == INDIRECT_REF); |
| 1848 | decl2 = TREE_OPERAND (decl2, 0); |
| 1849 | gcc_assert (DECL_P (decl2)); |
| 1850 | install_var_field (decl2, true, 3, ctx); |
| 1851 | install_var_local (decl2, ctx); |
| 1852 | install_var_local (decl, ctx); |
| 1853 | } |
| 1854 | else |
| 1855 | { |
| 1856 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
| 1857 | && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER |
| 1858 | && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) |
| 1859 | && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) |
| 1860 | install_var_field (decl, true, 7, ctx); |
| 1861 | else |
| 1862 | install_var_field (decl, true, 3, ctx); |
| 1863 | if (is_gimple_omp_offloaded (ctx->stmt)) |
| 1864 | install_var_local (decl, ctx); |
| 1865 | } |
| 1866 | } |
| 1867 | else |
| 1868 | { |
| 1869 | tree base = get_base_address (decl); |
| 1870 | tree nc = OMP_CLAUSE_CHAIN (c); |
| 1871 | if (DECL_P (base) |
| 1872 | && nc != NULL_TREE |
| 1873 | && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP |
| 1874 | && OMP_CLAUSE_DECL (nc) == base |
| 1875 | && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER |
| 1876 | && integer_zerop (OMP_CLAUSE_SIZE (nc))) |
| 1877 | { |
| 1878 | OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1; |
| 1879 | OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1; |
| 1880 | } |
| 1881 | else |
| 1882 | { |
| 1883 | if (ctx->outer) |
| 1884 | { |
| 1885 | scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer); |
| 1886 | decl = OMP_CLAUSE_DECL (c); |
| 1887 | } |
| 1888 | gcc_assert (!splay_tree_lookup (ctx->field_map, |
| 1889 | (splay_tree_key) decl)); |
| 1890 | tree field |
| 1891 | = build_decl (OMP_CLAUSE_LOCATION (c), |
| 1892 | FIELD_DECL, NULL_TREE, ptr_type_node); |
| 1893 | DECL_ALIGN (field) = TYPE_ALIGN (ptr_type_node); |
| 1894 | insert_field_into_struct (ctx->record_type, field); |
| 1895 | splay_tree_insert (ctx->field_map, (splay_tree_key) decl, |
| 1896 | (splay_tree_value) field); |
| 1897 | } |
| 1898 | } |
| 1899 | break; |
| 1900 | |
| 1901 | case OMP_CLAUSE_NOWAIT: |
| 1902 | case OMP_CLAUSE_ORDERED: |
| 1903 | case OMP_CLAUSE_COLLAPSE: |
| 1904 | case OMP_CLAUSE_UNTIED: |
| 1905 | case OMP_CLAUSE_MERGEABLE: |
| 1906 | case OMP_CLAUSE_PROC_BIND: |
| 1907 | case OMP_CLAUSE_SAFELEN: |
| 1908 | case OMP_CLAUSE_ASYNC: |
| 1909 | case OMP_CLAUSE_WAIT: |
| 1910 | case OMP_CLAUSE_GANG: |
| 1911 | case OMP_CLAUSE_WORKER: |
| 1912 | case OMP_CLAUSE_VECTOR: |
| 1913 | break; |
| 1914 | |
| 1915 | case OMP_CLAUSE_ALIGNED: |
| 1916 | decl = OMP_CLAUSE_DECL (c); |
| 1917 | if (is_global_var (decl) |
| 1918 | && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) |
| 1919 | install_var_local (decl, ctx); |
| 1920 | break; |
| 1921 | |
| 1922 | case OMP_CLAUSE_DEVICE_RESIDENT: |
| 1923 | case OMP_CLAUSE_USE_DEVICE: |
| 1924 | case OMP_CLAUSE__CACHE_: |
| 1925 | case OMP_CLAUSE_INDEPENDENT: |
| 1926 | case OMP_CLAUSE_AUTO: |
| 1927 | case OMP_CLAUSE_SEQ: |
| 1928 | sorry ("Clause not supported yet"); |
| 1929 | break; |
| 1930 | |
| 1931 | default: |
| 1932 | gcc_unreachable (); |
| 1933 | } |
| 1934 | } |
| 1935 | |
| 1936 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
| 1937 | { |
| 1938 | switch (OMP_CLAUSE_CODE (c)) |
| 1939 | { |
| 1940 | case OMP_CLAUSE_LASTPRIVATE: |
| 1941 | /* Let the corresponding firstprivate clause create |
| 1942 | the variable. */ |
| 1943 | if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) |
| 1944 | scan_array_reductions = true; |
| 1945 | if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) |
| 1946 | break; |
| 1947 | /* FALLTHRU */ |
| 1948 | |
| 1949 | case OMP_CLAUSE_FIRSTPRIVATE: |
| 1950 | if (is_gimple_omp_oacc (ctx->stmt)) |
| 1951 | { |
| 1952 | sorry ("clause not supported yet"); |
| 1953 | break; |
| 1954 | } |
| 1955 | /* FALLTHRU */ |
| 1956 | case OMP_CLAUSE_PRIVATE: |
| 1957 | case OMP_CLAUSE_REDUCTION: |
| 1958 | case OMP_CLAUSE_LINEAR: |
| 1959 | decl = OMP_CLAUSE_DECL (c); |
| 1960 | if (is_variable_sized (decl)) |
| 1961 | install_var_local (decl, ctx); |
| 1962 | fixup_remapped_decl (decl, ctx, |
| 1963 | OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE |
| 1964 | && OMP_CLAUSE_PRIVATE_DEBUG (c)); |
| 1965 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
| 1966 | && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
| 1967 | scan_array_reductions = true; |
| 1968 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR |
| 1969 | && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) |
| 1970 | scan_array_reductions = true; |
| 1971 | break; |
| 1972 | |
| 1973 | case OMP_CLAUSE_SHARED: |
| 1974 | /* Ignore shared directives in teams construct. */ |
| 1975 | if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) |
| 1976 | break; |
| 1977 | decl = OMP_CLAUSE_DECL (c); |
| 1978 | if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))) |
| 1979 | fixup_remapped_decl (decl, ctx, false); |
| 1980 | break; |
| 1981 | |
| 1982 | case OMP_CLAUSE_MAP: |
| 1983 | if (!is_gimple_omp_offloaded (ctx->stmt)) |
| 1984 | break; |
| 1985 | decl = OMP_CLAUSE_DECL (c); |
| 1986 | if (DECL_P (decl) |
| 1987 | && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)) |
| 1988 | && varpool_node::get_create (decl)->offloadable) |
| 1989 | break; |
| 1990 | if (DECL_P (decl)) |
| 1991 | { |
| 1992 | if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER |
| 1993 | && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE |
| 1994 | && !COMPLETE_TYPE_P (TREE_TYPE (decl))) |
| 1995 | { |
| 1996 | tree new_decl = lookup_decl (decl, ctx); |
| 1997 | TREE_TYPE (new_decl) |
| 1998 | = remap_type (TREE_TYPE (decl), &ctx->cb); |
| 1999 | } |
| 2000 | else if (DECL_SIZE (decl) |
| 2001 | && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST) |
| 2002 | { |
| 2003 | tree decl2 = DECL_VALUE_EXPR (decl); |
| 2004 | gcc_assert (TREE_CODE (decl2) == INDIRECT_REF); |
| 2005 | decl2 = TREE_OPERAND (decl2, 0); |
| 2006 | gcc_assert (DECL_P (decl2)); |
| 2007 | fixup_remapped_decl (decl2, ctx, false); |
| 2008 | fixup_remapped_decl (decl, ctx, true); |
| 2009 | } |
| 2010 | else |
| 2011 | fixup_remapped_decl (decl, ctx, false); |
| 2012 | } |
| 2013 | break; |
| 2014 | |
| 2015 | case OMP_CLAUSE_COPYPRIVATE: |
| 2016 | case OMP_CLAUSE_COPYIN: |
| 2017 | case OMP_CLAUSE_DEFAULT: |
| 2018 | case OMP_CLAUSE_IF: |
| 2019 | case OMP_CLAUSE_NUM_THREADS: |
| 2020 | case OMP_CLAUSE_NUM_TEAMS: |
| 2021 | case OMP_CLAUSE_THREAD_LIMIT: |
| 2022 | case OMP_CLAUSE_DEVICE: |
| 2023 | case OMP_CLAUSE_SCHEDULE: |
| 2024 | case OMP_CLAUSE_DIST_SCHEDULE: |
| 2025 | case OMP_CLAUSE_NOWAIT: |
| 2026 | case OMP_CLAUSE_ORDERED: |
| 2027 | case OMP_CLAUSE_COLLAPSE: |
| 2028 | case OMP_CLAUSE_UNTIED: |
| 2029 | case OMP_CLAUSE_FINAL: |
| 2030 | case OMP_CLAUSE_MERGEABLE: |
| 2031 | case OMP_CLAUSE_PROC_BIND: |
| 2032 | case OMP_CLAUSE_SAFELEN: |
| 2033 | case OMP_CLAUSE_ALIGNED: |
| 2034 | case OMP_CLAUSE_DEPEND: |
| 2035 | case OMP_CLAUSE__LOOPTEMP_: |
| 2036 | case OMP_CLAUSE_TO: |
| 2037 | case OMP_CLAUSE_FROM: |
| 2038 | case OMP_CLAUSE__CILK_FOR_COUNT_: |
| 2039 | case OMP_CLAUSE_ASYNC: |
| 2040 | case OMP_CLAUSE_WAIT: |
| 2041 | case OMP_CLAUSE_NUM_GANGS: |
| 2042 | case OMP_CLAUSE_NUM_WORKERS: |
| 2043 | case OMP_CLAUSE_VECTOR_LENGTH: |
| 2044 | case OMP_CLAUSE_GANG: |
| 2045 | case OMP_CLAUSE_WORKER: |
| 2046 | case OMP_CLAUSE_VECTOR: |
| 2047 | break; |
| 2048 | |
| 2049 | case OMP_CLAUSE_DEVICE_RESIDENT: |
| 2050 | case OMP_CLAUSE_USE_DEVICE: |
| 2051 | case OMP_CLAUSE__CACHE_: |
| 2052 | case OMP_CLAUSE_INDEPENDENT: |
| 2053 | case OMP_CLAUSE_AUTO: |
| 2054 | case OMP_CLAUSE_SEQ: |
| 2055 | sorry ("Clause not supported yet"); |
| 2056 | break; |
| 2057 | |
| 2058 | default: |
| 2059 | gcc_unreachable (); |
| 2060 | } |
| 2061 | } |
| 2062 | |
| 2063 | gcc_checking_assert (!scan_array_reductions |
| 2064 | || !is_gimple_omp_oacc (ctx->stmt)); |
| 2065 | if (scan_array_reductions) |
| 2066 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
| 2067 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION |
| 2068 | && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
| 2069 | { |
| 2070 | scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx); |
| 2071 | scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx); |
| 2072 | } |
| 2073 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
| 2074 | && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) |
| 2075 | scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx); |
| 2076 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR |
| 2077 | && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) |
| 2078 | scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx); |
| 2079 | } |
| 2080 | |
| 2081 | /* Create a new name for omp child function. Returns an identifier. If |
| 2082 | IS_CILK_FOR is true then the suffix for the child function is |
| 2083 | "_cilk_for_fn." */ |
| 2084 | |
| 2085 | static tree |
| 2086 | create_omp_child_function_name (bool task_copy, bool is_cilk_for) |
| 2087 | { |
| 2088 | if (is_cilk_for) |
| 2089 | return clone_function_name (current_function_decl, "_cilk_for_fn"); |
| 2090 | return clone_function_name (current_function_decl, |
| 2091 | task_copy ? "_omp_cpyfn" : "_omp_fn"); |
| 2092 | } |
| 2093 | |
| 2094 | /* Returns the type of the induction variable for the child function for |
| 2095 | _Cilk_for and the types for _high and _low variables based on TYPE. */ |
| 2096 | |
| 2097 | static tree |
| 2098 | cilk_for_check_loop_diff_type (tree type) |
| 2099 | { |
| 2100 | if (TYPE_PRECISION (type) <= TYPE_PRECISION (uint32_type_node)) |
| 2101 | { |
| 2102 | if (TYPE_UNSIGNED (type)) |
| 2103 | return uint32_type_node; |
| 2104 | else |
| 2105 | return integer_type_node; |
| 2106 | } |
| 2107 | else |
| 2108 | { |
| 2109 | if (TYPE_UNSIGNED (type)) |
| 2110 | return uint64_type_node; |
| 2111 | else |
| 2112 | return long_long_integer_type_node; |
| 2113 | } |
| 2114 | } |
| 2115 | |
| 2116 | /* Build a decl for the omp child function. It'll not contain a body |
| 2117 | yet, just the bare decl. */ |
| 2118 | |
| 2119 | static void |
| 2120 | create_omp_child_function (omp_context *ctx, bool task_copy) |
| 2121 | { |
| 2122 | tree decl, type, name, t; |
| 2123 | |
| 2124 | tree cilk_for_count |
| 2125 | = (flag_cilkplus && gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL) |
| 2126 | ? find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt), |
| 2127 | OMP_CLAUSE__CILK_FOR_COUNT_) : NULL_TREE; |
| 2128 | tree cilk_var_type = NULL_TREE; |
| 2129 | |
| 2130 | name = create_omp_child_function_name (task_copy, |
| 2131 | cilk_for_count != NULL_TREE); |
| 2132 | if (task_copy) |
| 2133 | type = build_function_type_list (void_type_node, ptr_type_node, |
| 2134 | ptr_type_node, NULL_TREE); |
| 2135 | else if (cilk_for_count) |
| 2136 | { |
| 2137 | type = TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count, 0)); |
| 2138 | cilk_var_type = cilk_for_check_loop_diff_type (type); |
| 2139 | type = build_function_type_list (void_type_node, ptr_type_node, |
| 2140 | cilk_var_type, cilk_var_type, NULL_TREE); |
| 2141 | } |
| 2142 | else |
| 2143 | type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE); |
| 2144 | |
| 2145 | decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type); |
| 2146 | |
| 2147 | gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt) |
| 2148 | || !task_copy); |
| 2149 | if (!task_copy) |
| 2150 | ctx->cb.dst_fn = decl; |
| 2151 | else |
| 2152 | gimple_omp_task_set_copy_fn (ctx->stmt, decl); |
| 2153 | |
| 2154 | TREE_STATIC (decl) = 1; |
| 2155 | TREE_USED (decl) = 1; |
| 2156 | DECL_ARTIFICIAL (decl) = 1; |
| 2157 | DECL_IGNORED_P (decl) = 0; |
| 2158 | TREE_PUBLIC (decl) = 0; |
| 2159 | DECL_UNINLINABLE (decl) = 1; |
| 2160 | DECL_EXTERNAL (decl) = 0; |
| 2161 | DECL_CONTEXT (decl) = NULL_TREE; |
| 2162 | DECL_INITIAL (decl) = make_node (BLOCK); |
| 2163 | if (cgraph_node::get (current_function_decl)->offloadable) |
| 2164 | cgraph_node::get_create (decl)->offloadable = 1; |
| 2165 | else |
| 2166 | { |
| 2167 | omp_context *octx; |
| 2168 | for (octx = ctx; octx; octx = octx->outer) |
| 2169 | if (is_gimple_omp_offloaded (octx->stmt)) |
| 2170 | { |
| 2171 | cgraph_node::get_create (decl)->offloadable = 1; |
| 2172 | #ifdef ENABLE_OFFLOADING |
| 2173 | g->have_offload = true; |
| 2174 | #endif |
| 2175 | break; |
| 2176 | } |
| 2177 | } |
| 2178 | |
| 2179 | if (cgraph_node::get_create (decl)->offloadable |
| 2180 | && !lookup_attribute ("omp declare target", |
| 2181 | DECL_ATTRIBUTES (current_function_decl))) |
| 2182 | DECL_ATTRIBUTES (decl) |
| 2183 | = tree_cons (get_identifier ("omp target entrypoint"), |
| 2184 | NULL_TREE, DECL_ATTRIBUTES (decl)); |
| 2185 | |
| 2186 | t = build_decl (DECL_SOURCE_LOCATION (decl), |
| 2187 | RESULT_DECL, NULL_TREE, void_type_node); |
| 2188 | DECL_ARTIFICIAL (t) = 1; |
| 2189 | DECL_IGNORED_P (t) = 1; |
| 2190 | DECL_CONTEXT (t) = decl; |
| 2191 | DECL_RESULT (decl) = t; |
| 2192 | |
| 2193 | /* _Cilk_for's child function requires two extra parameters called |
| 2194 | __low and __high that are set the by Cilk runtime when it calls this |
| 2195 | function. */ |
| 2196 | if (cilk_for_count) |
| 2197 | { |
| 2198 | t = build_decl (DECL_SOURCE_LOCATION (decl), |
| 2199 | PARM_DECL, get_identifier ("__high"), cilk_var_type); |
| 2200 | DECL_ARTIFICIAL (t) = 1; |
| 2201 | DECL_NAMELESS (t) = 1; |
| 2202 | DECL_ARG_TYPE (t) = ptr_type_node; |
| 2203 | DECL_CONTEXT (t) = current_function_decl; |
| 2204 | TREE_USED (t) = 1; |
| 2205 | DECL_CHAIN (t) = DECL_ARGUMENTS (decl); |
| 2206 | DECL_ARGUMENTS (decl) = t; |
| 2207 | |
| 2208 | t = build_decl (DECL_SOURCE_LOCATION (decl), |
| 2209 | PARM_DECL, get_identifier ("__low"), cilk_var_type); |
| 2210 | DECL_ARTIFICIAL (t) = 1; |
| 2211 | DECL_NAMELESS (t) = 1; |
| 2212 | DECL_ARG_TYPE (t) = ptr_type_node; |
| 2213 | DECL_CONTEXT (t) = current_function_decl; |
| 2214 | TREE_USED (t) = 1; |
| 2215 | DECL_CHAIN (t) = DECL_ARGUMENTS (decl); |
| 2216 | DECL_ARGUMENTS (decl) = t; |
| 2217 | } |
| 2218 | |
| 2219 | tree data_name = get_identifier (".omp_data_i"); |
| 2220 | t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name, |
| 2221 | ptr_type_node); |
| 2222 | DECL_ARTIFICIAL (t) = 1; |
| 2223 | DECL_NAMELESS (t) = 1; |
| 2224 | DECL_ARG_TYPE (t) = ptr_type_node; |
| 2225 | DECL_CONTEXT (t) = current_function_decl; |
| 2226 | TREE_USED (t) = 1; |
| 2227 | if (cilk_for_count) |
| 2228 | DECL_CHAIN (t) = DECL_ARGUMENTS (decl); |
| 2229 | DECL_ARGUMENTS (decl) = t; |
| 2230 | if (!task_copy) |
| 2231 | ctx->receiver_decl = t; |
| 2232 | else |
| 2233 | { |
| 2234 | t = build_decl (DECL_SOURCE_LOCATION (decl), |
| 2235 | PARM_DECL, get_identifier (".omp_data_o"), |
| 2236 | ptr_type_node); |
| 2237 | DECL_ARTIFICIAL (t) = 1; |
| 2238 | DECL_NAMELESS (t) = 1; |
| 2239 | DECL_ARG_TYPE (t) = ptr_type_node; |
| 2240 | DECL_CONTEXT (t) = current_function_decl; |
| 2241 | TREE_USED (t) = 1; |
| 2242 | TREE_ADDRESSABLE (t) = 1; |
| 2243 | DECL_CHAIN (t) = DECL_ARGUMENTS (decl); |
| 2244 | DECL_ARGUMENTS (decl) = t; |
| 2245 | } |
| 2246 | |
| 2247 | /* Allocate memory for the function structure. The call to |
| 2248 | allocate_struct_function clobbers CFUN, so we need to restore |
| 2249 | it afterward. */ |
| 2250 | push_struct_function (decl); |
| 2251 | cfun->function_end_locus = gimple_location (ctx->stmt); |
| 2252 | pop_cfun (); |
| 2253 | } |
| 2254 | |
| 2255 | /* Callback for walk_gimple_seq. Check if combined parallel |
| 2256 | contains gimple_omp_for_combined_into_p OMP_FOR. */ |
| 2257 | |
| 2258 | static tree |
| 2259 | find_combined_for (gimple_stmt_iterator *gsi_p, |
| 2260 | bool *handled_ops_p, |
| 2261 | struct walk_stmt_info *wi) |
| 2262 | { |
| 2263 | gimple stmt = gsi_stmt (*gsi_p); |
| 2264 | |
| 2265 | *handled_ops_p = true; |
| 2266 | switch (gimple_code (stmt)) |
| 2267 | { |
| 2268 | WALK_SUBSTMTS; |
| 2269 | |
| 2270 | case GIMPLE_OMP_FOR: |
| 2271 | if (gimple_omp_for_combined_into_p (stmt) |
| 2272 | && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR) |
| 2273 | { |
| 2274 | wi->info = stmt; |
| 2275 | return integer_zero_node; |
| 2276 | } |
| 2277 | break; |
| 2278 | default: |
| 2279 | break; |
| 2280 | } |
| 2281 | return NULL; |
| 2282 | } |
| 2283 | |
| 2284 | /* Scan an OpenMP parallel directive. */ |
| 2285 | |
| 2286 | static void |
| 2287 | scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx) |
| 2288 | { |
| 2289 | omp_context *ctx; |
| 2290 | tree name; |
| 2291 | gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi)); |
| 2292 | |
| 2293 | /* Ignore parallel directives with empty bodies, unless there |
| 2294 | are copyin clauses. */ |
| 2295 | if (optimize > 0 |
| 2296 | && empty_body_p (gimple_omp_body (stmt)) |
| 2297 | && find_omp_clause (gimple_omp_parallel_clauses (stmt), |
| 2298 | OMP_CLAUSE_COPYIN) == NULL) |
| 2299 | { |
| 2300 | gsi_replace (gsi, gimple_build_nop (), false); |
| 2301 | return; |
| 2302 | } |
| 2303 | |
| 2304 | if (gimple_omp_parallel_combined_p (stmt)) |
| 2305 | { |
| 2306 | struct walk_stmt_info wi; |
| 2307 | |
| 2308 | memset (&wi, 0, sizeof (wi)); |
| 2309 | wi.val_only = true; |
| 2310 | walk_gimple_seq (gimple_omp_body (stmt), |
| 2311 | find_combined_for, NULL, &wi); |
| 2312 | if (wi.info) |
| 2313 | { |
| 2314 | gomp_for *for_stmt = as_a <gomp_for *> ((gimple) wi.info); |
| 2315 | struct omp_for_data fd; |
| 2316 | extract_omp_for_data (for_stmt, &fd, NULL); |
| 2317 | /* We need two temporaries with fd.loop.v type (istart/iend) |
| 2318 | and then (fd.collapse - 1) temporaries with the same |
| 2319 | type for count2 ... countN-1 vars if not constant. */ |
| 2320 | size_t count = 2, i; |
| 2321 | tree type = fd.iter_type; |
| 2322 | if (fd.collapse > 1 |
| 2323 | && TREE_CODE (fd.loop.n2) != INTEGER_CST) |
| 2324 | count += fd.collapse - 1; |
| 2325 | for (i = 0; i < count; i++) |
| 2326 | { |
| 2327 | tree temp = create_tmp_var (type); |
| 2328 | tree c = build_omp_clause (UNKNOWN_LOCATION, |
| 2329 | OMP_CLAUSE__LOOPTEMP_); |
| 2330 | insert_decl_map (&outer_ctx->cb, temp, temp); |
| 2331 | OMP_CLAUSE_DECL (c) = temp; |
| 2332 | OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt); |
| 2333 | gimple_omp_parallel_set_clauses (stmt, c); |
| 2334 | } |
| 2335 | } |
| 2336 | } |
| 2337 | |
| 2338 | ctx = new_omp_context (stmt, outer_ctx); |
| 2339 | taskreg_contexts.safe_push (ctx); |
| 2340 | if (taskreg_nesting_level > 1) |
| 2341 | ctx->is_nested = true; |
| 2342 | ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
| 2343 | ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED; |
| 2344 | ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); |
| 2345 | name = create_tmp_var_name (".omp_data_s"); |
| 2346 | name = build_decl (gimple_location (stmt), |
| 2347 | TYPE_DECL, name, ctx->record_type); |
| 2348 | DECL_ARTIFICIAL (name) = 1; |
| 2349 | DECL_NAMELESS (name) = 1; |
| 2350 | TYPE_NAME (ctx->record_type) = name; |
| 2351 | create_omp_child_function (ctx, false); |
| 2352 | gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn); |
| 2353 | |
| 2354 | scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx); |
| 2355 | scan_omp (gimple_omp_body_ptr (stmt), ctx); |
| 2356 | |
| 2357 | if (TYPE_FIELDS (ctx->record_type) == NULL) |
| 2358 | ctx->record_type = ctx->receiver_decl = NULL; |
| 2359 | } |
| 2360 | |
| 2361 | /* Scan an OpenMP task directive. */ |
| 2362 | |
| 2363 | static void |
| 2364 | scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx) |
| 2365 | { |
| 2366 | omp_context *ctx; |
| 2367 | tree name, t; |
| 2368 | gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi)); |
| 2369 | |
| 2370 | /* Ignore task directives with empty bodies. */ |
| 2371 | if (optimize > 0 |
| 2372 | && empty_body_p (gimple_omp_body (stmt))) |
| 2373 | { |
| 2374 | gsi_replace (gsi, gimple_build_nop (), false); |
| 2375 | return; |
| 2376 | } |
| 2377 | |
| 2378 | ctx = new_omp_context (stmt, outer_ctx); |
| 2379 | taskreg_contexts.safe_push (ctx); |
| 2380 | if (taskreg_nesting_level > 1) |
| 2381 | ctx->is_nested = true; |
| 2382 | ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
| 2383 | ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED; |
| 2384 | ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); |
| 2385 | name = create_tmp_var_name (".omp_data_s"); |
| 2386 | name = build_decl (gimple_location (stmt), |
| 2387 | TYPE_DECL, name, ctx->record_type); |
| 2388 | DECL_ARTIFICIAL (name) = 1; |
| 2389 | DECL_NAMELESS (name) = 1; |
| 2390 | TYPE_NAME (ctx->record_type) = name; |
| 2391 | create_omp_child_function (ctx, false); |
| 2392 | gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn); |
| 2393 | |
| 2394 | scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx); |
| 2395 | |
| 2396 | if (ctx->srecord_type) |
| 2397 | { |
| 2398 | name = create_tmp_var_name (".omp_data_a"); |
| 2399 | name = build_decl (gimple_location (stmt), |
| 2400 | TYPE_DECL, name, ctx->srecord_type); |
| 2401 | DECL_ARTIFICIAL (name) = 1; |
| 2402 | DECL_NAMELESS (name) = 1; |
| 2403 | TYPE_NAME (ctx->srecord_type) = name; |
| 2404 | create_omp_child_function (ctx, true); |
| 2405 | } |
| 2406 | |
| 2407 | scan_omp (gimple_omp_body_ptr (stmt), ctx); |
| 2408 | |
| 2409 | if (TYPE_FIELDS (ctx->record_type) == NULL) |
| 2410 | { |
| 2411 | ctx->record_type = ctx->receiver_decl = NULL; |
| 2412 | t = build_int_cst (long_integer_type_node, 0); |
| 2413 | gimple_omp_task_set_arg_size (stmt, t); |
| 2414 | t = build_int_cst (long_integer_type_node, 1); |
| 2415 | gimple_omp_task_set_arg_align (stmt, t); |
| 2416 | } |
| 2417 | } |
| 2418 | |
| 2419 | |
| 2420 | /* If any decls have been made addressable during scan_omp, |
| 2421 | adjust their fields if needed, and layout record types |
| 2422 | of parallel/task constructs. */ |
| 2423 | |
| 2424 | static void |
| 2425 | finish_taskreg_scan (omp_context *ctx) |
| 2426 | { |
| 2427 | if (ctx->record_type == NULL_TREE) |
| 2428 | return; |
| 2429 | |
| 2430 | /* If any task_shared_vars were needed, verify all |
| 2431 | OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK} |
| 2432 | statements if use_pointer_for_field hasn't changed |
| 2433 | because of that. If it did, update field types now. */ |
| 2434 | if (task_shared_vars) |
| 2435 | { |
| 2436 | tree c; |
| 2437 | |
| 2438 | for (c = gimple_omp_taskreg_clauses (ctx->stmt); |
| 2439 | c; c = OMP_CLAUSE_CHAIN (c)) |
| 2440 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED) |
| 2441 | { |
| 2442 | tree decl = OMP_CLAUSE_DECL (c); |
| 2443 | |
| 2444 | /* Global variables don't need to be copied, |
| 2445 | the receiver side will use them directly. */ |
| 2446 | if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))) |
| 2447 | continue; |
| 2448 | if (!bitmap_bit_p (task_shared_vars, DECL_UID (decl)) |
| 2449 | || !use_pointer_for_field (decl, ctx)) |
| 2450 | continue; |
| 2451 | tree field = lookup_field (decl, ctx); |
| 2452 | if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE |
| 2453 | && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl)) |
| 2454 | continue; |
| 2455 | TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl)); |
| 2456 | TREE_THIS_VOLATILE (field) = 0; |
| 2457 | DECL_USER_ALIGN (field) = 0; |
| 2458 | DECL_ALIGN (field) = TYPE_ALIGN (TREE_TYPE (field)); |
| 2459 | if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field)) |
| 2460 | TYPE_ALIGN (ctx->record_type) = DECL_ALIGN (field); |
| 2461 | if (ctx->srecord_type) |
| 2462 | { |
| 2463 | tree sfield = lookup_sfield (decl, ctx); |
| 2464 | TREE_TYPE (sfield) = TREE_TYPE (field); |
| 2465 | TREE_THIS_VOLATILE (sfield) = 0; |
| 2466 | DECL_USER_ALIGN (sfield) = 0; |
| 2467 | DECL_ALIGN (sfield) = DECL_ALIGN (field); |
| 2468 | if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield)) |
| 2469 | TYPE_ALIGN (ctx->srecord_type) = DECL_ALIGN (sfield); |
| 2470 | } |
| 2471 | } |
| 2472 | } |
| 2473 | |
| 2474 | if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL) |
| 2475 | { |
| 2476 | layout_type (ctx->record_type); |
| 2477 | fixup_child_record_type (ctx); |
| 2478 | } |
| 2479 | else |
| 2480 | { |
| 2481 | location_t loc = gimple_location (ctx->stmt); |
| 2482 | tree *p, vla_fields = NULL_TREE, *q = &vla_fields; |
| 2483 | /* Move VLA fields to the end. */ |
| 2484 | p = &TYPE_FIELDS (ctx->record_type); |
| 2485 | while (*p) |
| 2486 | if (!TYPE_SIZE_UNIT (TREE_TYPE (*p)) |
| 2487 | || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p)))) |
| 2488 | { |
| 2489 | *q = *p; |
| 2490 | *p = TREE_CHAIN (*p); |
| 2491 | TREE_CHAIN (*q) = NULL_TREE; |
| 2492 | q = &TREE_CHAIN (*q); |
| 2493 | } |
| 2494 | else |
| 2495 | p = &DECL_CHAIN (*p); |
| 2496 | *p = vla_fields; |
| 2497 | layout_type (ctx->record_type); |
| 2498 | fixup_child_record_type (ctx); |
| 2499 | if (ctx->srecord_type) |
| 2500 | layout_type (ctx->srecord_type); |
| 2501 | tree t = fold_convert_loc (loc, long_integer_type_node, |
| 2502 | TYPE_SIZE_UNIT (ctx->record_type)); |
| 2503 | gimple_omp_task_set_arg_size (ctx->stmt, t); |
| 2504 | t = build_int_cst (long_integer_type_node, |
| 2505 | TYPE_ALIGN_UNIT (ctx->record_type)); |
| 2506 | gimple_omp_task_set_arg_align (ctx->stmt, t); |
| 2507 | } |
| 2508 | } |
| 2509 | |
| 2510 | |
| 2511 | static omp_context * |
| 2512 | enclosing_target_ctx (omp_context *ctx) |
| 2513 | { |
| 2514 | while (ctx != NULL |
| 2515 | && gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET) |
| 2516 | ctx = ctx->outer; |
| 2517 | gcc_assert (ctx != NULL); |
| 2518 | return ctx; |
| 2519 | } |
| 2520 | |
| 2521 | static bool |
| 2522 | oacc_loop_or_target_p (gimple stmt) |
| 2523 | { |
| 2524 | enum gimple_code outer_type = gimple_code (stmt); |
| 2525 | return ((outer_type == GIMPLE_OMP_TARGET |
| 2526 | && ((gimple_omp_target_kind (stmt) |
| 2527 | == GF_OMP_TARGET_KIND_OACC_PARALLEL) |
| 2528 | || (gimple_omp_target_kind (stmt) |
| 2529 | == GF_OMP_TARGET_KIND_OACC_KERNELS))) |
| 2530 | || (outer_type == GIMPLE_OMP_FOR |
| 2531 | && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)); |
| 2532 | } |
| 2533 | |
| 2534 | /* Scan a GIMPLE_OMP_FOR. */ |
| 2535 | |
| 2536 | static void |
| 2537 | scan_omp_for (gomp_for *stmt, omp_context *outer_ctx) |
| 2538 | { |
| 2539 | enum gimple_code outer_type = GIMPLE_ERROR_MARK; |
| 2540 | omp_context *ctx; |
| 2541 | size_t i; |
| 2542 | tree clauses = gimple_omp_for_clauses (stmt); |
| 2543 | |
| 2544 | if (outer_ctx) |
| 2545 | outer_type = gimple_code (outer_ctx->stmt); |
| 2546 | |
| 2547 | ctx = new_omp_context (stmt, outer_ctx); |
| 2548 | |
| 2549 | if (is_gimple_omp_oacc (stmt)) |
| 2550 | { |
| 2551 | if (outer_ctx && outer_type == GIMPLE_OMP_FOR) |
| 2552 | ctx->gwv_this = outer_ctx->gwv_this; |
| 2553 | for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
| 2554 | { |
| 2555 | int val; |
| 2556 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_GANG) |
| 2557 | val = MASK_GANG; |
| 2558 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WORKER) |
| 2559 | val = MASK_WORKER; |
| 2560 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR) |
| 2561 | val = MASK_VECTOR; |
| 2562 | else |
| 2563 | continue; |
| 2564 | ctx->gwv_this |= val; |
| 2565 | if (!outer_ctx) |
| 2566 | { |
| 2567 | /* Skip; not nested inside a region. */ |
| 2568 | continue; |
| 2569 | } |
| 2570 | if (!oacc_loop_or_target_p (outer_ctx->stmt)) |
| 2571 | { |
| 2572 | /* Skip; not nested inside an OpenACC region. */ |
| 2573 | continue; |
| 2574 | } |
| 2575 | if (outer_type == GIMPLE_OMP_FOR) |
| 2576 | outer_ctx->gwv_below |= val; |
| 2577 | if (OMP_CLAUSE_OPERAND (c, 0) != NULL_TREE) |
| 2578 | { |
| 2579 | omp_context *enclosing = enclosing_target_ctx (outer_ctx); |
| 2580 | if (gimple_omp_target_kind (enclosing->stmt) |
| 2581 | == GF_OMP_TARGET_KIND_OACC_PARALLEL) |
| 2582 | error_at (gimple_location (stmt), |
| 2583 | "no arguments allowed to gang, worker and vector clauses inside parallel"); |
| 2584 | } |
| 2585 | } |
| 2586 | } |
| 2587 | |
| 2588 | scan_sharing_clauses (clauses, ctx); |
| 2589 | |
| 2590 | scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx); |
| 2591 | for (i = 0; i < gimple_omp_for_collapse (stmt); i++) |
| 2592 | { |
| 2593 | scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx); |
| 2594 | scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx); |
| 2595 | scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx); |
| 2596 | scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx); |
| 2597 | } |
| 2598 | scan_omp (gimple_omp_body_ptr (stmt), ctx); |
| 2599 | |
| 2600 | if (is_gimple_omp_oacc (stmt)) |
| 2601 | { |
| 2602 | if (ctx->gwv_this & ctx->gwv_below) |
| 2603 | error_at (gimple_location (stmt), |
| 2604 | "gang, worker and vector may occur only once in a loop nest"); |
| 2605 | else if (ctx->gwv_below != 0 |
| 2606 | && ctx->gwv_this > ctx->gwv_below) |
| 2607 | error_at (gimple_location (stmt), |
| 2608 | "gang, worker and vector must occur in this order in a loop nest"); |
| 2609 | if (outer_ctx && outer_type == GIMPLE_OMP_FOR) |
| 2610 | outer_ctx->gwv_below |= ctx->gwv_below; |
| 2611 | } |
| 2612 | } |
| 2613 | |
| 2614 | /* Scan an OpenMP sections directive. */ |
| 2615 | |
| 2616 | static void |
| 2617 | scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx) |
| 2618 | { |
| 2619 | omp_context *ctx; |
| 2620 | |
| 2621 | ctx = new_omp_context (stmt, outer_ctx); |
| 2622 | scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx); |
| 2623 | scan_omp (gimple_omp_body_ptr (stmt), ctx); |
| 2624 | } |
| 2625 | |
| 2626 | /* Scan an OpenMP single directive. */ |
| 2627 | |
| 2628 | static void |
| 2629 | scan_omp_single (gomp_single *stmt, omp_context *outer_ctx) |
| 2630 | { |
| 2631 | omp_context *ctx; |
| 2632 | tree name; |
| 2633 | |
| 2634 | ctx = new_omp_context (stmt, outer_ctx); |
| 2635 | ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
| 2636 | ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); |
| 2637 | name = create_tmp_var_name (".omp_copy_s"); |
| 2638 | name = build_decl (gimple_location (stmt), |
| 2639 | TYPE_DECL, name, ctx->record_type); |
| 2640 | TYPE_NAME (ctx->record_type) = name; |
| 2641 | |
| 2642 | scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx); |
| 2643 | scan_omp (gimple_omp_body_ptr (stmt), ctx); |
| 2644 | |
| 2645 | if (TYPE_FIELDS (ctx->record_type) == NULL) |
| 2646 | ctx->record_type = NULL; |
| 2647 | else |
| 2648 | layout_type (ctx->record_type); |
| 2649 | } |
| 2650 | |
| 2651 | /* Scan a GIMPLE_OMP_TARGET. */ |
| 2652 | |
| 2653 | static void |
| 2654 | scan_omp_target (gomp_target *stmt, omp_context *outer_ctx) |
| 2655 | { |
| 2656 | omp_context *ctx; |
| 2657 | tree name; |
| 2658 | bool offloaded = is_gimple_omp_offloaded (stmt); |
| 2659 | tree clauses = gimple_omp_target_clauses (stmt); |
| 2660 | |
| 2661 | ctx = new_omp_context (stmt, outer_ctx); |
| 2662 | ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
| 2663 | ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED; |
| 2664 | ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); |
| 2665 | name = create_tmp_var_name (".omp_data_t"); |
| 2666 | name = build_decl (gimple_location (stmt), |
| 2667 | TYPE_DECL, name, ctx->record_type); |
| 2668 | DECL_ARTIFICIAL (name) = 1; |
| 2669 | DECL_NAMELESS (name) = 1; |
| 2670 | TYPE_NAME (ctx->record_type) = name; |
| 2671 | if (offloaded) |
| 2672 | { |
| 2673 | if (is_gimple_omp_oacc (stmt)) |
| 2674 | ctx->reduction_map = splay_tree_new (splay_tree_compare_pointers, |
| 2675 | 0, 0); |
| 2676 | |
| 2677 | create_omp_child_function (ctx, false); |
| 2678 | gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn); |
| 2679 | } |
| 2680 | |
| 2681 | if (is_gimple_omp_oacc (stmt)) |
| 2682 | { |
| 2683 | for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
| 2684 | { |
| 2685 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_GANGS) |
| 2686 | ctx->gwv_this |= MASK_GANG; |
| 2687 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_WORKERS) |
| 2688 | ctx->gwv_this |= MASK_WORKER; |
| 2689 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR_LENGTH) |
| 2690 | ctx->gwv_this |= MASK_VECTOR; |
| 2691 | } |
| 2692 | } |
| 2693 | |
| 2694 | scan_sharing_clauses (clauses, ctx); |
| 2695 | scan_omp (gimple_omp_body_ptr (stmt), ctx); |
| 2696 | |
| 2697 | if (TYPE_FIELDS (ctx->record_type) == NULL) |
| 2698 | ctx->record_type = ctx->receiver_decl = NULL; |
| 2699 | else |
| 2700 | { |
| 2701 | TYPE_FIELDS (ctx->record_type) |
| 2702 | = nreverse (TYPE_FIELDS (ctx->record_type)); |
| 2703 | #ifdef ENABLE_CHECKING |
| 2704 | tree field; |
| 2705 | unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type)); |
| 2706 | for (field = TYPE_FIELDS (ctx->record_type); |
| 2707 | field; |
| 2708 | field = DECL_CHAIN (field)) |
| 2709 | gcc_assert (DECL_ALIGN (field) == align); |
| 2710 | #endif |
| 2711 | layout_type (ctx->record_type); |
| 2712 | if (offloaded) |
| 2713 | fixup_child_record_type (ctx); |
| 2714 | } |
| 2715 | } |
| 2716 | |
| 2717 | /* Scan an OpenMP teams directive. */ |
| 2718 | |
| 2719 | static void |
| 2720 | scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx) |
| 2721 | { |
| 2722 | omp_context *ctx = new_omp_context (stmt, outer_ctx); |
| 2723 | scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx); |
| 2724 | scan_omp (gimple_omp_body_ptr (stmt), ctx); |
| 2725 | } |
| 2726 | |
| 2727 | /* Check nesting restrictions. */ |
| 2728 | static bool |
| 2729 | check_omp_nesting_restrictions (gimple stmt, omp_context *ctx) |
| 2730 | { |
| 2731 | /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin) |
| 2732 | inside an OpenACC CTX. */ |
| 2733 | if (!(is_gimple_omp (stmt) |
| 2734 | && is_gimple_omp_oacc (stmt))) |
| 2735 | { |
| 2736 | for (omp_context *ctx_ = ctx; ctx_ != NULL; ctx_ = ctx_->outer) |
| 2737 | if (is_gimple_omp (ctx_->stmt) |
| 2738 | && is_gimple_omp_oacc (ctx_->stmt)) |
| 2739 | { |
| 2740 | error_at (gimple_location (stmt), |
| 2741 | "non-OpenACC construct inside of OpenACC region"); |
| 2742 | return false; |
| 2743 | } |
| 2744 | } |
| 2745 | |
| 2746 | if (ctx != NULL) |
| 2747 | { |
| 2748 | if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR |
| 2749 | && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) |
| 2750 | { |
| 2751 | error_at (gimple_location (stmt), |
| 2752 | "OpenMP constructs may not be nested inside simd region"); |
| 2753 | return false; |
| 2754 | } |
| 2755 | else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) |
| 2756 | { |
| 2757 | if ((gimple_code (stmt) != GIMPLE_OMP_FOR |
| 2758 | || (gimple_omp_for_kind (stmt) |
| 2759 | != GF_OMP_FOR_KIND_DISTRIBUTE)) |
| 2760 | && gimple_code (stmt) != GIMPLE_OMP_PARALLEL) |
| 2761 | { |
| 2762 | error_at (gimple_location (stmt), |
| 2763 | "only distribute or parallel constructs are allowed to " |
| 2764 | "be closely nested inside teams construct"); |
| 2765 | return false; |
| 2766 | } |
| 2767 | } |
| 2768 | } |
| 2769 | switch (gimple_code (stmt)) |
| 2770 | { |
| 2771 | case GIMPLE_OMP_FOR: |
| 2772 | if (gimple_omp_for_kind (stmt) & GF_OMP_FOR_SIMD) |
| 2773 | return true; |
| 2774 | if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE) |
| 2775 | { |
| 2776 | if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS) |
| 2777 | { |
| 2778 | error_at (gimple_location (stmt), |
| 2779 | "distribute construct must be closely nested inside " |
| 2780 | "teams construct"); |
| 2781 | return false; |
| 2782 | } |
| 2783 | return true; |
| 2784 | } |
| 2785 | /* FALLTHRU */ |
| 2786 | case GIMPLE_CALL: |
| 2787 | if (is_gimple_call (stmt) |
| 2788 | && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) |
| 2789 | == BUILT_IN_GOMP_CANCEL |
| 2790 | || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) |
| 2791 | == BUILT_IN_GOMP_CANCELLATION_POINT)) |
| 2792 | { |
| 2793 | const char *bad = NULL; |
| 2794 | const char *kind = NULL; |
| 2795 | if (ctx == NULL) |
| 2796 | { |
| 2797 | error_at (gimple_location (stmt), "orphaned %qs construct", |
| 2798 | DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) |
| 2799 | == BUILT_IN_GOMP_CANCEL |
| 2800 | ? "#pragma omp cancel" |
| 2801 | : "#pragma omp cancellation point"); |
| 2802 | return false; |
| 2803 | } |
| 2804 | switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0)) |
| 2805 | ? tree_to_shwi (gimple_call_arg (stmt, 0)) |
| 2806 | : 0) |
| 2807 | { |
| 2808 | case 1: |
| 2809 | if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL) |
| 2810 | bad = "#pragma omp parallel"; |
| 2811 | else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) |
| 2812 | == BUILT_IN_GOMP_CANCEL |
| 2813 | && !integer_zerop (gimple_call_arg (stmt, 1))) |
| 2814 | ctx->cancellable = true; |
| 2815 | kind = "parallel"; |
| 2816 | break; |
| 2817 | case 2: |
| 2818 | if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR |
| 2819 | || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR) |
| 2820 | bad = "#pragma omp for"; |
| 2821 | else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) |
| 2822 | == BUILT_IN_GOMP_CANCEL |
| 2823 | && !integer_zerop (gimple_call_arg (stmt, 1))) |
| 2824 | { |
| 2825 | ctx->cancellable = true; |
| 2826 | if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt), |
| 2827 | OMP_CLAUSE_NOWAIT)) |
| 2828 | warning_at (gimple_location (stmt), 0, |
| 2829 | "%<#pragma omp cancel for%> inside " |
| 2830 | "%<nowait%> for construct"); |
| 2831 | if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt), |
| 2832 | OMP_CLAUSE_ORDERED)) |
| 2833 | warning_at (gimple_location (stmt), 0, |
| 2834 | "%<#pragma omp cancel for%> inside " |
| 2835 | "%<ordered%> for construct"); |
| 2836 | } |
| 2837 | kind = "for"; |
| 2838 | break; |
| 2839 | case 4: |
| 2840 | if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS |
| 2841 | && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION) |
| 2842 | bad = "#pragma omp sections"; |
| 2843 | else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) |
| 2844 | == BUILT_IN_GOMP_CANCEL |
| 2845 | && !integer_zerop (gimple_call_arg (stmt, 1))) |
| 2846 | { |
| 2847 | if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS) |
| 2848 | { |
| 2849 | ctx->cancellable = true; |
| 2850 | if (find_omp_clause (gimple_omp_sections_clauses |
| 2851 | (ctx->stmt), |
| 2852 | OMP_CLAUSE_NOWAIT)) |
| 2853 | warning_at (gimple_location (stmt), 0, |
| 2854 | "%<#pragma omp cancel sections%> inside " |
| 2855 | "%<nowait%> sections construct"); |
| 2856 | } |
| 2857 | else |
| 2858 | { |
| 2859 | gcc_assert (ctx->outer |
| 2860 | && gimple_code (ctx->outer->stmt) |
| 2861 | == GIMPLE_OMP_SECTIONS); |
| 2862 | ctx->outer->cancellable = true; |
| 2863 | if (find_omp_clause (gimple_omp_sections_clauses |
| 2864 | (ctx->outer->stmt), |
| 2865 | OMP_CLAUSE_NOWAIT)) |
| 2866 | warning_at (gimple_location (stmt), 0, |
| 2867 | "%<#pragma omp cancel sections%> inside " |
| 2868 | "%<nowait%> sections construct"); |
| 2869 | } |
| 2870 | } |
| 2871 | kind = "sections"; |
| 2872 | break; |
| 2873 | case 8: |
| 2874 | if (gimple_code (ctx->stmt) != GIMPLE_OMP_TASK) |
| 2875 | bad = "#pragma omp task"; |
| 2876 | else |
| 2877 | ctx->cancellable = true; |
| 2878 | kind = "taskgroup"; |
| 2879 | break; |
| 2880 | default: |
| 2881 | error_at (gimple_location (stmt), "invalid arguments"); |
| 2882 | return false; |
| 2883 | } |
| 2884 | if (bad) |
| 2885 | { |
| 2886 | error_at (gimple_location (stmt), |
| 2887 | "%<%s %s%> construct not closely nested inside of %qs", |
| 2888 | DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) |
| 2889 | == BUILT_IN_GOMP_CANCEL |
| 2890 | ? "#pragma omp cancel" |
| 2891 | : "#pragma omp cancellation point", kind, bad); |
| 2892 | return false; |
| 2893 | } |
| 2894 | } |
| 2895 | /* FALLTHRU */ |
| 2896 | case GIMPLE_OMP_SECTIONS: |
| 2897 | case GIMPLE_OMP_SINGLE: |
| 2898 | for (; ctx != NULL; ctx = ctx->outer) |
| 2899 | switch (gimple_code (ctx->stmt)) |
| 2900 | { |
| 2901 | case GIMPLE_OMP_FOR: |
| 2902 | case GIMPLE_OMP_SECTIONS: |
| 2903 | case GIMPLE_OMP_SINGLE: |
| 2904 | case GIMPLE_OMP_ORDERED: |
| 2905 | case GIMPLE_OMP_MASTER: |
| 2906 | case GIMPLE_OMP_TASK: |
| 2907 | case GIMPLE_OMP_CRITICAL: |
| 2908 | if (is_gimple_call (stmt)) |
| 2909 | { |
| 2910 | if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) |
| 2911 | != BUILT_IN_GOMP_BARRIER) |
| 2912 | return true; |
| 2913 | error_at (gimple_location (stmt), |
| 2914 | "barrier region may not be closely nested inside " |
| 2915 | "of work-sharing, critical, ordered, master or " |
| 2916 | "explicit task region"); |
| 2917 | return false; |
| 2918 | } |
| 2919 | error_at (gimple_location (stmt), |
| 2920 | "work-sharing region may not be closely nested inside " |
| 2921 | "of work-sharing, critical, ordered, master or explicit " |
| 2922 | "task region"); |
| 2923 | return false; |
| 2924 | case GIMPLE_OMP_PARALLEL: |
| 2925 | return true; |
| 2926 | default: |
| 2927 | break; |
| 2928 | } |
| 2929 | break; |
| 2930 | case GIMPLE_OMP_MASTER: |
| 2931 | for (; ctx != NULL; ctx = ctx->outer) |
| 2932 | switch (gimple_code (ctx->stmt)) |
| 2933 | { |
| 2934 | case GIMPLE_OMP_FOR: |
| 2935 | case GIMPLE_OMP_SECTIONS: |
| 2936 | case GIMPLE_OMP_SINGLE: |
| 2937 | case GIMPLE_OMP_TASK: |
| 2938 | error_at (gimple_location (stmt), |
| 2939 | "master region may not be closely nested inside " |
| 2940 | "of work-sharing or explicit task region"); |
| 2941 | return false; |
| 2942 | case GIMPLE_OMP_PARALLEL: |
| 2943 | return true; |
| 2944 | default: |
| 2945 | break; |
| 2946 | } |
| 2947 | break; |
| 2948 | case GIMPLE_OMP_ORDERED: |
| 2949 | for (; ctx != NULL; ctx = ctx->outer) |
| 2950 | switch (gimple_code (ctx->stmt)) |
| 2951 | { |
| 2952 | case GIMPLE_OMP_CRITICAL: |
| 2953 | case GIMPLE_OMP_TASK: |
| 2954 | error_at (gimple_location (stmt), |
| 2955 | "ordered region may not be closely nested inside " |
| 2956 | "of critical or explicit task region"); |
| 2957 | return false; |
| 2958 | case GIMPLE_OMP_FOR: |
| 2959 | if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt), |
| 2960 | OMP_CLAUSE_ORDERED) == NULL) |
| 2961 | { |
| 2962 | error_at (gimple_location (stmt), |
| 2963 | "ordered region must be closely nested inside " |
| 2964 | "a loop region with an ordered clause"); |
| 2965 | return false; |
| 2966 | } |
| 2967 | return true; |
| 2968 | case GIMPLE_OMP_PARALLEL: |
| 2969 | error_at (gimple_location (stmt), |
| 2970 | "ordered region must be closely nested inside " |
| 2971 | "a loop region with an ordered clause"); |
| 2972 | return false; |
| 2973 | default: |
| 2974 | break; |
| 2975 | } |
| 2976 | break; |
| 2977 | case GIMPLE_OMP_CRITICAL: |
| 2978 | { |
| 2979 | tree this_stmt_name |
| 2980 | = gimple_omp_critical_name (as_a <gomp_critical *> (stmt)); |
| 2981 | for (; ctx != NULL; ctx = ctx->outer) |
| 2982 | if (gomp_critical *other_crit |
| 2983 | = dyn_cast <gomp_critical *> (ctx->stmt)) |
| 2984 | if (this_stmt_name == gimple_omp_critical_name (other_crit)) |
| 2985 | { |
| 2986 | error_at (gimple_location (stmt), |
| 2987 | "critical region may not be nested inside a critical " |
| 2988 | "region with the same name"); |
| 2989 | return false; |
| 2990 | } |
| 2991 | } |
| 2992 | break; |
| 2993 | case GIMPLE_OMP_TEAMS: |
| 2994 | if (ctx == NULL |
| 2995 | || gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET |
| 2996 | || gimple_omp_target_kind (ctx->stmt) != GF_OMP_TARGET_KIND_REGION) |
| 2997 | { |
| 2998 | error_at (gimple_location (stmt), |
| 2999 | "teams construct not closely nested inside of target " |
| 3000 | "region"); |
| 3001 | return false; |
| 3002 | } |
| 3003 | break; |
| 3004 | case GIMPLE_OMP_TARGET: |
| 3005 | for (; ctx != NULL; ctx = ctx->outer) |
| 3006 | { |
| 3007 | if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET) |
| 3008 | { |
| 3009 | if (is_gimple_omp (stmt) |
| 3010 | && is_gimple_omp_oacc (stmt) |
| 3011 | && is_gimple_omp (ctx->stmt)) |
| 3012 | { |
| 3013 | error_at (gimple_location (stmt), |
| 3014 | "OpenACC construct inside of non-OpenACC region"); |
| 3015 | return false; |
| 3016 | } |
| 3017 | continue; |
| 3018 | } |
| 3019 | |
| 3020 | const char *stmt_name, *ctx_stmt_name; |
| 3021 | switch (gimple_omp_target_kind (stmt)) |
| 3022 | { |
| 3023 | case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break; |
| 3024 | case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break; |
| 3025 | case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break; |
| 3026 | case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break; |
| 3027 | case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break; |
| 3028 | case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break; |
| 3029 | case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break; |
| 3030 | case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: stmt_name = "enter/exit data"; break; |
| 3031 | default: gcc_unreachable (); |
| 3032 | } |
| 3033 | switch (gimple_omp_target_kind (ctx->stmt)) |
| 3034 | { |
| 3035 | case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break; |
| 3036 | case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break; |
| 3037 | case GF_OMP_TARGET_KIND_OACC_PARALLEL: ctx_stmt_name = "parallel"; break; |
| 3038 | case GF_OMP_TARGET_KIND_OACC_KERNELS: ctx_stmt_name = "kernels"; break; |
| 3039 | case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break; |
| 3040 | default: gcc_unreachable (); |
| 3041 | } |
| 3042 | |
| 3043 | /* OpenACC/OpenMP mismatch? */ |
| 3044 | if (is_gimple_omp_oacc (stmt) |
| 3045 | != is_gimple_omp_oacc (ctx->stmt)) |
| 3046 | { |
| 3047 | error_at (gimple_location (stmt), |
| 3048 | "%s %s construct inside of %s %s region", |
| 3049 | (is_gimple_omp_oacc (stmt) |
| 3050 | ? "OpenACC" : "OpenMP"), stmt_name, |
| 3051 | (is_gimple_omp_oacc (ctx->stmt) |
| 3052 | ? "OpenACC" : "OpenMP"), ctx_stmt_name); |
| 3053 | return false; |
| 3054 | } |
| 3055 | if (is_gimple_omp_offloaded (ctx->stmt)) |
| 3056 | { |
| 3057 | /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */ |
| 3058 | if (is_gimple_omp_oacc (ctx->stmt)) |
| 3059 | { |
| 3060 | error_at (gimple_location (stmt), |
| 3061 | "%s construct inside of %s region", |
| 3062 | stmt_name, ctx_stmt_name); |
| 3063 | return false; |
| 3064 | } |
| 3065 | else |
| 3066 | { |
| 3067 | gcc_checking_assert (!is_gimple_omp_oacc (stmt)); |
| 3068 | warning_at (gimple_location (stmt), 0, |
| 3069 | "%s construct inside of %s region", |
| 3070 | stmt_name, ctx_stmt_name); |
| 3071 | } |
| 3072 | } |
| 3073 | } |
| 3074 | break; |
| 3075 | default: |
| 3076 | break; |
| 3077 | } |
| 3078 | return true; |
| 3079 | } |
| 3080 | |
| 3081 | |
| 3082 | /* Helper function scan_omp. |
| 3083 | |
| 3084 | Callback for walk_tree or operators in walk_gimple_stmt used to |
| 3085 | scan for OMP directives in TP. */ |
| 3086 | |
| 3087 | static tree |
| 3088 | scan_omp_1_op (tree *tp, int *walk_subtrees, void *data) |
| 3089 | { |
| 3090 | struct walk_stmt_info *wi = (struct walk_stmt_info *) data; |
| 3091 | omp_context *ctx = (omp_context *) wi->info; |
| 3092 | tree t = *tp; |
| 3093 | |
| 3094 | switch (TREE_CODE (t)) |
| 3095 | { |
| 3096 | case VAR_DECL: |
| 3097 | case PARM_DECL: |
| 3098 | case LABEL_DECL: |
| 3099 | case RESULT_DECL: |
| 3100 | if (ctx) |
| 3101 | *tp = remap_decl (t, &ctx->cb); |
| 3102 | break; |
| 3103 | |
| 3104 | default: |
| 3105 | if (ctx && TYPE_P (t)) |
| 3106 | *tp = remap_type (t, &ctx->cb); |
| 3107 | else if (!DECL_P (t)) |
| 3108 | { |
| 3109 | *walk_subtrees = 1; |
| 3110 | if (ctx) |
| 3111 | { |
| 3112 | tree tem = remap_type (TREE_TYPE (t), &ctx->cb); |
| 3113 | if (tem != TREE_TYPE (t)) |
| 3114 | { |
| 3115 | if (TREE_CODE (t) == INTEGER_CST) |
| 3116 | *tp = wide_int_to_tree (tem, t); |
| 3117 | else |
| 3118 | TREE_TYPE (t) = tem; |
| 3119 | } |
| 3120 | } |
| 3121 | } |
| 3122 | break; |
| 3123 | } |
| 3124 | |
| 3125 | return NULL_TREE; |
| 3126 | } |
| 3127 | |
| 3128 | /* Return true if FNDECL is a setjmp or a longjmp. */ |
| 3129 | |
| 3130 | static bool |
| 3131 | setjmp_or_longjmp_p (const_tree fndecl) |
| 3132 | { |
| 3133 | if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL |
| 3134 | && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_SETJMP |
| 3135 | || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LONGJMP)) |
| 3136 | return true; |
| 3137 | |
| 3138 | tree declname = DECL_NAME (fndecl); |
| 3139 | if (!declname) |
| 3140 | return false; |
| 3141 | const char *name = IDENTIFIER_POINTER (declname); |
| 3142 | return !strcmp (name, "setjmp") || !strcmp (name, "longjmp"); |
| 3143 | } |
| 3144 | |
| 3145 | |
| 3146 | /* Helper function for scan_omp. |
| 3147 | |
| 3148 | Callback for walk_gimple_stmt used to scan for OMP directives in |
| 3149 | the current statement in GSI. */ |
| 3150 | |
| 3151 | static tree |
| 3152 | scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p, |
| 3153 | struct walk_stmt_info *wi) |
| 3154 | { |
| 3155 | gimple stmt = gsi_stmt (*gsi); |
| 3156 | omp_context *ctx = (omp_context *) wi->info; |
| 3157 | |
| 3158 | if (gimple_has_location (stmt)) |
| 3159 | input_location = gimple_location (stmt); |
| 3160 | |
| 3161 | /* Check the nesting restrictions. */ |
| 3162 | bool remove = false; |
| 3163 | if (is_gimple_omp (stmt)) |
| 3164 | remove = !check_omp_nesting_restrictions (stmt, ctx); |
| 3165 | else if (is_gimple_call (stmt)) |
| 3166 | { |
| 3167 | tree fndecl = gimple_call_fndecl (stmt); |
| 3168 | if (fndecl) |
| 3169 | { |
| 3170 | if (setjmp_or_longjmp_p (fndecl) |
| 3171 | && ctx |
| 3172 | && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR |
| 3173 | && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) |
| 3174 | { |
| 3175 | remove = true; |
| 3176 | error_at (gimple_location (stmt), |
| 3177 | "setjmp/longjmp inside simd construct"); |
| 3178 | } |
| 3179 | else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) |
| 3180 | switch (DECL_FUNCTION_CODE (fndecl)) |
| 3181 | { |
| 3182 | case BUILT_IN_GOMP_BARRIER: |
| 3183 | case BUILT_IN_GOMP_CANCEL: |
| 3184 | case BUILT_IN_GOMP_CANCELLATION_POINT: |
| 3185 | case BUILT_IN_GOMP_TASKYIELD: |
| 3186 | case BUILT_IN_GOMP_TASKWAIT: |
| 3187 | case BUILT_IN_GOMP_TASKGROUP_START: |
| 3188 | case BUILT_IN_GOMP_TASKGROUP_END: |
| 3189 | remove = !check_omp_nesting_restrictions (stmt, ctx); |
| 3190 | break; |
| 3191 | default: |
| 3192 | break; |
| 3193 | } |
| 3194 | } |
| 3195 | } |
| 3196 | if (remove) |
| 3197 | { |
| 3198 | stmt = gimple_build_nop (); |
| 3199 | gsi_replace (gsi, stmt, false); |
| 3200 | } |
| 3201 | |
| 3202 | *handled_ops_p = true; |
| 3203 | |
| 3204 | switch (gimple_code (stmt)) |
| 3205 | { |
| 3206 | case GIMPLE_OMP_PARALLEL: |
| 3207 | taskreg_nesting_level++; |
| 3208 | scan_omp_parallel (gsi, ctx); |
| 3209 | taskreg_nesting_level--; |
| 3210 | break; |
| 3211 | |
| 3212 | case GIMPLE_OMP_TASK: |
| 3213 | taskreg_nesting_level++; |
| 3214 | scan_omp_task (gsi, ctx); |
| 3215 | taskreg_nesting_level--; |
| 3216 | break; |
| 3217 | |
| 3218 | case GIMPLE_OMP_FOR: |
| 3219 | scan_omp_for (as_a <gomp_for *> (stmt), ctx); |
| 3220 | break; |
| 3221 | |
| 3222 | case GIMPLE_OMP_SECTIONS: |
| 3223 | scan_omp_sections (as_a <gomp_sections *> (stmt), ctx); |
| 3224 | break; |
| 3225 | |
| 3226 | case GIMPLE_OMP_SINGLE: |
| 3227 | scan_omp_single (as_a <gomp_single *> (stmt), ctx); |
| 3228 | break; |
| 3229 | |
| 3230 | case GIMPLE_OMP_SECTION: |
| 3231 | case GIMPLE_OMP_MASTER: |
| 3232 | case GIMPLE_OMP_TASKGROUP: |
| 3233 | case GIMPLE_OMP_ORDERED: |
| 3234 | case GIMPLE_OMP_CRITICAL: |
| 3235 | ctx = new_omp_context (stmt, ctx); |
| 3236 | scan_omp (gimple_omp_body_ptr (stmt), ctx); |
| 3237 | break; |
| 3238 | |
| 3239 | case GIMPLE_OMP_TARGET: |
| 3240 | scan_omp_target (as_a <gomp_target *> (stmt), ctx); |
| 3241 | break; |
| 3242 | |
| 3243 | case GIMPLE_OMP_TEAMS: |
| 3244 | scan_omp_teams (as_a <gomp_teams *> (stmt), ctx); |
| 3245 | break; |
| 3246 | |
| 3247 | case GIMPLE_BIND: |
| 3248 | { |
| 3249 | tree var; |
| 3250 | |
| 3251 | *handled_ops_p = false; |
| 3252 | if (ctx) |
| 3253 | for (var = gimple_bind_vars (as_a <gbind *> (stmt)); |
| 3254 | var ; |
| 3255 | var = DECL_CHAIN (var)) |
| 3256 | insert_decl_map (&ctx->cb, var, var); |
| 3257 | } |
| 3258 | break; |
| 3259 | default: |
| 3260 | *handled_ops_p = false; |
| 3261 | break; |
| 3262 | } |
| 3263 | |
| 3264 | return NULL_TREE; |
| 3265 | } |
| 3266 | |
| 3267 | |
| 3268 | /* Scan all the statements starting at the current statement. CTX |
| 3269 | contains context information about the OMP directives and |
| 3270 | clauses found during the scan. */ |
| 3271 | |
| 3272 | static void |
| 3273 | scan_omp (gimple_seq *body_p, omp_context *ctx) |
| 3274 | { |
| 3275 | location_t saved_location; |
| 3276 | struct walk_stmt_info wi; |
| 3277 | |
| 3278 | memset (&wi, 0, sizeof (wi)); |
| 3279 | wi.info = ctx; |
| 3280 | wi.want_locations = true; |
| 3281 | |
| 3282 | saved_location = input_location; |
| 3283 | walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi); |
| 3284 | input_location = saved_location; |
| 3285 | } |
| 3286 | \f |
| 3287 | /* Re-gimplification and code generation routines. */ |
| 3288 | |
| 3289 | /* Build a call to GOMP_barrier. */ |
| 3290 | |
| 3291 | static gimple |
| 3292 | build_omp_barrier (tree lhs) |
| 3293 | { |
| 3294 | tree fndecl = builtin_decl_explicit (lhs ? BUILT_IN_GOMP_BARRIER_CANCEL |
| 3295 | : BUILT_IN_GOMP_BARRIER); |
| 3296 | gcall *g = gimple_build_call (fndecl, 0); |
| 3297 | if (lhs) |
| 3298 | gimple_call_set_lhs (g, lhs); |
| 3299 | return g; |
| 3300 | } |
| 3301 | |
| 3302 | /* If a context was created for STMT when it was scanned, return it. */ |
| 3303 | |
| 3304 | static omp_context * |
| 3305 | maybe_lookup_ctx (gimple stmt) |
| 3306 | { |
| 3307 | splay_tree_node n; |
| 3308 | n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt); |
| 3309 | return n ? (omp_context *) n->value : NULL; |
| 3310 | } |
| 3311 | |
| 3312 | |
| 3313 | /* Find the mapping for DECL in CTX or the immediately enclosing |
| 3314 | context that has a mapping for DECL. |
| 3315 | |
| 3316 | If CTX is a nested parallel directive, we may have to use the decl |
| 3317 | mappings created in CTX's parent context. Suppose that we have the |
| 3318 | following parallel nesting (variable UIDs showed for clarity): |
| 3319 | |
| 3320 | iD.1562 = 0; |
| 3321 | #omp parallel shared(iD.1562) -> outer parallel |
| 3322 | iD.1562 = iD.1562 + 1; |
| 3323 | |
| 3324 | #omp parallel shared (iD.1562) -> inner parallel |
| 3325 | iD.1562 = iD.1562 - 1; |
| 3326 | |
| 3327 | Each parallel structure will create a distinct .omp_data_s structure |
| 3328 | for copying iD.1562 in/out of the directive: |
| 3329 | |
| 3330 | outer parallel .omp_data_s.1.i -> iD.1562 |
| 3331 | inner parallel .omp_data_s.2.i -> iD.1562 |
| 3332 | |
| 3333 | A shared variable mapping will produce a copy-out operation before |
| 3334 | the parallel directive and a copy-in operation after it. So, in |
| 3335 | this case we would have: |
| 3336 | |
| 3337 | iD.1562 = 0; |
| 3338 | .omp_data_o.1.i = iD.1562; |
| 3339 | #omp parallel shared(iD.1562) -> outer parallel |
| 3340 | .omp_data_i.1 = &.omp_data_o.1 |
| 3341 | .omp_data_i.1->i = .omp_data_i.1->i + 1; |
| 3342 | |
| 3343 | .omp_data_o.2.i = iD.1562; -> ** |
| 3344 | #omp parallel shared(iD.1562) -> inner parallel |
| 3345 | .omp_data_i.2 = &.omp_data_o.2 |
| 3346 | .omp_data_i.2->i = .omp_data_i.2->i - 1; |
| 3347 | |
| 3348 | |
| 3349 | ** This is a problem. The symbol iD.1562 cannot be referenced |
| 3350 | inside the body of the outer parallel region. But since we are |
| 3351 | emitting this copy operation while expanding the inner parallel |
| 3352 | directive, we need to access the CTX structure of the outer |
| 3353 | parallel directive to get the correct mapping: |
| 3354 | |
| 3355 | .omp_data_o.2.i = .omp_data_i.1->i |
| 3356 | |
| 3357 | Since there may be other workshare or parallel directives enclosing |
| 3358 | the parallel directive, it may be necessary to walk up the context |
| 3359 | parent chain. This is not a problem in general because nested |
| 3360 | parallelism happens only rarely. */ |
| 3361 | |
| 3362 | static tree |
| 3363 | lookup_decl_in_outer_ctx (tree decl, omp_context *ctx) |
| 3364 | { |
| 3365 | tree t; |
| 3366 | omp_context *up; |
| 3367 | |
| 3368 | for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer) |
| 3369 | t = maybe_lookup_decl (decl, up); |
| 3370 | |
| 3371 | gcc_assert (!ctx->is_nested || t || is_global_var (decl)); |
| 3372 | |
| 3373 | return t ? t : decl; |
| 3374 | } |
| 3375 | |
| 3376 | |
| 3377 | /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found |
| 3378 | in outer contexts. */ |
| 3379 | |
| 3380 | static tree |
| 3381 | maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx) |
| 3382 | { |
| 3383 | tree t = NULL; |
| 3384 | omp_context *up; |
| 3385 | |
| 3386 | for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer) |
| 3387 | t = maybe_lookup_decl (decl, up); |
| 3388 | |
| 3389 | return t ? t : decl; |
| 3390 | } |
| 3391 | |
| 3392 | |
| 3393 | /* Construct the initialization value for reduction CLAUSE. */ |
| 3394 | |
| 3395 | tree |
| 3396 | omp_reduction_init (tree clause, tree type) |
| 3397 | { |
| 3398 | location_t loc = OMP_CLAUSE_LOCATION (clause); |
| 3399 | switch (OMP_CLAUSE_REDUCTION_CODE (clause)) |
| 3400 | { |
| 3401 | case PLUS_EXPR: |
| 3402 | case MINUS_EXPR: |
| 3403 | case BIT_IOR_EXPR: |
| 3404 | case BIT_XOR_EXPR: |
| 3405 | case TRUTH_OR_EXPR: |
| 3406 | case TRUTH_ORIF_EXPR: |
| 3407 | case TRUTH_XOR_EXPR: |
| 3408 | case NE_EXPR: |
| 3409 | return build_zero_cst (type); |
| 3410 | |
| 3411 | case MULT_EXPR: |
| 3412 | case TRUTH_AND_EXPR: |
| 3413 | case TRUTH_ANDIF_EXPR: |
| 3414 | case EQ_EXPR: |
| 3415 | return fold_convert_loc (loc, type, integer_one_node); |
| 3416 | |
| 3417 | case BIT_AND_EXPR: |
| 3418 | return fold_convert_loc (loc, type, integer_minus_one_node); |
| 3419 | |
| 3420 | case MAX_EXPR: |
| 3421 | if (SCALAR_FLOAT_TYPE_P (type)) |
| 3422 | { |
| 3423 | REAL_VALUE_TYPE max, min; |
| 3424 | if (HONOR_INFINITIES (type)) |
| 3425 | { |
| 3426 | real_inf (&max); |
| 3427 | real_arithmetic (&min, NEGATE_EXPR, &max, NULL); |
| 3428 | } |
| 3429 | else |
| 3430 | real_maxval (&min, 1, TYPE_MODE (type)); |
| 3431 | return build_real (type, min); |
| 3432 | } |
| 3433 | else |
| 3434 | { |
| 3435 | gcc_assert (INTEGRAL_TYPE_P (type)); |
| 3436 | return TYPE_MIN_VALUE (type); |
| 3437 | } |
| 3438 | |
| 3439 | case MIN_EXPR: |
| 3440 | if (SCALAR_FLOAT_TYPE_P (type)) |
| 3441 | { |
| 3442 | REAL_VALUE_TYPE max; |
| 3443 | if (HONOR_INFINITIES (type)) |
| 3444 | real_inf (&max); |
| 3445 | else |
| 3446 | real_maxval (&max, 0, TYPE_MODE (type)); |
| 3447 | return build_real (type, max); |
| 3448 | } |
| 3449 | else |
| 3450 | { |
| 3451 | gcc_assert (INTEGRAL_TYPE_P (type)); |
| 3452 | return TYPE_MAX_VALUE (type); |
| 3453 | } |
| 3454 | |
| 3455 | default: |
| 3456 | gcc_unreachable (); |
| 3457 | } |
| 3458 | } |
| 3459 | |
| 3460 | /* Return alignment to be assumed for var in CLAUSE, which should be |
| 3461 | OMP_CLAUSE_ALIGNED. */ |
| 3462 | |
| 3463 | static tree |
| 3464 | omp_clause_aligned_alignment (tree clause) |
| 3465 | { |
| 3466 | if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause)) |
| 3467 | return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause); |
| 3468 | |
| 3469 | /* Otherwise return implementation defined alignment. */ |
| 3470 | unsigned int al = 1; |
| 3471 | machine_mode mode, vmode; |
| 3472 | int vs = targetm.vectorize.autovectorize_vector_sizes (); |
| 3473 | if (vs) |
| 3474 | vs = 1 << floor_log2 (vs); |
| 3475 | static enum mode_class classes[] |
| 3476 | = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT }; |
| 3477 | for (int i = 0; i < 4; i += 2) |
| 3478 | for (mode = GET_CLASS_NARROWEST_MODE (classes[i]); |
| 3479 | mode != VOIDmode; |
| 3480 | mode = GET_MODE_WIDER_MODE (mode)) |
| 3481 | { |
| 3482 | vmode = targetm.vectorize.preferred_simd_mode (mode); |
| 3483 | if (GET_MODE_CLASS (vmode) != classes[i + 1]) |
| 3484 | continue; |
| 3485 | while (vs |
| 3486 | && GET_MODE_SIZE (vmode) < vs |
| 3487 | && GET_MODE_2XWIDER_MODE (vmode) != VOIDmode) |
| 3488 | vmode = GET_MODE_2XWIDER_MODE (vmode); |
| 3489 | |
| 3490 | tree type = lang_hooks.types.type_for_mode (mode, 1); |
| 3491 | if (type == NULL_TREE || TYPE_MODE (type) != mode) |
| 3492 | continue; |
| 3493 | type = build_vector_type (type, GET_MODE_SIZE (vmode) |
| 3494 | / GET_MODE_SIZE (mode)); |
| 3495 | if (TYPE_MODE (type) != vmode) |
| 3496 | continue; |
| 3497 | if (TYPE_ALIGN_UNIT (type) > al) |
| 3498 | al = TYPE_ALIGN_UNIT (type); |
| 3499 | } |
| 3500 | return build_int_cst (integer_type_node, al); |
| 3501 | } |
| 3502 | |
| 3503 | /* Return maximum possible vectorization factor for the target. */ |
| 3504 | |
| 3505 | static int |
| 3506 | omp_max_vf (void) |
| 3507 | { |
| 3508 | if (!optimize |
| 3509 | || optimize_debug |
| 3510 | || !flag_tree_loop_optimize |
| 3511 | || (!flag_tree_loop_vectorize |
| 3512 | && (global_options_set.x_flag_tree_loop_vectorize |
| 3513 | || global_options_set.x_flag_tree_vectorize))) |
| 3514 | return 1; |
| 3515 | |
| 3516 | int vs = targetm.vectorize.autovectorize_vector_sizes (); |
| 3517 | if (vs) |
| 3518 | { |
| 3519 | vs = 1 << floor_log2 (vs); |
| 3520 | return vs; |
| 3521 | } |
| 3522 | machine_mode vqimode = targetm.vectorize.preferred_simd_mode (QImode); |
| 3523 | if (GET_MODE_CLASS (vqimode) == MODE_VECTOR_INT) |
| 3524 | return GET_MODE_NUNITS (vqimode); |
| 3525 | return 1; |
| 3526 | } |
| 3527 | |
| 3528 | /* Helper function of lower_rec_input_clauses, used for #pragma omp simd |
| 3529 | privatization. */ |
| 3530 | |
| 3531 | static bool |
| 3532 | lower_rec_simd_input_clauses (tree new_var, omp_context *ctx, int &max_vf, |
| 3533 | tree &idx, tree &lane, tree &ivar, tree &lvar) |
| 3534 | { |
| 3535 | if (max_vf == 0) |
| 3536 | { |
| 3537 | max_vf = omp_max_vf (); |
| 3538 | if (max_vf > 1) |
| 3539 | { |
| 3540 | tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt), |
| 3541 | OMP_CLAUSE_SAFELEN); |
| 3542 | if (c && TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) != INTEGER_CST) |
| 3543 | max_vf = 1; |
| 3544 | else if (c && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c), |
| 3545 | max_vf) == -1) |
| 3546 | max_vf = tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c)); |
| 3547 | } |
| 3548 | if (max_vf > 1) |
| 3549 | { |
| 3550 | idx = create_tmp_var (unsigned_type_node); |
| 3551 | lane = create_tmp_var (unsigned_type_node); |
| 3552 | } |
| 3553 | } |
| 3554 | if (max_vf == 1) |
| 3555 | return false; |
| 3556 | |
| 3557 | tree atype = build_array_type_nelts (TREE_TYPE (new_var), max_vf); |
| 3558 | tree avar = create_tmp_var_raw (atype); |
| 3559 | if (TREE_ADDRESSABLE (new_var)) |
| 3560 | TREE_ADDRESSABLE (avar) = 1; |
| 3561 | DECL_ATTRIBUTES (avar) |
| 3562 | = tree_cons (get_identifier ("omp simd array"), NULL, |
| 3563 | DECL_ATTRIBUTES (avar)); |
| 3564 | gimple_add_tmp_var (avar); |
| 3565 | ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, idx, |
| 3566 | NULL_TREE, NULL_TREE); |
| 3567 | lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, lane, |
| 3568 | NULL_TREE, NULL_TREE); |
| 3569 | if (DECL_P (new_var)) |
| 3570 | { |
| 3571 | SET_DECL_VALUE_EXPR (new_var, lvar); |
| 3572 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
| 3573 | } |
| 3574 | return true; |
| 3575 | } |
| 3576 | |
| 3577 | /* Helper function of lower_rec_input_clauses. For a reference |
| 3578 | in simd reduction, add an underlying variable it will reference. */ |
| 3579 | |
| 3580 | static void |
| 3581 | handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist) |
| 3582 | { |
| 3583 | tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard))); |
| 3584 | if (TREE_CONSTANT (z)) |
| 3585 | { |
| 3586 | const char *name = NULL; |
| 3587 | if (DECL_NAME (new_vard)) |
| 3588 | name = IDENTIFIER_POINTER (DECL_NAME (new_vard)); |
| 3589 | |
| 3590 | z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)), name); |
| 3591 | gimple_add_tmp_var (z); |
| 3592 | TREE_ADDRESSABLE (z) = 1; |
| 3593 | z = build_fold_addr_expr_loc (loc, z); |
| 3594 | gimplify_assign (new_vard, z, ilist); |
| 3595 | } |
| 3596 | } |
| 3597 | |
| 3598 | /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN, |
| 3599 | from the receiver (aka child) side and initializers for REFERENCE_TYPE |
| 3600 | private variables. Initialization statements go in ILIST, while calls |
| 3601 | to destructors go in DLIST. */ |
| 3602 | |
| 3603 | static void |
| 3604 | lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist, |
| 3605 | omp_context *ctx, struct omp_for_data *fd) |
| 3606 | { |
| 3607 | tree c, dtor, copyin_seq, x, ptr; |
| 3608 | bool copyin_by_ref = false; |
| 3609 | bool lastprivate_firstprivate = false; |
| 3610 | bool reduction_omp_orig_ref = false; |
| 3611 | int pass; |
| 3612 | bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR |
| 3613 | && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD); |
| 3614 | int max_vf = 0; |
| 3615 | tree lane = NULL_TREE, idx = NULL_TREE; |
| 3616 | tree ivar = NULL_TREE, lvar = NULL_TREE; |
| 3617 | gimple_seq llist[2] = { NULL, NULL }; |
| 3618 | |
| 3619 | copyin_seq = NULL; |
| 3620 | |
| 3621 | /* Set max_vf=1 (which will later enforce safelen=1) in simd loops |
| 3622 | with data sharing clauses referencing variable sized vars. That |
| 3623 | is unnecessarily hard to support and very unlikely to result in |
| 3624 | vectorized code anyway. */ |
| 3625 | if (is_simd) |
| 3626 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
| 3627 | switch (OMP_CLAUSE_CODE (c)) |
| 3628 | { |
| 3629 | case OMP_CLAUSE_LINEAR: |
| 3630 | if (OMP_CLAUSE_LINEAR_ARRAY (c)) |
| 3631 | max_vf = 1; |
| 3632 | /* FALLTHRU */ |
| 3633 | case OMP_CLAUSE_REDUCTION: |
| 3634 | case OMP_CLAUSE_PRIVATE: |
| 3635 | case OMP_CLAUSE_FIRSTPRIVATE: |
| 3636 | case OMP_CLAUSE_LASTPRIVATE: |
| 3637 | if (is_variable_sized (OMP_CLAUSE_DECL (c))) |
| 3638 | max_vf = 1; |
| 3639 | break; |
| 3640 | default: |
| 3641 | continue; |
| 3642 | } |
| 3643 | |
| 3644 | /* Do all the fixed sized types in the first pass, and the variable sized |
| 3645 | types in the second pass. This makes sure that the scalar arguments to |
| 3646 | the variable sized types are processed before we use them in the |
| 3647 | variable sized operations. */ |
| 3648 | for (pass = 0; pass < 2; ++pass) |
| 3649 | { |
| 3650 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
| 3651 | { |
| 3652 | enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c); |
| 3653 | tree var, new_var; |
| 3654 | bool by_ref; |
| 3655 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
| 3656 | |
| 3657 | switch (c_kind) |
| 3658 | { |
| 3659 | case OMP_CLAUSE_PRIVATE: |
| 3660 | if (OMP_CLAUSE_PRIVATE_DEBUG (c)) |
| 3661 | continue; |
| 3662 | break; |
| 3663 | case OMP_CLAUSE_SHARED: |
| 3664 | /* Ignore shared directives in teams construct. */ |
| 3665 | if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) |
| 3666 | continue; |
| 3667 | if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL) |
| 3668 | { |
| 3669 | gcc_assert (is_global_var (OMP_CLAUSE_DECL (c))); |
| 3670 | continue; |
| 3671 | } |
| 3672 | case OMP_CLAUSE_FIRSTPRIVATE: |
| 3673 | case OMP_CLAUSE_COPYIN: |
| 3674 | case OMP_CLAUSE_LINEAR: |
| 3675 | break; |
| 3676 | case OMP_CLAUSE_REDUCTION: |
| 3677 | if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)) |
| 3678 | reduction_omp_orig_ref = true; |
| 3679 | break; |
| 3680 | case OMP_CLAUSE__LOOPTEMP_: |
| 3681 | /* Handle _looptemp_ clauses only on parallel. */ |
| 3682 | if (fd) |
| 3683 | continue; |
| 3684 | break; |
| 3685 | case OMP_CLAUSE_LASTPRIVATE: |
| 3686 | if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) |
| 3687 | { |
| 3688 | lastprivate_firstprivate = true; |
| 3689 | if (pass != 0) |
| 3690 | continue; |
| 3691 | } |
| 3692 | /* Even without corresponding firstprivate, if |
| 3693 | decl is Fortran allocatable, it needs outer var |
| 3694 | reference. */ |
| 3695 | else if (pass == 0 |
| 3696 | && lang_hooks.decls.omp_private_outer_ref |
| 3697 | (OMP_CLAUSE_DECL (c))) |
| 3698 | lastprivate_firstprivate = true; |
| 3699 | break; |
| 3700 | case OMP_CLAUSE_ALIGNED: |
| 3701 | if (pass == 0) |
| 3702 | continue; |
| 3703 | var = OMP_CLAUSE_DECL (c); |
| 3704 | if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE |
| 3705 | && !is_global_var (var)) |
| 3706 | { |
| 3707 | new_var = maybe_lookup_decl (var, ctx); |
| 3708 | if (new_var == NULL_TREE) |
| 3709 | new_var = maybe_lookup_decl_in_outer_ctx (var, ctx); |
| 3710 | x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED); |
| 3711 | x = build_call_expr_loc (clause_loc, x, 2, new_var, |
| 3712 | omp_clause_aligned_alignment (c)); |
| 3713 | x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); |
| 3714 | x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x); |
| 3715 | gimplify_and_add (x, ilist); |
| 3716 | } |
| 3717 | else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE |
| 3718 | && is_global_var (var)) |
| 3719 | { |
| 3720 | tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2; |
| 3721 | new_var = lookup_decl (var, ctx); |
| 3722 | t = maybe_lookup_decl_in_outer_ctx (var, ctx); |
| 3723 | t = build_fold_addr_expr_loc (clause_loc, t); |
| 3724 | t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED); |
| 3725 | t = build_call_expr_loc (clause_loc, t2, 2, t, |
| 3726 | omp_clause_aligned_alignment (c)); |
| 3727 | t = fold_convert_loc (clause_loc, ptype, t); |
| 3728 | x = create_tmp_var (ptype); |
| 3729 | t = build2 (MODIFY_EXPR, ptype, x, t); |
| 3730 | gimplify_and_add (t, ilist); |
| 3731 | t = build_simple_mem_ref_loc (clause_loc, x); |
| 3732 | SET_DECL_VALUE_EXPR (new_var, t); |
| 3733 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
| 3734 | } |
| 3735 | continue; |
| 3736 | default: |
| 3737 | continue; |
| 3738 | } |
| 3739 | |
| 3740 | new_var = var = OMP_CLAUSE_DECL (c); |
| 3741 | if (c_kind != OMP_CLAUSE_COPYIN) |
| 3742 | new_var = lookup_decl (var, ctx); |
| 3743 | |
| 3744 | if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN) |
| 3745 | { |
| 3746 | if (pass != 0) |
| 3747 | continue; |
| 3748 | } |
| 3749 | else if (is_variable_sized (var)) |
| 3750 | { |
| 3751 | /* For variable sized types, we need to allocate the |
| 3752 | actual storage here. Call alloca and store the |
| 3753 | result in the pointer decl that we created elsewhere. */ |
| 3754 | if (pass == 0) |
| 3755 | continue; |
| 3756 | |
| 3757 | if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx)) |
| 3758 | { |
| 3759 | gcall *stmt; |
| 3760 | tree tmp, atmp; |
| 3761 | |
| 3762 | ptr = DECL_VALUE_EXPR (new_var); |
| 3763 | gcc_assert (TREE_CODE (ptr) == INDIRECT_REF); |
| 3764 | ptr = TREE_OPERAND (ptr, 0); |
| 3765 | gcc_assert (DECL_P (ptr)); |
| 3766 | x = TYPE_SIZE_UNIT (TREE_TYPE (new_var)); |
| 3767 | |
| 3768 | /* void *tmp = __builtin_alloca */ |
| 3769 | atmp = builtin_decl_explicit (BUILT_IN_ALLOCA); |
| 3770 | stmt = gimple_build_call (atmp, 1, x); |
| 3771 | tmp = create_tmp_var_raw (ptr_type_node); |
| 3772 | gimple_add_tmp_var (tmp); |
| 3773 | gimple_call_set_lhs (stmt, tmp); |
| 3774 | |
| 3775 | gimple_seq_add_stmt (ilist, stmt); |
| 3776 | |
| 3777 | x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp); |
| 3778 | gimplify_assign (ptr, x, ilist); |
| 3779 | } |
| 3780 | } |
| 3781 | else if (is_reference (var)) |
| 3782 | { |
| 3783 | /* For references that are being privatized for Fortran, |
| 3784 | allocate new backing storage for the new pointer |
| 3785 | variable. This allows us to avoid changing all the |
| 3786 | code that expects a pointer to something that expects |
| 3787 | a direct variable. */ |
| 3788 | if (pass == 0) |
| 3789 | continue; |
| 3790 | |
| 3791 | x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var))); |
| 3792 | if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx)) |
| 3793 | { |
| 3794 | x = build_receiver_ref (var, false, ctx); |
| 3795 | x = build_fold_addr_expr_loc (clause_loc, x); |
| 3796 | } |
| 3797 | else if (TREE_CONSTANT (x)) |
| 3798 | { |
| 3799 | /* For reduction in SIMD loop, defer adding the |
| 3800 | initialization of the reference, because if we decide |
| 3801 | to use SIMD array for it, the initilization could cause |
| 3802 | expansion ICE. */ |
| 3803 | if (c_kind == OMP_CLAUSE_REDUCTION && is_simd) |
| 3804 | x = NULL_TREE; |
| 3805 | else |
| 3806 | { |
| 3807 | const char *name = NULL; |
| 3808 | if (DECL_NAME (var)) |
| 3809 | name = IDENTIFIER_POINTER (DECL_NAME (new_var)); |
| 3810 | |
| 3811 | x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)), |
| 3812 | name); |
| 3813 | gimple_add_tmp_var (x); |
| 3814 | TREE_ADDRESSABLE (x) = 1; |
| 3815 | x = build_fold_addr_expr_loc (clause_loc, x); |
| 3816 | } |
| 3817 | } |
| 3818 | else |
| 3819 | { |
| 3820 | tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA); |
| 3821 | x = build_call_expr_loc (clause_loc, atmp, 1, x); |
| 3822 | } |
| 3823 | |
| 3824 | if (x) |
| 3825 | { |
| 3826 | x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); |
| 3827 | gimplify_assign (new_var, x, ilist); |
| 3828 | } |
| 3829 | |
| 3830 | new_var = build_simple_mem_ref_loc (clause_loc, new_var); |
| 3831 | } |
| 3832 | else if (c_kind == OMP_CLAUSE_REDUCTION |
| 3833 | && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
| 3834 | { |
| 3835 | if (pass == 0) |
| 3836 | continue; |
| 3837 | } |
| 3838 | else if (pass != 0) |
| 3839 | continue; |
| 3840 | |
| 3841 | switch (OMP_CLAUSE_CODE (c)) |
| 3842 | { |
| 3843 | case OMP_CLAUSE_SHARED: |
| 3844 | /* Ignore shared directives in teams construct. */ |
| 3845 | if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) |
| 3846 | continue; |
| 3847 | /* Shared global vars are just accessed directly. */ |
| 3848 | if (is_global_var (new_var)) |
| 3849 | break; |
| 3850 | /* Set up the DECL_VALUE_EXPR for shared variables now. This |
| 3851 | needs to be delayed until after fixup_child_record_type so |
| 3852 | that we get the correct type during the dereference. */ |
| 3853 | by_ref = use_pointer_for_field (var, ctx); |
| 3854 | x = build_receiver_ref (var, by_ref, ctx); |
| 3855 | SET_DECL_VALUE_EXPR (new_var, x); |
| 3856 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
| 3857 | |
| 3858 | /* ??? If VAR is not passed by reference, and the variable |
| 3859 | hasn't been initialized yet, then we'll get a warning for |
| 3860 | the store into the omp_data_s structure. Ideally, we'd be |
| 3861 | able to notice this and not store anything at all, but |
| 3862 | we're generating code too early. Suppress the warning. */ |
| 3863 | if (!by_ref) |
| 3864 | TREE_NO_WARNING (var) = 1; |
| 3865 | break; |
| 3866 | |
| 3867 | case OMP_CLAUSE_LASTPRIVATE: |
| 3868 | if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) |
| 3869 | break; |
| 3870 | /* FALLTHRU */ |
| 3871 | |
| 3872 | case OMP_CLAUSE_PRIVATE: |
| 3873 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE) |
| 3874 | x = build_outer_var_ref (var, ctx); |
| 3875 | else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) |
| 3876 | { |
| 3877 | if (is_task_ctx (ctx)) |
| 3878 | x = build_receiver_ref (var, false, ctx); |
| 3879 | else |
| 3880 | x = build_outer_var_ref (var, ctx); |
| 3881 | } |
| 3882 | else |
| 3883 | x = NULL; |
| 3884 | do_private: |
| 3885 | tree nx; |
| 3886 | nx = lang_hooks.decls.omp_clause_default_ctor (c, new_var, x); |
| 3887 | if (is_simd) |
| 3888 | { |
| 3889 | tree y = lang_hooks.decls.omp_clause_dtor (c, new_var); |
| 3890 | if ((TREE_ADDRESSABLE (new_var) || nx || y |
| 3891 | || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE) |
| 3892 | && lower_rec_simd_input_clauses (new_var, ctx, max_vf, |
| 3893 | idx, lane, ivar, lvar)) |
| 3894 | { |
| 3895 | if (nx) |
| 3896 | x = lang_hooks.decls.omp_clause_default_ctor |
| 3897 | (c, unshare_expr (ivar), x); |
| 3898 | if (nx && x) |
| 3899 | gimplify_and_add (x, &llist[0]); |
| 3900 | if (y) |
| 3901 | { |
| 3902 | y = lang_hooks.decls.omp_clause_dtor (c, ivar); |
| 3903 | if (y) |
| 3904 | { |
| 3905 | gimple_seq tseq = NULL; |
| 3906 | |
| 3907 | dtor = y; |
| 3908 | gimplify_stmt (&dtor, &tseq); |
| 3909 | gimple_seq_add_seq (&llist[1], tseq); |
| 3910 | } |
| 3911 | } |
| 3912 | break; |
| 3913 | } |
| 3914 | } |
| 3915 | if (nx) |
| 3916 | gimplify_and_add (nx, ilist); |
| 3917 | /* FALLTHRU */ |
| 3918 | |
| 3919 | do_dtor: |
| 3920 | x = lang_hooks.decls.omp_clause_dtor (c, new_var); |
| 3921 | if (x) |
| 3922 | { |
| 3923 | gimple_seq tseq = NULL; |
| 3924 | |
| 3925 | dtor = x; |
| 3926 | gimplify_stmt (&dtor, &tseq); |
| 3927 | gimple_seq_add_seq (dlist, tseq); |
| 3928 | } |
| 3929 | break; |
| 3930 | |
| 3931 | case OMP_CLAUSE_LINEAR: |
| 3932 | if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)) |
| 3933 | goto do_firstprivate; |
| 3934 | if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c)) |
| 3935 | x = NULL; |
| 3936 | else |
| 3937 | x = build_outer_var_ref (var, ctx); |
| 3938 | goto do_private; |
| 3939 | |
| 3940 | case OMP_CLAUSE_FIRSTPRIVATE: |
| 3941 | if (is_task_ctx (ctx)) |
| 3942 | { |
| 3943 | if (is_reference (var) || is_variable_sized (var)) |
| 3944 | goto do_dtor; |
| 3945 | else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, |
| 3946 | ctx)) |
| 3947 | || use_pointer_for_field (var, NULL)) |
| 3948 | { |
| 3949 | x = build_receiver_ref (var, false, ctx); |
| 3950 | SET_DECL_VALUE_EXPR (new_var, x); |
| 3951 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
| 3952 | goto do_dtor; |
| 3953 | } |
| 3954 | } |
| 3955 | do_firstprivate: |
| 3956 | x = build_outer_var_ref (var, ctx); |
| 3957 | if (is_simd) |
| 3958 | { |
| 3959 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR |
| 3960 | && gimple_omp_for_combined_into_p (ctx->stmt)) |
| 3961 | { |
| 3962 | tree t = OMP_CLAUSE_LINEAR_STEP (c); |
| 3963 | tree stept = TREE_TYPE (t); |
| 3964 | tree ct = find_omp_clause (clauses, |
| 3965 | OMP_CLAUSE__LOOPTEMP_); |
| 3966 | gcc_assert (ct); |
| 3967 | tree l = OMP_CLAUSE_DECL (ct); |
| 3968 | tree n1 = fd->loop.n1; |
| 3969 | tree step = fd->loop.step; |
| 3970 | tree itype = TREE_TYPE (l); |
| 3971 | if (POINTER_TYPE_P (itype)) |
| 3972 | itype = signed_type_for (itype); |
| 3973 | l = fold_build2 (MINUS_EXPR, itype, l, n1); |
| 3974 | if (TYPE_UNSIGNED (itype) |
| 3975 | && fd->loop.cond_code == GT_EXPR) |
| 3976 | l = fold_build2 (TRUNC_DIV_EXPR, itype, |
| 3977 | fold_build1 (NEGATE_EXPR, itype, l), |
| 3978 | fold_build1 (NEGATE_EXPR, |
| 3979 | itype, step)); |
| 3980 | else |
| 3981 | l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step); |
| 3982 | t = fold_build2 (MULT_EXPR, stept, |
| 3983 | fold_convert (stept, l), t); |
| 3984 | |
| 3985 | if (OMP_CLAUSE_LINEAR_ARRAY (c)) |
| 3986 | { |
| 3987 | x = lang_hooks.decls.omp_clause_linear_ctor |
| 3988 | (c, new_var, x, t); |
| 3989 | gimplify_and_add (x, ilist); |
| 3990 | goto do_dtor; |
| 3991 | } |
| 3992 | |
| 3993 | if (POINTER_TYPE_P (TREE_TYPE (x))) |
| 3994 | x = fold_build2 (POINTER_PLUS_EXPR, |
| 3995 | TREE_TYPE (x), x, t); |
| 3996 | else |
| 3997 | x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x, t); |
| 3998 | } |
| 3999 | |
| 4000 | if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR |
| 4001 | || TREE_ADDRESSABLE (new_var)) |
| 4002 | && lower_rec_simd_input_clauses (new_var, ctx, max_vf, |
| 4003 | idx, lane, ivar, lvar)) |
| 4004 | { |
| 4005 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR) |
| 4006 | { |
| 4007 | tree iv = create_tmp_var (TREE_TYPE (new_var)); |
| 4008 | x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x); |
| 4009 | gimplify_and_add (x, ilist); |
| 4010 | gimple_stmt_iterator gsi |
| 4011 | = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt)); |
| 4012 | gassign *g |
| 4013 | = gimple_build_assign (unshare_expr (lvar), iv); |
| 4014 | gsi_insert_before_without_update (&gsi, g, |
| 4015 | GSI_SAME_STMT); |
| 4016 | tree t = OMP_CLAUSE_LINEAR_STEP (c); |
| 4017 | enum tree_code code = PLUS_EXPR; |
| 4018 | if (POINTER_TYPE_P (TREE_TYPE (new_var))) |
| 4019 | code = POINTER_PLUS_EXPR; |
| 4020 | g = gimple_build_assign (iv, code, iv, t); |
| 4021 | gsi_insert_before_without_update (&gsi, g, |
| 4022 | GSI_SAME_STMT); |
| 4023 | break; |
| 4024 | } |
| 4025 | x = lang_hooks.decls.omp_clause_copy_ctor |
| 4026 | (c, unshare_expr (ivar), x); |
| 4027 | gimplify_and_add (x, &llist[0]); |
| 4028 | x = lang_hooks.decls.omp_clause_dtor (c, ivar); |
| 4029 | if (x) |
| 4030 | { |
| 4031 | gimple_seq tseq = NULL; |
| 4032 | |
| 4033 | dtor = x; |
| 4034 | gimplify_stmt (&dtor, &tseq); |
| 4035 | gimple_seq_add_seq (&llist[1], tseq); |
| 4036 | } |
| 4037 | break; |
| 4038 | } |
| 4039 | } |
| 4040 | x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x); |
| 4041 | gimplify_and_add (x, ilist); |
| 4042 | goto do_dtor; |
| 4043 | |
| 4044 | case OMP_CLAUSE__LOOPTEMP_: |
| 4045 | gcc_assert (is_parallel_ctx (ctx)); |
| 4046 | x = build_outer_var_ref (var, ctx); |
| 4047 | x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x); |
| 4048 | gimplify_and_add (x, ilist); |
| 4049 | break; |
| 4050 | |
| 4051 | case OMP_CLAUSE_COPYIN: |
| 4052 | by_ref = use_pointer_for_field (var, NULL); |
| 4053 | x = build_receiver_ref (var, by_ref, ctx); |
| 4054 | x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x); |
| 4055 | append_to_statement_list (x, ©in_seq); |
| 4056 | copyin_by_ref |= by_ref; |
| 4057 | break; |
| 4058 | |
| 4059 | case OMP_CLAUSE_REDUCTION: |
| 4060 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
| 4061 | { |
| 4062 | tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); |
| 4063 | gimple tseq; |
| 4064 | x = build_outer_var_ref (var, ctx); |
| 4065 | |
| 4066 | if (is_reference (var) |
| 4067 | && !useless_type_conversion_p (TREE_TYPE (placeholder), |
| 4068 | TREE_TYPE (x))) |
| 4069 | x = build_fold_addr_expr_loc (clause_loc, x); |
| 4070 | SET_DECL_VALUE_EXPR (placeholder, x); |
| 4071 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
| 4072 | tree new_vard = new_var; |
| 4073 | if (is_reference (var)) |
| 4074 | { |
| 4075 | gcc_assert (TREE_CODE (new_var) == MEM_REF); |
| 4076 | new_vard = TREE_OPERAND (new_var, 0); |
| 4077 | gcc_assert (DECL_P (new_vard)); |
| 4078 | } |
| 4079 | if (is_simd |
| 4080 | && lower_rec_simd_input_clauses (new_var, ctx, max_vf, |
| 4081 | idx, lane, ivar, lvar)) |
| 4082 | { |
| 4083 | if (new_vard == new_var) |
| 4084 | { |
| 4085 | gcc_assert (DECL_VALUE_EXPR (new_var) == lvar); |
| 4086 | SET_DECL_VALUE_EXPR (new_var, ivar); |
| 4087 | } |
| 4088 | else |
| 4089 | { |
| 4090 | SET_DECL_VALUE_EXPR (new_vard, |
| 4091 | build_fold_addr_expr (ivar)); |
| 4092 | DECL_HAS_VALUE_EXPR_P (new_vard) = 1; |
| 4093 | } |
| 4094 | x = lang_hooks.decls.omp_clause_default_ctor |
| 4095 | (c, unshare_expr (ivar), |
| 4096 | build_outer_var_ref (var, ctx)); |
| 4097 | if (x) |
| 4098 | gimplify_and_add (x, &llist[0]); |
| 4099 | if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) |
| 4100 | { |
| 4101 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); |
| 4102 | lower_omp (&tseq, ctx); |
| 4103 | gimple_seq_add_seq (&llist[0], tseq); |
| 4104 | } |
| 4105 | OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; |
| 4106 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); |
| 4107 | lower_omp (&tseq, ctx); |
| 4108 | gimple_seq_add_seq (&llist[1], tseq); |
| 4109 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
| 4110 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
| 4111 | if (new_vard == new_var) |
| 4112 | SET_DECL_VALUE_EXPR (new_var, lvar); |
| 4113 | else |
| 4114 | SET_DECL_VALUE_EXPR (new_vard, |
| 4115 | build_fold_addr_expr (lvar)); |
| 4116 | x = lang_hooks.decls.omp_clause_dtor (c, ivar); |
| 4117 | if (x) |
| 4118 | { |
| 4119 | tseq = NULL; |
| 4120 | dtor = x; |
| 4121 | gimplify_stmt (&dtor, &tseq); |
| 4122 | gimple_seq_add_seq (&llist[1], tseq); |
| 4123 | } |
| 4124 | break; |
| 4125 | } |
| 4126 | /* If this is a reference to constant size reduction var |
| 4127 | with placeholder, we haven't emitted the initializer |
| 4128 | for it because it is undesirable if SIMD arrays are used. |
| 4129 | But if they aren't used, we need to emit the deferred |
| 4130 | initialization now. */ |
| 4131 | else if (is_reference (var) && is_simd) |
| 4132 | handle_simd_reference (clause_loc, new_vard, ilist); |
| 4133 | x = lang_hooks.decls.omp_clause_default_ctor |
| 4134 | (c, unshare_expr (new_var), |
| 4135 | build_outer_var_ref (var, ctx)); |
| 4136 | if (x) |
| 4137 | gimplify_and_add (x, ilist); |
| 4138 | if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) |
| 4139 | { |
| 4140 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); |
| 4141 | lower_omp (&tseq, ctx); |
| 4142 | gimple_seq_add_seq (ilist, tseq); |
| 4143 | } |
| 4144 | OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; |
| 4145 | if (is_simd) |
| 4146 | { |
| 4147 | tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); |
| 4148 | lower_omp (&tseq, ctx); |
| 4149 | gimple_seq_add_seq (dlist, tseq); |
| 4150 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
| 4151 | } |
| 4152 | DECL_HAS_VALUE_EXPR_P (placeholder) = 0; |
| 4153 | goto do_dtor; |
| 4154 | } |
| 4155 | else |
| 4156 | { |
| 4157 | x = omp_reduction_init (c, TREE_TYPE (new_var)); |
| 4158 | gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE); |
| 4159 | enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c); |
| 4160 | |
| 4161 | /* reduction(-:var) sums up the partial results, so it |
| 4162 | acts identically to reduction(+:var). */ |
| 4163 | if (code == MINUS_EXPR) |
| 4164 | code = PLUS_EXPR; |
| 4165 | |
| 4166 | tree new_vard = new_var; |
| 4167 | if (is_simd && is_reference (var)) |
| 4168 | { |
| 4169 | gcc_assert (TREE_CODE (new_var) == MEM_REF); |
| 4170 | new_vard = TREE_OPERAND (new_var, 0); |
| 4171 | gcc_assert (DECL_P (new_vard)); |
| 4172 | } |
| 4173 | if (is_simd |
| 4174 | && lower_rec_simd_input_clauses (new_var, ctx, max_vf, |
| 4175 | idx, lane, ivar, lvar)) |
| 4176 | { |
| 4177 | tree ref = build_outer_var_ref (var, ctx); |
| 4178 | |
| 4179 | gimplify_assign (unshare_expr (ivar), x, &llist[0]); |
| 4180 | |
| 4181 | x = build2 (code, TREE_TYPE (ref), ref, ivar); |
| 4182 | ref = build_outer_var_ref (var, ctx); |
| 4183 | gimplify_assign (ref, x, &llist[1]); |
| 4184 | |
| 4185 | if (new_vard != new_var) |
| 4186 | { |
| 4187 | SET_DECL_VALUE_EXPR (new_vard, |
| 4188 | build_fold_addr_expr (lvar)); |
| 4189 | DECL_HAS_VALUE_EXPR_P (new_vard) = 1; |
| 4190 | } |
| 4191 | } |
| 4192 | else |
| 4193 | { |
| 4194 | if (is_reference (var) && is_simd) |
| 4195 | handle_simd_reference (clause_loc, new_vard, ilist); |
| 4196 | gimplify_assign (new_var, x, ilist); |
| 4197 | if (is_simd) |
| 4198 | { |
| 4199 | tree ref = build_outer_var_ref (var, ctx); |
| 4200 | |
| 4201 | x = build2 (code, TREE_TYPE (ref), ref, new_var); |
| 4202 | ref = build_outer_var_ref (var, ctx); |
| 4203 | gimplify_assign (ref, x, dlist); |
| 4204 | } |
| 4205 | } |
| 4206 | } |
| 4207 | break; |
| 4208 | |
| 4209 | default: |
| 4210 | gcc_unreachable (); |
| 4211 | } |
| 4212 | } |
| 4213 | } |
| 4214 | |
| 4215 | if (lane) |
| 4216 | { |
| 4217 | tree uid = create_tmp_var (ptr_type_node, "simduid"); |
| 4218 | /* Don't want uninit warnings on simduid, it is always uninitialized, |
| 4219 | but we use it not for the value, but for the DECL_UID only. */ |
| 4220 | TREE_NO_WARNING (uid) = 1; |
| 4221 | gimple g |
| 4222 | = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 1, uid); |
| 4223 | gimple_call_set_lhs (g, lane); |
| 4224 | gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt)); |
| 4225 | gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT); |
| 4226 | c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_); |
| 4227 | OMP_CLAUSE__SIMDUID__DECL (c) = uid; |
| 4228 | OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt); |
| 4229 | gimple_omp_for_set_clauses (ctx->stmt, c); |
| 4230 | g = gimple_build_assign (lane, INTEGER_CST, |
| 4231 | build_int_cst (unsigned_type_node, 0)); |
| 4232 | gimple_seq_add_stmt (ilist, g); |
| 4233 | for (int i = 0; i < 2; i++) |
| 4234 | if (llist[i]) |
| 4235 | { |
| 4236 | tree vf = create_tmp_var (unsigned_type_node); |
| 4237 | g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid); |
| 4238 | gimple_call_set_lhs (g, vf); |
| 4239 | gimple_seq *seq = i == 0 ? ilist : dlist; |
| 4240 | gimple_seq_add_stmt (seq, g); |
| 4241 | tree t = build_int_cst (unsigned_type_node, 0); |
| 4242 | g = gimple_build_assign (idx, INTEGER_CST, t); |
| 4243 | gimple_seq_add_stmt (seq, g); |
| 4244 | tree body = create_artificial_label (UNKNOWN_LOCATION); |
| 4245 | tree header = create_artificial_label (UNKNOWN_LOCATION); |
| 4246 | tree end = create_artificial_label (UNKNOWN_LOCATION); |
| 4247 | gimple_seq_add_stmt (seq, gimple_build_goto (header)); |
| 4248 | gimple_seq_add_stmt (seq, gimple_build_label (body)); |
| 4249 | gimple_seq_add_seq (seq, llist[i]); |
| 4250 | t = build_int_cst (unsigned_type_node, 1); |
| 4251 | g = gimple_build_assign (idx, PLUS_EXPR, idx, t); |
| 4252 | gimple_seq_add_stmt (seq, g); |
| 4253 | gimple_seq_add_stmt (seq, gimple_build_label (header)); |
| 4254 | g = gimple_build_cond (LT_EXPR, idx, vf, body, end); |
| 4255 | gimple_seq_add_stmt (seq, g); |
| 4256 | gimple_seq_add_stmt (seq, gimple_build_label (end)); |
| 4257 | } |
| 4258 | } |
| 4259 | |
| 4260 | /* The copyin sequence is not to be executed by the main thread, since |
| 4261 | that would result in self-copies. Perhaps not visible to scalars, |
| 4262 | but it certainly is to C++ operator=. */ |
| 4263 | if (copyin_seq) |
| 4264 | { |
| 4265 | x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM), |
| 4266 | 0); |
| 4267 | x = build2 (NE_EXPR, boolean_type_node, x, |
| 4268 | build_int_cst (TREE_TYPE (x), 0)); |
| 4269 | x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL); |
| 4270 | gimplify_and_add (x, ilist); |
| 4271 | } |
| 4272 | |
| 4273 | /* If any copyin variable is passed by reference, we must ensure the |
| 4274 | master thread doesn't modify it before it is copied over in all |
| 4275 | threads. Similarly for variables in both firstprivate and |
| 4276 | lastprivate clauses we need to ensure the lastprivate copying |
| 4277 | happens after firstprivate copying in all threads. And similarly |
| 4278 | for UDRs if initializer expression refers to omp_orig. */ |
| 4279 | if (copyin_by_ref || lastprivate_firstprivate || reduction_omp_orig_ref) |
| 4280 | { |
| 4281 | /* Don't add any barrier for #pragma omp simd or |
| 4282 | #pragma omp distribute. */ |
| 4283 | if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR |
| 4284 | || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR) |
| 4285 | gimple_seq_add_stmt (ilist, build_omp_barrier (NULL_TREE)); |
| 4286 | } |
| 4287 | |
| 4288 | /* If max_vf is non-zero, then we can use only a vectorization factor |
| 4289 | up to the max_vf we chose. So stick it into the safelen clause. */ |
| 4290 | if (max_vf) |
| 4291 | { |
| 4292 | tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt), |
| 4293 | OMP_CLAUSE_SAFELEN); |
| 4294 | if (c == NULL_TREE |
| 4295 | || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) == INTEGER_CST |
| 4296 | && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c), |
| 4297 | max_vf) == 1)) |
| 4298 | { |
| 4299 | c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN); |
| 4300 | OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node, |
| 4301 | max_vf); |
| 4302 | OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt); |
| 4303 | gimple_omp_for_set_clauses (ctx->stmt, c); |
| 4304 | } |
| 4305 | } |
| 4306 | } |
| 4307 | |
| 4308 | |
| 4309 | /* Generate code to implement the LASTPRIVATE clauses. This is used for |
| 4310 | both parallel and workshare constructs. PREDICATE may be NULL if it's |
| 4311 | always true. */ |
| 4312 | |
| 4313 | static void |
| 4314 | lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list, |
| 4315 | omp_context *ctx) |
| 4316 | { |
| 4317 | tree x, c, label = NULL, orig_clauses = clauses; |
| 4318 | bool par_clauses = false; |
| 4319 | tree simduid = NULL, lastlane = NULL; |
| 4320 | |
| 4321 | /* Early exit if there are no lastprivate or linear clauses. */ |
| 4322 | for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses)) |
| 4323 | if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE |
| 4324 | || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR |
| 4325 | && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses))) |
| 4326 | break; |
| 4327 | if (clauses == NULL) |
| 4328 | { |
| 4329 | /* If this was a workshare clause, see if it had been combined |
| 4330 | with its parallel. In that case, look for the clauses on the |
| 4331 | parallel statement itself. */ |
| 4332 | if (is_parallel_ctx (ctx)) |
| 4333 | return; |
| 4334 | |
| 4335 | ctx = ctx->outer; |
| 4336 | if (ctx == NULL || !is_parallel_ctx (ctx)) |
| 4337 | return; |
| 4338 | |
| 4339 | clauses = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt), |
| 4340 | OMP_CLAUSE_LASTPRIVATE); |
| 4341 | if (clauses == NULL) |
| 4342 | return; |
| 4343 | par_clauses = true; |
| 4344 | } |
| 4345 | |
| 4346 | if (predicate) |
| 4347 | { |
| 4348 | gcond *stmt; |
| 4349 | tree label_true, arm1, arm2; |
| 4350 | |
| 4351 | label = create_artificial_label (UNKNOWN_LOCATION); |
| 4352 | label_true = create_artificial_label (UNKNOWN_LOCATION); |
| 4353 | arm1 = TREE_OPERAND (predicate, 0); |
| 4354 | arm2 = TREE_OPERAND (predicate, 1); |
| 4355 | gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue); |
| 4356 | gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue); |
| 4357 | stmt = gimple_build_cond (TREE_CODE (predicate), arm1, arm2, |
| 4358 | label_true, label); |
| 4359 | gimple_seq_add_stmt (stmt_list, stmt); |
| 4360 | gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true)); |
| 4361 | } |
| 4362 | |
| 4363 | if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR |
| 4364 | && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) |
| 4365 | { |
| 4366 | simduid = find_omp_clause (orig_clauses, OMP_CLAUSE__SIMDUID_); |
| 4367 | if (simduid) |
| 4368 | simduid = OMP_CLAUSE__SIMDUID__DECL (simduid); |
| 4369 | } |
| 4370 | |
| 4371 | for (c = clauses; c ;) |
| 4372 | { |
| 4373 | tree var, new_var; |
| 4374 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
| 4375 | |
| 4376 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
| 4377 | || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR |
| 4378 | && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))) |
| 4379 | { |
| 4380 | var = OMP_CLAUSE_DECL (c); |
| 4381 | new_var = lookup_decl (var, ctx); |
| 4382 | |
| 4383 | if (simduid && DECL_HAS_VALUE_EXPR_P (new_var)) |
| 4384 | { |
| 4385 | tree val = DECL_VALUE_EXPR (new_var); |
| 4386 | if (TREE_CODE (val) == ARRAY_REF |
| 4387 | && VAR_P (TREE_OPERAND (val, 0)) |
| 4388 | && lookup_attribute ("omp simd array", |
| 4389 | DECL_ATTRIBUTES (TREE_OPERAND (val, |
| 4390 | 0)))) |
| 4391 | { |
| 4392 | if (lastlane == NULL) |
| 4393 | { |
| 4394 | lastlane = create_tmp_var (unsigned_type_node); |
| 4395 | gcall *g |
| 4396 | = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE, |
| 4397 | 2, simduid, |
| 4398 | TREE_OPERAND (val, 1)); |
| 4399 | gimple_call_set_lhs (g, lastlane); |
| 4400 | gimple_seq_add_stmt (stmt_list, g); |
| 4401 | } |
| 4402 | new_var = build4 (ARRAY_REF, TREE_TYPE (val), |
| 4403 | TREE_OPERAND (val, 0), lastlane, |
| 4404 | NULL_TREE, NULL_TREE); |
| 4405 | } |
| 4406 | } |
| 4407 | |
| 4408 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE |
| 4409 | && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) |
| 4410 | { |
| 4411 | lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx); |
| 4412 | gimple_seq_add_seq (stmt_list, |
| 4413 | OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)); |
| 4414 | OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL; |
| 4415 | } |
| 4416 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR |
| 4417 | && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) |
| 4418 | { |
| 4419 | lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx); |
| 4420 | gimple_seq_add_seq (stmt_list, |
| 4421 | OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)); |
| 4422 | OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL; |
| 4423 | } |
| 4424 | |
| 4425 | x = build_outer_var_ref (var, ctx); |
| 4426 | if (is_reference (var)) |
| 4427 | new_var = build_simple_mem_ref_loc (clause_loc, new_var); |
| 4428 | x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var); |
| 4429 | gimplify_and_add (x, stmt_list); |
| 4430 | } |
| 4431 | c = OMP_CLAUSE_CHAIN (c); |
| 4432 | if (c == NULL && !par_clauses) |
| 4433 | { |
| 4434 | /* If this was a workshare clause, see if it had been combined |
| 4435 | with its parallel. In that case, continue looking for the |
| 4436 | clauses also on the parallel statement itself. */ |
| 4437 | if (is_parallel_ctx (ctx)) |
| 4438 | break; |
| 4439 | |
| 4440 | ctx = ctx->outer; |
| 4441 | if (ctx == NULL || !is_parallel_ctx (ctx)) |
| 4442 | break; |
| 4443 | |
| 4444 | c = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt), |
| 4445 | OMP_CLAUSE_LASTPRIVATE); |
| 4446 | par_clauses = true; |
| 4447 | } |
| 4448 | } |
| 4449 | |
| 4450 | if (label) |
| 4451 | gimple_seq_add_stmt (stmt_list, gimple_build_label (label)); |
| 4452 | } |
| 4453 | |
| 4454 | static void |
| 4455 | oacc_lower_reduction_var_helper (gimple_seq *stmt_seqp, omp_context *ctx, |
| 4456 | tree tid, tree var, tree new_var) |
| 4457 | { |
| 4458 | /* The atomic add at the end of the sum creates unnecessary |
| 4459 | write contention on accelerators. To work around this, |
| 4460 | create an array to store the partial reductions. Later, in |
| 4461 | lower_omp_for (for openacc), the values of array will be |
| 4462 | combined. */ |
| 4463 | |
| 4464 | tree t = NULL_TREE, array, x; |
| 4465 | tree type = get_base_type (var); |
| 4466 | gimple stmt; |
| 4467 | |
| 4468 | /* Now insert the partial reductions into the array. */ |
| 4469 | |
| 4470 | /* Find the reduction array. */ |
| 4471 | |
| 4472 | tree ptype = build_pointer_type (type); |
| 4473 | |
| 4474 | t = lookup_oacc_reduction (oacc_get_reduction_array_id (var), ctx); |
| 4475 | t = build_receiver_ref (t, false, ctx->outer); |
| 4476 | |
| 4477 | array = create_tmp_var (ptype); |
| 4478 | gimplify_assign (array, t, stmt_seqp); |
| 4479 | |
| 4480 | tree ptr = create_tmp_var (TREE_TYPE (array)); |
| 4481 | |
| 4482 | /* Find the reduction array. */ |
| 4483 | |
| 4484 | /* testing a unary conversion. */ |
| 4485 | tree offset = create_tmp_var (sizetype); |
| 4486 | gimplify_assign (offset, TYPE_SIZE_UNIT (type), |
| 4487 | stmt_seqp); |
| 4488 | t = create_tmp_var (sizetype); |
| 4489 | gimplify_assign (t, unshare_expr (fold_build1 (NOP_EXPR, sizetype, tid)), |
| 4490 | stmt_seqp); |
| 4491 | stmt = gimple_build_assign (offset, MULT_EXPR, offset, t); |
| 4492 | gimple_seq_add_stmt (stmt_seqp, stmt); |
| 4493 | |
| 4494 | /* Offset expression. Does the POINTER_PLUS_EXPR take care |
| 4495 | of adding sizeof(var) to the array? */ |
| 4496 | ptr = create_tmp_var (ptype); |
| 4497 | stmt = gimple_build_assign (unshare_expr (ptr), POINTER_PLUS_EXPR, array, |
| 4498 | offset); |
| 4499 | gimple_seq_add_stmt (stmt_seqp, stmt); |
| 4500 | |
| 4501 | /* Move the local sum to gfc$sum[i]. */ |
| 4502 | x = unshare_expr (build_simple_mem_ref (ptr)); |
| 4503 | stmt = gimplify_assign (x, new_var, stmt_seqp); |
| 4504 | } |
| 4505 | |
| 4506 | /* Generate code to implement the REDUCTION clauses. */ |
| 4507 | |
| 4508 | static void |
| 4509 | lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx) |
| 4510 | { |
| 4511 | gimple_seq sub_seq = NULL; |
| 4512 | gimple stmt; |
| 4513 | tree x, c, tid = NULL_TREE; |
| 4514 | int count = 0; |
| 4515 | |
| 4516 | /* SIMD reductions are handled in lower_rec_input_clauses. */ |
| 4517 | if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR |
| 4518 | && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) |
| 4519 | return; |
| 4520 | |
| 4521 | /* First see if there is exactly one reduction clause. Use OMP_ATOMIC |
| 4522 | update in that case, otherwise use a lock. */ |
| 4523 | for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c)) |
| 4524 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) |
| 4525 | { |
| 4526 | if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
| 4527 | { |
| 4528 | /* Never use OMP_ATOMIC for array reductions or UDRs. */ |
| 4529 | count = -1; |
| 4530 | break; |
| 4531 | } |
| 4532 | count++; |
| 4533 | } |
| 4534 | |
| 4535 | if (count == 0) |
| 4536 | return; |
| 4537 | |
| 4538 | /* Initialize thread info for OpenACC. */ |
| 4539 | if (is_gimple_omp_oacc (ctx->stmt)) |
| 4540 | { |
| 4541 | /* Get the current thread id. */ |
| 4542 | tree call = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM); |
| 4543 | tid = create_tmp_var (TREE_TYPE (TREE_TYPE (call))); |
| 4544 | gimple stmt = gimple_build_call (call, 0); |
| 4545 | gimple_call_set_lhs (stmt, tid); |
| 4546 | gimple_seq_add_stmt (stmt_seqp, stmt); |
| 4547 | } |
| 4548 | |
| 4549 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
| 4550 | { |
| 4551 | tree var, ref, new_var; |
| 4552 | enum tree_code code; |
| 4553 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
| 4554 | |
| 4555 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) |
| 4556 | continue; |
| 4557 | |
| 4558 | var = OMP_CLAUSE_DECL (c); |
| 4559 | new_var = lookup_decl (var, ctx); |
| 4560 | if (is_reference (var)) |
| 4561 | new_var = build_simple_mem_ref_loc (clause_loc, new_var); |
| 4562 | ref = build_outer_var_ref (var, ctx); |
| 4563 | code = OMP_CLAUSE_REDUCTION_CODE (c); |
| 4564 | |
| 4565 | /* reduction(-:var) sums up the partial results, so it acts |
| 4566 | identically to reduction(+:var). */ |
| 4567 | if (code == MINUS_EXPR) |
| 4568 | code = PLUS_EXPR; |
| 4569 | |
| 4570 | if (is_gimple_omp_oacc (ctx->stmt)) |
| 4571 | { |
| 4572 | gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)); |
| 4573 | |
| 4574 | oacc_lower_reduction_var_helper (stmt_seqp, ctx, tid, var, new_var); |
| 4575 | } |
| 4576 | else if (count == 1) |
| 4577 | { |
| 4578 | tree addr = build_fold_addr_expr_loc (clause_loc, ref); |
| 4579 | |
| 4580 | addr = save_expr (addr); |
| 4581 | ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr); |
| 4582 | x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var); |
| 4583 | x = build2 (OMP_ATOMIC, void_type_node, addr, x); |
| 4584 | gimplify_and_add (x, stmt_seqp); |
| 4585 | return; |
| 4586 | } |
| 4587 | else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) |
| 4588 | { |
| 4589 | tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); |
| 4590 | |
| 4591 | if (is_reference (var) |
| 4592 | && !useless_type_conversion_p (TREE_TYPE (placeholder), |
| 4593 | TREE_TYPE (ref))) |
| 4594 | ref = build_fold_addr_expr_loc (clause_loc, ref); |
| 4595 | SET_DECL_VALUE_EXPR (placeholder, ref); |
| 4596 | DECL_HAS_VALUE_EXPR_P (placeholder) = 1; |
| 4597 | lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx); |
| 4598 | gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c)); |
| 4599 | OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; |
| 4600 | OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL; |
| 4601 | } |
| 4602 | else |
| 4603 | { |
| 4604 | x = build2 (code, TREE_TYPE (ref), ref, new_var); |
| 4605 | ref = build_outer_var_ref (var, ctx); |
| 4606 | gimplify_assign (ref, x, &sub_seq); |
| 4607 | } |
| 4608 | } |
| 4609 | |
| 4610 | if (is_gimple_omp_oacc (ctx->stmt)) |
| 4611 | return; |
| 4612 | |
| 4613 | stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START), |
| 4614 | 0); |
| 4615 | gimple_seq_add_stmt (stmt_seqp, stmt); |
| 4616 | |
| 4617 | gimple_seq_add_seq (stmt_seqp, sub_seq); |
| 4618 | |
| 4619 | stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END), |
| 4620 | 0); |
| 4621 | gimple_seq_add_stmt (stmt_seqp, stmt); |
| 4622 | } |
| 4623 | |
| 4624 | |
| 4625 | /* Generate code to implement the COPYPRIVATE clauses. */ |
| 4626 | |
| 4627 | static void |
| 4628 | lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist, |
| 4629 | omp_context *ctx) |
| 4630 | { |
| 4631 | tree c; |
| 4632 | |
| 4633 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
| 4634 | { |
| 4635 | tree var, new_var, ref, x; |
| 4636 | bool by_ref; |
| 4637 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
| 4638 | |
| 4639 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE) |
| 4640 | continue; |
| 4641 | |
| 4642 | var = OMP_CLAUSE_DECL (c); |
| 4643 | by_ref = use_pointer_for_field (var, NULL); |
| 4644 | |
| 4645 | ref = build_sender_ref (var, ctx); |
| 4646 | x = new_var = lookup_decl_in_outer_ctx (var, ctx); |
| 4647 | if (by_ref) |
| 4648 | { |
| 4649 | x = build_fold_addr_expr_loc (clause_loc, new_var); |
| 4650 | x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x); |
| 4651 | } |
| 4652 | gimplify_assign (ref, x, slist); |
| 4653 | |
| 4654 | ref = build_receiver_ref (var, false, ctx); |
| 4655 | if (by_ref) |
| 4656 | { |
| 4657 | ref = fold_convert_loc (clause_loc, |
| 4658 | build_pointer_type (TREE_TYPE (new_var)), |
| 4659 | ref); |
| 4660 | ref = build_fold_indirect_ref_loc (clause_loc, ref); |
| 4661 | } |
| 4662 | if (is_reference (var)) |
| 4663 | { |
| 4664 | ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref); |
| 4665 | ref = build_simple_mem_ref_loc (clause_loc, ref); |
| 4666 | new_var = build_simple_mem_ref_loc (clause_loc, new_var); |
| 4667 | } |
| 4668 | x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref); |
| 4669 | gimplify_and_add (x, rlist); |
| 4670 | } |
| 4671 | } |
| 4672 | |
| 4673 | |
| 4674 | /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE, |
| 4675 | and REDUCTION from the sender (aka parent) side. */ |
| 4676 | |
| 4677 | static void |
| 4678 | lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist, |
| 4679 | omp_context *ctx) |
| 4680 | { |
| 4681 | tree c; |
| 4682 | |
| 4683 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
| 4684 | { |
| 4685 | tree val, ref, x, var; |
| 4686 | bool by_ref, do_in = false, do_out = false; |
| 4687 | location_t clause_loc = OMP_CLAUSE_LOCATION (c); |
| 4688 | |
| 4689 | switch (OMP_CLAUSE_CODE (c)) |
| 4690 | { |
| 4691 | case OMP_CLAUSE_PRIVATE: |
| 4692 | if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) |
| 4693 | break; |
| 4694 | continue; |
| 4695 | case OMP_CLAUSE_FIRSTPRIVATE: |
| 4696 | case OMP_CLAUSE_COPYIN: |
| 4697 | case OMP_CLAUSE_LASTPRIVATE: |
| 4698 | case OMP_CLAUSE_REDUCTION: |
| 4699 | case OMP_CLAUSE__LOOPTEMP_: |
| 4700 | break; |
| 4701 | default: |
| 4702 | continue; |
| 4703 | } |
| 4704 | |
| 4705 | val = OMP_CLAUSE_DECL (c); |
| 4706 | var = lookup_decl_in_outer_ctx (val, ctx); |
| 4707 | |
| 4708 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN |
| 4709 | && is_global_var (var)) |
| 4710 | continue; |
| 4711 | if (is_variable_sized (val)) |
| 4712 | continue; |
| 4713 | by_ref = use_pointer_for_field (val, NULL); |
| 4714 | |
| 4715 | switch (OMP_CLAUSE_CODE (c)) |
| 4716 | { |
| 4717 | case OMP_CLAUSE_PRIVATE: |
| 4718 | case OMP_CLAUSE_FIRSTPRIVATE: |
| 4719 | case OMP_CLAUSE_COPYIN: |
| 4720 | case OMP_CLAUSE__LOOPTEMP_: |
| 4721 | do_in = true; |
| 4722 | break; |
| 4723 | |
| 4724 | case OMP_CLAUSE_LASTPRIVATE: |
| 4725 | if (by_ref || is_reference (val)) |
| 4726 | { |
| 4727 | if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) |
| 4728 | continue; |
| 4729 | do_in = true; |
| 4730 | } |
| 4731 | else |
| 4732 | { |
| 4733 | do_out = true; |
| 4734 | if (lang_hooks.decls.omp_private_outer_ref (val)) |
| 4735 | do_in = true; |
| 4736 | } |
| 4737 | break; |
| 4738 | |
| 4739 | case OMP_CLAUSE_REDUCTION: |
| 4740 | do_in = true; |
| 4741 | do_out = !(by_ref || is_reference (val)); |
| 4742 | break; |
| 4743 | |
| 4744 | default: |
| 4745 | gcc_unreachable (); |
| 4746 | } |
| 4747 | |
| 4748 | if (do_in) |
| 4749 | { |
| 4750 | ref = build_sender_ref (val, ctx); |
| 4751 | x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var; |
| 4752 | gimplify_assign (ref, x, ilist); |
| 4753 | if (is_task_ctx (ctx)) |
| 4754 | DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL; |
| 4755 | } |
| 4756 | |
| 4757 | if (do_out) |
| 4758 | { |
| 4759 | ref = build_sender_ref (val, ctx); |
| 4760 | gimplify_assign (var, ref, olist); |
| 4761 | } |
| 4762 | } |
| 4763 | } |
| 4764 | |
| 4765 | /* Generate code to implement SHARED from the sender (aka parent) |
| 4766 | side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't |
| 4767 | list things that got automatically shared. */ |
| 4768 | |
| 4769 | static void |
| 4770 | lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx) |
| 4771 | { |
| 4772 | tree var, ovar, nvar, f, x, record_type; |
| 4773 | |
| 4774 | if (ctx->record_type == NULL) |
| 4775 | return; |
| 4776 | |
| 4777 | record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type; |
| 4778 | for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f)) |
| 4779 | { |
| 4780 | ovar = DECL_ABSTRACT_ORIGIN (f); |
| 4781 | nvar = maybe_lookup_decl (ovar, ctx); |
| 4782 | if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar)) |
| 4783 | continue; |
| 4784 | |
| 4785 | /* If CTX is a nested parallel directive. Find the immediately |
| 4786 | enclosing parallel or workshare construct that contains a |
| 4787 | mapping for OVAR. */ |
| 4788 | var = lookup_decl_in_outer_ctx (ovar, ctx); |
| 4789 | |
| 4790 | if (use_pointer_for_field (ovar, ctx)) |
| 4791 | { |
| 4792 | x = build_sender_ref (ovar, ctx); |
| 4793 | var = build_fold_addr_expr (var); |
| 4794 | gimplify_assign (x, var, ilist); |
| 4795 | } |
| 4796 | else |
| 4797 | { |
| 4798 | x = build_sender_ref (ovar, ctx); |
| 4799 | gimplify_assign (x, var, ilist); |
| 4800 | |
| 4801 | if (!TREE_READONLY (var) |
| 4802 | /* We don't need to receive a new reference to a result |
| 4803 | or parm decl. In fact we may not store to it as we will |
| 4804 | invalidate any pending RSO and generate wrong gimple |
| 4805 | during inlining. */ |
| 4806 | && !((TREE_CODE (var) == RESULT_DECL |
| 4807 | || TREE_CODE (var) == PARM_DECL) |
| 4808 | && DECL_BY_REFERENCE (var))) |
| 4809 | { |
| 4810 | x = build_sender_ref (ovar, ctx); |
| 4811 | gimplify_assign (var, x, olist); |
| 4812 | } |
| 4813 | } |
| 4814 | } |
| 4815 | } |
| 4816 | |
| 4817 | |
| 4818 | /* A convenience function to build an empty GIMPLE_COND with just the |
| 4819 | condition. */ |
| 4820 | |
| 4821 | static gcond * |
| 4822 | gimple_build_cond_empty (tree cond) |
| 4823 | { |
| 4824 | enum tree_code pred_code; |
| 4825 | tree lhs, rhs; |
| 4826 | |
| 4827 | gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs); |
| 4828 | return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE); |
| 4829 | } |
| 4830 | |
| 4831 | |
| 4832 | /* Build the function calls to GOMP_parallel_start etc to actually |
| 4833 | generate the parallel operation. REGION is the parallel region |
| 4834 | being expanded. BB is the block where to insert the code. WS_ARGS |
| 4835 | will be set if this is a call to a combined parallel+workshare |
| 4836 | construct, it contains the list of additional arguments needed by |
| 4837 | the workshare construct. */ |
| 4838 | |
| 4839 | static void |
| 4840 | expand_parallel_call (struct omp_region *region, basic_block bb, |
| 4841 | gomp_parallel *entry_stmt, |
| 4842 | vec<tree, va_gc> *ws_args) |
| 4843 | { |
| 4844 | tree t, t1, t2, val, cond, c, clauses, flags; |
| 4845 | gimple_stmt_iterator gsi; |
| 4846 | gimple stmt; |
| 4847 | enum built_in_function start_ix; |
| 4848 | int start_ix2; |
| 4849 | location_t clause_loc; |
| 4850 | vec<tree, va_gc> *args; |
| 4851 | |
| 4852 | clauses = gimple_omp_parallel_clauses (entry_stmt); |
| 4853 | |
| 4854 | /* Determine what flavor of GOMP_parallel we will be |
| 4855 | emitting. */ |
| 4856 | start_ix = BUILT_IN_GOMP_PARALLEL; |
| 4857 | if (is_combined_parallel (region)) |
| 4858 | { |
| 4859 | switch (region->inner->type) |
| 4860 | { |
| 4861 | case GIMPLE_OMP_FOR: |
| 4862 | gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO); |
| 4863 | start_ix2 = ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC |
| 4864 | + (region->inner->sched_kind |
| 4865 | == OMP_CLAUSE_SCHEDULE_RUNTIME |
| 4866 | ? 3 : region->inner->sched_kind)); |
| 4867 | start_ix = (enum built_in_function)start_ix2; |
| 4868 | break; |
| 4869 | case GIMPLE_OMP_SECTIONS: |
| 4870 | start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS; |
| 4871 | break; |
| 4872 | default: |
| 4873 | gcc_unreachable (); |
| 4874 | } |
| 4875 | } |
| 4876 | |
| 4877 | /* By default, the value of NUM_THREADS is zero (selected at run time) |
| 4878 | and there is no conditional. */ |
| 4879 | cond = NULL_TREE; |
| 4880 | val = build_int_cst (unsigned_type_node, 0); |
| 4881 | flags = build_int_cst (unsigned_type_node, 0); |
| 4882 | |
| 4883 | c = find_omp_clause (clauses, OMP_CLAUSE_IF); |
| 4884 | if (c) |
| 4885 | cond = OMP_CLAUSE_IF_EXPR (c); |
| 4886 | |
| 4887 | c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS); |
| 4888 | if (c) |
| 4889 | { |
| 4890 | val = OMP_CLAUSE_NUM_THREADS_EXPR (c); |
| 4891 | clause_loc = OMP_CLAUSE_LOCATION (c); |
| 4892 | } |
| 4893 | else |
| 4894 | clause_loc = gimple_location (entry_stmt); |
| 4895 | |
| 4896 | c = find_omp_clause (clauses, OMP_CLAUSE_PROC_BIND); |
| 4897 | if (c) |
| 4898 | flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c)); |
| 4899 | |
| 4900 | /* Ensure 'val' is of the correct type. */ |
| 4901 | val = fold_convert_loc (clause_loc, unsigned_type_node, val); |
| 4902 | |
| 4903 | /* If we found the clause 'if (cond)', build either |
| 4904 | (cond != 0) or (cond ? val : 1u). */ |
| 4905 | if (cond) |
| 4906 | { |
| 4907 | cond = gimple_boolify (cond); |
| 4908 | |
| 4909 | if (integer_zerop (val)) |
| 4910 | val = fold_build2_loc (clause_loc, |
| 4911 | EQ_EXPR, unsigned_type_node, cond, |
| 4912 | build_int_cst (TREE_TYPE (cond), 0)); |
| 4913 | else |
| 4914 | { |
| 4915 | basic_block cond_bb, then_bb, else_bb; |
| 4916 | edge e, e_then, e_else; |
| 4917 | tree tmp_then, tmp_else, tmp_join, tmp_var; |
| 4918 | |
| 4919 | tmp_var = create_tmp_var (TREE_TYPE (val)); |
| 4920 | if (gimple_in_ssa_p (cfun)) |
| 4921 | { |
| 4922 | tmp_then = make_ssa_name (tmp_var); |
| 4923 | tmp_else = make_ssa_name (tmp_var); |
| 4924 | tmp_join = make_ssa_name (tmp_var); |
| 4925 | } |
| 4926 | else |
| 4927 | { |
| 4928 | tmp_then = tmp_var; |
| 4929 | tmp_else = tmp_var; |
| 4930 | tmp_join = tmp_var; |
| 4931 | } |
| 4932 | |
| 4933 | e = split_block (bb, NULL); |
| 4934 | cond_bb = e->src; |
| 4935 | bb = e->dest; |
| 4936 | remove_edge (e); |
| 4937 | |
| 4938 | then_bb = create_empty_bb (cond_bb); |
| 4939 | else_bb = create_empty_bb (then_bb); |
| 4940 | set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb); |
| 4941 | set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb); |
| 4942 | |
| 4943 | stmt = gimple_build_cond_empty (cond); |
| 4944 | gsi = gsi_start_bb (cond_bb); |
| 4945 | gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); |
| 4946 | |
| 4947 | gsi = gsi_start_bb (then_bb); |
| 4948 | stmt = gimple_build_assign (tmp_then, val); |
| 4949 | gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); |
| 4950 | |
| 4951 | gsi = gsi_start_bb (else_bb); |
| 4952 | stmt = gimple_build_assign |
| 4953 | (tmp_else, build_int_cst (unsigned_type_node, 1)); |
| 4954 | gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); |
| 4955 | |
| 4956 | make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE); |
| 4957 | make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE); |
| 4958 | add_bb_to_loop (then_bb, cond_bb->loop_father); |
| 4959 | add_bb_to_loop (else_bb, cond_bb->loop_father); |
| 4960 | e_then = make_edge (then_bb, bb, EDGE_FALLTHRU); |
| 4961 | e_else = make_edge (else_bb, bb, EDGE_FALLTHRU); |
| 4962 | |
| 4963 | if (gimple_in_ssa_p (cfun)) |
| 4964 | { |
| 4965 | gphi *phi = create_phi_node (tmp_join, bb); |
| 4966 | add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION); |
| 4967 | add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION); |
| 4968 | } |
| 4969 | |
| 4970 | val = tmp_join; |
| 4971 | } |
| 4972 | |
| 4973 | gsi = gsi_start_bb (bb); |
| 4974 | val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE, |
| 4975 | false, GSI_CONTINUE_LINKING); |
| 4976 | } |
| 4977 | |
| 4978 | gsi = gsi_last_bb (bb); |
| 4979 | t = gimple_omp_parallel_data_arg (entry_stmt); |
| 4980 | if (t == NULL) |
| 4981 | t1 = null_pointer_node; |
| 4982 | else |
| 4983 | t1 = build_fold_addr_expr (t); |
| 4984 | t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt)); |
| 4985 | |
| 4986 | vec_alloc (args, 4 + vec_safe_length (ws_args)); |
| 4987 | args->quick_push (t2); |
| 4988 | args->quick_push (t1); |
| 4989 | args->quick_push (val); |
| 4990 | if (ws_args) |
| 4991 | args->splice (*ws_args); |
| 4992 | args->quick_push (flags); |
| 4993 | |
| 4994 | t = build_call_expr_loc_vec (UNKNOWN_LOCATION, |
| 4995 | builtin_decl_explicit (start_ix), args); |
| 4996 | |
| 4997 | force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 4998 | false, GSI_CONTINUE_LINKING); |
| 4999 | } |
| 5000 | |
| 5001 | /* Insert a function call whose name is FUNC_NAME with the information from |
| 5002 | ENTRY_STMT into the basic_block BB. */ |
| 5003 | |
| 5004 | static void |
| 5005 | expand_cilk_for_call (basic_block bb, gomp_parallel *entry_stmt, |
| 5006 | vec <tree, va_gc> *ws_args) |
| 5007 | { |
| 5008 | tree t, t1, t2; |
| 5009 | gimple_stmt_iterator gsi; |
| 5010 | vec <tree, va_gc> *args; |
| 5011 | |
| 5012 | gcc_assert (vec_safe_length (ws_args) == 2); |
| 5013 | tree func_name = (*ws_args)[0]; |
| 5014 | tree grain = (*ws_args)[1]; |
| 5015 | |
| 5016 | tree clauses = gimple_omp_parallel_clauses (entry_stmt); |
| 5017 | tree count = find_omp_clause (clauses, OMP_CLAUSE__CILK_FOR_COUNT_); |
| 5018 | gcc_assert (count != NULL_TREE); |
| 5019 | count = OMP_CLAUSE_OPERAND (count, 0); |
| 5020 | |
| 5021 | gsi = gsi_last_bb (bb); |
| 5022 | t = gimple_omp_parallel_data_arg (entry_stmt); |
| 5023 | if (t == NULL) |
| 5024 | t1 = null_pointer_node; |
| 5025 | else |
| 5026 | t1 = build_fold_addr_expr (t); |
| 5027 | t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt)); |
| 5028 | |
| 5029 | vec_alloc (args, 4); |
| 5030 | args->quick_push (t2); |
| 5031 | args->quick_push (t1); |
| 5032 | args->quick_push (count); |
| 5033 | args->quick_push (grain); |
| 5034 | t = build_call_expr_loc_vec (UNKNOWN_LOCATION, func_name, args); |
| 5035 | |
| 5036 | force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, false, |
| 5037 | GSI_CONTINUE_LINKING); |
| 5038 | } |
| 5039 | |
| 5040 | /* Build the function call to GOMP_task to actually |
| 5041 | generate the task operation. BB is the block where to insert the code. */ |
| 5042 | |
| 5043 | static void |
| 5044 | expand_task_call (basic_block bb, gomp_task *entry_stmt) |
| 5045 | { |
| 5046 | tree t, t1, t2, t3, flags, cond, c, c2, clauses, depend; |
| 5047 | gimple_stmt_iterator gsi; |
| 5048 | location_t loc = gimple_location (entry_stmt); |
| 5049 | |
| 5050 | clauses = gimple_omp_task_clauses (entry_stmt); |
| 5051 | |
| 5052 | c = find_omp_clause (clauses, OMP_CLAUSE_IF); |
| 5053 | if (c) |
| 5054 | cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (c)); |
| 5055 | else |
| 5056 | cond = boolean_true_node; |
| 5057 | |
| 5058 | c = find_omp_clause (clauses, OMP_CLAUSE_UNTIED); |
| 5059 | c2 = find_omp_clause (clauses, OMP_CLAUSE_MERGEABLE); |
| 5060 | depend = find_omp_clause (clauses, OMP_CLAUSE_DEPEND); |
| 5061 | flags = build_int_cst (unsigned_type_node, |
| 5062 | (c ? 1 : 0) + (c2 ? 4 : 0) + (depend ? 8 : 0)); |
| 5063 | |
| 5064 | c = find_omp_clause (clauses, OMP_CLAUSE_FINAL); |
| 5065 | if (c) |
| 5066 | { |
| 5067 | c = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c)); |
| 5068 | c = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, c, |
| 5069 | build_int_cst (unsigned_type_node, 2), |
| 5070 | build_int_cst (unsigned_type_node, 0)); |
| 5071 | flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, c); |
| 5072 | } |
| 5073 | if (depend) |
| 5074 | depend = OMP_CLAUSE_DECL (depend); |
| 5075 | else |
| 5076 | depend = build_int_cst (ptr_type_node, 0); |
| 5077 | |
| 5078 | gsi = gsi_last_bb (bb); |
| 5079 | t = gimple_omp_task_data_arg (entry_stmt); |
| 5080 | if (t == NULL) |
| 5081 | t2 = null_pointer_node; |
| 5082 | else |
| 5083 | t2 = build_fold_addr_expr_loc (loc, t); |
| 5084 | t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt)); |
| 5085 | t = gimple_omp_task_copy_fn (entry_stmt); |
| 5086 | if (t == NULL) |
| 5087 | t3 = null_pointer_node; |
| 5088 | else |
| 5089 | t3 = build_fold_addr_expr_loc (loc, t); |
| 5090 | |
| 5091 | t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK), |
| 5092 | 8, t1, t2, t3, |
| 5093 | gimple_omp_task_arg_size (entry_stmt), |
| 5094 | gimple_omp_task_arg_align (entry_stmt), cond, flags, |
| 5095 | depend); |
| 5096 | |
| 5097 | force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 5098 | false, GSI_CONTINUE_LINKING); |
| 5099 | } |
| 5100 | |
| 5101 | |
| 5102 | /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW |
| 5103 | catch handler and return it. This prevents programs from violating the |
| 5104 | structured block semantics with throws. */ |
| 5105 | |
| 5106 | static gimple_seq |
| 5107 | maybe_catch_exception (gimple_seq body) |
| 5108 | { |
| 5109 | gimple g; |
| 5110 | tree decl; |
| 5111 | |
| 5112 | if (!flag_exceptions) |
| 5113 | return body; |
| 5114 | |
| 5115 | if (lang_hooks.eh_protect_cleanup_actions != NULL) |
| 5116 | decl = lang_hooks.eh_protect_cleanup_actions (); |
| 5117 | else |
| 5118 | decl = builtin_decl_explicit (BUILT_IN_TRAP); |
| 5119 | |
| 5120 | g = gimple_build_eh_must_not_throw (decl); |
| 5121 | g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g), |
| 5122 | GIMPLE_TRY_CATCH); |
| 5123 | |
| 5124 | return gimple_seq_alloc_with_stmt (g); |
| 5125 | } |
| 5126 | |
| 5127 | /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */ |
| 5128 | |
| 5129 | static tree |
| 5130 | vec2chain (vec<tree, va_gc> *v) |
| 5131 | { |
| 5132 | tree chain = NULL_TREE, t; |
| 5133 | unsigned ix; |
| 5134 | |
| 5135 | FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t) |
| 5136 | { |
| 5137 | DECL_CHAIN (t) = chain; |
| 5138 | chain = t; |
| 5139 | } |
| 5140 | |
| 5141 | return chain; |
| 5142 | } |
| 5143 | |
| 5144 | |
| 5145 | /* Remove barriers in REGION->EXIT's block. Note that this is only |
| 5146 | valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region |
| 5147 | is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that |
| 5148 | left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be |
| 5149 | removed. */ |
| 5150 | |
| 5151 | static void |
| 5152 | remove_exit_barrier (struct omp_region *region) |
| 5153 | { |
| 5154 | gimple_stmt_iterator gsi; |
| 5155 | basic_block exit_bb; |
| 5156 | edge_iterator ei; |
| 5157 | edge e; |
| 5158 | gimple stmt; |
| 5159 | int any_addressable_vars = -1; |
| 5160 | |
| 5161 | exit_bb = region->exit; |
| 5162 | |
| 5163 | /* If the parallel region doesn't return, we don't have REGION->EXIT |
| 5164 | block at all. */ |
| 5165 | if (! exit_bb) |
| 5166 | return; |
| 5167 | |
| 5168 | /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The |
| 5169 | workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of |
| 5170 | statements that can appear in between are extremely limited -- no |
| 5171 | memory operations at all. Here, we allow nothing at all, so the |
| 5172 | only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */ |
| 5173 | gsi = gsi_last_bb (exit_bb); |
| 5174 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN); |
| 5175 | gsi_prev (&gsi); |
| 5176 | if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL) |
| 5177 | return; |
| 5178 | |
| 5179 | FOR_EACH_EDGE (e, ei, exit_bb->preds) |
| 5180 | { |
| 5181 | gsi = gsi_last_bb (e->src); |
| 5182 | if (gsi_end_p (gsi)) |
| 5183 | continue; |
| 5184 | stmt = gsi_stmt (gsi); |
| 5185 | if (gimple_code (stmt) == GIMPLE_OMP_RETURN |
| 5186 | && !gimple_omp_return_nowait_p (stmt)) |
| 5187 | { |
| 5188 | /* OpenMP 3.0 tasks unfortunately prevent this optimization |
| 5189 | in many cases. If there could be tasks queued, the barrier |
| 5190 | might be needed to let the tasks run before some local |
| 5191 | variable of the parallel that the task uses as shared |
| 5192 | runs out of scope. The task can be spawned either |
| 5193 | from within current function (this would be easy to check) |
| 5194 | or from some function it calls and gets passed an address |
| 5195 | of such a variable. */ |
| 5196 | if (any_addressable_vars < 0) |
| 5197 | { |
| 5198 | gomp_parallel *parallel_stmt |
| 5199 | = as_a <gomp_parallel *> (last_stmt (region->entry)); |
| 5200 | tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt); |
| 5201 | tree local_decls, block, decl; |
| 5202 | unsigned ix; |
| 5203 | |
| 5204 | any_addressable_vars = 0; |
| 5205 | FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl) |
| 5206 | if (TREE_ADDRESSABLE (decl)) |
| 5207 | { |
| 5208 | any_addressable_vars = 1; |
| 5209 | break; |
| 5210 | } |
| 5211 | for (block = gimple_block (stmt); |
| 5212 | !any_addressable_vars |
| 5213 | && block |
| 5214 | && TREE_CODE (block) == BLOCK; |
| 5215 | block = BLOCK_SUPERCONTEXT (block)) |
| 5216 | { |
| 5217 | for (local_decls = BLOCK_VARS (block); |
| 5218 | local_decls; |
| 5219 | local_decls = DECL_CHAIN (local_decls)) |
| 5220 | if (TREE_ADDRESSABLE (local_decls)) |
| 5221 | { |
| 5222 | any_addressable_vars = 1; |
| 5223 | break; |
| 5224 | } |
| 5225 | if (block == gimple_block (parallel_stmt)) |
| 5226 | break; |
| 5227 | } |
| 5228 | } |
| 5229 | if (!any_addressable_vars) |
| 5230 | gimple_omp_return_set_nowait (stmt); |
| 5231 | } |
| 5232 | } |
| 5233 | } |
| 5234 | |
| 5235 | static void |
| 5236 | remove_exit_barriers (struct omp_region *region) |
| 5237 | { |
| 5238 | if (region->type == GIMPLE_OMP_PARALLEL) |
| 5239 | remove_exit_barrier (region); |
| 5240 | |
| 5241 | if (region->inner) |
| 5242 | { |
| 5243 | region = region->inner; |
| 5244 | remove_exit_barriers (region); |
| 5245 | while (region->next) |
| 5246 | { |
| 5247 | region = region->next; |
| 5248 | remove_exit_barriers (region); |
| 5249 | } |
| 5250 | } |
| 5251 | } |
| 5252 | |
| 5253 | /* Optimize omp_get_thread_num () and omp_get_num_threads () |
| 5254 | calls. These can't be declared as const functions, but |
| 5255 | within one parallel body they are constant, so they can be |
| 5256 | transformed there into __builtin_omp_get_{thread_num,num_threads} () |
| 5257 | which are declared const. Similarly for task body, except |
| 5258 | that in untied task omp_get_thread_num () can change at any task |
| 5259 | scheduling point. */ |
| 5260 | |
| 5261 | static void |
| 5262 | optimize_omp_library_calls (gimple entry_stmt) |
| 5263 | { |
| 5264 | basic_block bb; |
| 5265 | gimple_stmt_iterator gsi; |
| 5266 | tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); |
| 5267 | tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree); |
| 5268 | tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); |
| 5269 | tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree); |
| 5270 | bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK |
| 5271 | && find_omp_clause (gimple_omp_task_clauses (entry_stmt), |
| 5272 | OMP_CLAUSE_UNTIED) != NULL); |
| 5273 | |
| 5274 | FOR_EACH_BB_FN (bb, cfun) |
| 5275 | for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) |
| 5276 | { |
| 5277 | gimple call = gsi_stmt (gsi); |
| 5278 | tree decl; |
| 5279 | |
| 5280 | if (is_gimple_call (call) |
| 5281 | && (decl = gimple_call_fndecl (call)) |
| 5282 | && DECL_EXTERNAL (decl) |
| 5283 | && TREE_PUBLIC (decl) |
| 5284 | && DECL_INITIAL (decl) == NULL) |
| 5285 | { |
| 5286 | tree built_in; |
| 5287 | |
| 5288 | if (DECL_NAME (decl) == thr_num_id) |
| 5289 | { |
| 5290 | /* In #pragma omp task untied omp_get_thread_num () can change |
| 5291 | during the execution of the task region. */ |
| 5292 | if (untied_task) |
| 5293 | continue; |
| 5294 | built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); |
| 5295 | } |
| 5296 | else if (DECL_NAME (decl) == num_thr_id) |
| 5297 | built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); |
| 5298 | else |
| 5299 | continue; |
| 5300 | |
| 5301 | if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in) |
| 5302 | || gimple_call_num_args (call) != 0) |
| 5303 | continue; |
| 5304 | |
| 5305 | if (flag_exceptions && !TREE_NOTHROW (decl)) |
| 5306 | continue; |
| 5307 | |
| 5308 | if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE |
| 5309 | || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)), |
| 5310 | TREE_TYPE (TREE_TYPE (built_in)))) |
| 5311 | continue; |
| 5312 | |
| 5313 | gimple_call_set_fndecl (call, built_in); |
| 5314 | } |
| 5315 | } |
| 5316 | } |
| 5317 | |
| 5318 | /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be |
| 5319 | regimplified. */ |
| 5320 | |
| 5321 | static tree |
| 5322 | expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *) |
| 5323 | { |
| 5324 | tree t = *tp; |
| 5325 | |
| 5326 | /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */ |
| 5327 | if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t)) |
| 5328 | return t; |
| 5329 | |
| 5330 | if (TREE_CODE (t) == ADDR_EXPR) |
| 5331 | recompute_tree_invariant_for_addr_expr (t); |
| 5332 | |
| 5333 | *walk_subtrees = !TYPE_P (t) && !DECL_P (t); |
| 5334 | return NULL_TREE; |
| 5335 | } |
| 5336 | |
| 5337 | /* Prepend TO = FROM assignment before *GSI_P. */ |
| 5338 | |
| 5339 | static void |
| 5340 | expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from) |
| 5341 | { |
| 5342 | bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to); |
| 5343 | from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE, |
| 5344 | true, GSI_SAME_STMT); |
| 5345 | gimple stmt = gimple_build_assign (to, from); |
| 5346 | gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT); |
| 5347 | if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL) |
| 5348 | || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL)) |
| 5349 | { |
| 5350 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); |
| 5351 | gimple_regimplify_operands (stmt, &gsi); |
| 5352 | } |
| 5353 | } |
| 5354 | |
| 5355 | /* Expand the OpenMP parallel or task directive starting at REGION. */ |
| 5356 | |
| 5357 | static void |
| 5358 | expand_omp_taskreg (struct omp_region *region) |
| 5359 | { |
| 5360 | basic_block entry_bb, exit_bb, new_bb; |
| 5361 | struct function *child_cfun; |
| 5362 | tree child_fn, block, t; |
| 5363 | gimple_stmt_iterator gsi; |
| 5364 | gimple entry_stmt, stmt; |
| 5365 | edge e; |
| 5366 | vec<tree, va_gc> *ws_args; |
| 5367 | |
| 5368 | entry_stmt = last_stmt (region->entry); |
| 5369 | child_fn = gimple_omp_taskreg_child_fn (entry_stmt); |
| 5370 | child_cfun = DECL_STRUCT_FUNCTION (child_fn); |
| 5371 | |
| 5372 | entry_bb = region->entry; |
| 5373 | exit_bb = region->exit; |
| 5374 | |
| 5375 | bool is_cilk_for |
| 5376 | = (flag_cilkplus |
| 5377 | && gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL |
| 5378 | && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt), |
| 5379 | OMP_CLAUSE__CILK_FOR_COUNT_) != NULL_TREE); |
| 5380 | |
| 5381 | if (is_cilk_for) |
| 5382 | /* If it is a _Cilk_for statement, it is modelled *like* a parallel for, |
| 5383 | and the inner statement contains the name of the built-in function |
| 5384 | and grain. */ |
| 5385 | ws_args = region->inner->ws_args; |
| 5386 | else if (is_combined_parallel (region)) |
| 5387 | ws_args = region->ws_args; |
| 5388 | else |
| 5389 | ws_args = NULL; |
| 5390 | |
| 5391 | if (child_cfun->cfg) |
| 5392 | { |
| 5393 | /* Due to inlining, it may happen that we have already outlined |
| 5394 | the region, in which case all we need to do is make the |
| 5395 | sub-graph unreachable and emit the parallel call. */ |
| 5396 | edge entry_succ_e, exit_succ_e; |
| 5397 | |
| 5398 | entry_succ_e = single_succ_edge (entry_bb); |
| 5399 | |
| 5400 | gsi = gsi_last_bb (entry_bb); |
| 5401 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL |
| 5402 | || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK); |
| 5403 | gsi_remove (&gsi, true); |
| 5404 | |
| 5405 | new_bb = entry_bb; |
| 5406 | if (exit_bb) |
| 5407 | { |
| 5408 | exit_succ_e = single_succ_edge (exit_bb); |
| 5409 | make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU); |
| 5410 | } |
| 5411 | remove_edge_and_dominated_blocks (entry_succ_e); |
| 5412 | } |
| 5413 | else |
| 5414 | { |
| 5415 | unsigned srcidx, dstidx, num; |
| 5416 | |
| 5417 | /* If the parallel region needs data sent from the parent |
| 5418 | function, then the very first statement (except possible |
| 5419 | tree profile counter updates) of the parallel body |
| 5420 | is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since |
| 5421 | &.OMP_DATA_O is passed as an argument to the child function, |
| 5422 | we need to replace it with the argument as seen by the child |
| 5423 | function. |
| 5424 | |
| 5425 | In most cases, this will end up being the identity assignment |
| 5426 | .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had |
| 5427 | a function call that has been inlined, the original PARM_DECL |
| 5428 | .OMP_DATA_I may have been converted into a different local |
| 5429 | variable. In which case, we need to keep the assignment. */ |
| 5430 | if (gimple_omp_taskreg_data_arg (entry_stmt)) |
| 5431 | { |
| 5432 | basic_block entry_succ_bb = single_succ (entry_bb); |
| 5433 | tree arg, narg; |
| 5434 | gimple parcopy_stmt = NULL; |
| 5435 | |
| 5436 | for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi)) |
| 5437 | { |
| 5438 | gimple stmt; |
| 5439 | |
| 5440 | gcc_assert (!gsi_end_p (gsi)); |
| 5441 | stmt = gsi_stmt (gsi); |
| 5442 | if (gimple_code (stmt) != GIMPLE_ASSIGN) |
| 5443 | continue; |
| 5444 | |
| 5445 | if (gimple_num_ops (stmt) == 2) |
| 5446 | { |
| 5447 | tree arg = gimple_assign_rhs1 (stmt); |
| 5448 | |
| 5449 | /* We're ignore the subcode because we're |
| 5450 | effectively doing a STRIP_NOPS. */ |
| 5451 | |
| 5452 | if (TREE_CODE (arg) == ADDR_EXPR |
| 5453 | && TREE_OPERAND (arg, 0) |
| 5454 | == gimple_omp_taskreg_data_arg (entry_stmt)) |
| 5455 | { |
| 5456 | parcopy_stmt = stmt; |
| 5457 | break; |
| 5458 | } |
| 5459 | } |
| 5460 | } |
| 5461 | |
| 5462 | gcc_assert (parcopy_stmt != NULL); |
| 5463 | arg = DECL_ARGUMENTS (child_fn); |
| 5464 | |
| 5465 | if (!gimple_in_ssa_p (cfun)) |
| 5466 | { |
| 5467 | if (gimple_assign_lhs (parcopy_stmt) == arg) |
| 5468 | gsi_remove (&gsi, true); |
| 5469 | else |
| 5470 | { |
| 5471 | /* ?? Is setting the subcode really necessary ?? */ |
| 5472 | gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg)); |
| 5473 | gimple_assign_set_rhs1 (parcopy_stmt, arg); |
| 5474 | } |
| 5475 | } |
| 5476 | else |
| 5477 | { |
| 5478 | /* If we are in ssa form, we must load the value from the default |
| 5479 | definition of the argument. That should not be defined now, |
| 5480 | since the argument is not used uninitialized. */ |
| 5481 | gcc_assert (ssa_default_def (cfun, arg) == NULL); |
| 5482 | narg = make_ssa_name (arg, gimple_build_nop ()); |
| 5483 | set_ssa_default_def (cfun, arg, narg); |
| 5484 | /* ?? Is setting the subcode really necessary ?? */ |
| 5485 | gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (narg)); |
| 5486 | gimple_assign_set_rhs1 (parcopy_stmt, narg); |
| 5487 | update_stmt (parcopy_stmt); |
| 5488 | } |
| 5489 | } |
| 5490 | |
| 5491 | /* Declare local variables needed in CHILD_CFUN. */ |
| 5492 | block = DECL_INITIAL (child_fn); |
| 5493 | BLOCK_VARS (block) = vec2chain (child_cfun->local_decls); |
| 5494 | /* The gimplifier could record temporaries in parallel/task block |
| 5495 | rather than in containing function's local_decls chain, |
| 5496 | which would mean cgraph missed finalizing them. Do it now. */ |
| 5497 | for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t)) |
| 5498 | if (TREE_CODE (t) == VAR_DECL |
| 5499 | && TREE_STATIC (t) |
| 5500 | && !DECL_EXTERNAL (t)) |
| 5501 | varpool_node::finalize_decl (t); |
| 5502 | DECL_SAVED_TREE (child_fn) = NULL; |
| 5503 | /* We'll create a CFG for child_fn, so no gimple body is needed. */ |
| 5504 | gimple_set_body (child_fn, NULL); |
| 5505 | TREE_USED (block) = 1; |
| 5506 | |
| 5507 | /* Reset DECL_CONTEXT on function arguments. */ |
| 5508 | for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t)) |
| 5509 | DECL_CONTEXT (t) = child_fn; |
| 5510 | |
| 5511 | /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK, |
| 5512 | so that it can be moved to the child function. */ |
| 5513 | gsi = gsi_last_bb (entry_bb); |
| 5514 | stmt = gsi_stmt (gsi); |
| 5515 | gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL |
| 5516 | || gimple_code (stmt) == GIMPLE_OMP_TASK)); |
| 5517 | e = split_block (entry_bb, stmt); |
| 5518 | gsi_remove (&gsi, true); |
| 5519 | entry_bb = e->dest; |
| 5520 | single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; |
| 5521 | |
| 5522 | /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */ |
| 5523 | if (exit_bb) |
| 5524 | { |
| 5525 | gsi = gsi_last_bb (exit_bb); |
| 5526 | gcc_assert (!gsi_end_p (gsi) |
| 5527 | && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN); |
| 5528 | stmt = gimple_build_return (NULL); |
| 5529 | gsi_insert_after (&gsi, stmt, GSI_SAME_STMT); |
| 5530 | gsi_remove (&gsi, true); |
| 5531 | } |
| 5532 | |
| 5533 | /* Move the parallel region into CHILD_CFUN. */ |
| 5534 | |
| 5535 | if (gimple_in_ssa_p (cfun)) |
| 5536 | { |
| 5537 | init_tree_ssa (child_cfun); |
| 5538 | init_ssa_operands (child_cfun); |
| 5539 | child_cfun->gimple_df->in_ssa_p = true; |
| 5540 | block = NULL_TREE; |
| 5541 | } |
| 5542 | else |
| 5543 | block = gimple_block (entry_stmt); |
| 5544 | |
| 5545 | new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block); |
| 5546 | if (exit_bb) |
| 5547 | single_succ_edge (new_bb)->flags = EDGE_FALLTHRU; |
| 5548 | /* When the OMP expansion process cannot guarantee an up-to-date |
| 5549 | loop tree arrange for the child function to fixup loops. */ |
| 5550 | if (loops_state_satisfies_p (LOOPS_NEED_FIXUP)) |
| 5551 | child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP; |
| 5552 | |
| 5553 | /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */ |
| 5554 | num = vec_safe_length (child_cfun->local_decls); |
| 5555 | for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++) |
| 5556 | { |
| 5557 | t = (*child_cfun->local_decls)[srcidx]; |
| 5558 | if (DECL_CONTEXT (t) == cfun->decl) |
| 5559 | continue; |
| 5560 | if (srcidx != dstidx) |
| 5561 | (*child_cfun->local_decls)[dstidx] = t; |
| 5562 | dstidx++; |
| 5563 | } |
| 5564 | if (dstidx != num) |
| 5565 | vec_safe_truncate (child_cfun->local_decls, dstidx); |
| 5566 | |
| 5567 | /* Inform the callgraph about the new function. */ |
| 5568 | DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties; |
| 5569 | cgraph_node::add_new_function (child_fn, true); |
| 5570 | |
| 5571 | /* Fix the callgraph edges for child_cfun. Those for cfun will be |
| 5572 | fixed in a following pass. */ |
| 5573 | push_cfun (child_cfun); |
| 5574 | if (optimize) |
| 5575 | optimize_omp_library_calls (entry_stmt); |
| 5576 | cgraph_edge::rebuild_edges (); |
| 5577 | |
| 5578 | /* Some EH regions might become dead, see PR34608. If |
| 5579 | pass_cleanup_cfg isn't the first pass to happen with the |
| 5580 | new child, these dead EH edges might cause problems. |
| 5581 | Clean them up now. */ |
| 5582 | if (flag_exceptions) |
| 5583 | { |
| 5584 | basic_block bb; |
| 5585 | bool changed = false; |
| 5586 | |
| 5587 | FOR_EACH_BB_FN (bb, cfun) |
| 5588 | changed |= gimple_purge_dead_eh_edges (bb); |
| 5589 | if (changed) |
| 5590 | cleanup_tree_cfg (); |
| 5591 | } |
| 5592 | if (gimple_in_ssa_p (cfun)) |
| 5593 | update_ssa (TODO_update_ssa); |
| 5594 | pop_cfun (); |
| 5595 | } |
| 5596 | |
| 5597 | /* Emit a library call to launch the children threads. */ |
| 5598 | if (is_cilk_for) |
| 5599 | expand_cilk_for_call (new_bb, |
| 5600 | as_a <gomp_parallel *> (entry_stmt), ws_args); |
| 5601 | else if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL) |
| 5602 | expand_parallel_call (region, new_bb, |
| 5603 | as_a <gomp_parallel *> (entry_stmt), ws_args); |
| 5604 | else |
| 5605 | expand_task_call (new_bb, as_a <gomp_task *> (entry_stmt)); |
| 5606 | if (gimple_in_ssa_p (cfun)) |
| 5607 | update_ssa (TODO_update_ssa_only_virtuals); |
| 5608 | } |
| 5609 | |
| 5610 | |
| 5611 | /* Helper function for expand_omp_{for_*,simd}. If this is the outermost |
| 5612 | of the combined collapse > 1 loop constructs, generate code like: |
| 5613 | if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB; |
| 5614 | if (cond3 is <) |
| 5615 | adj = STEP3 - 1; |
| 5616 | else |
| 5617 | adj = STEP3 + 1; |
| 5618 | count3 = (adj + N32 - N31) / STEP3; |
| 5619 | if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB; |
| 5620 | if (cond2 is <) |
| 5621 | adj = STEP2 - 1; |
| 5622 | else |
| 5623 | adj = STEP2 + 1; |
| 5624 | count2 = (adj + N22 - N21) / STEP2; |
| 5625 | if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB; |
| 5626 | if (cond1 is <) |
| 5627 | adj = STEP1 - 1; |
| 5628 | else |
| 5629 | adj = STEP1 + 1; |
| 5630 | count1 = (adj + N12 - N11) / STEP1; |
| 5631 | count = count1 * count2 * count3; |
| 5632 | Furthermore, if ZERO_ITER_BB is NULL, create a BB which does: |
| 5633 | count = 0; |
| 5634 | and set ZERO_ITER_BB to that bb. If this isn't the outermost |
| 5635 | of the combined loop constructs, just initialize COUNTS array |
| 5636 | from the _looptemp_ clauses. */ |
| 5637 | |
| 5638 | /* NOTE: It *could* be better to moosh all of the BBs together, |
| 5639 | creating one larger BB with all the computation and the unexpected |
| 5640 | jump at the end. I.e. |
| 5641 | |
| 5642 | bool zero3, zero2, zero1, zero; |
| 5643 | |
| 5644 | zero3 = N32 c3 N31; |
| 5645 | count3 = (N32 - N31) /[cl] STEP3; |
| 5646 | zero2 = N22 c2 N21; |
| 5647 | count2 = (N22 - N21) /[cl] STEP2; |
| 5648 | zero1 = N12 c1 N11; |
| 5649 | count1 = (N12 - N11) /[cl] STEP1; |
| 5650 | zero = zero3 || zero2 || zero1; |
| 5651 | count = count1 * count2 * count3; |
| 5652 | if (__builtin_expect(zero, false)) goto zero_iter_bb; |
| 5653 | |
| 5654 | After all, we expect the zero=false, and thus we expect to have to |
| 5655 | evaluate all of the comparison expressions, so short-circuiting |
| 5656 | oughtn't be a win. Since the condition isn't protecting a |
| 5657 | denominator, we're not concerned about divide-by-zero, so we can |
| 5658 | fully evaluate count even if a numerator turned out to be wrong. |
| 5659 | |
| 5660 | It seems like putting this all together would create much better |
| 5661 | scheduling opportunities, and less pressure on the chip's branch |
| 5662 | predictor. */ |
| 5663 | |
| 5664 | static void |
| 5665 | expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi, |
| 5666 | basic_block &entry_bb, tree *counts, |
| 5667 | basic_block &zero_iter_bb, int &first_zero_iter, |
| 5668 | basic_block &l2_dom_bb) |
| 5669 | { |
| 5670 | tree t, type = TREE_TYPE (fd->loop.v); |
| 5671 | edge e, ne; |
| 5672 | int i; |
| 5673 | |
| 5674 | /* Collapsed loops need work for expansion into SSA form. */ |
| 5675 | gcc_assert (!gimple_in_ssa_p (cfun)); |
| 5676 | |
| 5677 | if (gimple_omp_for_combined_into_p (fd->for_stmt) |
| 5678 | && TREE_CODE (fd->loop.n2) != INTEGER_CST) |
| 5679 | { |
| 5680 | /* First two _looptemp_ clauses are for istart/iend, counts[0] |
| 5681 | isn't supposed to be handled, as the inner loop doesn't |
| 5682 | use it. */ |
| 5683 | tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), |
| 5684 | OMP_CLAUSE__LOOPTEMP_); |
| 5685 | gcc_assert (innerc); |
| 5686 | for (i = 0; i < fd->collapse; i++) |
| 5687 | { |
| 5688 | innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), |
| 5689 | OMP_CLAUSE__LOOPTEMP_); |
| 5690 | gcc_assert (innerc); |
| 5691 | if (i) |
| 5692 | counts[i] = OMP_CLAUSE_DECL (innerc); |
| 5693 | else |
| 5694 | counts[0] = NULL_TREE; |
| 5695 | } |
| 5696 | return; |
| 5697 | } |
| 5698 | |
| 5699 | for (i = 0; i < fd->collapse; i++) |
| 5700 | { |
| 5701 | tree itype = TREE_TYPE (fd->loops[i].v); |
| 5702 | |
| 5703 | if (SSA_VAR_P (fd->loop.n2) |
| 5704 | && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node, |
| 5705 | fold_convert (itype, fd->loops[i].n1), |
| 5706 | fold_convert (itype, fd->loops[i].n2))) |
| 5707 | == NULL_TREE || !integer_onep (t))) |
| 5708 | { |
| 5709 | gcond *cond_stmt; |
| 5710 | tree n1, n2; |
| 5711 | n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1)); |
| 5712 | n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE, |
| 5713 | true, GSI_SAME_STMT); |
| 5714 | n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2)); |
| 5715 | n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE, |
| 5716 | true, GSI_SAME_STMT); |
| 5717 | cond_stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2, |
| 5718 | NULL_TREE, NULL_TREE); |
| 5719 | gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT); |
| 5720 | if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), |
| 5721 | expand_omp_regimplify_p, NULL, NULL) |
| 5722 | || walk_tree (gimple_cond_rhs_ptr (cond_stmt), |
| 5723 | expand_omp_regimplify_p, NULL, NULL)) |
| 5724 | { |
| 5725 | *gsi = gsi_for_stmt (cond_stmt); |
| 5726 | gimple_regimplify_operands (cond_stmt, gsi); |
| 5727 | } |
| 5728 | e = split_block (entry_bb, cond_stmt); |
| 5729 | if (zero_iter_bb == NULL) |
| 5730 | { |
| 5731 | gassign *assign_stmt; |
| 5732 | first_zero_iter = i; |
| 5733 | zero_iter_bb = create_empty_bb (entry_bb); |
| 5734 | add_bb_to_loop (zero_iter_bb, entry_bb->loop_father); |
| 5735 | *gsi = gsi_after_labels (zero_iter_bb); |
| 5736 | assign_stmt = gimple_build_assign (fd->loop.n2, |
| 5737 | build_zero_cst (type)); |
| 5738 | gsi_insert_before (gsi, assign_stmt, GSI_SAME_STMT); |
| 5739 | set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb, |
| 5740 | entry_bb); |
| 5741 | } |
| 5742 | ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE); |
| 5743 | ne->probability = REG_BR_PROB_BASE / 2000 - 1; |
| 5744 | e->flags = EDGE_TRUE_VALUE; |
| 5745 | e->probability = REG_BR_PROB_BASE - ne->probability; |
| 5746 | if (l2_dom_bb == NULL) |
| 5747 | l2_dom_bb = entry_bb; |
| 5748 | entry_bb = e->dest; |
| 5749 | *gsi = gsi_last_bb (entry_bb); |
| 5750 | } |
| 5751 | |
| 5752 | if (POINTER_TYPE_P (itype)) |
| 5753 | itype = signed_type_for (itype); |
| 5754 | t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR |
| 5755 | ? -1 : 1)); |
| 5756 | t = fold_build2 (PLUS_EXPR, itype, |
| 5757 | fold_convert (itype, fd->loops[i].step), t); |
| 5758 | t = fold_build2 (PLUS_EXPR, itype, t, |
| 5759 | fold_convert (itype, fd->loops[i].n2)); |
| 5760 | t = fold_build2 (MINUS_EXPR, itype, t, |
| 5761 | fold_convert (itype, fd->loops[i].n1)); |
| 5762 | /* ?? We could probably use CEIL_DIV_EXPR instead of |
| 5763 | TRUNC_DIV_EXPR and adjusting by hand. Unless we can't |
| 5764 | generate the same code in the end because generically we |
| 5765 | don't know that the values involved must be negative for |
| 5766 | GT?? */ |
| 5767 | if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR) |
| 5768 | t = fold_build2 (TRUNC_DIV_EXPR, itype, |
| 5769 | fold_build1 (NEGATE_EXPR, itype, t), |
| 5770 | fold_build1 (NEGATE_EXPR, itype, |
| 5771 | fold_convert (itype, |
| 5772 | fd->loops[i].step))); |
| 5773 | else |
| 5774 | t = fold_build2 (TRUNC_DIV_EXPR, itype, t, |
| 5775 | fold_convert (itype, fd->loops[i].step)); |
| 5776 | t = fold_convert (type, t); |
| 5777 | if (TREE_CODE (t) == INTEGER_CST) |
| 5778 | counts[i] = t; |
| 5779 | else |
| 5780 | { |
| 5781 | counts[i] = create_tmp_reg (type, ".count"); |
| 5782 | expand_omp_build_assign (gsi, counts[i], t); |
| 5783 | } |
| 5784 | if (SSA_VAR_P (fd->loop.n2)) |
| 5785 | { |
| 5786 | if (i == 0) |
| 5787 | t = counts[0]; |
| 5788 | else |
| 5789 | t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]); |
| 5790 | expand_omp_build_assign (gsi, fd->loop.n2, t); |
| 5791 | } |
| 5792 | } |
| 5793 | } |
| 5794 | |
| 5795 | |
| 5796 | /* Helper function for expand_omp_{for_*,simd}. Generate code like: |
| 5797 | T = V; |
| 5798 | V3 = N31 + (T % count3) * STEP3; |
| 5799 | T = T / count3; |
| 5800 | V2 = N21 + (T % count2) * STEP2; |
| 5801 | T = T / count2; |
| 5802 | V1 = N11 + T * STEP1; |
| 5803 | if this loop doesn't have an inner loop construct combined with it. |
| 5804 | If it does have an inner loop construct combined with it and the |
| 5805 | iteration count isn't known constant, store values from counts array |
| 5806 | into its _looptemp_ temporaries instead. */ |
| 5807 | |
| 5808 | static void |
| 5809 | expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi, |
| 5810 | tree *counts, gimple inner_stmt, tree startvar) |
| 5811 | { |
| 5812 | int i; |
| 5813 | if (gimple_omp_for_combined_p (fd->for_stmt)) |
| 5814 | { |
| 5815 | /* If fd->loop.n2 is constant, then no propagation of the counts |
| 5816 | is needed, they are constant. */ |
| 5817 | if (TREE_CODE (fd->loop.n2) == INTEGER_CST) |
| 5818 | return; |
| 5819 | |
| 5820 | tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL |
| 5821 | ? gimple_omp_parallel_clauses (inner_stmt) |
| 5822 | : gimple_omp_for_clauses (inner_stmt); |
| 5823 | /* First two _looptemp_ clauses are for istart/iend, counts[0] |
| 5824 | isn't supposed to be handled, as the inner loop doesn't |
| 5825 | use it. */ |
| 5826 | tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_); |
| 5827 | gcc_assert (innerc); |
| 5828 | for (i = 0; i < fd->collapse; i++) |
| 5829 | { |
| 5830 | innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), |
| 5831 | OMP_CLAUSE__LOOPTEMP_); |
| 5832 | gcc_assert (innerc); |
| 5833 | if (i) |
| 5834 | { |
| 5835 | tree tem = OMP_CLAUSE_DECL (innerc); |
| 5836 | tree t = fold_convert (TREE_TYPE (tem), counts[i]); |
| 5837 | t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE, |
| 5838 | false, GSI_CONTINUE_LINKING); |
| 5839 | gassign *stmt = gimple_build_assign (tem, t); |
| 5840 | gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); |
| 5841 | } |
| 5842 | } |
| 5843 | return; |
| 5844 | } |
| 5845 | |
| 5846 | tree type = TREE_TYPE (fd->loop.v); |
| 5847 | tree tem = create_tmp_reg (type, ".tem"); |
| 5848 | gassign *stmt = gimple_build_assign (tem, startvar); |
| 5849 | gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); |
| 5850 | |
| 5851 | for (i = fd->collapse - 1; i >= 0; i--) |
| 5852 | { |
| 5853 | tree vtype = TREE_TYPE (fd->loops[i].v), itype, t; |
| 5854 | itype = vtype; |
| 5855 | if (POINTER_TYPE_P (vtype)) |
| 5856 | itype = signed_type_for (vtype); |
| 5857 | if (i != 0) |
| 5858 | t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]); |
| 5859 | else |
| 5860 | t = tem; |
| 5861 | t = fold_convert (itype, t); |
| 5862 | t = fold_build2 (MULT_EXPR, itype, t, |
| 5863 | fold_convert (itype, fd->loops[i].step)); |
| 5864 | if (POINTER_TYPE_P (vtype)) |
| 5865 | t = fold_build_pointer_plus (fd->loops[i].n1, t); |
| 5866 | else |
| 5867 | t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t); |
| 5868 | t = force_gimple_operand_gsi (gsi, t, |
| 5869 | DECL_P (fd->loops[i].v) |
| 5870 | && TREE_ADDRESSABLE (fd->loops[i].v), |
| 5871 | NULL_TREE, false, |
| 5872 | GSI_CONTINUE_LINKING); |
| 5873 | stmt = gimple_build_assign (fd->loops[i].v, t); |
| 5874 | gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); |
| 5875 | if (i != 0) |
| 5876 | { |
| 5877 | t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]); |
| 5878 | t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE, |
| 5879 | false, GSI_CONTINUE_LINKING); |
| 5880 | stmt = gimple_build_assign (tem, t); |
| 5881 | gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); |
| 5882 | } |
| 5883 | } |
| 5884 | } |
| 5885 | |
| 5886 | |
| 5887 | /* Helper function for expand_omp_for_*. Generate code like: |
| 5888 | L10: |
| 5889 | V3 += STEP3; |
| 5890 | if (V3 cond3 N32) goto BODY_BB; else goto L11; |
| 5891 | L11: |
| 5892 | V3 = N31; |
| 5893 | V2 += STEP2; |
| 5894 | if (V2 cond2 N22) goto BODY_BB; else goto L12; |
| 5895 | L12: |
| 5896 | V2 = N21; |
| 5897 | V1 += STEP1; |
| 5898 | goto BODY_BB; */ |
| 5899 | |
| 5900 | static basic_block |
| 5901 | extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb, |
| 5902 | basic_block body_bb) |
| 5903 | { |
| 5904 | basic_block last_bb, bb, collapse_bb = NULL; |
| 5905 | int i; |
| 5906 | gimple_stmt_iterator gsi; |
| 5907 | edge e; |
| 5908 | tree t; |
| 5909 | gimple stmt; |
| 5910 | |
| 5911 | last_bb = cont_bb; |
| 5912 | for (i = fd->collapse - 1; i >= 0; i--) |
| 5913 | { |
| 5914 | tree vtype = TREE_TYPE (fd->loops[i].v); |
| 5915 | |
| 5916 | bb = create_empty_bb (last_bb); |
| 5917 | add_bb_to_loop (bb, last_bb->loop_father); |
| 5918 | gsi = gsi_start_bb (bb); |
| 5919 | |
| 5920 | if (i < fd->collapse - 1) |
| 5921 | { |
| 5922 | e = make_edge (last_bb, bb, EDGE_FALSE_VALUE); |
| 5923 | e->probability = REG_BR_PROB_BASE / 8; |
| 5924 | |
| 5925 | t = fd->loops[i + 1].n1; |
| 5926 | t = force_gimple_operand_gsi (&gsi, t, |
| 5927 | DECL_P (fd->loops[i + 1].v) |
| 5928 | && TREE_ADDRESSABLE (fd->loops[i |
| 5929 | + 1].v), |
| 5930 | NULL_TREE, false, |
| 5931 | GSI_CONTINUE_LINKING); |
| 5932 | stmt = gimple_build_assign (fd->loops[i + 1].v, t); |
| 5933 | gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); |
| 5934 | } |
| 5935 | else |
| 5936 | collapse_bb = bb; |
| 5937 | |
| 5938 | set_immediate_dominator (CDI_DOMINATORS, bb, last_bb); |
| 5939 | |
| 5940 | if (POINTER_TYPE_P (vtype)) |
| 5941 | t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step); |
| 5942 | else |
| 5943 | t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step); |
| 5944 | t = force_gimple_operand_gsi (&gsi, t, |
| 5945 | DECL_P (fd->loops[i].v) |
| 5946 | && TREE_ADDRESSABLE (fd->loops[i].v), |
| 5947 | NULL_TREE, false, GSI_CONTINUE_LINKING); |
| 5948 | stmt = gimple_build_assign (fd->loops[i].v, t); |
| 5949 | gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); |
| 5950 | |
| 5951 | if (i > 0) |
| 5952 | { |
| 5953 | t = fd->loops[i].n2; |
| 5954 | t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 5955 | false, GSI_CONTINUE_LINKING); |
| 5956 | tree v = fd->loops[i].v; |
| 5957 | if (DECL_P (v) && TREE_ADDRESSABLE (v)) |
| 5958 | v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE, |
| 5959 | false, GSI_CONTINUE_LINKING); |
| 5960 | t = fold_build2 (fd->loops[i].cond_code, boolean_type_node, v, t); |
| 5961 | stmt = gimple_build_cond_empty (t); |
| 5962 | gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); |
| 5963 | e = make_edge (bb, body_bb, EDGE_TRUE_VALUE); |
| 5964 | e->probability = REG_BR_PROB_BASE * 7 / 8; |
| 5965 | } |
| 5966 | else |
| 5967 | make_edge (bb, body_bb, EDGE_FALLTHRU); |
| 5968 | last_bb = bb; |
| 5969 | } |
| 5970 | |
| 5971 | return collapse_bb; |
| 5972 | } |
| 5973 | |
| 5974 | |
| 5975 | /* A subroutine of expand_omp_for. Generate code for a parallel |
| 5976 | loop with any schedule. Given parameters: |
| 5977 | |
| 5978 | for (V = N1; V cond N2; V += STEP) BODY; |
| 5979 | |
| 5980 | where COND is "<" or ">", we generate pseudocode |
| 5981 | |
| 5982 | more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0); |
| 5983 | if (more) goto L0; else goto L3; |
| 5984 | L0: |
| 5985 | V = istart0; |
| 5986 | iend = iend0; |
| 5987 | L1: |
| 5988 | BODY; |
| 5989 | V += STEP; |
| 5990 | if (V cond iend) goto L1; else goto L2; |
| 5991 | L2: |
| 5992 | if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3; |
| 5993 | L3: |
| 5994 | |
| 5995 | If this is a combined omp parallel loop, instead of the call to |
| 5996 | GOMP_loop_foo_start, we call GOMP_loop_foo_next. |
| 5997 | If this is gimple_omp_for_combined_p loop, then instead of assigning |
| 5998 | V and iend in L0 we assign the first two _looptemp_ clause decls of the |
| 5999 | inner GIMPLE_OMP_FOR and V += STEP; and |
| 6000 | if (V cond iend) goto L1; else goto L2; are removed. |
| 6001 | |
| 6002 | For collapsed loops, given parameters: |
| 6003 | collapse(3) |
| 6004 | for (V1 = N11; V1 cond1 N12; V1 += STEP1) |
| 6005 | for (V2 = N21; V2 cond2 N22; V2 += STEP2) |
| 6006 | for (V3 = N31; V3 cond3 N32; V3 += STEP3) |
| 6007 | BODY; |
| 6008 | |
| 6009 | we generate pseudocode |
| 6010 | |
| 6011 | if (__builtin_expect (N32 cond3 N31, 0)) goto Z0; |
| 6012 | if (cond3 is <) |
| 6013 | adj = STEP3 - 1; |
| 6014 | else |
| 6015 | adj = STEP3 + 1; |
| 6016 | count3 = (adj + N32 - N31) / STEP3; |
| 6017 | if (__builtin_expect (N22 cond2 N21, 0)) goto Z0; |
| 6018 | if (cond2 is <) |
| 6019 | adj = STEP2 - 1; |
| 6020 | else |
| 6021 | adj = STEP2 + 1; |
| 6022 | count2 = (adj + N22 - N21) / STEP2; |
| 6023 | if (__builtin_expect (N12 cond1 N11, 0)) goto Z0; |
| 6024 | if (cond1 is <) |
| 6025 | adj = STEP1 - 1; |
| 6026 | else |
| 6027 | adj = STEP1 + 1; |
| 6028 | count1 = (adj + N12 - N11) / STEP1; |
| 6029 | count = count1 * count2 * count3; |
| 6030 | goto Z1; |
| 6031 | Z0: |
| 6032 | count = 0; |
| 6033 | Z1: |
| 6034 | more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0); |
| 6035 | if (more) goto L0; else goto L3; |
| 6036 | L0: |
| 6037 | V = istart0; |
| 6038 | T = V; |
| 6039 | V3 = N31 + (T % count3) * STEP3; |
| 6040 | T = T / count3; |
| 6041 | V2 = N21 + (T % count2) * STEP2; |
| 6042 | T = T / count2; |
| 6043 | V1 = N11 + T * STEP1; |
| 6044 | iend = iend0; |
| 6045 | L1: |
| 6046 | BODY; |
| 6047 | V += 1; |
| 6048 | if (V < iend) goto L10; else goto L2; |
| 6049 | L10: |
| 6050 | V3 += STEP3; |
| 6051 | if (V3 cond3 N32) goto L1; else goto L11; |
| 6052 | L11: |
| 6053 | V3 = N31; |
| 6054 | V2 += STEP2; |
| 6055 | if (V2 cond2 N22) goto L1; else goto L12; |
| 6056 | L12: |
| 6057 | V2 = N21; |
| 6058 | V1 += STEP1; |
| 6059 | goto L1; |
| 6060 | L2: |
| 6061 | if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3; |
| 6062 | L3: |
| 6063 | |
| 6064 | */ |
| 6065 | |
| 6066 | static void |
| 6067 | expand_omp_for_generic (struct omp_region *region, |
| 6068 | struct omp_for_data *fd, |
| 6069 | enum built_in_function start_fn, |
| 6070 | enum built_in_function next_fn, |
| 6071 | gimple inner_stmt) |
| 6072 | { |
| 6073 | tree type, istart0, iend0, iend; |
| 6074 | tree t, vmain, vback, bias = NULL_TREE; |
| 6075 | basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb; |
| 6076 | basic_block l2_bb = NULL, l3_bb = NULL; |
| 6077 | gimple_stmt_iterator gsi; |
| 6078 | gassign *assign_stmt; |
| 6079 | bool in_combined_parallel = is_combined_parallel (region); |
| 6080 | bool broken_loop = region->cont == NULL; |
| 6081 | edge e, ne; |
| 6082 | tree *counts = NULL; |
| 6083 | int i; |
| 6084 | |
| 6085 | gcc_assert (!broken_loop || !in_combined_parallel); |
| 6086 | gcc_assert (fd->iter_type == long_integer_type_node |
| 6087 | || !in_combined_parallel); |
| 6088 | |
| 6089 | type = TREE_TYPE (fd->loop.v); |
| 6090 | istart0 = create_tmp_var (fd->iter_type, ".istart0"); |
| 6091 | iend0 = create_tmp_var (fd->iter_type, ".iend0"); |
| 6092 | TREE_ADDRESSABLE (istart0) = 1; |
| 6093 | TREE_ADDRESSABLE (iend0) = 1; |
| 6094 | |
| 6095 | /* See if we need to bias by LLONG_MIN. */ |
| 6096 | if (fd->iter_type == long_long_unsigned_type_node |
| 6097 | && TREE_CODE (type) == INTEGER_TYPE |
| 6098 | && !TYPE_UNSIGNED (type)) |
| 6099 | { |
| 6100 | tree n1, n2; |
| 6101 | |
| 6102 | if (fd->loop.cond_code == LT_EXPR) |
| 6103 | { |
| 6104 | n1 = fd->loop.n1; |
| 6105 | n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step); |
| 6106 | } |
| 6107 | else |
| 6108 | { |
| 6109 | n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step); |
| 6110 | n2 = fd->loop.n1; |
| 6111 | } |
| 6112 | if (TREE_CODE (n1) != INTEGER_CST |
| 6113 | || TREE_CODE (n2) != INTEGER_CST |
| 6114 | || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0))) |
| 6115 | bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type)); |
| 6116 | } |
| 6117 | |
| 6118 | entry_bb = region->entry; |
| 6119 | cont_bb = region->cont; |
| 6120 | collapse_bb = NULL; |
| 6121 | gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); |
| 6122 | gcc_assert (broken_loop |
| 6123 | || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest); |
| 6124 | l0_bb = split_edge (FALLTHRU_EDGE (entry_bb)); |
| 6125 | l1_bb = single_succ (l0_bb); |
| 6126 | if (!broken_loop) |
| 6127 | { |
| 6128 | l2_bb = create_empty_bb (cont_bb); |
| 6129 | gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb); |
| 6130 | gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); |
| 6131 | } |
| 6132 | else |
| 6133 | l2_bb = NULL; |
| 6134 | l3_bb = BRANCH_EDGE (entry_bb)->dest; |
| 6135 | exit_bb = region->exit; |
| 6136 | |
| 6137 | gsi = gsi_last_bb (entry_bb); |
| 6138 | |
| 6139 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); |
| 6140 | if (fd->collapse > 1) |
| 6141 | { |
| 6142 | int first_zero_iter = -1; |
| 6143 | basic_block zero_iter_bb = NULL, l2_dom_bb = NULL; |
| 6144 | |
| 6145 | counts = XALLOCAVEC (tree, fd->collapse); |
| 6146 | expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, |
| 6147 | zero_iter_bb, first_zero_iter, |
| 6148 | l2_dom_bb); |
| 6149 | |
| 6150 | if (zero_iter_bb) |
| 6151 | { |
| 6152 | /* Some counts[i] vars might be uninitialized if |
| 6153 | some loop has zero iterations. But the body shouldn't |
| 6154 | be executed in that case, so just avoid uninit warnings. */ |
| 6155 | for (i = first_zero_iter; i < fd->collapse; i++) |
| 6156 | if (SSA_VAR_P (counts[i])) |
| 6157 | TREE_NO_WARNING (counts[i]) = 1; |
| 6158 | gsi_prev (&gsi); |
| 6159 | e = split_block (entry_bb, gsi_stmt (gsi)); |
| 6160 | entry_bb = e->dest; |
| 6161 | make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU); |
| 6162 | gsi = gsi_last_bb (entry_bb); |
| 6163 | set_immediate_dominator (CDI_DOMINATORS, entry_bb, |
| 6164 | get_immediate_dominator (CDI_DOMINATORS, |
| 6165 | zero_iter_bb)); |
| 6166 | } |
| 6167 | } |
| 6168 | if (in_combined_parallel) |
| 6169 | { |
| 6170 | /* In a combined parallel loop, emit a call to |
| 6171 | GOMP_loop_foo_next. */ |
| 6172 | t = build_call_expr (builtin_decl_explicit (next_fn), 2, |
| 6173 | build_fold_addr_expr (istart0), |
| 6174 | build_fold_addr_expr (iend0)); |
| 6175 | } |
| 6176 | else |
| 6177 | { |
| 6178 | tree t0, t1, t2, t3, t4; |
| 6179 | /* If this is not a combined parallel loop, emit a call to |
| 6180 | GOMP_loop_foo_start in ENTRY_BB. */ |
| 6181 | t4 = build_fold_addr_expr (iend0); |
| 6182 | t3 = build_fold_addr_expr (istart0); |
| 6183 | t2 = fold_convert (fd->iter_type, fd->loop.step); |
| 6184 | t1 = fd->loop.n2; |
| 6185 | t0 = fd->loop.n1; |
| 6186 | if (gimple_omp_for_combined_into_p (fd->for_stmt)) |
| 6187 | { |
| 6188 | tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), |
| 6189 | OMP_CLAUSE__LOOPTEMP_); |
| 6190 | gcc_assert (innerc); |
| 6191 | t0 = OMP_CLAUSE_DECL (innerc); |
| 6192 | innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), |
| 6193 | OMP_CLAUSE__LOOPTEMP_); |
| 6194 | gcc_assert (innerc); |
| 6195 | t1 = OMP_CLAUSE_DECL (innerc); |
| 6196 | } |
| 6197 | if (POINTER_TYPE_P (TREE_TYPE (t0)) |
| 6198 | && TYPE_PRECISION (TREE_TYPE (t0)) |
| 6199 | != TYPE_PRECISION (fd->iter_type)) |
| 6200 | { |
| 6201 | /* Avoid casting pointers to integer of a different size. */ |
| 6202 | tree itype = signed_type_for (type); |
| 6203 | t1 = fold_convert (fd->iter_type, fold_convert (itype, t1)); |
| 6204 | t0 = fold_convert (fd->iter_type, fold_convert (itype, t0)); |
| 6205 | } |
| 6206 | else |
| 6207 | { |
| 6208 | t1 = fold_convert (fd->iter_type, t1); |
| 6209 | t0 = fold_convert (fd->iter_type, t0); |
| 6210 | } |
| 6211 | if (bias) |
| 6212 | { |
| 6213 | t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias); |
| 6214 | t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias); |
| 6215 | } |
| 6216 | if (fd->iter_type == long_integer_type_node) |
| 6217 | { |
| 6218 | if (fd->chunk_size) |
| 6219 | { |
| 6220 | t = fold_convert (fd->iter_type, fd->chunk_size); |
| 6221 | t = build_call_expr (builtin_decl_explicit (start_fn), |
| 6222 | 6, t0, t1, t2, t, t3, t4); |
| 6223 | } |
| 6224 | else |
| 6225 | t = build_call_expr (builtin_decl_explicit (start_fn), |
| 6226 | 5, t0, t1, t2, t3, t4); |
| 6227 | } |
| 6228 | else |
| 6229 | { |
| 6230 | tree t5; |
| 6231 | tree c_bool_type; |
| 6232 | tree bfn_decl; |
| 6233 | |
| 6234 | /* The GOMP_loop_ull_*start functions have additional boolean |
| 6235 | argument, true for < loops and false for > loops. |
| 6236 | In Fortran, the C bool type can be different from |
| 6237 | boolean_type_node. */ |
| 6238 | bfn_decl = builtin_decl_explicit (start_fn); |
| 6239 | c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl)); |
| 6240 | t5 = build_int_cst (c_bool_type, |
| 6241 | fd->loop.cond_code == LT_EXPR ? 1 : 0); |
| 6242 | if (fd->chunk_size) |
| 6243 | { |
| 6244 | tree bfn_decl = builtin_decl_explicit (start_fn); |
| 6245 | t = fold_convert (fd->iter_type, fd->chunk_size); |
| 6246 | t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4); |
| 6247 | } |
| 6248 | else |
| 6249 | t = build_call_expr (builtin_decl_explicit (start_fn), |
| 6250 | 6, t5, t0, t1, t2, t3, t4); |
| 6251 | } |
| 6252 | } |
| 6253 | if (TREE_TYPE (t) != boolean_type_node) |
| 6254 | t = fold_build2 (NE_EXPR, boolean_type_node, |
| 6255 | t, build_int_cst (TREE_TYPE (t), 0)); |
| 6256 | t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 6257 | true, GSI_SAME_STMT); |
| 6258 | gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); |
| 6259 | |
| 6260 | /* Remove the GIMPLE_OMP_FOR statement. */ |
| 6261 | gsi_remove (&gsi, true); |
| 6262 | |
| 6263 | /* Iteration setup for sequential loop goes in L0_BB. */ |
| 6264 | tree startvar = fd->loop.v; |
| 6265 | tree endvar = NULL_TREE; |
| 6266 | |
| 6267 | if (gimple_omp_for_combined_p (fd->for_stmt)) |
| 6268 | { |
| 6269 | gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR |
| 6270 | && gimple_omp_for_kind (inner_stmt) |
| 6271 | == GF_OMP_FOR_KIND_SIMD); |
| 6272 | tree innerc = find_omp_clause (gimple_omp_for_clauses (inner_stmt), |
| 6273 | OMP_CLAUSE__LOOPTEMP_); |
| 6274 | gcc_assert (innerc); |
| 6275 | startvar = OMP_CLAUSE_DECL (innerc); |
| 6276 | innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), |
| 6277 | OMP_CLAUSE__LOOPTEMP_); |
| 6278 | gcc_assert (innerc); |
| 6279 | endvar = OMP_CLAUSE_DECL (innerc); |
| 6280 | } |
| 6281 | |
| 6282 | gsi = gsi_start_bb (l0_bb); |
| 6283 | t = istart0; |
| 6284 | if (bias) |
| 6285 | t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias); |
| 6286 | if (POINTER_TYPE_P (TREE_TYPE (startvar))) |
| 6287 | t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t); |
| 6288 | t = fold_convert (TREE_TYPE (startvar), t); |
| 6289 | t = force_gimple_operand_gsi (&gsi, t, |
| 6290 | DECL_P (startvar) |
| 6291 | && TREE_ADDRESSABLE (startvar), |
| 6292 | NULL_TREE, false, GSI_CONTINUE_LINKING); |
| 6293 | assign_stmt = gimple_build_assign (startvar, t); |
| 6294 | gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); |
| 6295 | |
| 6296 | t = iend0; |
| 6297 | if (bias) |
| 6298 | t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias); |
| 6299 | if (POINTER_TYPE_P (TREE_TYPE (startvar))) |
| 6300 | t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t); |
| 6301 | t = fold_convert (TREE_TYPE (startvar), t); |
| 6302 | iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 6303 | false, GSI_CONTINUE_LINKING); |
| 6304 | if (endvar) |
| 6305 | { |
| 6306 | assign_stmt = gimple_build_assign (endvar, iend); |
| 6307 | gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); |
| 6308 | if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (iend))) |
| 6309 | assign_stmt = gimple_build_assign (fd->loop.v, iend); |
| 6310 | else |
| 6311 | assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, iend); |
| 6312 | gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); |
| 6313 | } |
| 6314 | if (fd->collapse > 1) |
| 6315 | expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar); |
| 6316 | |
| 6317 | if (!broken_loop) |
| 6318 | { |
| 6319 | /* Code to control the increment and predicate for the sequential |
| 6320 | loop goes in the CONT_BB. */ |
| 6321 | gsi = gsi_last_bb (cont_bb); |
| 6322 | gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi)); |
| 6323 | gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE); |
| 6324 | vmain = gimple_omp_continue_control_use (cont_stmt); |
| 6325 | vback = gimple_omp_continue_control_def (cont_stmt); |
| 6326 | |
| 6327 | if (!gimple_omp_for_combined_p (fd->for_stmt)) |
| 6328 | { |
| 6329 | if (POINTER_TYPE_P (type)) |
| 6330 | t = fold_build_pointer_plus (vmain, fd->loop.step); |
| 6331 | else |
| 6332 | t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step); |
| 6333 | t = force_gimple_operand_gsi (&gsi, t, |
| 6334 | DECL_P (vback) |
| 6335 | && TREE_ADDRESSABLE (vback), |
| 6336 | NULL_TREE, true, GSI_SAME_STMT); |
| 6337 | assign_stmt = gimple_build_assign (vback, t); |
| 6338 | gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); |
| 6339 | |
| 6340 | t = build2 (fd->loop.cond_code, boolean_type_node, |
| 6341 | DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback, |
| 6342 | iend); |
| 6343 | gcond *cond_stmt = gimple_build_cond_empty (t); |
| 6344 | gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); |
| 6345 | } |
| 6346 | |
| 6347 | /* Remove GIMPLE_OMP_CONTINUE. */ |
| 6348 | gsi_remove (&gsi, true); |
| 6349 | |
| 6350 | if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt)) |
| 6351 | collapse_bb = extract_omp_for_update_vars (fd, cont_bb, l1_bb); |
| 6352 | |
| 6353 | /* Emit code to get the next parallel iteration in L2_BB. */ |
| 6354 | gsi = gsi_start_bb (l2_bb); |
| 6355 | |
| 6356 | t = build_call_expr (builtin_decl_explicit (next_fn), 2, |
| 6357 | build_fold_addr_expr (istart0), |
| 6358 | build_fold_addr_expr (iend0)); |
| 6359 | t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 6360 | false, GSI_CONTINUE_LINKING); |
| 6361 | if (TREE_TYPE (t) != boolean_type_node) |
| 6362 | t = fold_build2 (NE_EXPR, boolean_type_node, |
| 6363 | t, build_int_cst (TREE_TYPE (t), 0)); |
| 6364 | gcond *cond_stmt = gimple_build_cond_empty (t); |
| 6365 | gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING); |
| 6366 | } |
| 6367 | |
| 6368 | /* Add the loop cleanup function. */ |
| 6369 | gsi = gsi_last_bb (exit_bb); |
| 6370 | if (gimple_omp_return_nowait_p (gsi_stmt (gsi))) |
| 6371 | t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT); |
| 6372 | else if (gimple_omp_return_lhs (gsi_stmt (gsi))) |
| 6373 | t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL); |
| 6374 | else |
| 6375 | t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END); |
| 6376 | gcall *call_stmt = gimple_build_call (t, 0); |
| 6377 | if (gimple_omp_return_lhs (gsi_stmt (gsi))) |
| 6378 | gimple_call_set_lhs (call_stmt, gimple_omp_return_lhs (gsi_stmt (gsi))); |
| 6379 | gsi_insert_after (&gsi, call_stmt, GSI_SAME_STMT); |
| 6380 | gsi_remove (&gsi, true); |
| 6381 | |
| 6382 | /* Connect the new blocks. */ |
| 6383 | find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE; |
| 6384 | find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE; |
| 6385 | |
| 6386 | if (!broken_loop) |
| 6387 | { |
| 6388 | gimple_seq phis; |
| 6389 | |
| 6390 | e = find_edge (cont_bb, l3_bb); |
| 6391 | ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE); |
| 6392 | |
| 6393 | phis = phi_nodes (l3_bb); |
| 6394 | for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi)) |
| 6395 | { |
| 6396 | gimple phi = gsi_stmt (gsi); |
| 6397 | SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne), |
| 6398 | PHI_ARG_DEF_FROM_EDGE (phi, e)); |
| 6399 | } |
| 6400 | remove_edge (e); |
| 6401 | |
| 6402 | make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE); |
| 6403 | add_bb_to_loop (l2_bb, cont_bb->loop_father); |
| 6404 | e = find_edge (cont_bb, l1_bb); |
| 6405 | if (gimple_omp_for_combined_p (fd->for_stmt)) |
| 6406 | { |
| 6407 | remove_edge (e); |
| 6408 | e = NULL; |
| 6409 | } |
| 6410 | else if (fd->collapse > 1) |
| 6411 | { |
| 6412 | remove_edge (e); |
| 6413 | e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE); |
| 6414 | } |
| 6415 | else |
| 6416 | e->flags = EDGE_TRUE_VALUE; |
| 6417 | if (e) |
| 6418 | { |
| 6419 | e->probability = REG_BR_PROB_BASE * 7 / 8; |
| 6420 | find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8; |
| 6421 | } |
| 6422 | else |
| 6423 | { |
| 6424 | e = find_edge (cont_bb, l2_bb); |
| 6425 | e->flags = EDGE_FALLTHRU; |
| 6426 | } |
| 6427 | make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE); |
| 6428 | |
| 6429 | set_immediate_dominator (CDI_DOMINATORS, l2_bb, |
| 6430 | recompute_dominator (CDI_DOMINATORS, l2_bb)); |
| 6431 | set_immediate_dominator (CDI_DOMINATORS, l3_bb, |
| 6432 | recompute_dominator (CDI_DOMINATORS, l3_bb)); |
| 6433 | set_immediate_dominator (CDI_DOMINATORS, l0_bb, |
| 6434 | recompute_dominator (CDI_DOMINATORS, l0_bb)); |
| 6435 | set_immediate_dominator (CDI_DOMINATORS, l1_bb, |
| 6436 | recompute_dominator (CDI_DOMINATORS, l1_bb)); |
| 6437 | |
| 6438 | struct loop *outer_loop = alloc_loop (); |
| 6439 | outer_loop->header = l0_bb; |
| 6440 | outer_loop->latch = l2_bb; |
| 6441 | add_loop (outer_loop, l0_bb->loop_father); |
| 6442 | |
| 6443 | if (!gimple_omp_for_combined_p (fd->for_stmt)) |
| 6444 | { |
| 6445 | struct loop *loop = alloc_loop (); |
| 6446 | loop->header = l1_bb; |
| 6447 | /* The loop may have multiple latches. */ |
| 6448 | add_loop (loop, outer_loop); |
| 6449 | } |
| 6450 | } |
| 6451 | } |
| 6452 | |
| 6453 | |
| 6454 | /* A subroutine of expand_omp_for. Generate code for a parallel |
| 6455 | loop with static schedule and no specified chunk size. Given |
| 6456 | parameters: |
| 6457 | |
| 6458 | for (V = N1; V cond N2; V += STEP) BODY; |
| 6459 | |
| 6460 | where COND is "<" or ">", we generate pseudocode |
| 6461 | |
| 6462 | if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2; |
| 6463 | if (cond is <) |
| 6464 | adj = STEP - 1; |
| 6465 | else |
| 6466 | adj = STEP + 1; |
| 6467 | if ((__typeof (V)) -1 > 0 && cond is >) |
| 6468 | n = -(adj + N2 - N1) / -STEP; |
| 6469 | else |
| 6470 | n = (adj + N2 - N1) / STEP; |
| 6471 | q = n / nthreads; |
| 6472 | tt = n % nthreads; |
| 6473 | if (threadid < tt) goto L3; else goto L4; |
| 6474 | L3: |
| 6475 | tt = 0; |
| 6476 | q = q + 1; |
| 6477 | L4: |
| 6478 | s0 = q * threadid + tt; |
| 6479 | e0 = s0 + q; |
| 6480 | V = s0 * STEP + N1; |
| 6481 | if (s0 >= e0) goto L2; else goto L0; |
| 6482 | L0: |
| 6483 | e = e0 * STEP + N1; |
| 6484 | L1: |
| 6485 | BODY; |
| 6486 | V += STEP; |
| 6487 | if (V cond e) goto L1; |
| 6488 | L2: |
| 6489 | */ |
| 6490 | |
| 6491 | static void |
| 6492 | expand_omp_for_static_nochunk (struct omp_region *region, |
| 6493 | struct omp_for_data *fd, |
| 6494 | gimple inner_stmt) |
| 6495 | { |
| 6496 | tree n, q, s0, e0, e, t, tt, nthreads, threadid; |
| 6497 | tree type, itype, vmain, vback; |
| 6498 | basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb; |
| 6499 | basic_block body_bb, cont_bb, collapse_bb = NULL; |
| 6500 | basic_block fin_bb; |
| 6501 | gimple_stmt_iterator gsi; |
| 6502 | edge ep; |
| 6503 | bool broken_loop = region->cont == NULL; |
| 6504 | tree *counts = NULL; |
| 6505 | tree n1, n2, step; |
| 6506 | |
| 6507 | gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt) |
| 6508 | != GF_OMP_FOR_KIND_OACC_LOOP) |
| 6509 | || !inner_stmt); |
| 6510 | |
| 6511 | itype = type = TREE_TYPE (fd->loop.v); |
| 6512 | if (POINTER_TYPE_P (type)) |
| 6513 | itype = signed_type_for (type); |
| 6514 | |
| 6515 | entry_bb = region->entry; |
| 6516 | cont_bb = region->cont; |
| 6517 | gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); |
| 6518 | fin_bb = BRANCH_EDGE (entry_bb)->dest; |
| 6519 | gcc_assert (broken_loop |
| 6520 | || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest)); |
| 6521 | seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb)); |
| 6522 | body_bb = single_succ (seq_start_bb); |
| 6523 | if (!broken_loop) |
| 6524 | { |
| 6525 | gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb); |
| 6526 | gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); |
| 6527 | } |
| 6528 | exit_bb = region->exit; |
| 6529 | |
| 6530 | /* Iteration space partitioning goes in ENTRY_BB. */ |
| 6531 | gsi = gsi_last_bb (entry_bb); |
| 6532 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); |
| 6533 | |
| 6534 | if (fd->collapse > 1) |
| 6535 | { |
| 6536 | int first_zero_iter = -1; |
| 6537 | basic_block l2_dom_bb = NULL; |
| 6538 | |
| 6539 | counts = XALLOCAVEC (tree, fd->collapse); |
| 6540 | expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, |
| 6541 | fin_bb, first_zero_iter, |
| 6542 | l2_dom_bb); |
| 6543 | t = NULL_TREE; |
| 6544 | } |
| 6545 | else if (gimple_omp_for_combined_into_p (fd->for_stmt)) |
| 6546 | t = integer_one_node; |
| 6547 | else |
| 6548 | t = fold_binary (fd->loop.cond_code, boolean_type_node, |
| 6549 | fold_convert (type, fd->loop.n1), |
| 6550 | fold_convert (type, fd->loop.n2)); |
| 6551 | if (fd->collapse == 1 |
| 6552 | && TYPE_UNSIGNED (type) |
| 6553 | && (t == NULL_TREE || !integer_onep (t))) |
| 6554 | { |
| 6555 | n1 = fold_convert (type, unshare_expr (fd->loop.n1)); |
| 6556 | n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE, |
| 6557 | true, GSI_SAME_STMT); |
| 6558 | n2 = fold_convert (type, unshare_expr (fd->loop.n2)); |
| 6559 | n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE, |
| 6560 | true, GSI_SAME_STMT); |
| 6561 | gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2, |
| 6562 | NULL_TREE, NULL_TREE); |
| 6563 | gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); |
| 6564 | if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), |
| 6565 | expand_omp_regimplify_p, NULL, NULL) |
| 6566 | || walk_tree (gimple_cond_rhs_ptr (cond_stmt), |
| 6567 | expand_omp_regimplify_p, NULL, NULL)) |
| 6568 | { |
| 6569 | gsi = gsi_for_stmt (cond_stmt); |
| 6570 | gimple_regimplify_operands (cond_stmt, &gsi); |
| 6571 | } |
| 6572 | ep = split_block (entry_bb, cond_stmt); |
| 6573 | ep->flags = EDGE_TRUE_VALUE; |
| 6574 | entry_bb = ep->dest; |
| 6575 | ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1); |
| 6576 | ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE); |
| 6577 | ep->probability = REG_BR_PROB_BASE / 2000 - 1; |
| 6578 | if (gimple_in_ssa_p (cfun)) |
| 6579 | { |
| 6580 | int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx; |
| 6581 | for (gphi_iterator gpi = gsi_start_phis (fin_bb); |
| 6582 | !gsi_end_p (gpi); gsi_next (&gpi)) |
| 6583 | { |
| 6584 | gphi *phi = gpi.phi (); |
| 6585 | add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx), |
| 6586 | ep, UNKNOWN_LOCATION); |
| 6587 | } |
| 6588 | } |
| 6589 | gsi = gsi_last_bb (entry_bb); |
| 6590 | } |
| 6591 | |
| 6592 | switch (gimple_omp_for_kind (fd->for_stmt)) |
| 6593 | { |
| 6594 | case GF_OMP_FOR_KIND_FOR: |
| 6595 | nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); |
| 6596 | threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); |
| 6597 | break; |
| 6598 | case GF_OMP_FOR_KIND_DISTRIBUTE: |
| 6599 | nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS); |
| 6600 | threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM); |
| 6601 | break; |
| 6602 | case GF_OMP_FOR_KIND_OACC_LOOP: |
| 6603 | nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS); |
| 6604 | threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM); |
| 6605 | break; |
| 6606 | default: |
| 6607 | gcc_unreachable (); |
| 6608 | } |
| 6609 | nthreads = build_call_expr (nthreads, 0); |
| 6610 | nthreads = fold_convert (itype, nthreads); |
| 6611 | nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE, |
| 6612 | true, GSI_SAME_STMT); |
| 6613 | threadid = build_call_expr (threadid, 0); |
| 6614 | threadid = fold_convert (itype, threadid); |
| 6615 | threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE, |
| 6616 | true, GSI_SAME_STMT); |
| 6617 | |
| 6618 | n1 = fd->loop.n1; |
| 6619 | n2 = fd->loop.n2; |
| 6620 | step = fd->loop.step; |
| 6621 | if (gimple_omp_for_combined_into_p (fd->for_stmt)) |
| 6622 | { |
| 6623 | tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), |
| 6624 | OMP_CLAUSE__LOOPTEMP_); |
| 6625 | gcc_assert (innerc); |
| 6626 | n1 = OMP_CLAUSE_DECL (innerc); |
| 6627 | innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), |
| 6628 | OMP_CLAUSE__LOOPTEMP_); |
| 6629 | gcc_assert (innerc); |
| 6630 | n2 = OMP_CLAUSE_DECL (innerc); |
| 6631 | } |
| 6632 | n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1), |
| 6633 | true, NULL_TREE, true, GSI_SAME_STMT); |
| 6634 | n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2), |
| 6635 | true, NULL_TREE, true, GSI_SAME_STMT); |
| 6636 | step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step), |
| 6637 | true, NULL_TREE, true, GSI_SAME_STMT); |
| 6638 | |
| 6639 | t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1)); |
| 6640 | t = fold_build2 (PLUS_EXPR, itype, step, t); |
| 6641 | t = fold_build2 (PLUS_EXPR, itype, t, n2); |
| 6642 | t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1)); |
| 6643 | if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR) |
| 6644 | t = fold_build2 (TRUNC_DIV_EXPR, itype, |
| 6645 | fold_build1 (NEGATE_EXPR, itype, t), |
| 6646 | fold_build1 (NEGATE_EXPR, itype, step)); |
| 6647 | else |
| 6648 | t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step); |
| 6649 | t = fold_convert (itype, t); |
| 6650 | n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT); |
| 6651 | |
| 6652 | q = create_tmp_reg (itype, "q"); |
| 6653 | t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads); |
| 6654 | t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT); |
| 6655 | gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT); |
| 6656 | |
| 6657 | tt = create_tmp_reg (itype, "tt"); |
| 6658 | t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads); |
| 6659 | t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT); |
| 6660 | gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT); |
| 6661 | |
| 6662 | t = build2 (LT_EXPR, boolean_type_node, threadid, tt); |
| 6663 | gcond *cond_stmt = gimple_build_cond_empty (t); |
| 6664 | gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); |
| 6665 | |
| 6666 | second_bb = split_block (entry_bb, cond_stmt)->dest; |
| 6667 | gsi = gsi_last_bb (second_bb); |
| 6668 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); |
| 6669 | |
| 6670 | gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)), |
| 6671 | GSI_SAME_STMT); |
| 6672 | gassign *assign_stmt |
| 6673 | = gimple_build_assign (q, PLUS_EXPR, q, build_int_cst (itype, 1)); |
| 6674 | gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); |
| 6675 | |
| 6676 | third_bb = split_block (second_bb, assign_stmt)->dest; |
| 6677 | gsi = gsi_last_bb (third_bb); |
| 6678 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); |
| 6679 | |
| 6680 | t = build2 (MULT_EXPR, itype, q, threadid); |
| 6681 | t = build2 (PLUS_EXPR, itype, t, tt); |
| 6682 | s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT); |
| 6683 | |
| 6684 | t = fold_build2 (PLUS_EXPR, itype, s0, q); |
| 6685 | e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT); |
| 6686 | |
| 6687 | t = build2 (GE_EXPR, boolean_type_node, s0, e0); |
| 6688 | gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); |
| 6689 | |
| 6690 | /* Remove the GIMPLE_OMP_FOR statement. */ |
| 6691 | gsi_remove (&gsi, true); |
| 6692 | |
| 6693 | /* Setup code for sequential iteration goes in SEQ_START_BB. */ |
| 6694 | gsi = gsi_start_bb (seq_start_bb); |
| 6695 | |
| 6696 | tree startvar = fd->loop.v; |
| 6697 | tree endvar = NULL_TREE; |
| 6698 | |
| 6699 | if (gimple_omp_for_combined_p (fd->for_stmt)) |
| 6700 | { |
| 6701 | tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL |
| 6702 | ? gimple_omp_parallel_clauses (inner_stmt) |
| 6703 | : gimple_omp_for_clauses (inner_stmt); |
| 6704 | tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_); |
| 6705 | gcc_assert (innerc); |
| 6706 | startvar = OMP_CLAUSE_DECL (innerc); |
| 6707 | innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), |
| 6708 | OMP_CLAUSE__LOOPTEMP_); |
| 6709 | gcc_assert (innerc); |
| 6710 | endvar = OMP_CLAUSE_DECL (innerc); |
| 6711 | } |
| 6712 | t = fold_convert (itype, s0); |
| 6713 | t = fold_build2 (MULT_EXPR, itype, t, step); |
| 6714 | if (POINTER_TYPE_P (type)) |
| 6715 | t = fold_build_pointer_plus (n1, t); |
| 6716 | else |
| 6717 | t = fold_build2 (PLUS_EXPR, type, t, n1); |
| 6718 | t = fold_convert (TREE_TYPE (startvar), t); |
| 6719 | t = force_gimple_operand_gsi (&gsi, t, |
| 6720 | DECL_P (startvar) |
| 6721 | && TREE_ADDRESSABLE (startvar), |
| 6722 | NULL_TREE, false, GSI_CONTINUE_LINKING); |
| 6723 | assign_stmt = gimple_build_assign (startvar, t); |
| 6724 | gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); |
| 6725 | |
| 6726 | t = fold_convert (itype, e0); |
| 6727 | t = fold_build2 (MULT_EXPR, itype, t, step); |
| 6728 | if (POINTER_TYPE_P (type)) |
| 6729 | t = fold_build_pointer_plus (n1, t); |
| 6730 | else |
| 6731 | t = fold_build2 (PLUS_EXPR, type, t, n1); |
| 6732 | t = fold_convert (TREE_TYPE (startvar), t); |
| 6733 | e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 6734 | false, GSI_CONTINUE_LINKING); |
| 6735 | if (endvar) |
| 6736 | { |
| 6737 | assign_stmt = gimple_build_assign (endvar, e); |
| 6738 | gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); |
| 6739 | if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e))) |
| 6740 | assign_stmt = gimple_build_assign (fd->loop.v, e); |
| 6741 | else |
| 6742 | assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e); |
| 6743 | gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); |
| 6744 | } |
| 6745 | if (fd->collapse > 1) |
| 6746 | expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar); |
| 6747 | |
| 6748 | if (!broken_loop) |
| 6749 | { |
| 6750 | /* The code controlling the sequential loop replaces the |
| 6751 | GIMPLE_OMP_CONTINUE. */ |
| 6752 | gsi = gsi_last_bb (cont_bb); |
| 6753 | gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi)); |
| 6754 | gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE); |
| 6755 | vmain = gimple_omp_continue_control_use (cont_stmt); |
| 6756 | vback = gimple_omp_continue_control_def (cont_stmt); |
| 6757 | |
| 6758 | if (!gimple_omp_for_combined_p (fd->for_stmt)) |
| 6759 | { |
| 6760 | if (POINTER_TYPE_P (type)) |
| 6761 | t = fold_build_pointer_plus (vmain, step); |
| 6762 | else |
| 6763 | t = fold_build2 (PLUS_EXPR, type, vmain, step); |
| 6764 | t = force_gimple_operand_gsi (&gsi, t, |
| 6765 | DECL_P (vback) |
| 6766 | && TREE_ADDRESSABLE (vback), |
| 6767 | NULL_TREE, true, GSI_SAME_STMT); |
| 6768 | assign_stmt = gimple_build_assign (vback, t); |
| 6769 | gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); |
| 6770 | |
| 6771 | t = build2 (fd->loop.cond_code, boolean_type_node, |
| 6772 | DECL_P (vback) && TREE_ADDRESSABLE (vback) |
| 6773 | ? t : vback, e); |
| 6774 | gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); |
| 6775 | } |
| 6776 | |
| 6777 | /* Remove the GIMPLE_OMP_CONTINUE statement. */ |
| 6778 | gsi_remove (&gsi, true); |
| 6779 | |
| 6780 | if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt)) |
| 6781 | collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb); |
| 6782 | } |
| 6783 | |
| 6784 | /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */ |
| 6785 | gsi = gsi_last_bb (exit_bb); |
| 6786 | if (!gimple_omp_return_nowait_p (gsi_stmt (gsi))) |
| 6787 | { |
| 6788 | t = gimple_omp_return_lhs (gsi_stmt (gsi)); |
| 6789 | if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP) |
| 6790 | gcc_checking_assert (t == NULL_TREE); |
| 6791 | else |
| 6792 | gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT); |
| 6793 | } |
| 6794 | gsi_remove (&gsi, true); |
| 6795 | |
| 6796 | /* Connect all the blocks. */ |
| 6797 | ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE); |
| 6798 | ep->probability = REG_BR_PROB_BASE / 4 * 3; |
| 6799 | ep = find_edge (entry_bb, second_bb); |
| 6800 | ep->flags = EDGE_TRUE_VALUE; |
| 6801 | ep->probability = REG_BR_PROB_BASE / 4; |
| 6802 | find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE; |
| 6803 | find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE; |
| 6804 | |
| 6805 | if (!broken_loop) |
| 6806 | { |
| 6807 | ep = find_edge (cont_bb, body_bb); |
| 6808 | if (gimple_omp_for_combined_p (fd->for_stmt)) |
| 6809 | { |
| 6810 | remove_edge (ep); |
| 6811 | ep = NULL; |
| 6812 | } |
| 6813 | else if (fd->collapse > 1) |
| 6814 | { |
| 6815 | remove_edge (ep); |
| 6816 | ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE); |
| 6817 | } |
| 6818 | else |
| 6819 | ep->flags = EDGE_TRUE_VALUE; |
| 6820 | find_edge (cont_bb, fin_bb)->flags |
| 6821 | = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU; |
| 6822 | } |
| 6823 | |
| 6824 | set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb); |
| 6825 | set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb); |
| 6826 | set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb); |
| 6827 | |
| 6828 | set_immediate_dominator (CDI_DOMINATORS, body_bb, |
| 6829 | recompute_dominator (CDI_DOMINATORS, body_bb)); |
| 6830 | set_immediate_dominator (CDI_DOMINATORS, fin_bb, |
| 6831 | recompute_dominator (CDI_DOMINATORS, fin_bb)); |
| 6832 | |
| 6833 | if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt)) |
| 6834 | { |
| 6835 | struct loop *loop = alloc_loop (); |
| 6836 | loop->header = body_bb; |
| 6837 | if (collapse_bb == NULL) |
| 6838 | loop->latch = cont_bb; |
| 6839 | add_loop (loop, body_bb->loop_father); |
| 6840 | } |
| 6841 | } |
| 6842 | |
| 6843 | |
| 6844 | /* A subroutine of expand_omp_for. Generate code for a parallel |
| 6845 | loop with static schedule and a specified chunk size. Given |
| 6846 | parameters: |
| 6847 | |
| 6848 | for (V = N1; V cond N2; V += STEP) BODY; |
| 6849 | |
| 6850 | where COND is "<" or ">", we generate pseudocode |
| 6851 | |
| 6852 | if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2; |
| 6853 | if (cond is <) |
| 6854 | adj = STEP - 1; |
| 6855 | else |
| 6856 | adj = STEP + 1; |
| 6857 | if ((__typeof (V)) -1 > 0 && cond is >) |
| 6858 | n = -(adj + N2 - N1) / -STEP; |
| 6859 | else |
| 6860 | n = (adj + N2 - N1) / STEP; |
| 6861 | trip = 0; |
| 6862 | V = threadid * CHUNK * STEP + N1; -- this extra definition of V is |
| 6863 | here so that V is defined |
| 6864 | if the loop is not entered |
| 6865 | L0: |
| 6866 | s0 = (trip * nthreads + threadid) * CHUNK; |
| 6867 | e0 = min(s0 + CHUNK, n); |
| 6868 | if (s0 < n) goto L1; else goto L4; |
| 6869 | L1: |
| 6870 | V = s0 * STEP + N1; |
| 6871 | e = e0 * STEP + N1; |
| 6872 | L2: |
| 6873 | BODY; |
| 6874 | V += STEP; |
| 6875 | if (V cond e) goto L2; else goto L3; |
| 6876 | L3: |
| 6877 | trip += 1; |
| 6878 | goto L0; |
| 6879 | L4: |
| 6880 | */ |
| 6881 | |
| 6882 | static void |
| 6883 | expand_omp_for_static_chunk (struct omp_region *region, |
| 6884 | struct omp_for_data *fd, gimple inner_stmt) |
| 6885 | { |
| 6886 | tree n, s0, e0, e, t; |
| 6887 | tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid; |
| 6888 | tree type, itype, vmain, vback, vextra; |
| 6889 | basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb; |
| 6890 | basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb; |
| 6891 | gimple_stmt_iterator gsi; |
| 6892 | edge se; |
| 6893 | bool broken_loop = region->cont == NULL; |
| 6894 | tree *counts = NULL; |
| 6895 | tree n1, n2, step; |
| 6896 | |
| 6897 | gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt) |
| 6898 | != GF_OMP_FOR_KIND_OACC_LOOP) |
| 6899 | || !inner_stmt); |
| 6900 | |
| 6901 | itype = type = TREE_TYPE (fd->loop.v); |
| 6902 | if (POINTER_TYPE_P (type)) |
| 6903 | itype = signed_type_for (type); |
| 6904 | |
| 6905 | entry_bb = region->entry; |
| 6906 | se = split_block (entry_bb, last_stmt (entry_bb)); |
| 6907 | entry_bb = se->src; |
| 6908 | iter_part_bb = se->dest; |
| 6909 | cont_bb = region->cont; |
| 6910 | gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2); |
| 6911 | fin_bb = BRANCH_EDGE (iter_part_bb)->dest; |
| 6912 | gcc_assert (broken_loop |
| 6913 | || fin_bb == FALLTHRU_EDGE (cont_bb)->dest); |
| 6914 | seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb)); |
| 6915 | body_bb = single_succ (seq_start_bb); |
| 6916 | if (!broken_loop) |
| 6917 | { |
| 6918 | gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb); |
| 6919 | gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); |
| 6920 | trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb)); |
| 6921 | } |
| 6922 | exit_bb = region->exit; |
| 6923 | |
| 6924 | /* Trip and adjustment setup goes in ENTRY_BB. */ |
| 6925 | gsi = gsi_last_bb (entry_bb); |
| 6926 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); |
| 6927 | |
| 6928 | if (fd->collapse > 1) |
| 6929 | { |
| 6930 | int first_zero_iter = -1; |
| 6931 | basic_block l2_dom_bb = NULL; |
| 6932 | |
| 6933 | counts = XALLOCAVEC (tree, fd->collapse); |
| 6934 | expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, |
| 6935 | fin_bb, first_zero_iter, |
| 6936 | l2_dom_bb); |
| 6937 | t = NULL_TREE; |
| 6938 | } |
| 6939 | else if (gimple_omp_for_combined_into_p (fd->for_stmt)) |
| 6940 | t = integer_one_node; |
| 6941 | else |
| 6942 | t = fold_binary (fd->loop.cond_code, boolean_type_node, |
| 6943 | fold_convert (type, fd->loop.n1), |
| 6944 | fold_convert (type, fd->loop.n2)); |
| 6945 | if (fd->collapse == 1 |
| 6946 | && TYPE_UNSIGNED (type) |
| 6947 | && (t == NULL_TREE || !integer_onep (t))) |
| 6948 | { |
| 6949 | n1 = fold_convert (type, unshare_expr (fd->loop.n1)); |
| 6950 | n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE, |
| 6951 | true, GSI_SAME_STMT); |
| 6952 | n2 = fold_convert (type, unshare_expr (fd->loop.n2)); |
| 6953 | n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE, |
| 6954 | true, GSI_SAME_STMT); |
| 6955 | gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2, |
| 6956 | NULL_TREE, NULL_TREE); |
| 6957 | gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); |
| 6958 | if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), |
| 6959 | expand_omp_regimplify_p, NULL, NULL) |
| 6960 | || walk_tree (gimple_cond_rhs_ptr (cond_stmt), |
| 6961 | expand_omp_regimplify_p, NULL, NULL)) |
| 6962 | { |
| 6963 | gsi = gsi_for_stmt (cond_stmt); |
| 6964 | gimple_regimplify_operands (cond_stmt, &gsi); |
| 6965 | } |
| 6966 | se = split_block (entry_bb, cond_stmt); |
| 6967 | se->flags = EDGE_TRUE_VALUE; |
| 6968 | entry_bb = se->dest; |
| 6969 | se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1); |
| 6970 | se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE); |
| 6971 | se->probability = REG_BR_PROB_BASE / 2000 - 1; |
| 6972 | if (gimple_in_ssa_p (cfun)) |
| 6973 | { |
| 6974 | int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx; |
| 6975 | for (gphi_iterator gpi = gsi_start_phis (fin_bb); |
| 6976 | !gsi_end_p (gpi); gsi_next (&gpi)) |
| 6977 | { |
| 6978 | gphi *phi = gpi.phi (); |
| 6979 | add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx), |
| 6980 | se, UNKNOWN_LOCATION); |
| 6981 | } |
| 6982 | } |
| 6983 | gsi = gsi_last_bb (entry_bb); |
| 6984 | } |
| 6985 | |
| 6986 | switch (gimple_omp_for_kind (fd->for_stmt)) |
| 6987 | { |
| 6988 | case GF_OMP_FOR_KIND_FOR: |
| 6989 | nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); |
| 6990 | threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); |
| 6991 | break; |
| 6992 | case GF_OMP_FOR_KIND_DISTRIBUTE: |
| 6993 | nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS); |
| 6994 | threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM); |
| 6995 | break; |
| 6996 | case GF_OMP_FOR_KIND_OACC_LOOP: |
| 6997 | nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS); |
| 6998 | threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM); |
| 6999 | break; |
| 7000 | default: |
| 7001 | gcc_unreachable (); |
| 7002 | } |
| 7003 | nthreads = build_call_expr (nthreads, 0); |
| 7004 | nthreads = fold_convert (itype, nthreads); |
| 7005 | nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE, |
| 7006 | true, GSI_SAME_STMT); |
| 7007 | threadid = build_call_expr (threadid, 0); |
| 7008 | threadid = fold_convert (itype, threadid); |
| 7009 | threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE, |
| 7010 | true, GSI_SAME_STMT); |
| 7011 | |
| 7012 | n1 = fd->loop.n1; |
| 7013 | n2 = fd->loop.n2; |
| 7014 | step = fd->loop.step; |
| 7015 | if (gimple_omp_for_combined_into_p (fd->for_stmt)) |
| 7016 | { |
| 7017 | tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), |
| 7018 | OMP_CLAUSE__LOOPTEMP_); |
| 7019 | gcc_assert (innerc); |
| 7020 | n1 = OMP_CLAUSE_DECL (innerc); |
| 7021 | innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), |
| 7022 | OMP_CLAUSE__LOOPTEMP_); |
| 7023 | gcc_assert (innerc); |
| 7024 | n2 = OMP_CLAUSE_DECL (innerc); |
| 7025 | } |
| 7026 | n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1), |
| 7027 | true, NULL_TREE, true, GSI_SAME_STMT); |
| 7028 | n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2), |
| 7029 | true, NULL_TREE, true, GSI_SAME_STMT); |
| 7030 | step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step), |
| 7031 | true, NULL_TREE, true, GSI_SAME_STMT); |
| 7032 | fd->chunk_size |
| 7033 | = force_gimple_operand_gsi (&gsi, fold_convert (itype, fd->chunk_size), |
| 7034 | true, NULL_TREE, true, GSI_SAME_STMT); |
| 7035 | |
| 7036 | t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1)); |
| 7037 | t = fold_build2 (PLUS_EXPR, itype, step, t); |
| 7038 | t = fold_build2 (PLUS_EXPR, itype, t, n2); |
| 7039 | t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1)); |
| 7040 | if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR) |
| 7041 | t = fold_build2 (TRUNC_DIV_EXPR, itype, |
| 7042 | fold_build1 (NEGATE_EXPR, itype, t), |
| 7043 | fold_build1 (NEGATE_EXPR, itype, step)); |
| 7044 | else |
| 7045 | t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step); |
| 7046 | t = fold_convert (itype, t); |
| 7047 | n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 7048 | true, GSI_SAME_STMT); |
| 7049 | |
| 7050 | trip_var = create_tmp_reg (itype, ".trip"); |
| 7051 | if (gimple_in_ssa_p (cfun)) |
| 7052 | { |
| 7053 | trip_init = make_ssa_name (trip_var); |
| 7054 | trip_main = make_ssa_name (trip_var); |
| 7055 | trip_back = make_ssa_name (trip_var); |
| 7056 | } |
| 7057 | else |
| 7058 | { |
| 7059 | trip_init = trip_var; |
| 7060 | trip_main = trip_var; |
| 7061 | trip_back = trip_var; |
| 7062 | } |
| 7063 | |
| 7064 | gassign *assign_stmt |
| 7065 | = gimple_build_assign (trip_init, build_int_cst (itype, 0)); |
| 7066 | gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); |
| 7067 | |
| 7068 | t = fold_build2 (MULT_EXPR, itype, threadid, fd->chunk_size); |
| 7069 | t = fold_build2 (MULT_EXPR, itype, t, step); |
| 7070 | if (POINTER_TYPE_P (type)) |
| 7071 | t = fold_build_pointer_plus (n1, t); |
| 7072 | else |
| 7073 | t = fold_build2 (PLUS_EXPR, type, t, n1); |
| 7074 | vextra = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 7075 | true, GSI_SAME_STMT); |
| 7076 | |
| 7077 | /* Remove the GIMPLE_OMP_FOR. */ |
| 7078 | gsi_remove (&gsi, true); |
| 7079 | |
| 7080 | /* Iteration space partitioning goes in ITER_PART_BB. */ |
| 7081 | gsi = gsi_last_bb (iter_part_bb); |
| 7082 | |
| 7083 | t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads); |
| 7084 | t = fold_build2 (PLUS_EXPR, itype, t, threadid); |
| 7085 | t = fold_build2 (MULT_EXPR, itype, t, fd->chunk_size); |
| 7086 | s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 7087 | false, GSI_CONTINUE_LINKING); |
| 7088 | |
| 7089 | t = fold_build2 (PLUS_EXPR, itype, s0, fd->chunk_size); |
| 7090 | t = fold_build2 (MIN_EXPR, itype, t, n); |
| 7091 | e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 7092 | false, GSI_CONTINUE_LINKING); |
| 7093 | |
| 7094 | t = build2 (LT_EXPR, boolean_type_node, s0, n); |
| 7095 | gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING); |
| 7096 | |
| 7097 | /* Setup code for sequential iteration goes in SEQ_START_BB. */ |
| 7098 | gsi = gsi_start_bb (seq_start_bb); |
| 7099 | |
| 7100 | tree startvar = fd->loop.v; |
| 7101 | tree endvar = NULL_TREE; |
| 7102 | |
| 7103 | if (gimple_omp_for_combined_p (fd->for_stmt)) |
| 7104 | { |
| 7105 | tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL |
| 7106 | ? gimple_omp_parallel_clauses (inner_stmt) |
| 7107 | : gimple_omp_for_clauses (inner_stmt); |
| 7108 | tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_); |
| 7109 | gcc_assert (innerc); |
| 7110 | startvar = OMP_CLAUSE_DECL (innerc); |
| 7111 | innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), |
| 7112 | OMP_CLAUSE__LOOPTEMP_); |
| 7113 | gcc_assert (innerc); |
| 7114 | endvar = OMP_CLAUSE_DECL (innerc); |
| 7115 | } |
| 7116 | |
| 7117 | t = fold_convert (itype, s0); |
| 7118 | t = fold_build2 (MULT_EXPR, itype, t, step); |
| 7119 | if (POINTER_TYPE_P (type)) |
| 7120 | t = fold_build_pointer_plus (n1, t); |
| 7121 | else |
| 7122 | t = fold_build2 (PLUS_EXPR, type, t, n1); |
| 7123 | t = fold_convert (TREE_TYPE (startvar), t); |
| 7124 | t = force_gimple_operand_gsi (&gsi, t, |
| 7125 | DECL_P (startvar) |
| 7126 | && TREE_ADDRESSABLE (startvar), |
| 7127 | NULL_TREE, false, GSI_CONTINUE_LINKING); |
| 7128 | assign_stmt = gimple_build_assign (startvar, t); |
| 7129 | gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); |
| 7130 | |
| 7131 | t = fold_convert (itype, e0); |
| 7132 | t = fold_build2 (MULT_EXPR, itype, t, step); |
| 7133 | if (POINTER_TYPE_P (type)) |
| 7134 | t = fold_build_pointer_plus (n1, t); |
| 7135 | else |
| 7136 | t = fold_build2 (PLUS_EXPR, type, t, n1); |
| 7137 | t = fold_convert (TREE_TYPE (startvar), t); |
| 7138 | e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 7139 | false, GSI_CONTINUE_LINKING); |
| 7140 | if (endvar) |
| 7141 | { |
| 7142 | assign_stmt = gimple_build_assign (endvar, e); |
| 7143 | gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); |
| 7144 | if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e))) |
| 7145 | assign_stmt = gimple_build_assign (fd->loop.v, e); |
| 7146 | else |
| 7147 | assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e); |
| 7148 | gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); |
| 7149 | } |
| 7150 | if (fd->collapse > 1) |
| 7151 | expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar); |
| 7152 | |
| 7153 | if (!broken_loop) |
| 7154 | { |
| 7155 | /* The code controlling the sequential loop goes in CONT_BB, |
| 7156 | replacing the GIMPLE_OMP_CONTINUE. */ |
| 7157 | gsi = gsi_last_bb (cont_bb); |
| 7158 | gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi)); |
| 7159 | vmain = gimple_omp_continue_control_use (cont_stmt); |
| 7160 | vback = gimple_omp_continue_control_def (cont_stmt); |
| 7161 | |
| 7162 | if (!gimple_omp_for_combined_p (fd->for_stmt)) |
| 7163 | { |
| 7164 | if (POINTER_TYPE_P (type)) |
| 7165 | t = fold_build_pointer_plus (vmain, step); |
| 7166 | else |
| 7167 | t = fold_build2 (PLUS_EXPR, type, vmain, step); |
| 7168 | if (DECL_P (vback) && TREE_ADDRESSABLE (vback)) |
| 7169 | t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 7170 | true, GSI_SAME_STMT); |
| 7171 | assign_stmt = gimple_build_assign (vback, t); |
| 7172 | gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); |
| 7173 | |
| 7174 | t = build2 (fd->loop.cond_code, boolean_type_node, |
| 7175 | DECL_P (vback) && TREE_ADDRESSABLE (vback) |
| 7176 | ? t : vback, e); |
| 7177 | gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); |
| 7178 | } |
| 7179 | |
| 7180 | /* Remove GIMPLE_OMP_CONTINUE. */ |
| 7181 | gsi_remove (&gsi, true); |
| 7182 | |
| 7183 | if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt)) |
| 7184 | collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb); |
| 7185 | |
| 7186 | /* Trip update code goes into TRIP_UPDATE_BB. */ |
| 7187 | gsi = gsi_start_bb (trip_update_bb); |
| 7188 | |
| 7189 | t = build_int_cst (itype, 1); |
| 7190 | t = build2 (PLUS_EXPR, itype, trip_main, t); |
| 7191 | assign_stmt = gimple_build_assign (trip_back, t); |
| 7192 | gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); |
| 7193 | } |
| 7194 | |
| 7195 | /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */ |
| 7196 | gsi = gsi_last_bb (exit_bb); |
| 7197 | if (!gimple_omp_return_nowait_p (gsi_stmt (gsi))) |
| 7198 | { |
| 7199 | t = gimple_omp_return_lhs (gsi_stmt (gsi)); |
| 7200 | if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP) |
| 7201 | gcc_checking_assert (t == NULL_TREE); |
| 7202 | else |
| 7203 | gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT); |
| 7204 | } |
| 7205 | gsi_remove (&gsi, true); |
| 7206 | |
| 7207 | /* Connect the new blocks. */ |
| 7208 | find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE; |
| 7209 | find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE; |
| 7210 | |
| 7211 | if (!broken_loop) |
| 7212 | { |
| 7213 | se = find_edge (cont_bb, body_bb); |
| 7214 | if (gimple_omp_for_combined_p (fd->for_stmt)) |
| 7215 | { |
| 7216 | remove_edge (se); |
| 7217 | se = NULL; |
| 7218 | } |
| 7219 | else if (fd->collapse > 1) |
| 7220 | { |
| 7221 | remove_edge (se); |
| 7222 | se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE); |
| 7223 | } |
| 7224 | else |
| 7225 | se->flags = EDGE_TRUE_VALUE; |
| 7226 | find_edge (cont_bb, trip_update_bb)->flags |
| 7227 | = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU; |
| 7228 | |
| 7229 | redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb); |
| 7230 | } |
| 7231 | |
| 7232 | if (gimple_in_ssa_p (cfun)) |
| 7233 | { |
| 7234 | gphi_iterator psi; |
| 7235 | gphi *phi; |
| 7236 | edge re, ene; |
| 7237 | edge_var_map *vm; |
| 7238 | size_t i; |
| 7239 | |
| 7240 | gcc_assert (fd->collapse == 1 && !broken_loop); |
| 7241 | |
| 7242 | /* When we redirect the edge from trip_update_bb to iter_part_bb, we |
| 7243 | remove arguments of the phi nodes in fin_bb. We need to create |
| 7244 | appropriate phi nodes in iter_part_bb instead. */ |
| 7245 | se = single_pred_edge (fin_bb); |
| 7246 | re = single_succ_edge (trip_update_bb); |
| 7247 | vec<edge_var_map> *head = redirect_edge_var_map_vector (re); |
| 7248 | ene = single_succ_edge (entry_bb); |
| 7249 | |
| 7250 | psi = gsi_start_phis (fin_bb); |
| 7251 | for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm); |
| 7252 | gsi_next (&psi), ++i) |
| 7253 | { |
| 7254 | gphi *nphi; |
| 7255 | source_location locus; |
| 7256 | |
| 7257 | phi = psi.phi (); |
| 7258 | t = gimple_phi_result (phi); |
| 7259 | gcc_assert (t == redirect_edge_var_map_result (vm)); |
| 7260 | nphi = create_phi_node (t, iter_part_bb); |
| 7261 | |
| 7262 | t = PHI_ARG_DEF_FROM_EDGE (phi, se); |
| 7263 | locus = gimple_phi_arg_location_from_edge (phi, se); |
| 7264 | |
| 7265 | /* A special case -- fd->loop.v is not yet computed in |
| 7266 | iter_part_bb, we need to use vextra instead. */ |
| 7267 | if (t == fd->loop.v) |
| 7268 | t = vextra; |
| 7269 | add_phi_arg (nphi, t, ene, locus); |
| 7270 | locus = redirect_edge_var_map_location (vm); |
| 7271 | add_phi_arg (nphi, redirect_edge_var_map_def (vm), re, locus); |
| 7272 | } |
| 7273 | gcc_assert (gsi_end_p (psi) && i == head->length ()); |
| 7274 | redirect_edge_var_map_clear (re); |
| 7275 | while (1) |
| 7276 | { |
| 7277 | psi = gsi_start_phis (fin_bb); |
| 7278 | if (gsi_end_p (psi)) |
| 7279 | break; |
| 7280 | remove_phi_node (&psi, false); |
| 7281 | } |
| 7282 | |
| 7283 | /* Make phi node for trip. */ |
| 7284 | phi = create_phi_node (trip_main, iter_part_bb); |
| 7285 | add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb), |
| 7286 | UNKNOWN_LOCATION); |
| 7287 | add_phi_arg (phi, trip_init, single_succ_edge (entry_bb), |
| 7288 | UNKNOWN_LOCATION); |
| 7289 | } |
| 7290 | |
| 7291 | if (!broken_loop) |
| 7292 | set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb); |
| 7293 | set_immediate_dominator (CDI_DOMINATORS, iter_part_bb, |
| 7294 | recompute_dominator (CDI_DOMINATORS, iter_part_bb)); |
| 7295 | set_immediate_dominator (CDI_DOMINATORS, fin_bb, |
| 7296 | recompute_dominator (CDI_DOMINATORS, fin_bb)); |
| 7297 | set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, |
| 7298 | recompute_dominator (CDI_DOMINATORS, seq_start_bb)); |
| 7299 | set_immediate_dominator (CDI_DOMINATORS, body_bb, |
| 7300 | recompute_dominator (CDI_DOMINATORS, body_bb)); |
| 7301 | |
| 7302 | if (!broken_loop) |
| 7303 | { |
| 7304 | struct loop *trip_loop = alloc_loop (); |
| 7305 | trip_loop->header = iter_part_bb; |
| 7306 | trip_loop->latch = trip_update_bb; |
| 7307 | add_loop (trip_loop, iter_part_bb->loop_father); |
| 7308 | |
| 7309 | if (!gimple_omp_for_combined_p (fd->for_stmt)) |
| 7310 | { |
| 7311 | struct loop *loop = alloc_loop (); |
| 7312 | loop->header = body_bb; |
| 7313 | if (collapse_bb == NULL) |
| 7314 | loop->latch = cont_bb; |
| 7315 | add_loop (loop, trip_loop); |
| 7316 | } |
| 7317 | } |
| 7318 | } |
| 7319 | |
| 7320 | /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop. |
| 7321 | Given parameters: |
| 7322 | for (V = N1; V cond N2; V += STEP) BODY; |
| 7323 | |
| 7324 | where COND is "<" or ">" or "!=", we generate pseudocode |
| 7325 | |
| 7326 | for (ind_var = low; ind_var < high; ind_var++) |
| 7327 | { |
| 7328 | V = n1 + (ind_var * STEP) |
| 7329 | |
| 7330 | <BODY> |
| 7331 | } |
| 7332 | |
| 7333 | In the above pseudocode, low and high are function parameters of the |
| 7334 | child function. In the function below, we are inserting a temp. |
| 7335 | variable that will be making a call to two OMP functions that will not be |
| 7336 | found in the body of _Cilk_for (since OMP_FOR cannot be mixed |
| 7337 | with _Cilk_for). These functions are replaced with low and high |
| 7338 | by the function that handles taskreg. */ |
| 7339 | |
| 7340 | |
| 7341 | static void |
| 7342 | expand_cilk_for (struct omp_region *region, struct omp_for_data *fd) |
| 7343 | { |
| 7344 | bool broken_loop = region->cont == NULL; |
| 7345 | basic_block entry_bb = region->entry; |
| 7346 | basic_block cont_bb = region->cont; |
| 7347 | |
| 7348 | gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); |
| 7349 | gcc_assert (broken_loop |
| 7350 | || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest); |
| 7351 | basic_block l0_bb = FALLTHRU_EDGE (entry_bb)->dest; |
| 7352 | basic_block l1_bb, l2_bb; |
| 7353 | |
| 7354 | if (!broken_loop) |
| 7355 | { |
| 7356 | gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb); |
| 7357 | gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); |
| 7358 | l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest; |
| 7359 | l2_bb = BRANCH_EDGE (entry_bb)->dest; |
| 7360 | } |
| 7361 | else |
| 7362 | { |
| 7363 | BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL; |
| 7364 | l1_bb = split_edge (BRANCH_EDGE (entry_bb)); |
| 7365 | l2_bb = single_succ (l1_bb); |
| 7366 | } |
| 7367 | basic_block exit_bb = region->exit; |
| 7368 | basic_block l2_dom_bb = NULL; |
| 7369 | |
| 7370 | gimple_stmt_iterator gsi = gsi_last_bb (entry_bb); |
| 7371 | |
| 7372 | /* Below statements until the "tree high_val = ..." are pseudo statements |
| 7373 | used to pass information to be used by expand_omp_taskreg. |
| 7374 | low_val and high_val will be replaced by the __low and __high |
| 7375 | parameter from the child function. |
| 7376 | |
| 7377 | The call_exprs part is a place-holder, it is mainly used |
| 7378 | to distinctly identify to the top-level part that this is |
| 7379 | where we should put low and high (reasoning given in header |
| 7380 | comment). */ |
| 7381 | |
| 7382 | tree child_fndecl |
| 7383 | = gimple_omp_parallel_child_fn ( |
| 7384 | as_a <gomp_parallel *> (last_stmt (region->outer->entry))); |
| 7385 | tree t, low_val = NULL_TREE, high_val = NULL_TREE; |
| 7386 | for (t = DECL_ARGUMENTS (child_fndecl); t; t = TREE_CHAIN (t)) |
| 7387 | { |
| 7388 | if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__high")) |
| 7389 | high_val = t; |
| 7390 | else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__low")) |
| 7391 | low_val = t; |
| 7392 | } |
| 7393 | gcc_assert (low_val && high_val); |
| 7394 | |
| 7395 | tree type = TREE_TYPE (low_val); |
| 7396 | tree ind_var = create_tmp_reg (type, "__cilk_ind_var"); |
| 7397 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); |
| 7398 | |
| 7399 | /* Not needed in SSA form right now. */ |
| 7400 | gcc_assert (!gimple_in_ssa_p (cfun)); |
| 7401 | if (l2_dom_bb == NULL) |
| 7402 | l2_dom_bb = l1_bb; |
| 7403 | |
| 7404 | tree n1 = low_val; |
| 7405 | tree n2 = high_val; |
| 7406 | |
| 7407 | gimple stmt = gimple_build_assign (ind_var, n1); |
| 7408 | |
| 7409 | /* Replace the GIMPLE_OMP_FOR statement. */ |
| 7410 | gsi_replace (&gsi, stmt, true); |
| 7411 | |
| 7412 | if (!broken_loop) |
| 7413 | { |
| 7414 | /* Code to control the increment goes in the CONT_BB. */ |
| 7415 | gsi = gsi_last_bb (cont_bb); |
| 7416 | stmt = gsi_stmt (gsi); |
| 7417 | gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE); |
| 7418 | stmt = gimple_build_assign (ind_var, PLUS_EXPR, ind_var, |
| 7419 | build_one_cst (type)); |
| 7420 | |
| 7421 | /* Replace GIMPLE_OMP_CONTINUE. */ |
| 7422 | gsi_replace (&gsi, stmt, true); |
| 7423 | } |
| 7424 | |
| 7425 | /* Emit the condition in L1_BB. */ |
| 7426 | gsi = gsi_after_labels (l1_bb); |
| 7427 | t = fold_build2 (MULT_EXPR, TREE_TYPE (fd->loop.step), |
| 7428 | fold_convert (TREE_TYPE (fd->loop.step), ind_var), |
| 7429 | fd->loop.step); |
| 7430 | if (POINTER_TYPE_P (TREE_TYPE (fd->loop.n1))) |
| 7431 | t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (fd->loop.n1), |
| 7432 | fd->loop.n1, fold_convert (sizetype, t)); |
| 7433 | else |
| 7434 | t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loop.n1), |
| 7435 | fd->loop.n1, fold_convert (TREE_TYPE (fd->loop.n1), t)); |
| 7436 | t = fold_convert (TREE_TYPE (fd->loop.v), t); |
| 7437 | expand_omp_build_assign (&gsi, fd->loop.v, t); |
| 7438 | |
| 7439 | /* The condition is always '<' since the runtime will fill in the low |
| 7440 | and high values. */ |
| 7441 | stmt = gimple_build_cond (LT_EXPR, ind_var, n2, NULL_TREE, NULL_TREE); |
| 7442 | gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); |
| 7443 | |
| 7444 | /* Remove GIMPLE_OMP_RETURN. */ |
| 7445 | gsi = gsi_last_bb (exit_bb); |
| 7446 | gsi_remove (&gsi, true); |
| 7447 | |
| 7448 | /* Connect the new blocks. */ |
| 7449 | remove_edge (FALLTHRU_EDGE (entry_bb)); |
| 7450 | |
| 7451 | edge e, ne; |
| 7452 | if (!broken_loop) |
| 7453 | { |
| 7454 | remove_edge (BRANCH_EDGE (entry_bb)); |
| 7455 | make_edge (entry_bb, l1_bb, EDGE_FALLTHRU); |
| 7456 | |
| 7457 | e = BRANCH_EDGE (l1_bb); |
| 7458 | ne = FALLTHRU_EDGE (l1_bb); |
| 7459 | e->flags = EDGE_TRUE_VALUE; |
| 7460 | } |
| 7461 | else |
| 7462 | { |
| 7463 | single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; |
| 7464 | |
| 7465 | ne = single_succ_edge (l1_bb); |
| 7466 | e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE); |
| 7467 | |
| 7468 | } |
| 7469 | ne->flags = EDGE_FALSE_VALUE; |
| 7470 | e->probability = REG_BR_PROB_BASE * 7 / 8; |
| 7471 | ne->probability = REG_BR_PROB_BASE / 8; |
| 7472 | |
| 7473 | set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb); |
| 7474 | set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb); |
| 7475 | set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb); |
| 7476 | |
| 7477 | if (!broken_loop) |
| 7478 | { |
| 7479 | struct loop *loop = alloc_loop (); |
| 7480 | loop->header = l1_bb; |
| 7481 | loop->latch = cont_bb; |
| 7482 | add_loop (loop, l1_bb->loop_father); |
| 7483 | loop->safelen = INT_MAX; |
| 7484 | } |
| 7485 | |
| 7486 | /* Pick the correct library function based on the precision of the |
| 7487 | induction variable type. */ |
| 7488 | tree lib_fun = NULL_TREE; |
| 7489 | if (TYPE_PRECISION (type) == 32) |
| 7490 | lib_fun = cilk_for_32_fndecl; |
| 7491 | else if (TYPE_PRECISION (type) == 64) |
| 7492 | lib_fun = cilk_for_64_fndecl; |
| 7493 | else |
| 7494 | gcc_unreachable (); |
| 7495 | |
| 7496 | gcc_assert (fd->sched_kind == OMP_CLAUSE_SCHEDULE_CILKFOR); |
| 7497 | |
| 7498 | /* WS_ARGS contains the library function flavor to call: |
| 7499 | __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the |
| 7500 | user-defined grain value. If the user does not define one, then zero |
| 7501 | is passed in by the parser. */ |
| 7502 | vec_alloc (region->ws_args, 2); |
| 7503 | region->ws_args->quick_push (lib_fun); |
| 7504 | region->ws_args->quick_push (fd->chunk_size); |
| 7505 | } |
| 7506 | |
| 7507 | /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing |
| 7508 | loop. Given parameters: |
| 7509 | |
| 7510 | for (V = N1; V cond N2; V += STEP) BODY; |
| 7511 | |
| 7512 | where COND is "<" or ">", we generate pseudocode |
| 7513 | |
| 7514 | V = N1; |
| 7515 | goto L1; |
| 7516 | L0: |
| 7517 | BODY; |
| 7518 | V += STEP; |
| 7519 | L1: |
| 7520 | if (V cond N2) goto L0; else goto L2; |
| 7521 | L2: |
| 7522 | |
| 7523 | For collapsed loops, given parameters: |
| 7524 | collapse(3) |
| 7525 | for (V1 = N11; V1 cond1 N12; V1 += STEP1) |
| 7526 | for (V2 = N21; V2 cond2 N22; V2 += STEP2) |
| 7527 | for (V3 = N31; V3 cond3 N32; V3 += STEP3) |
| 7528 | BODY; |
| 7529 | |
| 7530 | we generate pseudocode |
| 7531 | |
| 7532 | if (cond3 is <) |
| 7533 | adj = STEP3 - 1; |
| 7534 | else |
| 7535 | adj = STEP3 + 1; |
| 7536 | count3 = (adj + N32 - N31) / STEP3; |
| 7537 | if (cond2 is <) |
| 7538 | adj = STEP2 - 1; |
| 7539 | else |
| 7540 | adj = STEP2 + 1; |
| 7541 | count2 = (adj + N22 - N21) / STEP2; |
| 7542 | if (cond1 is <) |
| 7543 | adj = STEP1 - 1; |
| 7544 | else |
| 7545 | adj = STEP1 + 1; |
| 7546 | count1 = (adj + N12 - N11) / STEP1; |
| 7547 | count = count1 * count2 * count3; |
| 7548 | V = 0; |
| 7549 | V1 = N11; |
| 7550 | V2 = N21; |
| 7551 | V3 = N31; |
| 7552 | goto L1; |
| 7553 | L0: |
| 7554 | BODY; |
| 7555 | V += 1; |
| 7556 | V3 += STEP3; |
| 7557 | V2 += (V3 cond3 N32) ? 0 : STEP2; |
| 7558 | V3 = (V3 cond3 N32) ? V3 : N31; |
| 7559 | V1 += (V2 cond2 N22) ? 0 : STEP1; |
| 7560 | V2 = (V2 cond2 N22) ? V2 : N21; |
| 7561 | L1: |
| 7562 | if (V < count) goto L0; else goto L2; |
| 7563 | L2: |
| 7564 | |
| 7565 | */ |
| 7566 | |
| 7567 | static void |
| 7568 | expand_omp_simd (struct omp_region *region, struct omp_for_data *fd) |
| 7569 | { |
| 7570 | tree type, t; |
| 7571 | basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb; |
| 7572 | gimple_stmt_iterator gsi; |
| 7573 | gimple stmt; |
| 7574 | gcond *cond_stmt; |
| 7575 | bool broken_loop = region->cont == NULL; |
| 7576 | edge e, ne; |
| 7577 | tree *counts = NULL; |
| 7578 | int i; |
| 7579 | tree safelen = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), |
| 7580 | OMP_CLAUSE_SAFELEN); |
| 7581 | tree simduid = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), |
| 7582 | OMP_CLAUSE__SIMDUID_); |
| 7583 | tree n1, n2; |
| 7584 | |
| 7585 | type = TREE_TYPE (fd->loop.v); |
| 7586 | entry_bb = region->entry; |
| 7587 | cont_bb = region->cont; |
| 7588 | gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); |
| 7589 | gcc_assert (broken_loop |
| 7590 | || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest); |
| 7591 | l0_bb = FALLTHRU_EDGE (entry_bb)->dest; |
| 7592 | if (!broken_loop) |
| 7593 | { |
| 7594 | gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb); |
| 7595 | gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); |
| 7596 | l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest; |
| 7597 | l2_bb = BRANCH_EDGE (entry_bb)->dest; |
| 7598 | } |
| 7599 | else |
| 7600 | { |
| 7601 | BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL; |
| 7602 | l1_bb = split_edge (BRANCH_EDGE (entry_bb)); |
| 7603 | l2_bb = single_succ (l1_bb); |
| 7604 | } |
| 7605 | exit_bb = region->exit; |
| 7606 | l2_dom_bb = NULL; |
| 7607 | |
| 7608 | gsi = gsi_last_bb (entry_bb); |
| 7609 | |
| 7610 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); |
| 7611 | /* Not needed in SSA form right now. */ |
| 7612 | gcc_assert (!gimple_in_ssa_p (cfun)); |
| 7613 | if (fd->collapse > 1) |
| 7614 | { |
| 7615 | int first_zero_iter = -1; |
| 7616 | basic_block zero_iter_bb = l2_bb; |
| 7617 | |
| 7618 | counts = XALLOCAVEC (tree, fd->collapse); |
| 7619 | expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, |
| 7620 | zero_iter_bb, first_zero_iter, |
| 7621 | l2_dom_bb); |
| 7622 | } |
| 7623 | if (l2_dom_bb == NULL) |
| 7624 | l2_dom_bb = l1_bb; |
| 7625 | |
| 7626 | n1 = fd->loop.n1; |
| 7627 | n2 = fd->loop.n2; |
| 7628 | if (gimple_omp_for_combined_into_p (fd->for_stmt)) |
| 7629 | { |
| 7630 | tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), |
| 7631 | OMP_CLAUSE__LOOPTEMP_); |
| 7632 | gcc_assert (innerc); |
| 7633 | n1 = OMP_CLAUSE_DECL (innerc); |
| 7634 | innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), |
| 7635 | OMP_CLAUSE__LOOPTEMP_); |
| 7636 | gcc_assert (innerc); |
| 7637 | n2 = OMP_CLAUSE_DECL (innerc); |
| 7638 | expand_omp_build_assign (&gsi, fd->loop.v, |
| 7639 | fold_convert (type, n1)); |
| 7640 | if (fd->collapse > 1) |
| 7641 | { |
| 7642 | gsi_prev (&gsi); |
| 7643 | expand_omp_for_init_vars (fd, &gsi, counts, NULL, n1); |
| 7644 | gsi_next (&gsi); |
| 7645 | } |
| 7646 | } |
| 7647 | else |
| 7648 | { |
| 7649 | expand_omp_build_assign (&gsi, fd->loop.v, |
| 7650 | fold_convert (type, fd->loop.n1)); |
| 7651 | if (fd->collapse > 1) |
| 7652 | for (i = 0; i < fd->collapse; i++) |
| 7653 | { |
| 7654 | tree itype = TREE_TYPE (fd->loops[i].v); |
| 7655 | if (POINTER_TYPE_P (itype)) |
| 7656 | itype = signed_type_for (itype); |
| 7657 | t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1); |
| 7658 | expand_omp_build_assign (&gsi, fd->loops[i].v, t); |
| 7659 | } |
| 7660 | } |
| 7661 | |
| 7662 | /* Remove the GIMPLE_OMP_FOR statement. */ |
| 7663 | gsi_remove (&gsi, true); |
| 7664 | |
| 7665 | if (!broken_loop) |
| 7666 | { |
| 7667 | /* Code to control the increment goes in the CONT_BB. */ |
| 7668 | gsi = gsi_last_bb (cont_bb); |
| 7669 | stmt = gsi_stmt (gsi); |
| 7670 | gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE); |
| 7671 | |
| 7672 | if (POINTER_TYPE_P (type)) |
| 7673 | t = fold_build_pointer_plus (fd->loop.v, fd->loop.step); |
| 7674 | else |
| 7675 | t = fold_build2 (PLUS_EXPR, type, fd->loop.v, fd->loop.step); |
| 7676 | expand_omp_build_assign (&gsi, fd->loop.v, t); |
| 7677 | |
| 7678 | if (fd->collapse > 1) |
| 7679 | { |
| 7680 | i = fd->collapse - 1; |
| 7681 | if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v))) |
| 7682 | { |
| 7683 | t = fold_convert (sizetype, fd->loops[i].step); |
| 7684 | t = fold_build_pointer_plus (fd->loops[i].v, t); |
| 7685 | } |
| 7686 | else |
| 7687 | { |
| 7688 | t = fold_convert (TREE_TYPE (fd->loops[i].v), |
| 7689 | fd->loops[i].step); |
| 7690 | t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v), |
| 7691 | fd->loops[i].v, t); |
| 7692 | } |
| 7693 | expand_omp_build_assign (&gsi, fd->loops[i].v, t); |
| 7694 | |
| 7695 | for (i = fd->collapse - 1; i > 0; i--) |
| 7696 | { |
| 7697 | tree itype = TREE_TYPE (fd->loops[i].v); |
| 7698 | tree itype2 = TREE_TYPE (fd->loops[i - 1].v); |
| 7699 | if (POINTER_TYPE_P (itype2)) |
| 7700 | itype2 = signed_type_for (itype2); |
| 7701 | t = build3 (COND_EXPR, itype2, |
| 7702 | build2 (fd->loops[i].cond_code, boolean_type_node, |
| 7703 | fd->loops[i].v, |
| 7704 | fold_convert (itype, fd->loops[i].n2)), |
| 7705 | build_int_cst (itype2, 0), |
| 7706 | fold_convert (itype2, fd->loops[i - 1].step)); |
| 7707 | if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i - 1].v))) |
| 7708 | t = fold_build_pointer_plus (fd->loops[i - 1].v, t); |
| 7709 | else |
| 7710 | t = fold_build2 (PLUS_EXPR, itype2, fd->loops[i - 1].v, t); |
| 7711 | expand_omp_build_assign (&gsi, fd->loops[i - 1].v, t); |
| 7712 | |
| 7713 | t = build3 (COND_EXPR, itype, |
| 7714 | build2 (fd->loops[i].cond_code, boolean_type_node, |
| 7715 | fd->loops[i].v, |
| 7716 | fold_convert (itype, fd->loops[i].n2)), |
| 7717 | fd->loops[i].v, |
| 7718 | fold_convert (itype, fd->loops[i].n1)); |
| 7719 | expand_omp_build_assign (&gsi, fd->loops[i].v, t); |
| 7720 | } |
| 7721 | } |
| 7722 | |
| 7723 | /* Remove GIMPLE_OMP_CONTINUE. */ |
| 7724 | gsi_remove (&gsi, true); |
| 7725 | } |
| 7726 | |
| 7727 | /* Emit the condition in L1_BB. */ |
| 7728 | gsi = gsi_start_bb (l1_bb); |
| 7729 | |
| 7730 | t = fold_convert (type, n2); |
| 7731 | t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, |
| 7732 | false, GSI_CONTINUE_LINKING); |
| 7733 | t = build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t); |
| 7734 | cond_stmt = gimple_build_cond_empty (t); |
| 7735 | gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING); |
| 7736 | if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), expand_omp_regimplify_p, |
| 7737 | NULL, NULL) |
| 7738 | || walk_tree (gimple_cond_rhs_ptr (cond_stmt), expand_omp_regimplify_p, |
| 7739 | NULL, NULL)) |
| 7740 | { |
| 7741 | gsi = gsi_for_stmt (cond_stmt); |
| 7742 | gimple_regimplify_operands (cond_stmt, &gsi); |
| 7743 | } |
| 7744 | |
| 7745 | /* Remove GIMPLE_OMP_RETURN. */ |
| 7746 | gsi = gsi_last_bb (exit_bb); |
| 7747 | gsi_remove (&gsi, true); |
| 7748 | |
| 7749 | /* Connect the new blocks. */ |
| 7750 | remove_edge (FALLTHRU_EDGE (entry_bb)); |
| 7751 | |
| 7752 | if (!broken_loop) |
| 7753 | { |
| 7754 | remove_edge (BRANCH_EDGE (entry_bb)); |
| 7755 | make_edge (entry_bb, l1_bb, EDGE_FALLTHRU); |
| 7756 | |
| 7757 | e = BRANCH_EDGE (l1_bb); |
| 7758 | ne = FALLTHRU_EDGE (l1_bb); |
| 7759 | e->flags = EDGE_TRUE_VALUE; |
| 7760 | } |
| 7761 | else |
| 7762 | { |
| 7763 | single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; |
| 7764 | |
| 7765 | ne = single_succ_edge (l1_bb); |
| 7766 | e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE); |
| 7767 | |
| 7768 | } |
| 7769 | ne->flags = EDGE_FALSE_VALUE; |
| 7770 | e->probability = REG_BR_PROB_BASE * 7 / 8; |
| 7771 | ne->probability = REG_BR_PROB_BASE / 8; |
| 7772 | |
| 7773 | set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb); |
| 7774 | set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb); |
| 7775 | set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb); |
| 7776 | |
| 7777 | if (!broken_loop) |
| 7778 | { |
| 7779 | struct loop *loop = alloc_loop (); |
| 7780 | loop->header = l1_bb; |
| 7781 | loop->latch = cont_bb; |
| 7782 | add_loop (loop, l1_bb->loop_father); |
| 7783 | if (safelen == NULL_TREE) |
| 7784 | loop->safelen = INT_MAX; |
| 7785 | else |
| 7786 | { |
| 7787 | safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen); |
| 7788 | if (TREE_CODE (safelen) != INTEGER_CST) |
| 7789 | loop->safelen = 0; |
| 7790 | else if (!tree_fits_uhwi_p (safelen) |
| 7791 | || tree_to_uhwi (safelen) > INT_MAX) |
| 7792 | loop->safelen = INT_MAX; |
| 7793 | else |
| 7794 | loop->safelen = tree_to_uhwi (safelen); |
| 7795 | if (loop->safelen == 1) |
| 7796 | loop->safelen = 0; |
| 7797 | } |
| 7798 | if (simduid) |
| 7799 | { |
| 7800 | loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid); |
| 7801 | cfun->has_simduid_loops = true; |
| 7802 | } |
| 7803 | /* If not -fno-tree-loop-vectorize, hint that we want to vectorize |
| 7804 | the loop. */ |
| 7805 | if ((flag_tree_loop_vectorize |
| 7806 | || (!global_options_set.x_flag_tree_loop_vectorize |
| 7807 | && !global_options_set.x_flag_tree_vectorize)) |
| 7808 | && flag_tree_loop_optimize |
| 7809 | && loop->safelen > 1) |
| 7810 | { |
| 7811 | loop->force_vectorize = true; |
| 7812 | cfun->has_force_vectorize_loops = true; |
| 7813 | } |
| 7814 | } |
| 7815 | } |
| 7816 | |
| 7817 | |
| 7818 | /* Expand the OMP loop defined by REGION. */ |
| 7819 | |
| 7820 | static void |
| 7821 | expand_omp_for (struct omp_region *region, gimple inner_stmt) |
| 7822 | { |
| 7823 | struct omp_for_data fd; |
| 7824 | struct omp_for_data_loop *loops; |
| 7825 | |
| 7826 | loops |
| 7827 | = (struct omp_for_data_loop *) |
| 7828 | alloca (gimple_omp_for_collapse (last_stmt (region->entry)) |
| 7829 | * sizeof (struct omp_for_data_loop)); |
| 7830 | extract_omp_for_data (as_a <gomp_for *> (last_stmt (region->entry)), |
| 7831 | &fd, loops); |
| 7832 | region->sched_kind = fd.sched_kind; |
| 7833 | |
| 7834 | gcc_assert (EDGE_COUNT (region->entry->succs) == 2); |
| 7835 | BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL; |
| 7836 | FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL; |
| 7837 | if (region->cont) |
| 7838 | { |
| 7839 | gcc_assert (EDGE_COUNT (region->cont->succs) == 2); |
| 7840 | BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL; |
| 7841 | FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL; |
| 7842 | } |
| 7843 | else |
| 7844 | /* If there isn't a continue then this is a degerate case where |
| 7845 | the introduction of abnormal edges during lowering will prevent |
| 7846 | original loops from being detected. Fix that up. */ |
| 7847 | loops_state_set (LOOPS_NEED_FIXUP); |
| 7848 | |
| 7849 | if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_SIMD) |
| 7850 | expand_omp_simd (region, &fd); |
| 7851 | else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_CILKFOR) |
| 7852 | expand_cilk_for (region, &fd); |
| 7853 | else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC |
| 7854 | && !fd.have_ordered) |
| 7855 | { |
| 7856 | if (fd.chunk_size == NULL) |
| 7857 | expand_omp_for_static_nochunk (region, &fd, inner_stmt); |
| 7858 | else |
| 7859 | expand_omp_for_static_chunk (region, &fd, inner_stmt); |
| 7860 | } |
| 7861 | else |
| 7862 | { |
| 7863 | int fn_index, start_ix, next_ix; |
| 7864 | |
| 7865 | gcc_assert (gimple_omp_for_kind (fd.for_stmt) |
| 7866 | == GF_OMP_FOR_KIND_FOR); |
| 7867 | if (fd.chunk_size == NULL |
| 7868 | && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC) |
| 7869 | fd.chunk_size = integer_zero_node; |
| 7870 | gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO); |
| 7871 | fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME) |
| 7872 | ? 3 : fd.sched_kind; |
| 7873 | fn_index += fd.have_ordered * 4; |
| 7874 | start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index; |
| 7875 | next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index; |
| 7876 | if (fd.iter_type == long_long_unsigned_type_node) |
| 7877 | { |
| 7878 | start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START |
| 7879 | - (int)BUILT_IN_GOMP_LOOP_STATIC_START); |
| 7880 | next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT |
| 7881 | - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT); |
| 7882 | } |
| 7883 | expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix, |
| 7884 | (enum built_in_function) next_ix, inner_stmt); |
| 7885 | } |
| 7886 | |
| 7887 | if (gimple_in_ssa_p (cfun)) |
| 7888 | update_ssa (TODO_update_ssa_only_virtuals); |
| 7889 | } |
| 7890 | |
| 7891 | |
| 7892 | /* Expand code for an OpenMP sections directive. In pseudo code, we generate |
| 7893 | |
| 7894 | v = GOMP_sections_start (n); |
| 7895 | L0: |
| 7896 | switch (v) |
| 7897 | { |
| 7898 | case 0: |
| 7899 | goto L2; |
| 7900 | case 1: |
| 7901 | section 1; |
| 7902 | goto L1; |
| 7903 | case 2: |
| 7904 | ... |
| 7905 | case n: |
| 7906 | ... |
| 7907 | default: |
| 7908 | abort (); |
| 7909 | } |
| 7910 | L1: |
| 7911 | v = GOMP_sections_next (); |
| 7912 | goto L0; |
| 7913 | L2: |
| 7914 | reduction; |
| 7915 | |
| 7916 | If this is a combined parallel sections, replace the call to |
| 7917 | GOMP_sections_start with call to GOMP_sections_next. */ |
| 7918 | |
| 7919 | static void |
| 7920 | expand_omp_sections (struct omp_region *region) |
| 7921 | { |
| 7922 | tree t, u, vin = NULL, vmain, vnext, l2; |
| 7923 | unsigned len; |
| 7924 | basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb; |
| 7925 | gimple_stmt_iterator si, switch_si; |
| 7926 | gomp_sections *sections_stmt; |
| 7927 | gimple stmt; |
| 7928 | gomp_continue *cont; |
| 7929 | edge_iterator ei; |
| 7930 | edge e; |
| 7931 | struct omp_region *inner; |
| 7932 | unsigned i, casei; |
| 7933 | bool exit_reachable = region->cont != NULL; |
| 7934 | |
| 7935 | gcc_assert (region->exit != NULL); |
| 7936 | entry_bb = region->entry; |
| 7937 | l0_bb = single_succ (entry_bb); |
| 7938 | l1_bb = region->cont; |
| 7939 | l2_bb = region->exit; |
| 7940 | if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb) |
| 7941 | l2 = gimple_block_label (l2_bb); |
| 7942 | else |
| 7943 | { |
| 7944 | /* This can happen if there are reductions. */ |
| 7945 | len = EDGE_COUNT (l0_bb->succs); |
| 7946 | gcc_assert (len > 0); |
| 7947 | e = EDGE_SUCC (l0_bb, len - 1); |
| 7948 | si = gsi_last_bb (e->dest); |
| 7949 | l2 = NULL_TREE; |
| 7950 | if (gsi_end_p (si) |
| 7951 | || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION) |
| 7952 | l2 = gimple_block_label (e->dest); |
| 7953 | else |
| 7954 | FOR_EACH_EDGE (e, ei, l0_bb->succs) |
| 7955 | { |
| 7956 | si = gsi_last_bb (e->dest); |
| 7957 | if (gsi_end_p (si) |
| 7958 | || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION) |
| 7959 | { |
| 7960 | l2 = gimple_block_label (e->dest); |
| 7961 | break; |
| 7962 | } |
| 7963 | } |
| 7964 | } |
| 7965 | if (exit_reachable) |
| 7966 | default_bb = create_empty_bb (l1_bb->prev_bb); |
| 7967 | else |
| 7968 | default_bb = create_empty_bb (l0_bb); |
| 7969 | |
| 7970 | /* We will build a switch() with enough cases for all the |
| 7971 | GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work |
| 7972 | and a default case to abort if something goes wrong. */ |
| 7973 | len = EDGE_COUNT (l0_bb->succs); |
| 7974 | |
| 7975 | /* Use vec::quick_push on label_vec throughout, since we know the size |
| 7976 | in advance. */ |
| 7977 | auto_vec<tree> label_vec (len); |
| 7978 | |
| 7979 | /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the |
| 7980 | GIMPLE_OMP_SECTIONS statement. */ |
| 7981 | si = gsi_last_bb (entry_bb); |
| 7982 | sections_stmt = as_a <gomp_sections *> (gsi_stmt (si)); |
| 7983 | gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS); |
| 7984 | vin = gimple_omp_sections_control (sections_stmt); |
| 7985 | if (!is_combined_parallel (region)) |
| 7986 | { |
| 7987 | /* If we are not inside a combined parallel+sections region, |
| 7988 | call GOMP_sections_start. */ |
| 7989 | t = build_int_cst (unsigned_type_node, len - 1); |
| 7990 | u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START); |
| 7991 | stmt = gimple_build_call (u, 1, t); |
| 7992 | } |
| 7993 | else |
| 7994 | { |
| 7995 | /* Otherwise, call GOMP_sections_next. */ |
| 7996 | u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT); |
| 7997 | stmt = gimple_build_call (u, 0); |
| 7998 | } |
| 7999 | gimple_call_set_lhs (stmt, vin); |
| 8000 | gsi_insert_after (&si, stmt, GSI_SAME_STMT); |
| 8001 | gsi_remove (&si, true); |
| 8002 | |
| 8003 | /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in |
| 8004 | L0_BB. */ |
| 8005 | switch_si = gsi_last_bb (l0_bb); |
| 8006 | gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH); |
| 8007 | if (exit_reachable) |
| 8008 | { |
| 8009 | cont = as_a <gomp_continue *> (last_stmt (l1_bb)); |
| 8010 | gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE); |
| 8011 | vmain = gimple_omp_continue_control_use (cont); |
| 8012 | vnext = gimple_omp_continue_control_def (cont); |
| 8013 | } |
| 8014 | else |
| 8015 | { |
| 8016 | vmain = vin; |
| 8017 | vnext = NULL_TREE; |
| 8018 | } |
| 8019 | |
| 8020 | t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2); |
| 8021 | label_vec.quick_push (t); |
| 8022 | i = 1; |
| 8023 | |
| 8024 | /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */ |
| 8025 | for (inner = region->inner, casei = 1; |
| 8026 | inner; |
| 8027 | inner = inner->next, i++, casei++) |
| 8028 | { |
| 8029 | basic_block s_entry_bb, s_exit_bb; |
| 8030 | |
| 8031 | /* Skip optional reduction region. */ |
| 8032 | if (inner->type == GIMPLE_OMP_ATOMIC_LOAD) |
| 8033 | { |
| 8034 | --i; |
| 8035 | --casei; |
| 8036 | continue; |
| 8037 | } |
| 8038 | |
| 8039 | s_entry_bb = inner->entry; |
| 8040 | s_exit_bb = inner->exit; |
| 8041 | |
| 8042 | t = gimple_block_label (s_entry_bb); |
| 8043 | u = build_int_cst (unsigned_type_node, casei); |
| 8044 | u = build_case_label (u, NULL, t); |
| 8045 | label_vec.quick_push (u); |
| 8046 | |
| 8047 | si = gsi_last_bb (s_entry_bb); |
| 8048 | gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION); |
| 8049 | gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si))); |
| 8050 | gsi_remove (&si, true); |
| 8051 | single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU; |
| 8052 | |
| 8053 | if (s_exit_bb == NULL) |
| 8054 | continue; |
| 8055 | |
| 8056 | si = gsi_last_bb (s_exit_bb); |
| 8057 | gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN); |
| 8058 | gsi_remove (&si, true); |
| 8059 | |
| 8060 | single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU; |
| 8061 | } |
| 8062 | |
| 8063 | /* Error handling code goes in DEFAULT_BB. */ |
| 8064 | t = gimple_block_label (default_bb); |
| 8065 | u = build_case_label (NULL, NULL, t); |
| 8066 | make_edge (l0_bb, default_bb, 0); |
| 8067 | add_bb_to_loop (default_bb, current_loops->tree_root); |
| 8068 | |
| 8069 | stmt = gimple_build_switch (vmain, u, label_vec); |
| 8070 | gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT); |
| 8071 | gsi_remove (&switch_si, true); |
| 8072 | |
| 8073 | si = gsi_start_bb (default_bb); |
| 8074 | stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0); |
| 8075 | gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING); |
| 8076 | |
| 8077 | if (exit_reachable) |
| 8078 | { |
| 8079 | tree bfn_decl; |
| 8080 | |
| 8081 | /* Code to get the next section goes in L1_BB. */ |
| 8082 | si = gsi_last_bb (l1_bb); |
| 8083 | gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE); |
| 8084 | |
| 8085 | bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT); |
| 8086 | stmt = gimple_build_call (bfn_decl, 0); |
| 8087 | gimple_call_set_lhs (stmt, vnext); |
| 8088 | gsi_insert_after (&si, stmt, GSI_SAME_STMT); |
| 8089 | gsi_remove (&si, true); |
| 8090 | |
| 8091 | single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU; |
| 8092 | } |
| 8093 | |
| 8094 | /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */ |
| 8095 | si = gsi_last_bb (l2_bb); |
| 8096 | if (gimple_omp_return_nowait_p (gsi_stmt (si))) |
| 8097 | t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT); |
| 8098 | else if (gimple_omp_return_lhs (gsi_stmt (si))) |
| 8099 | t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL); |
| 8100 | else |
| 8101 | t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END); |
| 8102 | stmt = gimple_build_call (t, 0); |
| 8103 | if (gimple_omp_return_lhs (gsi_stmt (si))) |
| 8104 | gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si))); |
| 8105 | gsi_insert_after (&si, stmt, GSI_SAME_STMT); |
| 8106 | gsi_remove (&si, true); |
| 8107 | |
| 8108 | set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb); |
| 8109 | } |
| 8110 | |
| 8111 | |
| 8112 | /* Expand code for an OpenMP single directive. We've already expanded |
| 8113 | much of the code, here we simply place the GOMP_barrier call. */ |
| 8114 | |
| 8115 | static void |
| 8116 | expand_omp_single (struct omp_region *region) |
| 8117 | { |
| 8118 | basic_block entry_bb, exit_bb; |
| 8119 | gimple_stmt_iterator si; |
| 8120 | |
| 8121 | entry_bb = region->entry; |
| 8122 | exit_bb = region->exit; |
| 8123 | |
| 8124 | si = gsi_last_bb (entry_bb); |
| 8125 | gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE); |
| 8126 | gsi_remove (&si, true); |
| 8127 | single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; |
| 8128 | |
| 8129 | si = gsi_last_bb (exit_bb); |
| 8130 | if (!gimple_omp_return_nowait_p (gsi_stmt (si))) |
| 8131 | { |
| 8132 | tree t = gimple_omp_return_lhs (gsi_stmt (si)); |
| 8133 | gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT); |
| 8134 | } |
| 8135 | gsi_remove (&si, true); |
| 8136 | single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU; |
| 8137 | } |
| 8138 | |
| 8139 | |
| 8140 | /* Generic expansion for OpenMP synchronization directives: master, |
| 8141 | ordered and critical. All we need to do here is remove the entry |
| 8142 | and exit markers for REGION. */ |
| 8143 | |
| 8144 | static void |
| 8145 | expand_omp_synch (struct omp_region *region) |
| 8146 | { |
| 8147 | basic_block entry_bb, exit_bb; |
| 8148 | gimple_stmt_iterator si; |
| 8149 | |
| 8150 | entry_bb = region->entry; |
| 8151 | exit_bb = region->exit; |
| 8152 | |
| 8153 | si = gsi_last_bb (entry_bb); |
| 8154 | gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE |
| 8155 | || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER |
| 8156 | || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP |
| 8157 | || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED |
| 8158 | || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL |
| 8159 | || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS); |
| 8160 | gsi_remove (&si, true); |
| 8161 | single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; |
| 8162 | |
| 8163 | if (exit_bb) |
| 8164 | { |
| 8165 | si = gsi_last_bb (exit_bb); |
| 8166 | gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN); |
| 8167 | gsi_remove (&si, true); |
| 8168 | single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU; |
| 8169 | } |
| 8170 | } |
| 8171 | |
| 8172 | /* A subroutine of expand_omp_atomic. Attempt to implement the atomic |
| 8173 | operation as a normal volatile load. */ |
| 8174 | |
| 8175 | static bool |
| 8176 | expand_omp_atomic_load (basic_block load_bb, tree addr, |
| 8177 | tree loaded_val, int index) |
| 8178 | { |
| 8179 | enum built_in_function tmpbase; |
| 8180 | gimple_stmt_iterator gsi; |
| 8181 | basic_block store_bb; |
| 8182 | location_t loc; |
| 8183 | gimple stmt; |
| 8184 | tree decl, call, type, itype; |
| 8185 | |
| 8186 | gsi = gsi_last_bb (load_bb); |
| 8187 | stmt = gsi_stmt (gsi); |
| 8188 | gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD); |
| 8189 | loc = gimple_location (stmt); |
| 8190 | |
| 8191 | /* ??? If the target does not implement atomic_load_optab[mode], and mode |
| 8192 | is smaller than word size, then expand_atomic_load assumes that the load |
| 8193 | is atomic. We could avoid the builtin entirely in this case. */ |
| 8194 | |
| 8195 | tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1); |
| 8196 | decl = builtin_decl_explicit (tmpbase); |
| 8197 | if (decl == NULL_TREE) |
| 8198 | return false; |
| 8199 | |
| 8200 | type = TREE_TYPE (loaded_val); |
| 8201 | itype = TREE_TYPE (TREE_TYPE (decl)); |
| 8202 | |
| 8203 | call = build_call_expr_loc (loc, decl, 2, addr, |
| 8204 | build_int_cst (NULL, |
| 8205 | gimple_omp_atomic_seq_cst_p (stmt) |
| 8206 | ? MEMMODEL_SEQ_CST |
| 8207 | : MEMMODEL_RELAXED)); |
| 8208 | if (!useless_type_conversion_p (type, itype)) |
| 8209 | call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call); |
| 8210 | call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call); |
| 8211 | |
| 8212 | force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT); |
| 8213 | gsi_remove (&gsi, true); |
| 8214 | |
| 8215 | store_bb = single_succ (load_bb); |
| 8216 | gsi = gsi_last_bb (store_bb); |
| 8217 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE); |
| 8218 | gsi_remove (&gsi, true); |
| 8219 | |
| 8220 | if (gimple_in_ssa_p (cfun)) |
| 8221 | update_ssa (TODO_update_ssa_no_phi); |
| 8222 | |
| 8223 | return true; |
| 8224 | } |
| 8225 | |
| 8226 | /* A subroutine of expand_omp_atomic. Attempt to implement the atomic |
| 8227 | operation as a normal volatile store. */ |
| 8228 | |
| 8229 | static bool |
| 8230 | expand_omp_atomic_store (basic_block load_bb, tree addr, |
| 8231 | tree loaded_val, tree stored_val, int index) |
| 8232 | { |
| 8233 | enum built_in_function tmpbase; |
| 8234 | gimple_stmt_iterator gsi; |
| 8235 | basic_block store_bb = single_succ (load_bb); |
| 8236 | location_t loc; |
| 8237 | gimple stmt; |
| 8238 | tree decl, call, type, itype; |
| 8239 | machine_mode imode; |
| 8240 | bool exchange; |
| 8241 | |
| 8242 | gsi = gsi_last_bb (load_bb); |
| 8243 | stmt = gsi_stmt (gsi); |
| 8244 | gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD); |
| 8245 | |
| 8246 | /* If the load value is needed, then this isn't a store but an exchange. */ |
| 8247 | exchange = gimple_omp_atomic_need_value_p (stmt); |
| 8248 | |
| 8249 | gsi = gsi_last_bb (store_bb); |
| 8250 | stmt = gsi_stmt (gsi); |
| 8251 | gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE); |
| 8252 | loc = gimple_location (stmt); |
| 8253 | |
| 8254 | /* ??? If the target does not implement atomic_store_optab[mode], and mode |
| 8255 | is smaller than word size, then expand_atomic_store assumes that the store |
| 8256 | is atomic. We could avoid the builtin entirely in this case. */ |
| 8257 | |
| 8258 | tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N); |
| 8259 | tmpbase = (enum built_in_function) ((int) tmpbase + index + 1); |
| 8260 | decl = builtin_decl_explicit (tmpbase); |
| 8261 | if (decl == NULL_TREE) |
| 8262 | return false; |
| 8263 | |
| 8264 | type = TREE_TYPE (stored_val); |
| 8265 | |
| 8266 | /* Dig out the type of the function's second argument. */ |
| 8267 | itype = TREE_TYPE (decl); |
| 8268 | itype = TYPE_ARG_TYPES (itype); |
| 8269 | itype = TREE_CHAIN (itype); |
| 8270 | itype = TREE_VALUE (itype); |
| 8271 | imode = TYPE_MODE (itype); |
| 8272 | |
| 8273 | if (exchange && !can_atomic_exchange_p (imode, true)) |
| 8274 | return false; |
| 8275 | |
| 8276 | if (!useless_type_conversion_p (itype, type)) |
| 8277 | stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val); |
| 8278 | call = build_call_expr_loc (loc, decl, 3, addr, stored_val, |
| 8279 | build_int_cst (NULL, |
| 8280 | gimple_omp_atomic_seq_cst_p (stmt) |
| 8281 | ? MEMMODEL_SEQ_CST |
| 8282 | : MEMMODEL_RELAXED)); |
| 8283 | if (exchange) |
| 8284 | { |
| 8285 | if (!useless_type_conversion_p (type, itype)) |
| 8286 | call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call); |
| 8287 | call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call); |
| 8288 | } |
| 8289 | |
| 8290 | force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT); |
| 8291 | gsi_remove (&gsi, true); |
| 8292 | |
| 8293 | /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */ |
| 8294 | gsi = gsi_last_bb (load_bb); |
| 8295 | gsi_remove (&gsi, true); |
| 8296 | |
| 8297 | if (gimple_in_ssa_p (cfun)) |
| 8298 | update_ssa (TODO_update_ssa_no_phi); |
| 8299 | |
| 8300 | return true; |
| 8301 | } |
| 8302 | |
| 8303 | /* A subroutine of expand_omp_atomic. Attempt to implement the atomic |
| 8304 | operation as a __atomic_fetch_op builtin. INDEX is log2 of the |
| 8305 | size of the data type, and thus usable to find the index of the builtin |
| 8306 | decl. Returns false if the expression is not of the proper form. */ |
| 8307 | |
| 8308 | static bool |
| 8309 | expand_omp_atomic_fetch_op (basic_block load_bb, |
| 8310 | tree addr, tree loaded_val, |
| 8311 | tree stored_val, int index) |
| 8312 | { |
| 8313 | enum built_in_function oldbase, newbase, tmpbase; |
| 8314 | tree decl, itype, call; |
| 8315 | tree lhs, rhs; |
| 8316 | basic_block store_bb = single_succ (load_bb); |
| 8317 | gimple_stmt_iterator gsi; |
| 8318 | gimple stmt; |
| 8319 | location_t loc; |
| 8320 | enum tree_code code; |
| 8321 | bool need_old, need_new; |
| 8322 | machine_mode imode; |
| 8323 | bool seq_cst; |
| 8324 | |
| 8325 | /* We expect to find the following sequences: |
| 8326 | |
| 8327 | load_bb: |
| 8328 | GIMPLE_OMP_ATOMIC_LOAD (tmp, mem) |
| 8329 | |
| 8330 | store_bb: |
| 8331 | val = tmp OP something; (or: something OP tmp) |
| 8332 | GIMPLE_OMP_STORE (val) |
| 8333 | |
| 8334 | ???FIXME: Allow a more flexible sequence. |
| 8335 | Perhaps use data flow to pick the statements. |
| 8336 | |
| 8337 | */ |
| 8338 | |
| 8339 | gsi = gsi_after_labels (store_bb); |
| 8340 | stmt = gsi_stmt (gsi); |
| 8341 | loc = gimple_location (stmt); |
| 8342 | if (!is_gimple_assign (stmt)) |
| 8343 | return false; |
| 8344 | gsi_next (&gsi); |
| 8345 | if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE) |
| 8346 | return false; |
| 8347 | need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi)); |
| 8348 | need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb)); |
| 8349 | seq_cst = gimple_omp_atomic_seq_cst_p (last_stmt (load_bb)); |
| 8350 | gcc_checking_assert (!need_old || !need_new); |
| 8351 | |
| 8352 | if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0)) |
| 8353 | return false; |
| 8354 | |
| 8355 | /* Check for one of the supported fetch-op operations. */ |
| 8356 | code = gimple_assign_rhs_code (stmt); |
| 8357 | switch (code) |
| 8358 | { |
| 8359 | case PLUS_EXPR: |
| 8360 | case POINTER_PLUS_EXPR: |
| 8361 | oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N; |
| 8362 | newbase = BUILT_IN_ATOMIC_ADD_FETCH_N; |
| 8363 | break; |
| 8364 | case MINUS_EXPR: |
| 8365 | oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N; |
| 8366 | newbase = BUILT_IN_ATOMIC_SUB_FETCH_N; |
| 8367 | break; |
| 8368 | case BIT_AND_EXPR: |
| 8369 | oldbase = BUILT_IN_ATOMIC_FETCH_AND_N; |
| 8370 | newbase = BUILT_IN_ATOMIC_AND_FETCH_N; |
| 8371 | break; |
| 8372 | case BIT_IOR_EXPR: |
| 8373 | oldbase = BUILT_IN_ATOMIC_FETCH_OR_N; |
| 8374 | newbase = BUILT_IN_ATOMIC_OR_FETCH_N; |
| 8375 | break; |
| 8376 | case BIT_XOR_EXPR: |
| 8377 | oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N; |
| 8378 | newbase = BUILT_IN_ATOMIC_XOR_FETCH_N; |
| 8379 | break; |
| 8380 | default: |
| 8381 | return false; |
| 8382 | } |
| 8383 | |
| 8384 | /* Make sure the expression is of the proper form. */ |
| 8385 | if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0)) |
| 8386 | rhs = gimple_assign_rhs2 (stmt); |
| 8387 | else if (commutative_tree_code (gimple_assign_rhs_code (stmt)) |
| 8388 | && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0)) |
| 8389 | rhs = gimple_assign_rhs1 (stmt); |
| 8390 | else |
| 8391 | return false; |
| 8392 | |
| 8393 | tmpbase = ((enum built_in_function) |
| 8394 | ((need_new ? newbase : oldbase) + index + 1)); |
| 8395 | decl = builtin_decl_explicit (tmpbase); |
| 8396 | if (decl == NULL_TREE) |
| 8397 | return false; |
| 8398 | itype = TREE_TYPE (TREE_TYPE (decl)); |
| 8399 | imode = TYPE_MODE (itype); |
| 8400 | |
| 8401 | /* We could test all of the various optabs involved, but the fact of the |
| 8402 | matter is that (with the exception of i486 vs i586 and xadd) all targets |
| 8403 | that support any atomic operaton optab also implements compare-and-swap. |
| 8404 | Let optabs.c take care of expanding any compare-and-swap loop. */ |
| 8405 | if (!can_compare_and_swap_p (imode, true)) |
| 8406 | return false; |
| 8407 | |
| 8408 | gsi = gsi_last_bb (load_bb); |
| 8409 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD); |
| 8410 | |
| 8411 | /* OpenMP does not imply any barrier-like semantics on its atomic ops. |
| 8412 | It only requires that the operation happen atomically. Thus we can |
| 8413 | use the RELAXED memory model. */ |
| 8414 | call = build_call_expr_loc (loc, decl, 3, addr, |
| 8415 | fold_convert_loc (loc, itype, rhs), |
| 8416 | build_int_cst (NULL, |
| 8417 | seq_cst ? MEMMODEL_SEQ_CST |
| 8418 | : MEMMODEL_RELAXED)); |
| 8419 | |
| 8420 | if (need_old || need_new) |
| 8421 | { |
| 8422 | lhs = need_old ? loaded_val : stored_val; |
| 8423 | call = fold_convert_loc (loc, TREE_TYPE (lhs), call); |
| 8424 | call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call); |
| 8425 | } |
| 8426 | else |
| 8427 | call = fold_convert_loc (loc, void_type_node, call); |
| 8428 | force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT); |
| 8429 | gsi_remove (&gsi, true); |
| 8430 | |
| 8431 | gsi = gsi_last_bb (store_bb); |
| 8432 | gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE); |
| 8433 | gsi_remove (&gsi, true); |
| 8434 | gsi = gsi_last_bb (store_bb); |
| 8435 | gsi_remove (&gsi, true); |
| 8436 | |
| 8437 | if (gimple_in_ssa_p (cfun)) |
| 8438 | update_ssa (TODO_update_ssa_no_phi); |
| 8439 | |
| 8440 | return true; |
| 8441 | } |
| 8442 | |
| 8443 | /* A subroutine of expand_omp_atomic. Implement the atomic operation as: |
| 8444 | |
| 8445 | oldval = *addr; |
| 8446 | repeat: |
| 8447 | newval = rhs; // with oldval replacing *addr in rhs |
| 8448 | oldval = __sync_val_compare_and_swap (addr, oldval, newval); |
| 8449 | if (oldval != newval) |
| 8450 | goto repeat; |
| 8451 | |
| 8452 | INDEX is log2 of the size of the data type, and thus usable to find the |
| 8453 | index of the builtin decl. */ |
| 8454 | |
| 8455 | static bool |
| 8456 | expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb, |
| 8457 | tree addr, tree loaded_val, tree stored_val, |
| 8458 | int index) |
| 8459 | { |
| 8460 | tree loadedi, storedi, initial, new_storedi, old_vali; |
| 8461 | tree type, itype, cmpxchg, iaddr; |
| 8462 | gimple_stmt_iterator si; |
| 8463 | basic_block loop_header = single_succ (load_bb); |
| 8464 | gimple phi, stmt; |
| 8465 | edge e; |
| 8466 | enum built_in_function fncode; |
| 8467 | |
| 8468 | /* ??? We need a non-pointer interface to __atomic_compare_exchange in |
| 8469 | order to use the RELAXED memory model effectively. */ |
| 8470 | fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N |
| 8471 | + index + 1); |
| 8472 | cmpxchg = builtin_decl_explicit (fncode); |
| 8473 | if (cmpxchg == NULL_TREE) |
| 8474 | return false; |
| 8475 | type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr))); |
| 8476 | itype = TREE_TYPE (TREE_TYPE (cmpxchg)); |
| 8477 | |
| 8478 | if (!can_compare_and_swap_p (TYPE_MODE (itype), true)) |
| 8479 | return false; |
| 8480 | |
| 8481 | /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */ |
| 8482 | si = gsi_last_bb (load_bb); |
| 8483 | gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD); |
| 8484 | |
| 8485 | /* For floating-point values, we'll need to view-convert them to integers |
| 8486 | so that we can perform the atomic compare and swap. Simplify the |
| 8487 | following code by always setting up the "i"ntegral variables. */ |
| 8488 | if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type)) |
| 8489 | { |
| 8490 | tree iaddr_val; |
| 8491 | |
| 8492 | iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode, |
| 8493 | true)); |
| 8494 | iaddr_val |
| 8495 | = force_gimple_operand_gsi (&si, |
| 8496 | fold_convert (TREE_TYPE (iaddr), addr), |
| 8497 | false, NULL_TREE, true, GSI_SAME_STMT); |
| 8498 | stmt = gimple_build_assign (iaddr, iaddr_val); |
| 8499 | gsi_insert_before (&si, stmt, GSI_SAME_STMT); |
| 8500 | loadedi = create_tmp_var (itype); |
| 8501 | if (gimple_in_ssa_p (cfun)) |
| 8502 | loadedi = make_ssa_name (loadedi); |
| 8503 | } |
| 8504 | else |
| 8505 | { |
| 8506 | iaddr = addr; |
| 8507 | loadedi = loaded_val; |
| 8508 | } |
| 8509 | |
| 8510 | fncode = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1); |
| 8511 | tree loaddecl = builtin_decl_explicit (fncode); |
| 8512 | if (loaddecl) |
| 8513 | initial |
| 8514 | = fold_convert (TREE_TYPE (TREE_TYPE (iaddr)), |
| 8515 | build_call_expr (loaddecl, 2, iaddr, |
| 8516 | build_int_cst (NULL_TREE, |
| 8517 | MEMMODEL_RELAXED))); |
| 8518 | else |
| 8519 | initial = build2 (MEM_REF, TREE_TYPE (TREE_TYPE (iaddr)), iaddr, |
| 8520 | build_int_cst (TREE_TYPE (iaddr), 0)); |
| 8521 | |
| 8522 | initial |
| 8523 | = force_gimple_operand_gsi (&si, initial, true, NULL_TREE, true, |
| 8524 | GSI_SAME_STMT); |
| 8525 | |
| 8526 | /* Move the value to the LOADEDI temporary. */ |
| 8527 | if (gimple_in_ssa_p (cfun)) |
| 8528 | { |
| 8529 | gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header))); |
| 8530 | phi = create_phi_node (loadedi, loop_header); |
| 8531 | SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)), |
| 8532 | initial); |
| 8533 | } |
| 8534 | else |
| 8535 | gsi_insert_before (&si, |
| 8536 | gimple_build_assign (loadedi, initial), |
| 8537 | GSI_SAME_STMT); |
| 8538 | if (loadedi != loaded_val) |
| 8539 | { |
| 8540 | gimple_stmt_iterator gsi2; |
| 8541 | tree x; |
| 8542 | |
| 8543 | x = build1 (VIEW_CONVERT_EXPR, type, loadedi); |
| 8544 | gsi2 = gsi_start_bb (loop_header); |
| 8545 | if (gimple_in_ssa_p (cfun)) |
| 8546 | { |
| 8547 | gassign *stmt; |
| 8548 | x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE, |
| 8549 | true, GSI_SAME_STMT); |
| 8550 | stmt = gimple_build_assign (loaded_val, x); |
| 8551 | gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT); |
| 8552 | } |
| 8553 | else |
| 8554 | { |
| 8555 | x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x); |
| 8556 | force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE, |
| 8557 | true, GSI_SAME_STMT); |
| 8558 | } |
| 8559 | } |
| 8560 | gsi_remove (&si, true); |
| 8561 | |
| 8562 | si = gsi_last_bb (store_bb); |
| 8563 | gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE); |
| 8564 | |
| 8565 | if (iaddr == addr) |
| 8566 | storedi = stored_val; |
| 8567 | else |
| 8568 | storedi = |
| 8569 | force_gimple_operand_gsi (&si, |
| 8570 | build1 (VIEW_CONVERT_EXPR, itype, |
| 8571 | stored_val), true, NULL_TREE, true, |
| 8572 | GSI_SAME_STMT); |
| 8573 | |
| 8574 | /* Build the compare&swap statement. */ |
| 8575 | new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi); |
| 8576 | new_storedi = force_gimple_operand_gsi (&si, |
| 8577 | fold_convert (TREE_TYPE (loadedi), |
| 8578 | new_storedi), |
| 8579 | true, NULL_TREE, |
| 8580 | true, GSI_SAME_STMT); |
| 8581 | |
| 8582 | if (gimple_in_ssa_p (cfun)) |
| 8583 | old_vali = loadedi; |
| 8584 | else |
| 8585 | { |
| 8586 | old_vali = create_tmp_var (TREE_TYPE (loadedi)); |
| 8587 | stmt = gimple_build_assign (old_vali, loadedi); |
| 8588 | gsi_insert_before (&si, stmt, GSI_SAME_STMT); |
| 8589 | |
| 8590 | stmt = gimple_build_assign (loadedi, new_storedi); |
| 8591 | gsi_insert_before (&si, stmt, GSI_SAME_STMT); |
| 8592 | } |
| 8593 | |
| 8594 | /* Note that we always perform the comparison as an integer, even for |
| 8595 | floating point. This allows the atomic operation to properly |
| 8596 | succeed even with NaNs and -0.0. */ |
| 8597 | stmt = gimple_build_cond_empty |
| 8598 | (build2 (NE_EXPR, boolean_type_node, |
| 8599 | new_storedi, old_vali)); |
| 8600 | gsi_insert_before (&si, stmt, GSI_SAME_STMT); |
| 8601 | |
| 8602 | /* Update cfg. */ |
| 8603 | e = single_succ_edge (store_bb); |
| 8604 | e->flags &= ~EDGE_FALLTHRU; |
| 8605 | e->flags |= EDGE_FALSE_VALUE; |
| 8606 | |
| 8607 | e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE); |
| 8608 | |
| 8609 | /* Copy the new value to loadedi (we already did that before the condition |
| 8610 | if we are not in SSA). */ |
| 8611 | if (gimple_in_ssa_p (cfun)) |
| 8612 | { |
| 8613 | phi = gimple_seq_first_stmt (phi_nodes (loop_header)); |
| 8614 | SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi); |
| 8615 | } |
| 8616 | |
| 8617 | /* Remove GIMPLE_OMP_ATOMIC_STORE. */ |
| 8618 | gsi_remove (&si, true); |
| 8619 | |
| 8620 | struct loop *loop = alloc_loop (); |
| 8621 | loop->header = loop_header; |
| 8622 | loop->latch = store_bb; |
| 8623 | add_loop (loop, loop_header->loop_father); |
| 8624 | |
| 8625 | if (gimple_in_ssa_p (cfun)) |
| 8626 | update_ssa (TODO_update_ssa_no_phi); |
| 8627 | |
| 8628 | return true; |
| 8629 | } |
| 8630 | |
| 8631 | /* A subroutine of expand_omp_atomic. Implement the atomic operation as: |
| 8632 | |
| 8633 | GOMP_atomic_start (); |
| 8634 | *addr = rhs; |
| 8635 | GOMP_atomic_end (); |
| 8636 | |
| 8637 | The result is not globally atomic, but works so long as all parallel |
| 8638 | references are within #pragma omp atomic directives. According to |
| 8639 | responses received from omp@openmp.org, appears to be within spec. |
| 8640 | Which makes sense, since that's how several other compilers handle |
| 8641 | this situation as well. |
| 8642 | LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're |
| 8643 | expanding. STORED_VAL is the operand of the matching |
| 8644 | GIMPLE_OMP_ATOMIC_STORE. |
| 8645 | |
| 8646 | We replace |
| 8647 | GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with |
| 8648 | loaded_val = *addr; |
| 8649 | |
| 8650 | and replace |
| 8651 | GIMPLE_OMP_ATOMIC_STORE (stored_val) with |
| 8652 | *addr = stored_val; |
| 8653 | */ |
| 8654 | |
| 8655 | static bool |
| 8656 | expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb, |
| 8657 | tree addr, tree loaded_val, tree stored_val) |
| 8658 | { |
| 8659 | gimple_stmt_iterator si; |
| 8660 | gassign *stmt; |
| 8661 | tree t; |
| 8662 | |
| 8663 | si = gsi_last_bb (load_bb); |
| 8664 | gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD); |
| 8665 | |
| 8666 | t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START); |
| 8667 | t = build_call_expr (t, 0); |
| 8668 | force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT); |
| 8669 | |
| 8670 | stmt = gimple_build_assign (loaded_val, build_simple_mem_ref (addr)); |
| 8671 | gsi_insert_before (&si, stmt, GSI_SAME_STMT); |
| 8672 | gsi_remove (&si, true); |
| 8673 | |
| 8674 | si = gsi_last_bb (store_bb); |
| 8675 | gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE); |
| 8676 | |
| 8677 | stmt = gimple_build_assign (build_simple_mem_ref (unshare_expr (addr)), |
| 8678 | stored_val); |
| 8679 | gsi_insert_before (&si, stmt, GSI_SAME_STMT); |
| 8680 | |
| 8681 | t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END); |
| 8682 | t = build_call_expr (t, 0); |
| 8683 | force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT); |
| 8684 | gsi_remove (&si, true); |
| 8685 | |
| 8686 | if (gimple_in_ssa_p (cfun)) |
| 8687 | update_ssa (TODO_update_ssa_no_phi); |
| 8688 | return true; |
| 8689 | } |
| 8690 | |
| 8691 | /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand |
| 8692 | using expand_omp_atomic_fetch_op. If it failed, we try to |
| 8693 | call expand_omp_atomic_pipeline, and if it fails too, the |
| 8694 | ultimate fallback is wrapping the operation in a mutex |
| 8695 | (expand_omp_atomic_mutex). REGION is the atomic region built |
| 8696 | by build_omp_regions_1(). */ |
| 8697 | |
| 8698 | static void |
| 8699 | expand_omp_atomic (struct omp_region *region) |
| 8700 | { |
| 8701 | basic_block load_bb = region->entry, store_bb = region->exit; |
| 8702 | gomp_atomic_load *load = as_a <gomp_atomic_load *> (last_stmt (load_bb)); |
| 8703 | gomp_atomic_store *store = as_a <gomp_atomic_store *> (last_stmt (store_bb)); |
| 8704 | tree loaded_val = gimple_omp_atomic_load_lhs (load); |
| 8705 | tree addr = gimple_omp_atomic_load_rhs (load); |
| 8706 | tree stored_val = gimple_omp_atomic_store_val (store); |
| 8707 | tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr))); |
| 8708 | HOST_WIDE_INT index; |
| 8709 | |
| 8710 | /* Make sure the type is one of the supported sizes. */ |
| 8711 | index = tree_to_uhwi (TYPE_SIZE_UNIT (type)); |
| 8712 | index = exact_log2 (index); |
| 8713 | if (index >= 0 && index <= 4) |
| 8714 | { |
| 8715 | unsigned int align = TYPE_ALIGN_UNIT (type); |
| 8716 | |
| 8717 | /* __sync builtins require strict data alignment. */ |
| 8718 | if (exact_log2 (align) >= index) |
| 8719 | { |
| 8720 | /* Atomic load. */ |
| 8721 | if (loaded_val == stored_val |
| 8722 | && (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT |
| 8723 | || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT) |
| 8724 | && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD |
| 8725 | && expand_omp_atomic_load (load_bb, addr, loaded_val, index)) |
| 8726 | return; |
| 8727 | |
| 8728 | /* Atomic store. */ |
| 8729 | if ((GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT |
| 8730 | || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT) |
| 8731 | && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD |
| 8732 | && store_bb == single_succ (load_bb) |
| 8733 | && first_stmt (store_bb) == store |
| 8734 | && expand_omp_atomic_store (load_bb, addr, loaded_val, |
| 8735 | stored_val, index)) |
| 8736 | return; |
| 8737 | |
| 8738 | /* When possible, use specialized atomic update functions. */ |
| 8739 | if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type)) |
| 8740 | && store_bb == single_succ (load_bb) |
| 8741 | && expand_omp_atomic_fetch_op (load_bb, addr, |
| 8742 | loaded_val, stored_val, index)) |
| 8743 | return; |
| 8744 | |
| 8745 | /* If we don't have specialized __sync builtins, try and implement |
| 8746 | as a compare and swap loop. */ |
| 8747 | if (expand_omp_atomic_pipeline (load_bb, store_bb, addr, |
| 8748 | loaded_val, stored_val, index)) |
| 8749 | return; |
| 8750 | } |
| 8751 | } |
| 8752 | |
| 8753 | /* The ultimate fallback is wrapping the operation in a mutex. */ |
| 8754 | expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val); |
| 8755 | } |
| 8756 | |
| 8757 | |
| 8758 | /* Expand the GIMPLE_OMP_TARGET starting at REGION. */ |
| 8759 | |
| 8760 | static void |
| 8761 | expand_omp_target (struct omp_region *region) |
| 8762 | { |
| 8763 | basic_block entry_bb, exit_bb, new_bb; |
| 8764 | struct function *child_cfun; |
| 8765 | tree child_fn, block, t; |
| 8766 | gimple_stmt_iterator gsi; |
| 8767 | gomp_target *entry_stmt; |
| 8768 | gimple stmt; |
| 8769 | edge e; |
| 8770 | bool offloaded, data_region; |
| 8771 | |
| 8772 | entry_stmt = as_a <gomp_target *> (last_stmt (region->entry)); |
| 8773 | new_bb = region->entry; |
| 8774 | |
| 8775 | offloaded = is_gimple_omp_offloaded (entry_stmt); |
| 8776 | switch (gimple_omp_target_kind (entry_stmt)) |
| 8777 | { |
| 8778 | case GF_OMP_TARGET_KIND_REGION: |
| 8779 | case GF_OMP_TARGET_KIND_UPDATE: |
| 8780 | case GF_OMP_TARGET_KIND_OACC_PARALLEL: |
| 8781 | case GF_OMP_TARGET_KIND_OACC_KERNELS: |
| 8782 | case GF_OMP_TARGET_KIND_OACC_UPDATE: |
| 8783 | case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: |
| 8784 | data_region = false; |
| 8785 | break; |
| 8786 | case GF_OMP_TARGET_KIND_DATA: |
| 8787 | case GF_OMP_TARGET_KIND_OACC_DATA: |
| 8788 | data_region = true; |
| 8789 | break; |
| 8790 | default: |
| 8791 | gcc_unreachable (); |
| 8792 | } |
| 8793 | |
| 8794 | child_fn = NULL_TREE; |
| 8795 | child_cfun = NULL; |
| 8796 | if (offloaded) |
| 8797 | { |
| 8798 | child_fn = gimple_omp_target_child_fn (entry_stmt); |
| 8799 | child_cfun = DECL_STRUCT_FUNCTION (child_fn); |
| 8800 | } |
| 8801 | |
| 8802 | /* Supported by expand_omp_taskreg, but not here. */ |
| 8803 | if (child_cfun != NULL) |
| 8804 | gcc_checking_assert (!child_cfun->cfg); |
| 8805 | gcc_checking_assert (!gimple_in_ssa_p (cfun)); |
| 8806 | |
| 8807 | entry_bb = region->entry; |
| 8808 | exit_bb = region->exit; |
| 8809 | |
| 8810 | if (offloaded) |
| 8811 | { |
| 8812 | unsigned srcidx, dstidx, num; |
| 8813 | |
| 8814 | /* If the offloading region needs data sent from the parent |
| 8815 | function, then the very first statement (except possible |
| 8816 | tree profile counter updates) of the offloading body |
| 8817 | is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since |
| 8818 | &.OMP_DATA_O is passed as an argument to the child function, |
| 8819 | we need to replace it with the argument as seen by the child |
| 8820 | function. |
| 8821 | |
| 8822 | In most cases, this will end up being the identity assignment |
| 8823 | .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had |
| 8824 | a function call that has been inlined, the original PARM_DECL |
| 8825 | .OMP_DATA_I may have been converted into a different local |
| 8826 | variable. In which case, we need to keep the assignment. */ |
| 8827 | tree data_arg = gimple_omp_target_data_arg (entry_stmt); |
| 8828 | if (data_arg) |
| 8829 | { |
| 8830 | basic_block entry_succ_bb = single_succ (entry_bb); |
| 8831 | gimple_stmt_iterator gsi; |
| 8832 | tree arg; |
| 8833 | gimple tgtcopy_stmt = NULL; |
| 8834 | tree sender = TREE_VEC_ELT (data_arg, 0); |
| 8835 | |
| 8836 | for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi)) |
| 8837 | { |
| 8838 | gcc_assert (!gsi_end_p (gsi)); |
| 8839 | stmt = gsi_stmt (gsi); |
| 8840 | if (gimple_code (stmt) != GIMPLE_ASSIGN) |
| 8841 | continue; |
| 8842 | |
| 8843 | if (gimple_num_ops (stmt) == 2) |
| 8844 | { |
| 8845 | tree arg = gimple_assign_rhs1 (stmt); |
| 8846 | |
| 8847 | /* We're ignoring the subcode because we're |
| 8848 | effectively doing a STRIP_NOPS. */ |
| 8849 | |
| 8850 | if (TREE_CODE (arg) == ADDR_EXPR |
| 8851 | && TREE_OPERAND (arg, 0) == sender) |
| 8852 | { |
| 8853 | tgtcopy_stmt = stmt; |
| 8854 | break; |
| 8855 | } |
| 8856 | } |
| 8857 | } |
| 8858 | |
| 8859 | gcc_assert (tgtcopy_stmt != NULL); |
| 8860 | arg = DECL_ARGUMENTS (child_fn); |
| 8861 | |
| 8862 | gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg); |
| 8863 | gsi_remove (&gsi, true); |
| 8864 | } |
| 8865 | |
| 8866 | /* Declare local variables needed in CHILD_CFUN. */ |
| 8867 | block = DECL_INITIAL (child_fn); |
| 8868 | BLOCK_VARS (block) = vec2chain (child_cfun->local_decls); |
| 8869 | /* The gimplifier could record temporaries in the offloading block |
| 8870 | rather than in containing function's local_decls chain, |
| 8871 | which would mean cgraph missed finalizing them. Do it now. */ |
| 8872 | for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t)) |
| 8873 | if (TREE_CODE (t) == VAR_DECL |
| 8874 | && TREE_STATIC (t) |
| 8875 | && !DECL_EXTERNAL (t)) |
| 8876 | varpool_node::finalize_decl (t); |
| 8877 | DECL_SAVED_TREE (child_fn) = NULL; |
| 8878 | /* We'll create a CFG for child_fn, so no gimple body is needed. */ |
| 8879 | gimple_set_body (child_fn, NULL); |
| 8880 | TREE_USED (block) = 1; |
| 8881 | |
| 8882 | /* Reset DECL_CONTEXT on function arguments. */ |
| 8883 | for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t)) |
| 8884 | DECL_CONTEXT (t) = child_fn; |
| 8885 | |
| 8886 | /* Split ENTRY_BB at GIMPLE_*, |
| 8887 | so that it can be moved to the child function. */ |
| 8888 | gsi = gsi_last_bb (entry_bb); |
| 8889 | stmt = gsi_stmt (gsi); |
| 8890 | gcc_assert (stmt |
| 8891 | && gimple_code (stmt) == gimple_code (entry_stmt)); |
| 8892 | e = split_block (entry_bb, stmt); |
| 8893 | gsi_remove (&gsi, true); |
| 8894 | entry_bb = e->dest; |
| 8895 | single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; |
| 8896 | |
| 8897 | /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */ |
| 8898 | if (exit_bb) |
| 8899 | { |
| 8900 | gsi = gsi_last_bb (exit_bb); |
| 8901 | gcc_assert (!gsi_end_p (gsi) |
| 8902 | && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN); |
| 8903 | stmt = gimple_build_return (NULL); |
| 8904 | gsi_insert_after (&gsi, stmt, GSI_SAME_STMT); |
| 8905 | gsi_remove (&gsi, true); |
| 8906 | } |
| 8907 | |
| 8908 | /* Move the offloading region into CHILD_CFUN. */ |
| 8909 | |
| 8910 | block = gimple_block (entry_stmt); |
| 8911 | |
| 8912 | new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block); |
| 8913 | if (exit_bb) |
| 8914 | single_succ_edge (new_bb)->flags = EDGE_FALLTHRU; |
| 8915 | /* When the OMP expansion process cannot guarantee an up-to-date |
| 8916 | loop tree arrange for the child function to fixup loops. */ |
| 8917 | if (loops_state_satisfies_p (LOOPS_NEED_FIXUP)) |
| 8918 | child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP; |
| 8919 | |
| 8920 | /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */ |
| 8921 | num = vec_safe_length (child_cfun->local_decls); |
| 8922 | for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++) |
| 8923 | { |
| 8924 | t = (*child_cfun->local_decls)[srcidx]; |
| 8925 | if (DECL_CONTEXT (t) == cfun->decl) |
| 8926 | continue; |
| 8927 | if (srcidx != dstidx) |
| 8928 | (*child_cfun->local_decls)[dstidx] = t; |
| 8929 | dstidx++; |
| 8930 | } |
| 8931 | if (dstidx != num) |
| 8932 | vec_safe_truncate (child_cfun->local_decls, dstidx); |
| 8933 | |
| 8934 | /* Inform the callgraph about the new function. */ |
| 8935 | DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties; |
| 8936 | cgraph_node::add_new_function (child_fn, true); |
| 8937 | |
| 8938 | #ifdef ENABLE_OFFLOADING |
| 8939 | /* Add the new function to the offload table. */ |
| 8940 | vec_safe_push (offload_funcs, child_fn); |
| 8941 | #endif |
| 8942 | |
| 8943 | /* Fix the callgraph edges for child_cfun. Those for cfun will be |
| 8944 | fixed in a following pass. */ |
| 8945 | push_cfun (child_cfun); |
| 8946 | cgraph_edge::rebuild_edges (); |
| 8947 | |
| 8948 | #ifdef ENABLE_OFFLOADING |
| 8949 | /* Prevent IPA from removing child_fn as unreachable, since there are no |
| 8950 | refs from the parent function to child_fn in offload LTO mode. */ |
| 8951 | struct cgraph_node *node = cgraph_node::get (child_fn); |
| 8952 | node->mark_force_output (); |
| 8953 | #endif |
| 8954 | |
| 8955 | /* Some EH regions might become dead, see PR34608. If |
| 8956 | pass_cleanup_cfg isn't the first pass to happen with the |
| 8957 | new child, these dead EH edges might cause problems. |
| 8958 | Clean them up now. */ |
| 8959 | if (flag_exceptions) |
| 8960 | { |
| 8961 | basic_block bb; |
| 8962 | bool changed = false; |
| 8963 | |
| 8964 | FOR_EACH_BB_FN (bb, cfun) |
| 8965 | changed |= gimple_purge_dead_eh_edges (bb); |
| 8966 | if (changed) |
| 8967 | cleanup_tree_cfg (); |
| 8968 | } |
| 8969 | pop_cfun (); |
| 8970 | } |
| 8971 | |
| 8972 | /* Emit a library call to launch the offloading region, or do data |
| 8973 | transfers. */ |
| 8974 | tree t1, t2, t3, t4, device, cond, c, clauses; |
| 8975 | enum built_in_function start_ix; |
| 8976 | location_t clause_loc; |
| 8977 | |
| 8978 | switch (gimple_omp_target_kind (entry_stmt)) |
| 8979 | { |
| 8980 | case GF_OMP_TARGET_KIND_REGION: |
| 8981 | start_ix = BUILT_IN_GOMP_TARGET; |
| 8982 | break; |
| 8983 | case GF_OMP_TARGET_KIND_DATA: |
| 8984 | start_ix = BUILT_IN_GOMP_TARGET_DATA; |
| 8985 | break; |
| 8986 | case GF_OMP_TARGET_KIND_UPDATE: |
| 8987 | start_ix = BUILT_IN_GOMP_TARGET_UPDATE; |
| 8988 | break; |
| 8989 | case GF_OMP_TARGET_KIND_OACC_PARALLEL: |
| 8990 | case GF_OMP_TARGET_KIND_OACC_KERNELS: |
| 8991 | start_ix = BUILT_IN_GOACC_PARALLEL; |
| 8992 | break; |
| 8993 | case GF_OMP_TARGET_KIND_OACC_DATA: |
| 8994 | start_ix = BUILT_IN_GOACC_DATA_START; |
| 8995 | break; |
| 8996 | case GF_OMP_TARGET_KIND_OACC_UPDATE: |
| 8997 | start_ix = BUILT_IN_GOACC_UPDATE; |
| 8998 | break; |
| 8999 | case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: |
| 9000 | start_ix = BUILT_IN_GOACC_ENTER_EXIT_DATA; |
| 9001 | break; |
| 9002 | default: |
| 9003 | gcc_unreachable (); |
| 9004 | } |
| 9005 | |
| 9006 | clauses = gimple_omp_target_clauses (entry_stmt); |
| 9007 | |
| 9008 | /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime |
| 9009 | library choose) and there is no conditional. */ |
| 9010 | cond = NULL_TREE; |
| 9011 | device = build_int_cst (integer_type_node, GOMP_DEVICE_ICV); |
| 9012 | |
| 9013 | c = find_omp_clause (clauses, OMP_CLAUSE_IF); |
| 9014 | if (c) |
| 9015 | cond = OMP_CLAUSE_IF_EXPR (c); |
| 9016 | |
| 9017 | c = find_omp_clause (clauses, OMP_CLAUSE_DEVICE); |
| 9018 | if (c) |
| 9019 | { |
| 9020 | /* Even if we pass it to all library function calls, it is currently only |
| 9021 | defined/used for the OpenMP target ones. */ |
| 9022 | gcc_checking_assert (start_ix == BUILT_IN_GOMP_TARGET |
| 9023 | || start_ix == BUILT_IN_GOMP_TARGET_DATA |
| 9024 | || start_ix == BUILT_IN_GOMP_TARGET_UPDATE); |
| 9025 | |
| 9026 | device = OMP_CLAUSE_DEVICE_ID (c); |
| 9027 | clause_loc = OMP_CLAUSE_LOCATION (c); |
| 9028 | } |
| 9029 | else |
| 9030 | clause_loc = gimple_location (entry_stmt); |
| 9031 | |
| 9032 | /* Ensure 'device' is of the correct type. */ |
| 9033 | device = fold_convert_loc (clause_loc, integer_type_node, device); |
| 9034 | |
| 9035 | /* If we found the clause 'if (cond)', build |
| 9036 | (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */ |
| 9037 | if (cond) |
| 9038 | { |
| 9039 | cond = gimple_boolify (cond); |
| 9040 | |
| 9041 | basic_block cond_bb, then_bb, else_bb; |
| 9042 | edge e; |
| 9043 | tree tmp_var; |
| 9044 | |
| 9045 | tmp_var = create_tmp_var (TREE_TYPE (device)); |
| 9046 | if (offloaded) |
| 9047 | e = split_block (new_bb, NULL); |
| 9048 | else |
| 9049 | { |
| 9050 | gsi = gsi_last_bb (new_bb); |
| 9051 | gsi_prev (&gsi); |
| 9052 | e = split_block (new_bb, gsi_stmt (gsi)); |
| 9053 | } |
| 9054 | cond_bb = e->src; |
| 9055 | new_bb = e->dest; |
| 9056 | remove_edge (e); |
| 9057 | |
| 9058 | then_bb = create_empty_bb (cond_bb); |
| 9059 | else_bb = create_empty_bb (then_bb); |
| 9060 | set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb); |
| 9061 | set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb); |
| 9062 | |
| 9063 | stmt = gimple_build_cond_empty (cond); |
| 9064 | gsi = gsi_last_bb (cond_bb); |
| 9065 | gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); |
| 9066 | |
| 9067 | gsi = gsi_start_bb (then_bb); |
| 9068 | stmt = gimple_build_assign (tmp_var, device); |
| 9069 | gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); |
| 9070 | |
| 9071 | gsi = gsi_start_bb (else_bb); |
| 9072 | stmt = gimple_build_assign (tmp_var, |
| 9073 | build_int_cst (integer_type_node, |
| 9074 | GOMP_DEVICE_HOST_FALLBACK)); |
| 9075 | gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); |
| 9076 | |
| 9077 | make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE); |
| 9078 | make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE); |
| 9079 | add_bb_to_loop (then_bb, cond_bb->loop_father); |
| 9080 | add_bb_to_loop (else_bb, cond_bb->loop_father); |
| 9081 | make_edge (then_bb, new_bb, EDGE_FALLTHRU); |
| 9082 | make_edge (else_bb, new_bb, EDGE_FALLTHRU); |
| 9083 | |
| 9084 | device = tmp_var; |
| 9085 | } |
| 9086 | |
| 9087 | gsi = gsi_last_bb (new_bb); |
| 9088 | t = gimple_omp_target_data_arg (entry_stmt); |
| 9089 | if (t == NULL) |
| 9090 | { |
| 9091 | t1 = size_zero_node; |
| 9092 | t2 = build_zero_cst (ptr_type_node); |
| 9093 | t3 = t2; |
| 9094 | t4 = t2; |
| 9095 | } |
| 9096 | else |
| 9097 | { |
| 9098 | t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1)))); |
| 9099 | t1 = size_binop (PLUS_EXPR, t1, size_int (1)); |
| 9100 | t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0)); |
| 9101 | t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1)); |
| 9102 | t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2)); |
| 9103 | } |
| 9104 | |
| 9105 | gimple g; |
| 9106 | vec<tree> *args; |
| 9107 | /* The maximum number used by any start_ix, without varargs. */ |
| 9108 | unsigned int argcnt = 11; |
| 9109 | |
| 9110 | vec_alloc (args, argcnt); |
| 9111 | args->quick_push (device); |
| 9112 | if (offloaded) |
| 9113 | args->quick_push (build_fold_addr_expr (child_fn)); |
| 9114 | switch (start_ix) |
| 9115 | { |
| 9116 | case BUILT_IN_GOMP_TARGET: |
| 9117 | case BUILT_IN_GOMP_TARGET_DATA: |
| 9118 | case BUILT_IN_GOMP_TARGET_UPDATE: |
| 9119 | /* This const void * is part of the current ABI, but we're not actually |
| 9120 | using it. */ |
| 9121 | args->quick_push (build_zero_cst (ptr_type_node)); |
| 9122 | break; |
| 9123 | case BUILT_IN_GOACC_DATA_START: |
| 9124 | case BUILT_IN_GOACC_ENTER_EXIT_DATA: |
| 9125 | case BUILT_IN_GOACC_PARALLEL: |
| 9126 | case BUILT_IN_GOACC_UPDATE: |
| 9127 | break; |
| 9128 | default: |
| 9129 | gcc_unreachable (); |
| 9130 | } |
| 9131 | args->quick_push (t1); |
| 9132 | args->quick_push (t2); |
| 9133 | args->quick_push (t3); |
| 9134 | args->quick_push (t4); |
| 9135 | switch (start_ix) |
| 9136 | { |
| 9137 | case BUILT_IN_GOACC_DATA_START: |
| 9138 | case BUILT_IN_GOMP_TARGET: |
| 9139 | case BUILT_IN_GOMP_TARGET_DATA: |
| 9140 | case BUILT_IN_GOMP_TARGET_UPDATE: |
| 9141 | break; |
| 9142 | case BUILT_IN_GOACC_PARALLEL: |
| 9143 | { |
| 9144 | tree t_num_gangs, t_num_workers, t_vector_length; |
| 9145 | |
| 9146 | /* Default values for num_gangs, num_workers, and vector_length. */ |
| 9147 | t_num_gangs = t_num_workers = t_vector_length |
| 9148 | = fold_convert_loc (gimple_location (entry_stmt), |
| 9149 | integer_type_node, integer_one_node); |
| 9150 | /* ..., but if present, use the value specified by the respective |
| 9151 | clause, making sure that are of the correct type. */ |
| 9152 | c = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS); |
| 9153 | if (c) |
| 9154 | t_num_gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (c), |
| 9155 | integer_type_node, |
| 9156 | OMP_CLAUSE_NUM_GANGS_EXPR (c)); |
| 9157 | c = find_omp_clause (clauses, OMP_CLAUSE_NUM_WORKERS); |
| 9158 | if (c) |
| 9159 | t_num_workers = fold_convert_loc (OMP_CLAUSE_LOCATION (c), |
| 9160 | integer_type_node, |
| 9161 | OMP_CLAUSE_NUM_WORKERS_EXPR (c)); |
| 9162 | c = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH); |
| 9163 | if (c) |
| 9164 | t_vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (c), |
| 9165 | integer_type_node, |
| 9166 | OMP_CLAUSE_VECTOR_LENGTH_EXPR (c)); |
| 9167 | args->quick_push (t_num_gangs); |
| 9168 | args->quick_push (t_num_workers); |
| 9169 | args->quick_push (t_vector_length); |
| 9170 | } |
| 9171 | /* FALLTHRU */ |
| 9172 | case BUILT_IN_GOACC_ENTER_EXIT_DATA: |
| 9173 | case BUILT_IN_GOACC_UPDATE: |
| 9174 | { |
| 9175 | tree t_async; |
| 9176 | int t_wait_idx; |
| 9177 | |
| 9178 | /* Default values for t_async. */ |
| 9179 | t_async = fold_convert_loc (gimple_location (entry_stmt), |
| 9180 | integer_type_node, |
| 9181 | build_int_cst (integer_type_node, |
| 9182 | GOMP_ASYNC_SYNC)); |
| 9183 | /* ..., but if present, use the value specified by the respective |
| 9184 | clause, making sure that is of the correct type. */ |
| 9185 | c = find_omp_clause (clauses, OMP_CLAUSE_ASYNC); |
| 9186 | if (c) |
| 9187 | t_async = fold_convert_loc (OMP_CLAUSE_LOCATION (c), |
| 9188 | integer_type_node, |
| 9189 | OMP_CLAUSE_ASYNC_EXPR (c)); |
| 9190 | |
| 9191 | args->quick_push (t_async); |
| 9192 | /* Save the index, and... */ |
| 9193 | t_wait_idx = args->length (); |
| 9194 | /* ... push a default value. */ |
| 9195 | args->quick_push (fold_convert_loc (gimple_location (entry_stmt), |
| 9196 | integer_type_node, |
| 9197 | integer_zero_node)); |
| 9198 | c = find_omp_clause (clauses, OMP_CLAUSE_WAIT); |
| 9199 | if (c) |
| 9200 | { |
| 9201 | int n = 0; |
| 9202 | |
| 9203 | for (; c; c = OMP_CLAUSE_CHAIN (c)) |
| 9204 | { |
| 9205 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WAIT) |
| 9206 | { |
| 9207 | args->safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c), |
| 9208 | integer_type_node, |
| 9209 | OMP_CLAUSE_WAIT_EXPR (c))); |
| 9210 | n++; |
| 9211 | } |
| 9212 | } |
| 9213 | |
| 9214 | /* Now that we know the number, replace the default value. */ |
| 9215 | args->ordered_remove (t_wait_idx); |
| 9216 | args->quick_insert (t_wait_idx, |
| 9217 | fold_convert_loc (gimple_location (entry_stmt), |
| 9218 | integer_type_node, |
| 9219 | build_int_cst (integer_type_node, n))); |
| 9220 | } |
| 9221 | } |
| 9222 | break; |
| 9223 | default: |
| 9224 | gcc_unreachable (); |
| 9225 | } |
| 9226 | |
| 9227 | g = gimple_build_call_vec (builtin_decl_explicit (start_ix), *args); |
| 9228 | args->release (); |
| 9229 | gimple_set_location (g, gimple_location (entry_stmt)); |
| 9230 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); |
| 9231 | if (!offloaded) |
| 9232 | { |
| 9233 | g = gsi_stmt (gsi); |
| 9234 | gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET); |
| 9235 | gsi_remove (&gsi, true); |
| 9236 | } |
| 9237 | if (data_region |
| 9238 | && region->exit) |
| 9239 | { |
| 9240 | gsi = gsi_last_bb (region->exit); |
| 9241 | g = gsi_stmt (gsi); |
| 9242 | gcc_assert (g && gimple_code (g) == GIMPLE_OMP_RETURN); |
| 9243 | gsi_remove (&gsi, true); |
| 9244 | } |
| 9245 | } |
| 9246 | |
| 9247 | |
| 9248 | /* Expand the parallel region tree rooted at REGION. Expansion |
| 9249 | proceeds in depth-first order. Innermost regions are expanded |
| 9250 | first. This way, parallel regions that require a new function to |
| 9251 | be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any |
| 9252 | internal dependencies in their body. */ |
| 9253 | |
| 9254 | static void |
| 9255 | expand_omp (struct omp_region *region) |
| 9256 | { |
| 9257 | while (region) |
| 9258 | { |
| 9259 | location_t saved_location; |
| 9260 | gimple inner_stmt = NULL; |
| 9261 | |
| 9262 | /* First, determine whether this is a combined parallel+workshare |
| 9263 | region. */ |
| 9264 | if (region->type == GIMPLE_OMP_PARALLEL) |
| 9265 | determine_parallel_type (region); |
| 9266 | |
| 9267 | if (region->type == GIMPLE_OMP_FOR |
| 9268 | && gimple_omp_for_combined_p (last_stmt (region->entry))) |
| 9269 | inner_stmt = last_stmt (region->inner->entry); |
| 9270 | |
| 9271 | if (region->inner) |
| 9272 | expand_omp (region->inner); |
| 9273 | |
| 9274 | saved_location = input_location; |
| 9275 | if (gimple_has_location (last_stmt (region->entry))) |
| 9276 | input_location = gimple_location (last_stmt (region->entry)); |
| 9277 | |
| 9278 | switch (region->type) |
| 9279 | { |
| 9280 | case GIMPLE_OMP_PARALLEL: |
| 9281 | case GIMPLE_OMP_TASK: |
| 9282 | expand_omp_taskreg (region); |
| 9283 | break; |
| 9284 | |
| 9285 | case GIMPLE_OMP_FOR: |
| 9286 | expand_omp_for (region, inner_stmt); |
| 9287 | break; |
| 9288 | |
| 9289 | case GIMPLE_OMP_SECTIONS: |
| 9290 | expand_omp_sections (region); |
| 9291 | break; |
| 9292 | |
| 9293 | case GIMPLE_OMP_SECTION: |
| 9294 | /* Individual omp sections are handled together with their |
| 9295 | parent GIMPLE_OMP_SECTIONS region. */ |
| 9296 | break; |
| 9297 | |
| 9298 | case GIMPLE_OMP_SINGLE: |
| 9299 | expand_omp_single (region); |
| 9300 | break; |
| 9301 | |
| 9302 | case GIMPLE_OMP_MASTER: |
| 9303 | case GIMPLE_OMP_TASKGROUP: |
| 9304 | case GIMPLE_OMP_ORDERED: |
| 9305 | case GIMPLE_OMP_CRITICAL: |
| 9306 | case GIMPLE_OMP_TEAMS: |
| 9307 | expand_omp_synch (region); |
| 9308 | break; |
| 9309 | |
| 9310 | case GIMPLE_OMP_ATOMIC_LOAD: |
| 9311 | expand_omp_atomic (region); |
| 9312 | break; |
| 9313 | |
| 9314 | case GIMPLE_OMP_TARGET: |
| 9315 | expand_omp_target (region); |
| 9316 | break; |
| 9317 | |
| 9318 | default: |
| 9319 | gcc_unreachable (); |
| 9320 | } |
| 9321 | |
| 9322 | input_location = saved_location; |
| 9323 | region = region->next; |
| 9324 | } |
| 9325 | } |
| 9326 | |
| 9327 | |
| 9328 | /* Helper for build_omp_regions. Scan the dominator tree starting at |
| 9329 | block BB. PARENT is the region that contains BB. If SINGLE_TREE is |
| 9330 | true, the function ends once a single tree is built (otherwise, whole |
| 9331 | forest of OMP constructs may be built). */ |
| 9332 | |
| 9333 | static void |
| 9334 | build_omp_regions_1 (basic_block bb, struct omp_region *parent, |
| 9335 | bool single_tree) |
| 9336 | { |
| 9337 | gimple_stmt_iterator gsi; |
| 9338 | gimple stmt; |
| 9339 | basic_block son; |
| 9340 | |
| 9341 | gsi = gsi_last_bb (bb); |
| 9342 | if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi))) |
| 9343 | { |
| 9344 | struct omp_region *region; |
| 9345 | enum gimple_code code; |
| 9346 | |
| 9347 | stmt = gsi_stmt (gsi); |
| 9348 | code = gimple_code (stmt); |
| 9349 | if (code == GIMPLE_OMP_RETURN) |
| 9350 | { |
| 9351 | /* STMT is the return point out of region PARENT. Mark it |
| 9352 | as the exit point and make PARENT the immediately |
| 9353 | enclosing region. */ |
| 9354 | gcc_assert (parent); |
| 9355 | region = parent; |
| 9356 | region->exit = bb; |
| 9357 | parent = parent->outer; |
| 9358 | } |
| 9359 | else if (code == GIMPLE_OMP_ATOMIC_STORE) |
| 9360 | { |
| 9361 | /* GIMPLE_OMP_ATOMIC_STORE is analoguous to |
| 9362 | GIMPLE_OMP_RETURN, but matches with |
| 9363 | GIMPLE_OMP_ATOMIC_LOAD. */ |
| 9364 | gcc_assert (parent); |
| 9365 | gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD); |
| 9366 | region = parent; |
| 9367 | region->exit = bb; |
| 9368 | parent = parent->outer; |
| 9369 | } |
| 9370 | else if (code == GIMPLE_OMP_CONTINUE) |
| 9371 | { |
| 9372 | gcc_assert (parent); |
| 9373 | parent->cont = bb; |
| 9374 | } |
| 9375 | else if (code == GIMPLE_OMP_SECTIONS_SWITCH) |
| 9376 | { |
| 9377 | /* GIMPLE_OMP_SECTIONS_SWITCH is part of |
| 9378 | GIMPLE_OMP_SECTIONS, and we do nothing for it. */ |
| 9379 | } |
| 9380 | else |
| 9381 | { |
| 9382 | region = new_omp_region (bb, code, parent); |
| 9383 | /* Otherwise... */ |
| 9384 | if (code == GIMPLE_OMP_TARGET) |
| 9385 | { |
| 9386 | switch (gimple_omp_target_kind (stmt)) |
| 9387 | { |
| 9388 | case GF_OMP_TARGET_KIND_REGION: |
| 9389 | case GF_OMP_TARGET_KIND_DATA: |
| 9390 | case GF_OMP_TARGET_KIND_OACC_PARALLEL: |
| 9391 | case GF_OMP_TARGET_KIND_OACC_KERNELS: |
| 9392 | case GF_OMP_TARGET_KIND_OACC_DATA: |
| 9393 | break; |
| 9394 | case GF_OMP_TARGET_KIND_UPDATE: |
| 9395 | case GF_OMP_TARGET_KIND_OACC_UPDATE: |
| 9396 | case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: |
| 9397 | /* ..., other than for those stand-alone directives... */ |
| 9398 | region = NULL; |
| 9399 | break; |
| 9400 | default: |
| 9401 | gcc_unreachable (); |
| 9402 | } |
| 9403 | } |
| 9404 | /* ..., this directive becomes the parent for a new region. */ |
| 9405 | if (region) |
| 9406 | parent = region; |
| 9407 | } |
| 9408 | } |
| 9409 | |
| 9410 | if (single_tree && !parent) |
| 9411 | return; |
| 9412 | |
| 9413 | for (son = first_dom_son (CDI_DOMINATORS, bb); |
| 9414 | son; |
| 9415 | son = next_dom_son (CDI_DOMINATORS, son)) |
| 9416 | build_omp_regions_1 (son, parent, single_tree); |
| 9417 | } |
| 9418 | |
| 9419 | /* Builds the tree of OMP regions rooted at ROOT, storing it to |
| 9420 | root_omp_region. */ |
| 9421 | |
| 9422 | static void |
| 9423 | build_omp_regions_root (basic_block root) |
| 9424 | { |
| 9425 | gcc_assert (root_omp_region == NULL); |
| 9426 | build_omp_regions_1 (root, NULL, true); |
| 9427 | gcc_assert (root_omp_region != NULL); |
| 9428 | } |
| 9429 | |
| 9430 | /* Expands omp construct (and its subconstructs) starting in HEAD. */ |
| 9431 | |
| 9432 | void |
| 9433 | omp_expand_local (basic_block head) |
| 9434 | { |
| 9435 | build_omp_regions_root (head); |
| 9436 | if (dump_file && (dump_flags & TDF_DETAILS)) |
| 9437 | { |
| 9438 | fprintf (dump_file, "\nOMP region tree\n\n"); |
| 9439 | dump_omp_region (dump_file, root_omp_region, 0); |
| 9440 | fprintf (dump_file, "\n"); |
| 9441 | } |
| 9442 | |
| 9443 | remove_exit_barriers (root_omp_region); |
| 9444 | expand_omp (root_omp_region); |
| 9445 | |
| 9446 | free_omp_regions (); |
| 9447 | } |
| 9448 | |
| 9449 | /* Scan the CFG and build a tree of OMP regions. Return the root of |
| 9450 | the OMP region tree. */ |
| 9451 | |
| 9452 | static void |
| 9453 | build_omp_regions (void) |
| 9454 | { |
| 9455 | gcc_assert (root_omp_region == NULL); |
| 9456 | calculate_dominance_info (CDI_DOMINATORS); |
| 9457 | build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false); |
| 9458 | } |
| 9459 | |
| 9460 | /* Main entry point for expanding OMP-GIMPLE into runtime calls. */ |
| 9461 | |
| 9462 | static unsigned int |
| 9463 | execute_expand_omp (void) |
| 9464 | { |
| 9465 | build_omp_regions (); |
| 9466 | |
| 9467 | if (!root_omp_region) |
| 9468 | return 0; |
| 9469 | |
| 9470 | if (dump_file) |
| 9471 | { |
| 9472 | fprintf (dump_file, "\nOMP region tree\n\n"); |
| 9473 | dump_omp_region (dump_file, root_omp_region, 0); |
| 9474 | fprintf (dump_file, "\n"); |
| 9475 | } |
| 9476 | |
| 9477 | remove_exit_barriers (root_omp_region); |
| 9478 | |
| 9479 | expand_omp (root_omp_region); |
| 9480 | |
| 9481 | cleanup_tree_cfg (); |
| 9482 | |
| 9483 | free_omp_regions (); |
| 9484 | |
| 9485 | return 0; |
| 9486 | } |
| 9487 | |
| 9488 | /* OMP expansion -- the default pass, run before creation of SSA form. */ |
| 9489 | |
| 9490 | namespace { |
| 9491 | |
| 9492 | const pass_data pass_data_expand_omp = |
| 9493 | { |
| 9494 | GIMPLE_PASS, /* type */ |
| 9495 | "ompexp", /* name */ |
| 9496 | OPTGROUP_NONE, /* optinfo_flags */ |
| 9497 | TV_NONE, /* tv_id */ |
| 9498 | PROP_gimple_any, /* properties_required */ |
| 9499 | PROP_gimple_eomp, /* properties_provided */ |
| 9500 | 0, /* properties_destroyed */ |
| 9501 | 0, /* todo_flags_start */ |
| 9502 | 0, /* todo_flags_finish */ |
| 9503 | }; |
| 9504 | |
| 9505 | class pass_expand_omp : public gimple_opt_pass |
| 9506 | { |
| 9507 | public: |
| 9508 | pass_expand_omp (gcc::context *ctxt) |
| 9509 | : gimple_opt_pass (pass_data_expand_omp, ctxt) |
| 9510 | {} |
| 9511 | |
| 9512 | /* opt_pass methods: */ |
| 9513 | virtual unsigned int execute (function *) |
| 9514 | { |
| 9515 | bool gate = ((flag_cilkplus != 0 || flag_openacc != 0 || flag_openmp != 0 |
| 9516 | || flag_openmp_simd != 0) |
| 9517 | && !seen_error ()); |
| 9518 | |
| 9519 | /* This pass always runs, to provide PROP_gimple_eomp. |
| 9520 | But often, there is nothing to do. */ |
| 9521 | if (!gate) |
| 9522 | return 0; |
| 9523 | |
| 9524 | return execute_expand_omp (); |
| 9525 | } |
| 9526 | |
| 9527 | }; // class pass_expand_omp |
| 9528 | |
| 9529 | } // anon namespace |
| 9530 | |
| 9531 | gimple_opt_pass * |
| 9532 | make_pass_expand_omp (gcc::context *ctxt) |
| 9533 | { |
| 9534 | return new pass_expand_omp (ctxt); |
| 9535 | } |
| 9536 | |
| 9537 | namespace { |
| 9538 | |
| 9539 | const pass_data pass_data_expand_omp_ssa = |
| 9540 | { |
| 9541 | GIMPLE_PASS, /* type */ |
| 9542 | "ompexpssa", /* name */ |
| 9543 | OPTGROUP_NONE, /* optinfo_flags */ |
| 9544 | TV_NONE, /* tv_id */ |
| 9545 | PROP_cfg | PROP_ssa, /* properties_required */ |
| 9546 | PROP_gimple_eomp, /* properties_provided */ |
| 9547 | 0, /* properties_destroyed */ |
| 9548 | 0, /* todo_flags_start */ |
| 9549 | TODO_cleanup_cfg | TODO_rebuild_alias, /* todo_flags_finish */ |
| 9550 | }; |
| 9551 | |
| 9552 | class pass_expand_omp_ssa : public gimple_opt_pass |
| 9553 | { |
| 9554 | public: |
| 9555 | pass_expand_omp_ssa (gcc::context *ctxt) |
| 9556 | : gimple_opt_pass (pass_data_expand_omp_ssa, ctxt) |
| 9557 | {} |
| 9558 | |
| 9559 | /* opt_pass methods: */ |
| 9560 | virtual bool gate (function *fun) |
| 9561 | { |
| 9562 | return !(fun->curr_properties & PROP_gimple_eomp); |
| 9563 | } |
| 9564 | virtual unsigned int execute (function *) { return execute_expand_omp (); } |
| 9565 | |
| 9566 | }; // class pass_expand_omp_ssa |
| 9567 | |
| 9568 | } // anon namespace |
| 9569 | |
| 9570 | gimple_opt_pass * |
| 9571 | make_pass_expand_omp_ssa (gcc::context *ctxt) |
| 9572 | { |
| 9573 | return new pass_expand_omp_ssa (ctxt); |
| 9574 | } |
| 9575 | \f |
| 9576 | /* Routines to lower OMP directives into OMP-GIMPLE. */ |
| 9577 | |
| 9578 | /* Helper function to preform, potentially COMPLEX_TYPE, operation and |
| 9579 | convert it to gimple. */ |
| 9580 | static void |
| 9581 | oacc_gimple_assign (tree dest, tree_code op, tree src, gimple_seq *seq) |
| 9582 | { |
| 9583 | gimple stmt; |
| 9584 | |
| 9585 | if (TREE_CODE (TREE_TYPE (dest)) != COMPLEX_TYPE) |
| 9586 | { |
| 9587 | stmt = gimple_build_assign (dest, op, dest, src); |
| 9588 | gimple_seq_add_stmt (seq, stmt); |
| 9589 | return; |
| 9590 | } |
| 9591 | |
| 9592 | tree t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); |
| 9593 | tree rdest = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest); |
| 9594 | gimplify_assign (t, rdest, seq); |
| 9595 | rdest = t; |
| 9596 | |
| 9597 | t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); |
| 9598 | tree idest = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest); |
| 9599 | gimplify_assign (t, idest, seq); |
| 9600 | idest = t; |
| 9601 | |
| 9602 | t = create_tmp_var (TREE_TYPE (TREE_TYPE (src))); |
| 9603 | tree rsrc = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src); |
| 9604 | gimplify_assign (t, rsrc, seq); |
| 9605 | rsrc = t; |
| 9606 | |
| 9607 | t = create_tmp_var (TREE_TYPE (TREE_TYPE (src))); |
| 9608 | tree isrc = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src); |
| 9609 | gimplify_assign (t, isrc, seq); |
| 9610 | isrc = t; |
| 9611 | |
| 9612 | tree r = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); |
| 9613 | tree i = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); |
| 9614 | tree result; |
| 9615 | |
| 9616 | if (op == PLUS_EXPR) |
| 9617 | { |
| 9618 | stmt = gimple_build_assign (r, op, rdest, rsrc); |
| 9619 | gimple_seq_add_stmt (seq, stmt); |
| 9620 | |
| 9621 | stmt = gimple_build_assign (i, op, idest, isrc); |
| 9622 | gimple_seq_add_stmt (seq, stmt); |
| 9623 | } |
| 9624 | else if (op == MULT_EXPR) |
| 9625 | { |
| 9626 | /* Let x = a + ib = dest, y = c + id = src. |
| 9627 | x * y = (ac - bd) + i(ad + bc) */ |
| 9628 | tree ac = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); |
| 9629 | tree bd = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); |
| 9630 | tree ad = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); |
| 9631 | tree bc = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); |
| 9632 | |
| 9633 | stmt = gimple_build_assign (ac, MULT_EXPR, rdest, rsrc); |
| 9634 | gimple_seq_add_stmt (seq, stmt); |
| 9635 | |
| 9636 | stmt = gimple_build_assign (bd, MULT_EXPR, idest, isrc); |
| 9637 | gimple_seq_add_stmt (seq, stmt); |
| 9638 | |
| 9639 | stmt = gimple_build_assign (r, MINUS_EXPR, ac, bd); |
| 9640 | gimple_seq_add_stmt (seq, stmt); |
| 9641 | |
| 9642 | stmt = gimple_build_assign (ad, MULT_EXPR, rdest, isrc); |
| 9643 | gimple_seq_add_stmt (seq, stmt); |
| 9644 | |
| 9645 | stmt = gimple_build_assign (bd, MULT_EXPR, idest, rsrc); |
| 9646 | gimple_seq_add_stmt (seq, stmt); |
| 9647 | |
| 9648 | stmt = gimple_build_assign (i, PLUS_EXPR, ad, bc); |
| 9649 | gimple_seq_add_stmt (seq, stmt); |
| 9650 | } |
| 9651 | else |
| 9652 | gcc_unreachable (); |
| 9653 | |
| 9654 | result = build2 (COMPLEX_EXPR, TREE_TYPE (dest), r, i); |
| 9655 | gimplify_assign (dest, result, seq); |
| 9656 | } |
| 9657 | |
| 9658 | /* Helper function to initialize local data for the reduction arrays. |
| 9659 | The reduction arrays need to be placed inside the calling function |
| 9660 | for accelerators, or else the host won't be able to preform the final |
| 9661 | reduction. */ |
| 9662 | |
| 9663 | static void |
| 9664 | oacc_initialize_reduction_data (tree clauses, tree nthreads, |
| 9665 | gimple_seq *stmt_seqp, omp_context *ctx) |
| 9666 | { |
| 9667 | tree c, t, oc; |
| 9668 | gimple stmt; |
| 9669 | omp_context *octx; |
| 9670 | |
| 9671 | /* Find the innermost OpenACC parallel context. */ |
| 9672 | if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET |
| 9673 | && (gimple_omp_target_kind (ctx->stmt) |
| 9674 | == GF_OMP_TARGET_KIND_OACC_PARALLEL)) |
| 9675 | octx = ctx; |
| 9676 | else |
| 9677 | octx = ctx->outer; |
| 9678 | gcc_checking_assert (gimple_code (octx->stmt) == GIMPLE_OMP_TARGET |
| 9679 | && (gimple_omp_target_kind (octx->stmt) |
| 9680 | == GF_OMP_TARGET_KIND_OACC_PARALLEL)); |
| 9681 | |
| 9682 | /* Extract the clauses. */ |
| 9683 | oc = gimple_omp_target_clauses (octx->stmt); |
| 9684 | |
| 9685 | /* Find the last outer clause. */ |
| 9686 | for (; oc && OMP_CLAUSE_CHAIN (oc); oc = OMP_CLAUSE_CHAIN (oc)) |
| 9687 | ; |
| 9688 | |
| 9689 | /* Allocate arrays for each reduction variable. */ |
| 9690 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
| 9691 | { |
| 9692 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) |
| 9693 | continue; |
| 9694 | |
| 9695 | tree var = OMP_CLAUSE_DECL (c); |
| 9696 | tree type = get_base_type (var); |
| 9697 | tree array = lookup_oacc_reduction (oacc_get_reduction_array_id (var), |
| 9698 | ctx); |
| 9699 | tree size, call; |
| 9700 | |
| 9701 | /* Calculate size of the reduction array. */ |
| 9702 | t = create_tmp_var (TREE_TYPE (nthreads)); |
| 9703 | stmt = gimple_build_assign (t, MULT_EXPR, nthreads, |
| 9704 | fold_convert (TREE_TYPE (nthreads), |
| 9705 | TYPE_SIZE_UNIT (type))); |
| 9706 | gimple_seq_add_stmt (stmt_seqp, stmt); |
| 9707 | |
| 9708 | size = create_tmp_var (sizetype); |
| 9709 | gimplify_assign (size, fold_build1 (NOP_EXPR, sizetype, t), stmt_seqp); |
| 9710 | |
| 9711 | /* Now allocate memory for it. */ |
| 9712 | call = unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA)); |
| 9713 | stmt = gimple_build_call (call, 1, size); |
| 9714 | gimple_call_set_lhs (stmt, array); |
| 9715 | gimple_seq_add_stmt (stmt_seqp, stmt); |
| 9716 | |
| 9717 | /* Map this array into the accelerator. */ |
| 9718 | |
| 9719 | /* Add the reduction array to the list of clauses. */ |
| 9720 | tree x = array; |
| 9721 | t = build_omp_clause (gimple_location (ctx->stmt), OMP_CLAUSE_MAP); |
| 9722 | OMP_CLAUSE_SET_MAP_KIND (t, GOMP_MAP_FORCE_FROM); |
| 9723 | OMP_CLAUSE_DECL (t) = x; |
| 9724 | OMP_CLAUSE_CHAIN (t) = NULL; |
| 9725 | if (oc) |
| 9726 | OMP_CLAUSE_CHAIN (oc) = t; |
| 9727 | else |
| 9728 | gimple_omp_target_set_clauses (as_a <gomp_target *> (octx->stmt), t); |
| 9729 | OMP_CLAUSE_SIZE (t) = size; |
| 9730 | oc = t; |
| 9731 | } |
| 9732 | } |
| 9733 | |
| 9734 | /* Helper function to process the array of partial reductions. Nthreads |
| 9735 | indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS |
| 9736 | cannot be used here, because nthreads on the host may be different than |
| 9737 | on the accelerator. */ |
| 9738 | |
| 9739 | static void |
| 9740 | oacc_finalize_reduction_data (tree clauses, tree nthreads, |
| 9741 | gimple_seq *stmt_seqp, omp_context *ctx) |
| 9742 | { |
| 9743 | tree c, x, var, array, loop_header, loop_body, loop_exit, type; |
| 9744 | gimple stmt; |
| 9745 | |
| 9746 | /* Create for loop. |
| 9747 | |
| 9748 | let var = the original reduction variable |
| 9749 | let array = reduction variable array |
| 9750 | |
| 9751 | for (i = 0; i < nthreads; i++) |
| 9752 | var op= array[i] |
| 9753 | */ |
| 9754 | |
| 9755 | loop_header = create_artificial_label (UNKNOWN_LOCATION); |
| 9756 | loop_body = create_artificial_label (UNKNOWN_LOCATION); |
| 9757 | loop_exit = create_artificial_label (UNKNOWN_LOCATION); |
| 9758 | |
| 9759 | /* Create and initialize an index variable. */ |
| 9760 | tree ix = create_tmp_var (sizetype); |
| 9761 | gimplify_assign (ix, fold_build1 (NOP_EXPR, sizetype, integer_zero_node), |
| 9762 | stmt_seqp); |
| 9763 | |
| 9764 | /* Insert the loop header label here. */ |
| 9765 | gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_header)); |
| 9766 | |
| 9767 | /* Exit loop if ix >= nthreads. */ |
| 9768 | x = create_tmp_var (sizetype); |
| 9769 | gimplify_assign (x, fold_build1 (NOP_EXPR, sizetype, nthreads), stmt_seqp); |
| 9770 | stmt = gimple_build_cond (GE_EXPR, ix, x, loop_exit, loop_body); |
| 9771 | gimple_seq_add_stmt (stmt_seqp, stmt); |
| 9772 | |
| 9773 | /* Insert the loop body label here. */ |
| 9774 | gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_body)); |
| 9775 | |
| 9776 | /* Collapse each reduction array, one element at a time. */ |
| 9777 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
| 9778 | { |
| 9779 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) |
| 9780 | continue; |
| 9781 | |
| 9782 | tree_code reduction_code = OMP_CLAUSE_REDUCTION_CODE (c); |
| 9783 | |
| 9784 | /* reduction(-:var) sums up the partial results, so it acts |
| 9785 | identically to reduction(+:var). */ |
| 9786 | if (reduction_code == MINUS_EXPR) |
| 9787 | reduction_code = PLUS_EXPR; |
| 9788 | |
| 9789 | /* Set up reduction variable var. */ |
| 9790 | var = OMP_CLAUSE_DECL (c); |
| 9791 | type = get_base_type (var); |
| 9792 | array = lookup_oacc_reduction (oacc_get_reduction_array_id |
| 9793 | (OMP_CLAUSE_DECL (c)), ctx); |
| 9794 | |
| 9795 | /* Calculate the array offset. */ |
| 9796 | tree offset = create_tmp_var (sizetype); |
| 9797 | gimplify_assign (offset, TYPE_SIZE_UNIT (type), stmt_seqp); |
| 9798 | stmt = gimple_build_assign (offset, MULT_EXPR, offset, ix); |
| 9799 | gimple_seq_add_stmt (stmt_seqp, stmt); |
| 9800 | |
| 9801 | tree ptr = create_tmp_var (TREE_TYPE (array)); |
| 9802 | stmt = gimple_build_assign (ptr, POINTER_PLUS_EXPR, array, offset); |
| 9803 | gimple_seq_add_stmt (stmt_seqp, stmt); |
| 9804 | |
| 9805 | /* Extract array[ix] into mem. */ |
| 9806 | tree mem = create_tmp_var (type); |
| 9807 | gimplify_assign (mem, build_simple_mem_ref (ptr), stmt_seqp); |
| 9808 | |
| 9809 | /* Find the original reduction variable. */ |
| 9810 | if (is_reference (var)) |
| 9811 | var = build_simple_mem_ref (var); |
| 9812 | |
| 9813 | tree t = create_tmp_var (type); |
| 9814 | |
| 9815 | x = lang_hooks.decls.omp_clause_assign_op (c, t, var); |
| 9816 | gimplify_and_add (unshare_expr(x), stmt_seqp); |
| 9817 | |
| 9818 | /* var = var op mem */ |
| 9819 | switch (OMP_CLAUSE_REDUCTION_CODE (c)) |
| 9820 | { |
| 9821 | case TRUTH_ANDIF_EXPR: |
| 9822 | case TRUTH_ORIF_EXPR: |
| 9823 | t = fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c), integer_type_node, |
| 9824 | t, mem); |
| 9825 | gimplify_and_add (t, stmt_seqp); |
| 9826 | break; |
| 9827 | default: |
| 9828 | /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */ |
| 9829 | oacc_gimple_assign (t, OMP_CLAUSE_REDUCTION_CODE (c), mem, |
| 9830 | stmt_seqp); |
| 9831 | } |
| 9832 | |
| 9833 | t = fold_build1 (NOP_EXPR, TREE_TYPE (var), t); |
| 9834 | x = lang_hooks.decls.omp_clause_assign_op (c, var, t); |
| 9835 | gimplify_and_add (unshare_expr(x), stmt_seqp); |
| 9836 | } |
| 9837 | |
| 9838 | /* Increment the induction variable. */ |
| 9839 | tree one = fold_build1 (NOP_EXPR, sizetype, integer_one_node); |
| 9840 | stmt = gimple_build_assign (ix, PLUS_EXPR, ix, one); |
| 9841 | gimple_seq_add_stmt (stmt_seqp, stmt); |
| 9842 | |
| 9843 | /* Go back to the top of the loop. */ |
| 9844 | gimple_seq_add_stmt (stmt_seqp, gimple_build_goto (loop_header)); |
| 9845 | |
| 9846 | /* Place the loop exit label here. */ |
| 9847 | gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_exit)); |
| 9848 | } |
| 9849 | |
| 9850 | /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and |
| 9851 | scan that for reductions. */ |
| 9852 | |
| 9853 | static void |
| 9854 | oacc_process_reduction_data (gimple_seq *body, gimple_seq *in_stmt_seqp, |
| 9855 | gimple_seq *out_stmt_seqp, omp_context *ctx) |
| 9856 | { |
| 9857 | gimple_stmt_iterator gsi; |
| 9858 | gimple_seq inner = NULL; |
| 9859 | |
| 9860 | /* A collapse clause may have inserted a new bind block. */ |
| 9861 | gsi = gsi_start (*body); |
| 9862 | while (!gsi_end_p (gsi)) |
| 9863 | { |
| 9864 | gimple stmt = gsi_stmt (gsi); |
| 9865 | if (gbind *bind_stmt = dyn_cast <gbind *> (stmt)) |
| 9866 | { |
| 9867 | inner = gimple_bind_body (bind_stmt); |
| 9868 | body = &inner; |
| 9869 | gsi = gsi_start (*body); |
| 9870 | } |
| 9871 | else if (dyn_cast <gomp_for *> (stmt)) |
| 9872 | break; |
| 9873 | else |
| 9874 | gsi_next (&gsi); |
| 9875 | } |
| 9876 | |
| 9877 | for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi)) |
| 9878 | { |
| 9879 | tree clauses, nthreads, t, c, acc_device, acc_device_host, call, |
| 9880 | enter, exit; |
| 9881 | bool reduction_found = false; |
| 9882 | |
| 9883 | gimple stmt = gsi_stmt (gsi); |
| 9884 | |
| 9885 | switch (gimple_code (stmt)) |
| 9886 | { |
| 9887 | case GIMPLE_OMP_FOR: |
| 9888 | clauses = gimple_omp_for_clauses (stmt); |
| 9889 | |
| 9890 | /* Search for a reduction clause. */ |
| 9891 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
| 9892 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) |
| 9893 | { |
| 9894 | reduction_found = true; |
| 9895 | break; |
| 9896 | } |
| 9897 | |
| 9898 | if (!reduction_found) |
| 9899 | break; |
| 9900 | |
| 9901 | ctx = maybe_lookup_ctx (stmt); |
| 9902 | t = NULL_TREE; |
| 9903 | |
| 9904 | /* Extract the number of threads. */ |
| 9905 | nthreads = create_tmp_var (sizetype); |
| 9906 | t = oacc_max_threads (ctx); |
| 9907 | gimplify_assign (nthreads, t, in_stmt_seqp); |
| 9908 | |
| 9909 | /* Determine if this is kernel will be executed on the host. */ |
| 9910 | call = builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE); |
| 9911 | acc_device = create_tmp_var (integer_type_node, ".acc_device_type"); |
| 9912 | stmt = gimple_build_call (call, 0); |
| 9913 | gimple_call_set_lhs (stmt, acc_device); |
| 9914 | gimple_seq_add_stmt (in_stmt_seqp, stmt); |
| 9915 | |
| 9916 | /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */ |
| 9917 | acc_device_host = create_tmp_var (integer_type_node, |
| 9918 | ".acc_device_host"); |
| 9919 | gimplify_assign (acc_device_host, |
| 9920 | build_int_cst (integer_type_node, |
| 9921 | GOMP_DEVICE_HOST), |
| 9922 | in_stmt_seqp); |
| 9923 | |
| 9924 | enter = create_artificial_label (UNKNOWN_LOCATION); |
| 9925 | exit = create_artificial_label (UNKNOWN_LOCATION); |
| 9926 | |
| 9927 | stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host, |
| 9928 | enter, exit); |
| 9929 | gimple_seq_add_stmt (in_stmt_seqp, stmt); |
| 9930 | gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter)); |
| 9931 | gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype, |
| 9932 | integer_one_node), |
| 9933 | in_stmt_seqp); |
| 9934 | gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit)); |
| 9935 | |
| 9936 | /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */ |
| 9937 | gimplify_assign (acc_device_host, |
| 9938 | build_int_cst (integer_type_node, |
| 9939 | GOMP_DEVICE_HOST_NONSHM), |
| 9940 | in_stmt_seqp); |
| 9941 | |
| 9942 | enter = create_artificial_label (UNKNOWN_LOCATION); |
| 9943 | exit = create_artificial_label (UNKNOWN_LOCATION); |
| 9944 | |
| 9945 | stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host, |
| 9946 | enter, exit); |
| 9947 | gimple_seq_add_stmt (in_stmt_seqp, stmt); |
| 9948 | gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter)); |
| 9949 | gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype, |
| 9950 | integer_one_node), |
| 9951 | in_stmt_seqp); |
| 9952 | gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit)); |
| 9953 | |
| 9954 | oacc_initialize_reduction_data (clauses, nthreads, in_stmt_seqp, |
| 9955 | ctx); |
| 9956 | oacc_finalize_reduction_data (clauses, nthreads, out_stmt_seqp, ctx); |
| 9957 | break; |
| 9958 | default: |
| 9959 | // Scan for other directives which support reduction here. |
| 9960 | break; |
| 9961 | } |
| 9962 | } |
| 9963 | } |
| 9964 | |
| 9965 | /* If ctx is a worksharing context inside of a cancellable parallel |
| 9966 | region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN |
| 9967 | and conditional branch to parallel's cancel_label to handle |
| 9968 | cancellation in the implicit barrier. */ |
| 9969 | |
| 9970 | static void |
| 9971 | maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple_seq *body) |
| 9972 | { |
| 9973 | gimple omp_return = gimple_seq_last_stmt (*body); |
| 9974 | gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN); |
| 9975 | if (gimple_omp_return_nowait_p (omp_return)) |
| 9976 | return; |
| 9977 | if (ctx->outer |
| 9978 | && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_PARALLEL |
| 9979 | && ctx->outer->cancellable) |
| 9980 | { |
| 9981 | tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL); |
| 9982 | tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl)); |
| 9983 | tree lhs = create_tmp_var (c_bool_type); |
| 9984 | gimple_omp_return_set_lhs (omp_return, lhs); |
| 9985 | tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION); |
| 9986 | gimple g = gimple_build_cond (NE_EXPR, lhs, |
| 9987 | fold_convert (c_bool_type, |
| 9988 | boolean_false_node), |
| 9989 | ctx->outer->cancel_label, fallthru_label); |
| 9990 | gimple_seq_add_stmt (body, g); |
| 9991 | gimple_seq_add_stmt (body, gimple_build_label (fallthru_label)); |
| 9992 | } |
| 9993 | } |
| 9994 | |
| 9995 | /* Lower the OpenMP sections directive in the current statement in GSI_P. |
| 9996 | CTX is the enclosing OMP context for the current statement. */ |
| 9997 | |
| 9998 | static void |
| 9999 | lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
| 10000 | { |
| 10001 | tree block, control; |
| 10002 | gimple_stmt_iterator tgsi; |
| 10003 | gomp_sections *stmt; |
| 10004 | gimple t; |
| 10005 | gbind *new_stmt, *bind; |
| 10006 | gimple_seq ilist, dlist, olist, new_body; |
| 10007 | |
| 10008 | stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p)); |
| 10009 | |
| 10010 | push_gimplify_context (); |
| 10011 | |
| 10012 | dlist = NULL; |
| 10013 | ilist = NULL; |
| 10014 | lower_rec_input_clauses (gimple_omp_sections_clauses (stmt), |
| 10015 | &ilist, &dlist, ctx, NULL); |
| 10016 | |
| 10017 | new_body = gimple_omp_body (stmt); |
| 10018 | gimple_omp_set_body (stmt, NULL); |
| 10019 | tgsi = gsi_start (new_body); |
| 10020 | for (; !gsi_end_p (tgsi); gsi_next (&tgsi)) |
| 10021 | { |
| 10022 | omp_context *sctx; |
| 10023 | gimple sec_start; |
| 10024 | |
| 10025 | sec_start = gsi_stmt (tgsi); |
| 10026 | sctx = maybe_lookup_ctx (sec_start); |
| 10027 | gcc_assert (sctx); |
| 10028 | |
| 10029 | lower_omp (gimple_omp_body_ptr (sec_start), sctx); |
| 10030 | gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start), |
| 10031 | GSI_CONTINUE_LINKING); |
| 10032 | gimple_omp_set_body (sec_start, NULL); |
| 10033 | |
| 10034 | if (gsi_one_before_end_p (tgsi)) |
| 10035 | { |
| 10036 | gimple_seq l = NULL; |
| 10037 | lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL, |
| 10038 | &l, ctx); |
| 10039 | gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING); |
| 10040 | gimple_omp_section_set_last (sec_start); |
| 10041 | } |
| 10042 | |
| 10043 | gsi_insert_after (&tgsi, gimple_build_omp_return (false), |
| 10044 | GSI_CONTINUE_LINKING); |
| 10045 | } |
| 10046 | |
| 10047 | block = make_node (BLOCK); |
| 10048 | bind = gimple_build_bind (NULL, new_body, block); |
| 10049 | |
| 10050 | olist = NULL; |
| 10051 | lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx); |
| 10052 | |
| 10053 | block = make_node (BLOCK); |
| 10054 | new_stmt = gimple_build_bind (NULL, NULL, block); |
| 10055 | gsi_replace (gsi_p, new_stmt, true); |
| 10056 | |
| 10057 | pop_gimplify_context (new_stmt); |
| 10058 | gimple_bind_append_vars (new_stmt, ctx->block_vars); |
| 10059 | BLOCK_VARS (block) = gimple_bind_vars (bind); |
| 10060 | if (BLOCK_VARS (block)) |
| 10061 | TREE_USED (block) = 1; |
| 10062 | |
| 10063 | new_body = NULL; |
| 10064 | gimple_seq_add_seq (&new_body, ilist); |
| 10065 | gimple_seq_add_stmt (&new_body, stmt); |
| 10066 | gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ()); |
| 10067 | gimple_seq_add_stmt (&new_body, bind); |
| 10068 | |
| 10069 | control = create_tmp_var (unsigned_type_node, ".section"); |
| 10070 | t = gimple_build_omp_continue (control, control); |
| 10071 | gimple_omp_sections_set_control (stmt, control); |
| 10072 | gimple_seq_add_stmt (&new_body, t); |
| 10073 | |
| 10074 | gimple_seq_add_seq (&new_body, olist); |
| 10075 | if (ctx->cancellable) |
| 10076 | gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label)); |
| 10077 | gimple_seq_add_seq (&new_body, dlist); |
| 10078 | |
| 10079 | new_body = maybe_catch_exception (new_body); |
| 10080 | |
| 10081 | t = gimple_build_omp_return |
| 10082 | (!!find_omp_clause (gimple_omp_sections_clauses (stmt), |
| 10083 | OMP_CLAUSE_NOWAIT)); |
| 10084 | gimple_seq_add_stmt (&new_body, t); |
| 10085 | maybe_add_implicit_barrier_cancel (ctx, &new_body); |
| 10086 | |
| 10087 | gimple_bind_set_body (new_stmt, new_body); |
| 10088 | } |
| 10089 | |
| 10090 | |
| 10091 | /* A subroutine of lower_omp_single. Expand the simple form of |
| 10092 | a GIMPLE_OMP_SINGLE, without a copyprivate clause: |
| 10093 | |
| 10094 | if (GOMP_single_start ()) |
| 10095 | BODY; |
| 10096 | [ GOMP_barrier (); ] -> unless 'nowait' is present. |
| 10097 | |
| 10098 | FIXME. It may be better to delay expanding the logic of this until |
| 10099 | pass_expand_omp. The expanded logic may make the job more difficult |
| 10100 | to a synchronization analysis pass. */ |
| 10101 | |
| 10102 | static void |
| 10103 | lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p) |
| 10104 | { |
| 10105 | location_t loc = gimple_location (single_stmt); |
| 10106 | tree tlabel = create_artificial_label (loc); |
| 10107 | tree flabel = create_artificial_label (loc); |
| 10108 | gimple call, cond; |
| 10109 | tree lhs, decl; |
| 10110 | |
| 10111 | decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START); |
| 10112 | lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl))); |
| 10113 | call = gimple_build_call (decl, 0); |
| 10114 | gimple_call_set_lhs (call, lhs); |
| 10115 | gimple_seq_add_stmt (pre_p, call); |
| 10116 | |
| 10117 | cond = gimple_build_cond (EQ_EXPR, lhs, |
| 10118 | fold_convert_loc (loc, TREE_TYPE (lhs), |
| 10119 | boolean_true_node), |
| 10120 | tlabel, flabel); |
| 10121 | gimple_seq_add_stmt (pre_p, cond); |
| 10122 | gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel)); |
| 10123 | gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt)); |
| 10124 | gimple_seq_add_stmt (pre_p, gimple_build_label (flabel)); |
| 10125 | } |
| 10126 | |
| 10127 | |
| 10128 | /* A subroutine of lower_omp_single. Expand the simple form of |
| 10129 | a GIMPLE_OMP_SINGLE, with a copyprivate clause: |
| 10130 | |
| 10131 | #pragma omp single copyprivate (a, b, c) |
| 10132 | |
| 10133 | Create a new structure to hold copies of 'a', 'b' and 'c' and emit: |
| 10134 | |
| 10135 | { |
| 10136 | if ((copyout_p = GOMP_single_copy_start ()) == NULL) |
| 10137 | { |
| 10138 | BODY; |
| 10139 | copyout.a = a; |
| 10140 | copyout.b = b; |
| 10141 | copyout.c = c; |
| 10142 | GOMP_single_copy_end (©out); |
| 10143 | } |
| 10144 | else |
| 10145 | { |
| 10146 | a = copyout_p->a; |
| 10147 | b = copyout_p->b; |
| 10148 | c = copyout_p->c; |
| 10149 | } |
| 10150 | GOMP_barrier (); |
| 10151 | } |
| 10152 | |
| 10153 | FIXME. It may be better to delay expanding the logic of this until |
| 10154 | pass_expand_omp. The expanded logic may make the job more difficult |
| 10155 | to a synchronization analysis pass. */ |
| 10156 | |
| 10157 | static void |
| 10158 | lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p, |
| 10159 | omp_context *ctx) |
| 10160 | { |
| 10161 | tree ptr_type, t, l0, l1, l2, bfn_decl; |
| 10162 | gimple_seq copyin_seq; |
| 10163 | location_t loc = gimple_location (single_stmt); |
| 10164 | |
| 10165 | ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o"); |
| 10166 | |
| 10167 | ptr_type = build_pointer_type (ctx->record_type); |
| 10168 | ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i"); |
| 10169 | |
| 10170 | l0 = create_artificial_label (loc); |
| 10171 | l1 = create_artificial_label (loc); |
| 10172 | l2 = create_artificial_label (loc); |
| 10173 | |
| 10174 | bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START); |
| 10175 | t = build_call_expr_loc (loc, bfn_decl, 0); |
| 10176 | t = fold_convert_loc (loc, ptr_type, t); |
| 10177 | gimplify_assign (ctx->receiver_decl, t, pre_p); |
| 10178 | |
| 10179 | t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl, |
| 10180 | build_int_cst (ptr_type, 0)); |
| 10181 | t = build3 (COND_EXPR, void_type_node, t, |
| 10182 | build_and_jump (&l0), build_and_jump (&l1)); |
| 10183 | gimplify_and_add (t, pre_p); |
| 10184 | |
| 10185 | gimple_seq_add_stmt (pre_p, gimple_build_label (l0)); |
| 10186 | |
| 10187 | gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt)); |
| 10188 | |
| 10189 | copyin_seq = NULL; |
| 10190 | lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p, |
| 10191 | ©in_seq, ctx); |
| 10192 | |
| 10193 | t = build_fold_addr_expr_loc (loc, ctx->sender_decl); |
| 10194 | bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END); |
| 10195 | t = build_call_expr_loc (loc, bfn_decl, 1, t); |
| 10196 | gimplify_and_add (t, pre_p); |
| 10197 | |
| 10198 | t = build_and_jump (&l2); |
| 10199 | gimplify_and_add (t, pre_p); |
| 10200 | |
| 10201 | gimple_seq_add_stmt (pre_p, gimple_build_label (l1)); |
| 10202 | |
| 10203 | gimple_seq_add_seq (pre_p, copyin_seq); |
| 10204 | |
| 10205 | gimple_seq_add_stmt (pre_p, gimple_build_label (l2)); |
| 10206 | } |
| 10207 | |
| 10208 | |
| 10209 | /* Expand code for an OpenMP single directive. */ |
| 10210 | |
| 10211 | static void |
| 10212 | lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
| 10213 | { |
| 10214 | tree block; |
| 10215 | gimple t; |
| 10216 | gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p)); |
| 10217 | gbind *bind; |
| 10218 | gimple_seq bind_body, bind_body_tail = NULL, dlist; |
| 10219 | |
| 10220 | push_gimplify_context (); |
| 10221 | |
| 10222 | block = make_node (BLOCK); |
| 10223 | bind = gimple_build_bind (NULL, NULL, block); |
| 10224 | gsi_replace (gsi_p, bind, true); |
| 10225 | bind_body = NULL; |
| 10226 | dlist = NULL; |
| 10227 | lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt), |
| 10228 | &bind_body, &dlist, ctx, NULL); |
| 10229 | lower_omp (gimple_omp_body_ptr (single_stmt), ctx); |
| 10230 | |
| 10231 | gimple_seq_add_stmt (&bind_body, single_stmt); |
| 10232 | |
| 10233 | if (ctx->record_type) |
| 10234 | lower_omp_single_copy (single_stmt, &bind_body, ctx); |
| 10235 | else |
| 10236 | lower_omp_single_simple (single_stmt, &bind_body); |
| 10237 | |
| 10238 | gimple_omp_set_body (single_stmt, NULL); |
| 10239 | |
| 10240 | gimple_seq_add_seq (&bind_body, dlist); |
| 10241 | |
| 10242 | bind_body = maybe_catch_exception (bind_body); |
| 10243 | |
| 10244 | t = gimple_build_omp_return |
| 10245 | (!!find_omp_clause (gimple_omp_single_clauses (single_stmt), |
| 10246 | OMP_CLAUSE_NOWAIT)); |
| 10247 | gimple_seq_add_stmt (&bind_body_tail, t); |
| 10248 | maybe_add_implicit_barrier_cancel (ctx, &bind_body_tail); |
| 10249 | if (ctx->record_type) |
| 10250 | { |
| 10251 | gimple_stmt_iterator gsi = gsi_start (bind_body_tail); |
| 10252 | tree clobber = build_constructor (ctx->record_type, NULL); |
| 10253 | TREE_THIS_VOLATILE (clobber) = 1; |
| 10254 | gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl, |
| 10255 | clobber), GSI_SAME_STMT); |
| 10256 | } |
| 10257 | gimple_seq_add_seq (&bind_body, bind_body_tail); |
| 10258 | gimple_bind_set_body (bind, bind_body); |
| 10259 | |
| 10260 | pop_gimplify_context (bind); |
| 10261 | |
| 10262 | gimple_bind_append_vars (bind, ctx->block_vars); |
| 10263 | BLOCK_VARS (block) = ctx->block_vars; |
| 10264 | if (BLOCK_VARS (block)) |
| 10265 | TREE_USED (block) = 1; |
| 10266 | } |
| 10267 | |
| 10268 | |
| 10269 | /* Expand code for an OpenMP master directive. */ |
| 10270 | |
| 10271 | static void |
| 10272 | lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
| 10273 | { |
| 10274 | tree block, lab = NULL, x, bfn_decl; |
| 10275 | gimple stmt = gsi_stmt (*gsi_p); |
| 10276 | gbind *bind; |
| 10277 | location_t loc = gimple_location (stmt); |
| 10278 | gimple_seq tseq; |
| 10279 | |
| 10280 | push_gimplify_context (); |
| 10281 | |
| 10282 | block = make_node (BLOCK); |
| 10283 | bind = gimple_build_bind (NULL, NULL, block); |
| 10284 | gsi_replace (gsi_p, bind, true); |
| 10285 | gimple_bind_add_stmt (bind, stmt); |
| 10286 | |
| 10287 | bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); |
| 10288 | x = build_call_expr_loc (loc, bfn_decl, 0); |
| 10289 | x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node); |
| 10290 | x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab)); |
| 10291 | tseq = NULL; |
| 10292 | gimplify_and_add (x, &tseq); |
| 10293 | gimple_bind_add_seq (bind, tseq); |
| 10294 | |
| 10295 | lower_omp (gimple_omp_body_ptr (stmt), ctx); |
| 10296 | gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt))); |
| 10297 | gimple_bind_add_seq (bind, gimple_omp_body (stmt)); |
| 10298 | gimple_omp_set_body (stmt, NULL); |
| 10299 | |
| 10300 | gimple_bind_add_stmt (bind, gimple_build_label (lab)); |
| 10301 | |
| 10302 | gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); |
| 10303 | |
| 10304 | pop_gimplify_context (bind); |
| 10305 | |
| 10306 | gimple_bind_append_vars (bind, ctx->block_vars); |
| 10307 | BLOCK_VARS (block) = ctx->block_vars; |
| 10308 | } |
| 10309 | |
| 10310 | |
| 10311 | /* Expand code for an OpenMP taskgroup directive. */ |
| 10312 | |
| 10313 | static void |
| 10314 | lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
| 10315 | { |
| 10316 | gimple stmt = gsi_stmt (*gsi_p); |
| 10317 | gcall *x; |
| 10318 | gbind *bind; |
| 10319 | tree block = make_node (BLOCK); |
| 10320 | |
| 10321 | bind = gimple_build_bind (NULL, NULL, block); |
| 10322 | gsi_replace (gsi_p, bind, true); |
| 10323 | gimple_bind_add_stmt (bind, stmt); |
| 10324 | |
| 10325 | x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START), |
| 10326 | 0); |
| 10327 | gimple_bind_add_stmt (bind, x); |
| 10328 | |
| 10329 | lower_omp (gimple_omp_body_ptr (stmt), ctx); |
| 10330 | gimple_bind_add_seq (bind, gimple_omp_body (stmt)); |
| 10331 | gimple_omp_set_body (stmt, NULL); |
| 10332 | |
| 10333 | gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); |
| 10334 | |
| 10335 | gimple_bind_append_vars (bind, ctx->block_vars); |
| 10336 | BLOCK_VARS (block) = ctx->block_vars; |
| 10337 | } |
| 10338 | |
| 10339 | |
| 10340 | /* Expand code for an OpenMP ordered directive. */ |
| 10341 | |
| 10342 | static void |
| 10343 | lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
| 10344 | { |
| 10345 | tree block; |
| 10346 | gimple stmt = gsi_stmt (*gsi_p); |
| 10347 | gcall *x; |
| 10348 | gbind *bind; |
| 10349 | |
| 10350 | push_gimplify_context (); |
| 10351 | |
| 10352 | block = make_node (BLOCK); |
| 10353 | bind = gimple_build_bind (NULL, NULL, block); |
| 10354 | gsi_replace (gsi_p, bind, true); |
| 10355 | gimple_bind_add_stmt (bind, stmt); |
| 10356 | |
| 10357 | x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START), |
| 10358 | 0); |
| 10359 | gimple_bind_add_stmt (bind, x); |
| 10360 | |
| 10361 | lower_omp (gimple_omp_body_ptr (stmt), ctx); |
| 10362 | gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt))); |
| 10363 | gimple_bind_add_seq (bind, gimple_omp_body (stmt)); |
| 10364 | gimple_omp_set_body (stmt, NULL); |
| 10365 | |
| 10366 | x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END), 0); |
| 10367 | gimple_bind_add_stmt (bind, x); |
| 10368 | |
| 10369 | gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); |
| 10370 | |
| 10371 | pop_gimplify_context (bind); |
| 10372 | |
| 10373 | gimple_bind_append_vars (bind, ctx->block_vars); |
| 10374 | BLOCK_VARS (block) = gimple_bind_vars (bind); |
| 10375 | } |
| 10376 | |
| 10377 | |
| 10378 | /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple |
| 10379 | substitution of a couple of function calls. But in the NAMED case, |
| 10380 | requires that languages coordinate a symbol name. It is therefore |
| 10381 | best put here in common code. */ |
| 10382 | |
| 10383 | static GTY(()) hash_map<tree, tree> *critical_name_mutexes; |
| 10384 | |
| 10385 | static void |
| 10386 | lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
| 10387 | { |
| 10388 | tree block; |
| 10389 | tree name, lock, unlock; |
| 10390 | gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p)); |
| 10391 | gbind *bind; |
| 10392 | location_t loc = gimple_location (stmt); |
| 10393 | gimple_seq tbody; |
| 10394 | |
| 10395 | name = gimple_omp_critical_name (stmt); |
| 10396 | if (name) |
| 10397 | { |
| 10398 | tree decl; |
| 10399 | |
| 10400 | if (!critical_name_mutexes) |
| 10401 | critical_name_mutexes = hash_map<tree, tree>::create_ggc (10); |
| 10402 | |
| 10403 | tree *n = critical_name_mutexes->get (name); |
| 10404 | if (n == NULL) |
| 10405 | { |
| 10406 | char *new_str; |
| 10407 | |
| 10408 | decl = create_tmp_var_raw (ptr_type_node); |
| 10409 | |
| 10410 | new_str = ACONCAT ((".gomp_critical_user_", |
| 10411 | IDENTIFIER_POINTER (name), NULL)); |
| 10412 | DECL_NAME (decl) = get_identifier (new_str); |
| 10413 | TREE_PUBLIC (decl) = 1; |
| 10414 | TREE_STATIC (decl) = 1; |
| 10415 | DECL_COMMON (decl) = 1; |
| 10416 | DECL_ARTIFICIAL (decl) = 1; |
| 10417 | DECL_IGNORED_P (decl) = 1; |
| 10418 | |
| 10419 | varpool_node::finalize_decl (decl); |
| 10420 | |
| 10421 | critical_name_mutexes->put (name, decl); |
| 10422 | } |
| 10423 | else |
| 10424 | decl = *n; |
| 10425 | |
| 10426 | /* If '#pragma omp critical' is inside offloaded region or |
| 10427 | inside function marked as offloadable, the symbol must be |
| 10428 | marked as offloadable too. */ |
| 10429 | omp_context *octx; |
| 10430 | if (cgraph_node::get (current_function_decl)->offloadable) |
| 10431 | varpool_node::get_create (decl)->offloadable = 1; |
| 10432 | else |
| 10433 | for (octx = ctx->outer; octx; octx = octx->outer) |
| 10434 | if (is_gimple_omp_offloaded (octx->stmt)) |
| 10435 | { |
| 10436 | varpool_node::get_create (decl)->offloadable = 1; |
| 10437 | break; |
| 10438 | } |
| 10439 | |
| 10440 | lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START); |
| 10441 | lock = build_call_expr_loc (loc, lock, 1, build_fold_addr_expr_loc (loc, decl)); |
| 10442 | |
| 10443 | unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END); |
| 10444 | unlock = build_call_expr_loc (loc, unlock, 1, |
| 10445 | build_fold_addr_expr_loc (loc, decl)); |
| 10446 | } |
| 10447 | else |
| 10448 | { |
| 10449 | lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START); |
| 10450 | lock = build_call_expr_loc (loc, lock, 0); |
| 10451 | |
| 10452 | unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END); |
| 10453 | unlock = build_call_expr_loc (loc, unlock, 0); |
| 10454 | } |
| 10455 | |
| 10456 | push_gimplify_context (); |
| 10457 | |
| 10458 | block = make_node (BLOCK); |
| 10459 | bind = gimple_build_bind (NULL, NULL, block); |
| 10460 | gsi_replace (gsi_p, bind, true); |
| 10461 | gimple_bind_add_stmt (bind, stmt); |
| 10462 | |
| 10463 | tbody = gimple_bind_body (bind); |
| 10464 | gimplify_and_add (lock, &tbody); |
| 10465 | gimple_bind_set_body (bind, tbody); |
| 10466 | |
| 10467 | lower_omp (gimple_omp_body_ptr (stmt), ctx); |
| 10468 | gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt))); |
| 10469 | gimple_bind_add_seq (bind, gimple_omp_body (stmt)); |
| 10470 | gimple_omp_set_body (stmt, NULL); |
| 10471 | |
| 10472 | tbody = gimple_bind_body (bind); |
| 10473 | gimplify_and_add (unlock, &tbody); |
| 10474 | gimple_bind_set_body (bind, tbody); |
| 10475 | |
| 10476 | gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); |
| 10477 | |
| 10478 | pop_gimplify_context (bind); |
| 10479 | gimple_bind_append_vars (bind, ctx->block_vars); |
| 10480 | BLOCK_VARS (block) = gimple_bind_vars (bind); |
| 10481 | } |
| 10482 | |
| 10483 | |
| 10484 | /* A subroutine of lower_omp_for. Generate code to emit the predicate |
| 10485 | for a lastprivate clause. Given a loop control predicate of (V |
| 10486 | cond N2), we gate the clause on (!(V cond N2)). The lowered form |
| 10487 | is appended to *DLIST, iterator initialization is appended to |
| 10488 | *BODY_P. */ |
| 10489 | |
| 10490 | static void |
| 10491 | lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p, |
| 10492 | gimple_seq *dlist, struct omp_context *ctx) |
| 10493 | { |
| 10494 | tree clauses, cond, vinit; |
| 10495 | enum tree_code cond_code; |
| 10496 | gimple_seq stmts; |
| 10497 | |
| 10498 | cond_code = fd->loop.cond_code; |
| 10499 | cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR; |
| 10500 | |
| 10501 | /* When possible, use a strict equality expression. This can let VRP |
| 10502 | type optimizations deduce the value and remove a copy. */ |
| 10503 | if (tree_fits_shwi_p (fd->loop.step)) |
| 10504 | { |
| 10505 | HOST_WIDE_INT step = tree_to_shwi (fd->loop.step); |
| 10506 | if (step == 1 || step == -1) |
| 10507 | cond_code = EQ_EXPR; |
| 10508 | } |
| 10509 | |
| 10510 | cond = build2 (cond_code, boolean_type_node, fd->loop.v, fd->loop.n2); |
| 10511 | |
| 10512 | clauses = gimple_omp_for_clauses (fd->for_stmt); |
| 10513 | stmts = NULL; |
| 10514 | lower_lastprivate_clauses (clauses, cond, &stmts, ctx); |
| 10515 | if (!gimple_seq_empty_p (stmts)) |
| 10516 | { |
| 10517 | gimple_seq_add_seq (&stmts, *dlist); |
| 10518 | *dlist = stmts; |
| 10519 | |
| 10520 | /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */ |
| 10521 | vinit = fd->loop.n1; |
| 10522 | if (cond_code == EQ_EXPR |
| 10523 | && tree_fits_shwi_p (fd->loop.n2) |
| 10524 | && ! integer_zerop (fd->loop.n2)) |
| 10525 | vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0); |
| 10526 | else |
| 10527 | vinit = unshare_expr (vinit); |
| 10528 | |
| 10529 | /* Initialize the iterator variable, so that threads that don't execute |
| 10530 | any iterations don't execute the lastprivate clauses by accident. */ |
| 10531 | gimplify_assign (fd->loop.v, vinit, body_p); |
| 10532 | } |
| 10533 | } |
| 10534 | |
| 10535 | |
| 10536 | /* Lower code for an OMP loop directive. */ |
| 10537 | |
| 10538 | static void |
| 10539 | lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
| 10540 | { |
| 10541 | tree *rhs_p, block; |
| 10542 | struct omp_for_data fd, *fdp = NULL; |
| 10543 | gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p)); |
| 10544 | gbind *new_stmt; |
| 10545 | gimple_seq omp_for_body, body, dlist; |
| 10546 | size_t i; |
| 10547 | |
| 10548 | push_gimplify_context (); |
| 10549 | |
| 10550 | lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx); |
| 10551 | |
| 10552 | block = make_node (BLOCK); |
| 10553 | new_stmt = gimple_build_bind (NULL, NULL, block); |
| 10554 | /* Replace at gsi right away, so that 'stmt' is no member |
| 10555 | of a sequence anymore as we're going to add to to a different |
| 10556 | one below. */ |
| 10557 | gsi_replace (gsi_p, new_stmt, true); |
| 10558 | |
| 10559 | /* Move declaration of temporaries in the loop body before we make |
| 10560 | it go away. */ |
| 10561 | omp_for_body = gimple_omp_body (stmt); |
| 10562 | if (!gimple_seq_empty_p (omp_for_body) |
| 10563 | && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND) |
| 10564 | { |
| 10565 | gbind *inner_bind |
| 10566 | = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body)); |
| 10567 | tree vars = gimple_bind_vars (inner_bind); |
| 10568 | gimple_bind_append_vars (new_stmt, vars); |
| 10569 | /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't |
| 10570 | keep them on the inner_bind and it's block. */ |
| 10571 | gimple_bind_set_vars (inner_bind, NULL_TREE); |
| 10572 | if (gimple_bind_block (inner_bind)) |
| 10573 | BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE; |
| 10574 | } |
| 10575 | |
| 10576 | if (gimple_omp_for_combined_into_p (stmt)) |
| 10577 | { |
| 10578 | extract_omp_for_data (stmt, &fd, NULL); |
| 10579 | fdp = &fd; |
| 10580 | |
| 10581 | /* We need two temporaries with fd.loop.v type (istart/iend) |
| 10582 | and then (fd.collapse - 1) temporaries with the same |
| 10583 | type for count2 ... countN-1 vars if not constant. */ |
| 10584 | size_t count = 2; |
| 10585 | tree type = fd.iter_type; |
| 10586 | if (fd.collapse > 1 |
| 10587 | && TREE_CODE (fd.loop.n2) != INTEGER_CST) |
| 10588 | count += fd.collapse - 1; |
| 10589 | bool parallel_for = gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR; |
| 10590 | tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt); |
| 10591 | tree clauses = *pc; |
| 10592 | if (parallel_for) |
| 10593 | outerc |
| 10594 | = find_omp_clause (gimple_omp_parallel_clauses (ctx->outer->stmt), |
| 10595 | OMP_CLAUSE__LOOPTEMP_); |
| 10596 | for (i = 0; i < count; i++) |
| 10597 | { |
| 10598 | tree temp; |
| 10599 | if (parallel_for) |
| 10600 | { |
| 10601 | gcc_assert (outerc); |
| 10602 | temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer); |
| 10603 | outerc = find_omp_clause (OMP_CLAUSE_CHAIN (outerc), |
| 10604 | OMP_CLAUSE__LOOPTEMP_); |
| 10605 | } |
| 10606 | else |
| 10607 | { |
| 10608 | temp = create_tmp_var (type); |
| 10609 | insert_decl_map (&ctx->outer->cb, temp, temp); |
| 10610 | } |
| 10611 | *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_); |
| 10612 | OMP_CLAUSE_DECL (*pc) = temp; |
| 10613 | pc = &OMP_CLAUSE_CHAIN (*pc); |
| 10614 | } |
| 10615 | *pc = clauses; |
| 10616 | } |
| 10617 | |
| 10618 | /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */ |
| 10619 | dlist = NULL; |
| 10620 | body = NULL; |
| 10621 | lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx, |
| 10622 | fdp); |
| 10623 | gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt)); |
| 10624 | |
| 10625 | lower_omp (gimple_omp_body_ptr (stmt), ctx); |
| 10626 | |
| 10627 | /* Lower the header expressions. At this point, we can assume that |
| 10628 | the header is of the form: |
| 10629 | |
| 10630 | #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3) |
| 10631 | |
| 10632 | We just need to make sure that VAL1, VAL2 and VAL3 are lowered |
| 10633 | using the .omp_data_s mapping, if needed. */ |
| 10634 | for (i = 0; i < gimple_omp_for_collapse (stmt); i++) |
| 10635 | { |
| 10636 | rhs_p = gimple_omp_for_initial_ptr (stmt, i); |
| 10637 | if (!is_gimple_min_invariant (*rhs_p)) |
| 10638 | *rhs_p = get_formal_tmp_var (*rhs_p, &body); |
| 10639 | |
| 10640 | rhs_p = gimple_omp_for_final_ptr (stmt, i); |
| 10641 | if (!is_gimple_min_invariant (*rhs_p)) |
| 10642 | *rhs_p = get_formal_tmp_var (*rhs_p, &body); |
| 10643 | |
| 10644 | rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1); |
| 10645 | if (!is_gimple_min_invariant (*rhs_p)) |
| 10646 | *rhs_p = get_formal_tmp_var (*rhs_p, &body); |
| 10647 | } |
| 10648 | |
| 10649 | /* Once lowered, extract the bounds and clauses. */ |
| 10650 | extract_omp_for_data (stmt, &fd, NULL); |
| 10651 | |
| 10652 | lower_omp_for_lastprivate (&fd, &body, &dlist, ctx); |
| 10653 | |
| 10654 | gimple_seq_add_stmt (&body, stmt); |
| 10655 | gimple_seq_add_seq (&body, gimple_omp_body (stmt)); |
| 10656 | |
| 10657 | gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v, |
| 10658 | fd.loop.v)); |
| 10659 | |
| 10660 | /* After the loop, add exit clauses. */ |
| 10661 | lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx); |
| 10662 | |
| 10663 | if (ctx->cancellable) |
| 10664 | gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label)); |
| 10665 | |
| 10666 | gimple_seq_add_seq (&body, dlist); |
| 10667 | |
| 10668 | body = maybe_catch_exception (body); |
| 10669 | |
| 10670 | /* Region exit marker goes at the end of the loop body. */ |
| 10671 | gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait)); |
| 10672 | maybe_add_implicit_barrier_cancel (ctx, &body); |
| 10673 | pop_gimplify_context (new_stmt); |
| 10674 | |
| 10675 | gimple_bind_append_vars (new_stmt, ctx->block_vars); |
| 10676 | BLOCK_VARS (block) = gimple_bind_vars (new_stmt); |
| 10677 | if (BLOCK_VARS (block)) |
| 10678 | TREE_USED (block) = 1; |
| 10679 | |
| 10680 | gimple_bind_set_body (new_stmt, body); |
| 10681 | gimple_omp_set_body (stmt, NULL); |
| 10682 | gimple_omp_for_set_pre_body (stmt, NULL); |
| 10683 | } |
| 10684 | |
| 10685 | /* Callback for walk_stmts. Check if the current statement only contains |
| 10686 | GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */ |
| 10687 | |
| 10688 | static tree |
| 10689 | check_combined_parallel (gimple_stmt_iterator *gsi_p, |
| 10690 | bool *handled_ops_p, |
| 10691 | struct walk_stmt_info *wi) |
| 10692 | { |
| 10693 | int *info = (int *) wi->info; |
| 10694 | gimple stmt = gsi_stmt (*gsi_p); |
| 10695 | |
| 10696 | *handled_ops_p = true; |
| 10697 | switch (gimple_code (stmt)) |
| 10698 | { |
| 10699 | WALK_SUBSTMTS; |
| 10700 | |
| 10701 | case GIMPLE_OMP_FOR: |
| 10702 | case GIMPLE_OMP_SECTIONS: |
| 10703 | *info = *info == 0 ? 1 : -1; |
| 10704 | break; |
| 10705 | default: |
| 10706 | *info = -1; |
| 10707 | break; |
| 10708 | } |
| 10709 | return NULL; |
| 10710 | } |
| 10711 | |
| 10712 | struct omp_taskcopy_context |
| 10713 | { |
| 10714 | /* This field must be at the beginning, as we do "inheritance": Some |
| 10715 | callback functions for tree-inline.c (e.g., omp_copy_decl) |
| 10716 | receive a copy_body_data pointer that is up-casted to an |
| 10717 | omp_context pointer. */ |
| 10718 | copy_body_data cb; |
| 10719 | omp_context *ctx; |
| 10720 | }; |
| 10721 | |
| 10722 | static tree |
| 10723 | task_copyfn_copy_decl (tree var, copy_body_data *cb) |
| 10724 | { |
| 10725 | struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb; |
| 10726 | |
| 10727 | if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var)) |
| 10728 | return create_tmp_var (TREE_TYPE (var)); |
| 10729 | |
| 10730 | return var; |
| 10731 | } |
| 10732 | |
| 10733 | static tree |
| 10734 | task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type) |
| 10735 | { |
| 10736 | tree name, new_fields = NULL, type, f; |
| 10737 | |
| 10738 | type = lang_hooks.types.make_type (RECORD_TYPE); |
| 10739 | name = DECL_NAME (TYPE_NAME (orig_type)); |
| 10740 | name = build_decl (gimple_location (tcctx->ctx->stmt), |
| 10741 | TYPE_DECL, name, type); |
| 10742 | TYPE_NAME (type) = name; |
| 10743 | |
| 10744 | for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f)) |
| 10745 | { |
| 10746 | tree new_f = copy_node (f); |
| 10747 | DECL_CONTEXT (new_f) = type; |
| 10748 | TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb); |
| 10749 | TREE_CHAIN (new_f) = new_fields; |
| 10750 | walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL); |
| 10751 | walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL); |
| 10752 | walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r, |
| 10753 | &tcctx->cb, NULL); |
| 10754 | new_fields = new_f; |
| 10755 | tcctx->cb.decl_map->put (f, new_f); |
| 10756 | } |
| 10757 | TYPE_FIELDS (type) = nreverse (new_fields); |
| 10758 | layout_type (type); |
| 10759 | return type; |
| 10760 | } |
| 10761 | |
| 10762 | /* Create task copyfn. */ |
| 10763 | |
| 10764 | static void |
| 10765 | create_task_copyfn (gomp_task *task_stmt, omp_context *ctx) |
| 10766 | { |
| 10767 | struct function *child_cfun; |
| 10768 | tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl; |
| 10769 | tree record_type, srecord_type, bind, list; |
| 10770 | bool record_needs_remap = false, srecord_needs_remap = false; |
| 10771 | splay_tree_node n; |
| 10772 | struct omp_taskcopy_context tcctx; |
| 10773 | location_t loc = gimple_location (task_stmt); |
| 10774 | |
| 10775 | child_fn = gimple_omp_task_copy_fn (task_stmt); |
| 10776 | child_cfun = DECL_STRUCT_FUNCTION (child_fn); |
| 10777 | gcc_assert (child_cfun->cfg == NULL); |
| 10778 | DECL_SAVED_TREE (child_fn) = alloc_stmt_list (); |
| 10779 | |
| 10780 | /* Reset DECL_CONTEXT on function arguments. */ |
| 10781 | for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t)) |
| 10782 | DECL_CONTEXT (t) = child_fn; |
| 10783 | |
| 10784 | /* Populate the function. */ |
| 10785 | push_gimplify_context (); |
| 10786 | push_cfun (child_cfun); |
| 10787 | |
| 10788 | bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL); |
| 10789 | TREE_SIDE_EFFECTS (bind) = 1; |
| 10790 | list = NULL; |
| 10791 | DECL_SAVED_TREE (child_fn) = bind; |
| 10792 | DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt); |
| 10793 | |
| 10794 | /* Remap src and dst argument types if needed. */ |
| 10795 | record_type = ctx->record_type; |
| 10796 | srecord_type = ctx->srecord_type; |
| 10797 | for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f)) |
| 10798 | if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) |
| 10799 | { |
| 10800 | record_needs_remap = true; |
| 10801 | break; |
| 10802 | } |
| 10803 | for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f)) |
| 10804 | if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) |
| 10805 | { |
| 10806 | srecord_needs_remap = true; |
| 10807 | break; |
| 10808 | } |
| 10809 | |
| 10810 | if (record_needs_remap || srecord_needs_remap) |
| 10811 | { |
| 10812 | memset (&tcctx, '\0', sizeof (tcctx)); |
| 10813 | tcctx.cb.src_fn = ctx->cb.src_fn; |
| 10814 | tcctx.cb.dst_fn = child_fn; |
| 10815 | tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn); |
| 10816 | gcc_checking_assert (tcctx.cb.src_node); |
| 10817 | tcctx.cb.dst_node = tcctx.cb.src_node; |
| 10818 | tcctx.cb.src_cfun = ctx->cb.src_cfun; |
| 10819 | tcctx.cb.copy_decl = task_copyfn_copy_decl; |
| 10820 | tcctx.cb.eh_lp_nr = 0; |
| 10821 | tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE; |
| 10822 | tcctx.cb.decl_map = new hash_map<tree, tree>; |
| 10823 | tcctx.ctx = ctx; |
| 10824 | |
| 10825 | if (record_needs_remap) |
| 10826 | record_type = task_copyfn_remap_type (&tcctx, record_type); |
| 10827 | if (srecord_needs_remap) |
| 10828 | srecord_type = task_copyfn_remap_type (&tcctx, srecord_type); |
| 10829 | } |
| 10830 | else |
| 10831 | tcctx.cb.decl_map = NULL; |
| 10832 | |
| 10833 | arg = DECL_ARGUMENTS (child_fn); |
| 10834 | TREE_TYPE (arg) = build_pointer_type (record_type); |
| 10835 | sarg = DECL_CHAIN (arg); |
| 10836 | TREE_TYPE (sarg) = build_pointer_type (srecord_type); |
| 10837 | |
| 10838 | /* First pass: initialize temporaries used in record_type and srecord_type |
| 10839 | sizes and field offsets. */ |
| 10840 | if (tcctx.cb.decl_map) |
| 10841 | for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
| 10842 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) |
| 10843 | { |
| 10844 | tree *p; |
| 10845 | |
| 10846 | decl = OMP_CLAUSE_DECL (c); |
| 10847 | p = tcctx.cb.decl_map->get (decl); |
| 10848 | if (p == NULL) |
| 10849 | continue; |
| 10850 | n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); |
| 10851 | sf = (tree) n->value; |
| 10852 | sf = *tcctx.cb.decl_map->get (sf); |
| 10853 | src = build_simple_mem_ref_loc (loc, sarg); |
| 10854 | src = omp_build_component_ref (src, sf); |
| 10855 | t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src); |
| 10856 | append_to_statement_list (t, &list); |
| 10857 | } |
| 10858 | |
| 10859 | /* Second pass: copy shared var pointers and copy construct non-VLA |
| 10860 | firstprivate vars. */ |
| 10861 | for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
| 10862 | switch (OMP_CLAUSE_CODE (c)) |
| 10863 | { |
| 10864 | case OMP_CLAUSE_SHARED: |
| 10865 | decl = OMP_CLAUSE_DECL (c); |
| 10866 | n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); |
| 10867 | if (n == NULL) |
| 10868 | break; |
| 10869 | f = (tree) n->value; |
| 10870 | if (tcctx.cb.decl_map) |
| 10871 | f = *tcctx.cb.decl_map->get (f); |
| 10872 | n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); |
| 10873 | sf = (tree) n->value; |
| 10874 | if (tcctx.cb.decl_map) |
| 10875 | sf = *tcctx.cb.decl_map->get (sf); |
| 10876 | src = build_simple_mem_ref_loc (loc, sarg); |
| 10877 | src = omp_build_component_ref (src, sf); |
| 10878 | dst = build_simple_mem_ref_loc (loc, arg); |
| 10879 | dst = omp_build_component_ref (dst, f); |
| 10880 | t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src); |
| 10881 | append_to_statement_list (t, &list); |
| 10882 | break; |
| 10883 | case OMP_CLAUSE_FIRSTPRIVATE: |
| 10884 | decl = OMP_CLAUSE_DECL (c); |
| 10885 | if (is_variable_sized (decl)) |
| 10886 | break; |
| 10887 | n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); |
| 10888 | if (n == NULL) |
| 10889 | break; |
| 10890 | f = (tree) n->value; |
| 10891 | if (tcctx.cb.decl_map) |
| 10892 | f = *tcctx.cb.decl_map->get (f); |
| 10893 | n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); |
| 10894 | if (n != NULL) |
| 10895 | { |
| 10896 | sf = (tree) n->value; |
| 10897 | if (tcctx.cb.decl_map) |
| 10898 | sf = *tcctx.cb.decl_map->get (sf); |
| 10899 | src = build_simple_mem_ref_loc (loc, sarg); |
| 10900 | src = omp_build_component_ref (src, sf); |
| 10901 | if (use_pointer_for_field (decl, NULL) || is_reference (decl)) |
| 10902 | src = build_simple_mem_ref_loc (loc, src); |
| 10903 | } |
| 10904 | else |
| 10905 | src = decl; |
| 10906 | dst = build_simple_mem_ref_loc (loc, arg); |
| 10907 | dst = omp_build_component_ref (dst, f); |
| 10908 | t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src); |
| 10909 | append_to_statement_list (t, &list); |
| 10910 | break; |
| 10911 | case OMP_CLAUSE_PRIVATE: |
| 10912 | if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c)) |
| 10913 | break; |
| 10914 | decl = OMP_CLAUSE_DECL (c); |
| 10915 | n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); |
| 10916 | f = (tree) n->value; |
| 10917 | if (tcctx.cb.decl_map) |
| 10918 | f = *tcctx.cb.decl_map->get (f); |
| 10919 | n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); |
| 10920 | if (n != NULL) |
| 10921 | { |
| 10922 | sf = (tree) n->value; |
| 10923 | if (tcctx.cb.decl_map) |
| 10924 | sf = *tcctx.cb.decl_map->get (sf); |
| 10925 | src = build_simple_mem_ref_loc (loc, sarg); |
| 10926 | src = omp_build_component_ref (src, sf); |
| 10927 | if (use_pointer_for_field (decl, NULL)) |
| 10928 | src = build_simple_mem_ref_loc (loc, src); |
| 10929 | } |
| 10930 | else |
| 10931 | src = decl; |
| 10932 | dst = build_simple_mem_ref_loc (loc, arg); |
| 10933 | dst = omp_build_component_ref (dst, f); |
| 10934 | t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src); |
| 10935 | append_to_statement_list (t, &list); |
| 10936 | break; |
| 10937 | default: |
| 10938 | break; |
| 10939 | } |
| 10940 | |
| 10941 | /* Last pass: handle VLA firstprivates. */ |
| 10942 | if (tcctx.cb.decl_map) |
| 10943 | for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) |
| 10944 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) |
| 10945 | { |
| 10946 | tree ind, ptr, df; |
| 10947 | |
| 10948 | decl = OMP_CLAUSE_DECL (c); |
| 10949 | if (!is_variable_sized (decl)) |
| 10950 | continue; |
| 10951 | n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); |
| 10952 | if (n == NULL) |
| 10953 | continue; |
| 10954 | f = (tree) n->value; |
| 10955 | f = *tcctx.cb.decl_map->get (f); |
| 10956 | gcc_assert (DECL_HAS_VALUE_EXPR_P (decl)); |
| 10957 | ind = DECL_VALUE_EXPR (decl); |
| 10958 | gcc_assert (TREE_CODE (ind) == INDIRECT_REF); |
| 10959 | gcc_assert (DECL_P (TREE_OPERAND (ind, 0))); |
| 10960 | n = splay_tree_lookup (ctx->sfield_map, |
| 10961 | (splay_tree_key) TREE_OPERAND (ind, 0)); |
| 10962 | sf = (tree) n->value; |
| 10963 | sf = *tcctx.cb.decl_map->get (sf); |
| 10964 | src = build_simple_mem_ref_loc (loc, sarg); |
| 10965 | src = omp_build_component_ref (src, sf); |
| 10966 | src = build_simple_mem_ref_loc (loc, src); |
| 10967 | dst = build_simple_mem_ref_loc (loc, arg); |
| 10968 | dst = omp_build_component_ref (dst, f); |
| 10969 | t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src); |
| 10970 | append_to_statement_list (t, &list); |
| 10971 | n = splay_tree_lookup (ctx->field_map, |
| 10972 | (splay_tree_key) TREE_OPERAND (ind, 0)); |
| 10973 | df = (tree) n->value; |
| 10974 | df = *tcctx.cb.decl_map->get (df); |
| 10975 | ptr = build_simple_mem_ref_loc (loc, arg); |
| 10976 | ptr = omp_build_component_ref (ptr, df); |
| 10977 | t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr, |
| 10978 | build_fold_addr_expr_loc (loc, dst)); |
| 10979 | append_to_statement_list (t, &list); |
| 10980 | } |
| 10981 | |
| 10982 | t = build1 (RETURN_EXPR, void_type_node, NULL); |
| 10983 | append_to_statement_list (t, &list); |
| 10984 | |
| 10985 | if (tcctx.cb.decl_map) |
| 10986 | delete tcctx.cb.decl_map; |
| 10987 | pop_gimplify_context (NULL); |
| 10988 | BIND_EXPR_BODY (bind) = list; |
| 10989 | pop_cfun (); |
| 10990 | } |
| 10991 | |
| 10992 | static void |
| 10993 | lower_depend_clauses (gimple stmt, gimple_seq *iseq, gimple_seq *oseq) |
| 10994 | { |
| 10995 | tree c, clauses; |
| 10996 | gimple g; |
| 10997 | size_t n_in = 0, n_out = 0, idx = 2, i; |
| 10998 | |
| 10999 | clauses = find_omp_clause (gimple_omp_task_clauses (stmt), |
| 11000 | OMP_CLAUSE_DEPEND); |
| 11001 | gcc_assert (clauses); |
| 11002 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
| 11003 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND) |
| 11004 | switch (OMP_CLAUSE_DEPEND_KIND (c)) |
| 11005 | { |
| 11006 | case OMP_CLAUSE_DEPEND_IN: |
| 11007 | n_in++; |
| 11008 | break; |
| 11009 | case OMP_CLAUSE_DEPEND_OUT: |
| 11010 | case OMP_CLAUSE_DEPEND_INOUT: |
| 11011 | n_out++; |
| 11012 | break; |
| 11013 | default: |
| 11014 | gcc_unreachable (); |
| 11015 | } |
| 11016 | tree type = build_array_type_nelts (ptr_type_node, n_in + n_out + 2); |
| 11017 | tree array = create_tmp_var (type); |
| 11018 | tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE, |
| 11019 | NULL_TREE); |
| 11020 | g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_in + n_out)); |
| 11021 | gimple_seq_add_stmt (iseq, g); |
| 11022 | r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE, |
| 11023 | NULL_TREE); |
| 11024 | g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_out)); |
| 11025 | gimple_seq_add_stmt (iseq, g); |
| 11026 | for (i = 0; i < 2; i++) |
| 11027 | { |
| 11028 | if ((i ? n_in : n_out) == 0) |
| 11029 | continue; |
| 11030 | for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) |
| 11031 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND |
| 11032 | && ((OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_IN) ^ i)) |
| 11033 | { |
| 11034 | tree t = OMP_CLAUSE_DECL (c); |
| 11035 | t = fold_convert (ptr_type_node, t); |
| 11036 | gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue); |
| 11037 | r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++), |
| 11038 | NULL_TREE, NULL_TREE); |
| 11039 | g = gimple_build_assign (r, t); |
| 11040 | gimple_seq_add_stmt (iseq, g); |
| 11041 | } |
| 11042 | } |
| 11043 | tree *p = gimple_omp_task_clauses_ptr (stmt); |
| 11044 | c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND); |
| 11045 | OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array); |
| 11046 | OMP_CLAUSE_CHAIN (c) = *p; |
| 11047 | *p = c; |
| 11048 | tree clobber = build_constructor (type, NULL); |
| 11049 | TREE_THIS_VOLATILE (clobber) = 1; |
| 11050 | g = gimple_build_assign (array, clobber); |
| 11051 | gimple_seq_add_stmt (oseq, g); |
| 11052 | } |
| 11053 | |
| 11054 | /* Lower the OpenMP parallel or task directive in the current statement |
| 11055 | in GSI_P. CTX holds context information for the directive. */ |
| 11056 | |
| 11057 | static void |
| 11058 | lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
| 11059 | { |
| 11060 | tree clauses; |
| 11061 | tree child_fn, t; |
| 11062 | gimple stmt = gsi_stmt (*gsi_p); |
| 11063 | gbind *par_bind, *bind, *dep_bind = NULL; |
| 11064 | gimple_seq par_body, olist, ilist, par_olist, par_rlist, par_ilist, new_body; |
| 11065 | location_t loc = gimple_location (stmt); |
| 11066 | |
| 11067 | clauses = gimple_omp_taskreg_clauses (stmt); |
| 11068 | par_bind |
| 11069 | = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt))); |
| 11070 | par_body = gimple_bind_body (par_bind); |
| 11071 | child_fn = ctx->cb.dst_fn; |
| 11072 | if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL |
| 11073 | && !gimple_omp_parallel_combined_p (stmt)) |
| 11074 | { |
| 11075 | struct walk_stmt_info wi; |
| 11076 | int ws_num = 0; |
| 11077 | |
| 11078 | memset (&wi, 0, sizeof (wi)); |
| 11079 | wi.info = &ws_num; |
| 11080 | wi.val_only = true; |
| 11081 | walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi); |
| 11082 | if (ws_num == 1) |
| 11083 | gimple_omp_parallel_set_combined_p (stmt, true); |
| 11084 | } |
| 11085 | gimple_seq dep_ilist = NULL; |
| 11086 | gimple_seq dep_olist = NULL; |
| 11087 | if (gimple_code (stmt) == GIMPLE_OMP_TASK |
| 11088 | && find_omp_clause (clauses, OMP_CLAUSE_DEPEND)) |
| 11089 | { |
| 11090 | push_gimplify_context (); |
| 11091 | dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK)); |
| 11092 | lower_depend_clauses (stmt, &dep_ilist, &dep_olist); |
| 11093 | } |
| 11094 | |
| 11095 | if (ctx->srecord_type) |
| 11096 | create_task_copyfn (as_a <gomp_task *> (stmt), ctx); |
| 11097 | |
| 11098 | push_gimplify_context (); |
| 11099 | |
| 11100 | par_olist = NULL; |
| 11101 | par_ilist = NULL; |
| 11102 | par_rlist = NULL; |
| 11103 | lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL); |
| 11104 | lower_omp (&par_body, ctx); |
| 11105 | if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL) |
| 11106 | lower_reduction_clauses (clauses, &par_rlist, ctx); |
| 11107 | |
| 11108 | /* Declare all the variables created by mapping and the variables |
| 11109 | declared in the scope of the parallel body. */ |
| 11110 | record_vars_into (ctx->block_vars, child_fn); |
| 11111 | record_vars_into (gimple_bind_vars (par_bind), child_fn); |
| 11112 | |
| 11113 | if (ctx->record_type) |
| 11114 | { |
| 11115 | ctx->sender_decl |
| 11116 | = create_tmp_var (ctx->srecord_type ? ctx->srecord_type |
| 11117 | : ctx->record_type, ".omp_data_o"); |
| 11118 | DECL_NAMELESS (ctx->sender_decl) = 1; |
| 11119 | TREE_ADDRESSABLE (ctx->sender_decl) = 1; |
| 11120 | gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl); |
| 11121 | } |
| 11122 | |
| 11123 | olist = NULL; |
| 11124 | ilist = NULL; |
| 11125 | lower_send_clauses (clauses, &ilist, &olist, ctx); |
| 11126 | lower_send_shared_vars (&ilist, &olist, ctx); |
| 11127 | |
| 11128 | if (ctx->record_type) |
| 11129 | { |
| 11130 | tree clobber = build_constructor (TREE_TYPE (ctx->sender_decl), NULL); |
| 11131 | TREE_THIS_VOLATILE (clobber) = 1; |
| 11132 | gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl, |
| 11133 | clobber)); |
| 11134 | } |
| 11135 | |
| 11136 | /* Once all the expansions are done, sequence all the different |
| 11137 | fragments inside gimple_omp_body. */ |
| 11138 | |
| 11139 | new_body = NULL; |
| 11140 | |
| 11141 | if (ctx->record_type) |
| 11142 | { |
| 11143 | t = build_fold_addr_expr_loc (loc, ctx->sender_decl); |
| 11144 | /* fixup_child_record_type might have changed receiver_decl's type. */ |
| 11145 | t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t); |
| 11146 | gimple_seq_add_stmt (&new_body, |
| 11147 | gimple_build_assign (ctx->receiver_decl, t)); |
| 11148 | } |
| 11149 | |
| 11150 | gimple_seq_add_seq (&new_body, par_ilist); |
| 11151 | gimple_seq_add_seq (&new_body, par_body); |
| 11152 | gimple_seq_add_seq (&new_body, par_rlist); |
| 11153 | if (ctx->cancellable) |
| 11154 | gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label)); |
| 11155 | gimple_seq_add_seq (&new_body, par_olist); |
| 11156 | new_body = maybe_catch_exception (new_body); |
| 11157 | gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false)); |
| 11158 | gimple_omp_set_body (stmt, new_body); |
| 11159 | |
| 11160 | bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind)); |
| 11161 | gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true); |
| 11162 | gimple_bind_add_seq (bind, ilist); |
| 11163 | gimple_bind_add_stmt (bind, stmt); |
| 11164 | gimple_bind_add_seq (bind, olist); |
| 11165 | |
| 11166 | pop_gimplify_context (NULL); |
| 11167 | |
| 11168 | if (dep_bind) |
| 11169 | { |
| 11170 | gimple_bind_add_seq (dep_bind, dep_ilist); |
| 11171 | gimple_bind_add_stmt (dep_bind, bind); |
| 11172 | gimple_bind_add_seq (dep_bind, dep_olist); |
| 11173 | pop_gimplify_context (dep_bind); |
| 11174 | } |
| 11175 | } |
| 11176 | |
| 11177 | /* Lower the GIMPLE_OMP_TARGET in the current statement |
| 11178 | in GSI_P. CTX holds context information for the directive. */ |
| 11179 | |
| 11180 | static void |
| 11181 | lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
| 11182 | { |
| 11183 | tree clauses; |
| 11184 | tree child_fn, t, c; |
| 11185 | gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p)); |
| 11186 | gbind *tgt_bind, *bind; |
| 11187 | gimple_seq tgt_body, olist, ilist, orlist, irlist, new_body; |
| 11188 | location_t loc = gimple_location (stmt); |
| 11189 | bool offloaded, data_region; |
| 11190 | unsigned int map_cnt = 0; |
| 11191 | |
| 11192 | offloaded = is_gimple_omp_offloaded (stmt); |
| 11193 | switch (gimple_omp_target_kind (stmt)) |
| 11194 | { |
| 11195 | case GF_OMP_TARGET_KIND_REGION: |
| 11196 | case GF_OMP_TARGET_KIND_UPDATE: |
| 11197 | case GF_OMP_TARGET_KIND_OACC_PARALLEL: |
| 11198 | case GF_OMP_TARGET_KIND_OACC_KERNELS: |
| 11199 | case GF_OMP_TARGET_KIND_OACC_UPDATE: |
| 11200 | case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: |
| 11201 | data_region = false; |
| 11202 | break; |
| 11203 | case GF_OMP_TARGET_KIND_DATA: |
| 11204 | case GF_OMP_TARGET_KIND_OACC_DATA: |
| 11205 | data_region = true; |
| 11206 | break; |
| 11207 | default: |
| 11208 | gcc_unreachable (); |
| 11209 | } |
| 11210 | |
| 11211 | clauses = gimple_omp_target_clauses (stmt); |
| 11212 | |
| 11213 | tgt_bind = NULL; |
| 11214 | tgt_body = NULL; |
| 11215 | if (offloaded) |
| 11216 | { |
| 11217 | tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt)); |
| 11218 | tgt_body = gimple_bind_body (tgt_bind); |
| 11219 | } |
| 11220 | else if (data_region) |
| 11221 | tgt_body = gimple_omp_body (stmt); |
| 11222 | child_fn = ctx->cb.dst_fn; |
| 11223 | |
| 11224 | push_gimplify_context (); |
| 11225 | |
| 11226 | irlist = NULL; |
| 11227 | orlist = NULL; |
| 11228 | if (offloaded |
| 11229 | && is_gimple_omp_oacc (stmt)) |
| 11230 | oacc_process_reduction_data (&tgt_body, &irlist, &orlist, ctx); |
| 11231 | |
| 11232 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
| 11233 | switch (OMP_CLAUSE_CODE (c)) |
| 11234 | { |
| 11235 | tree var, x; |
| 11236 | |
| 11237 | default: |
| 11238 | break; |
| 11239 | case OMP_CLAUSE_MAP: |
| 11240 | #ifdef ENABLE_CHECKING |
| 11241 | /* First check what we're prepared to handle in the following. */ |
| 11242 | switch (OMP_CLAUSE_MAP_KIND (c)) |
| 11243 | { |
| 11244 | case GOMP_MAP_ALLOC: |
| 11245 | case GOMP_MAP_TO: |
| 11246 | case GOMP_MAP_FROM: |
| 11247 | case GOMP_MAP_TOFROM: |
| 11248 | case GOMP_MAP_POINTER: |
| 11249 | case GOMP_MAP_TO_PSET: |
| 11250 | break; |
| 11251 | case GOMP_MAP_FORCE_ALLOC: |
| 11252 | case GOMP_MAP_FORCE_TO: |
| 11253 | case GOMP_MAP_FORCE_FROM: |
| 11254 | case GOMP_MAP_FORCE_TOFROM: |
| 11255 | case GOMP_MAP_FORCE_PRESENT: |
| 11256 | case GOMP_MAP_FORCE_DEALLOC: |
| 11257 | case GOMP_MAP_FORCE_DEVICEPTR: |
| 11258 | gcc_assert (is_gimple_omp_oacc (stmt)); |
| 11259 | break; |
| 11260 | default: |
| 11261 | gcc_unreachable (); |
| 11262 | } |
| 11263 | #endif |
| 11264 | /* FALLTHRU */ |
| 11265 | case OMP_CLAUSE_TO: |
| 11266 | case OMP_CLAUSE_FROM: |
| 11267 | var = OMP_CLAUSE_DECL (c); |
| 11268 | if (!DECL_P (var)) |
| 11269 | { |
| 11270 | if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP |
| 11271 | || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)) |
| 11272 | map_cnt++; |
| 11273 | continue; |
| 11274 | } |
| 11275 | |
| 11276 | if (DECL_SIZE (var) |
| 11277 | && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST) |
| 11278 | { |
| 11279 | tree var2 = DECL_VALUE_EXPR (var); |
| 11280 | gcc_assert (TREE_CODE (var2) == INDIRECT_REF); |
| 11281 | var2 = TREE_OPERAND (var2, 0); |
| 11282 | gcc_assert (DECL_P (var2)); |
| 11283 | var = var2; |
| 11284 | } |
| 11285 | |
| 11286 | if (!maybe_lookup_field (var, ctx)) |
| 11287 | continue; |
| 11288 | |
| 11289 | if (offloaded) |
| 11290 | { |
| 11291 | x = build_receiver_ref (var, true, ctx); |
| 11292 | tree new_var = lookup_decl (var, ctx); |
| 11293 | if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER |
| 11294 | && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) |
| 11295 | && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE) |
| 11296 | x = build_simple_mem_ref (x); |
| 11297 | SET_DECL_VALUE_EXPR (new_var, x); |
| 11298 | DECL_HAS_VALUE_EXPR_P (new_var) = 1; |
| 11299 | } |
| 11300 | map_cnt++; |
| 11301 | } |
| 11302 | |
| 11303 | if (offloaded) |
| 11304 | { |
| 11305 | target_nesting_level++; |
| 11306 | lower_omp (&tgt_body, ctx); |
| 11307 | target_nesting_level--; |
| 11308 | } |
| 11309 | else if (data_region) |
| 11310 | lower_omp (&tgt_body, ctx); |
| 11311 | |
| 11312 | if (offloaded) |
| 11313 | { |
| 11314 | /* Declare all the variables created by mapping and the variables |
| 11315 | declared in the scope of the target body. */ |
| 11316 | record_vars_into (ctx->block_vars, child_fn); |
| 11317 | record_vars_into (gimple_bind_vars (tgt_bind), child_fn); |
| 11318 | } |
| 11319 | |
| 11320 | olist = NULL; |
| 11321 | ilist = NULL; |
| 11322 | if (ctx->record_type) |
| 11323 | { |
| 11324 | ctx->sender_decl |
| 11325 | = create_tmp_var (ctx->record_type, ".omp_data_arr"); |
| 11326 | DECL_NAMELESS (ctx->sender_decl) = 1; |
| 11327 | TREE_ADDRESSABLE (ctx->sender_decl) = 1; |
| 11328 | t = make_tree_vec (3); |
| 11329 | TREE_VEC_ELT (t, 0) = ctx->sender_decl; |
| 11330 | TREE_VEC_ELT (t, 1) |
| 11331 | = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt), |
| 11332 | ".omp_data_sizes"); |
| 11333 | DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1; |
| 11334 | TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1; |
| 11335 | TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1; |
| 11336 | tree tkind_type; |
| 11337 | int talign_shift; |
| 11338 | if (is_gimple_omp_oacc (stmt)) |
| 11339 | { |
| 11340 | tkind_type = short_unsigned_type_node; |
| 11341 | talign_shift = 8; |
| 11342 | } |
| 11343 | else |
| 11344 | { |
| 11345 | tkind_type = unsigned_char_type_node; |
| 11346 | talign_shift = 3; |
| 11347 | } |
| 11348 | TREE_VEC_ELT (t, 2) |
| 11349 | = create_tmp_var (build_array_type_nelts (tkind_type, map_cnt), |
| 11350 | ".omp_data_kinds"); |
| 11351 | DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1; |
| 11352 | TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1; |
| 11353 | TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1; |
| 11354 | gimple_omp_target_set_data_arg (stmt, t); |
| 11355 | |
| 11356 | vec<constructor_elt, va_gc> *vsize; |
| 11357 | vec<constructor_elt, va_gc> *vkind; |
| 11358 | vec_alloc (vsize, map_cnt); |
| 11359 | vec_alloc (vkind, map_cnt); |
| 11360 | unsigned int map_idx = 0; |
| 11361 | |
| 11362 | for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) |
| 11363 | switch (OMP_CLAUSE_CODE (c)) |
| 11364 | { |
| 11365 | tree ovar, nc; |
| 11366 | |
| 11367 | default: |
| 11368 | break; |
| 11369 | case OMP_CLAUSE_MAP: |
| 11370 | case OMP_CLAUSE_TO: |
| 11371 | case OMP_CLAUSE_FROM: |
| 11372 | nc = c; |
| 11373 | ovar = OMP_CLAUSE_DECL (c); |
| 11374 | if (!DECL_P (ovar)) |
| 11375 | { |
| 11376 | if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
| 11377 | && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)) |
| 11378 | { |
| 11379 | gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c)) |
| 11380 | == get_base_address (ovar)); |
| 11381 | nc = OMP_CLAUSE_CHAIN (c); |
| 11382 | ovar = OMP_CLAUSE_DECL (nc); |
| 11383 | } |
| 11384 | else |
| 11385 | { |
| 11386 | tree x = build_sender_ref (ovar, ctx); |
| 11387 | tree v |
| 11388 | = build_fold_addr_expr_with_type (ovar, ptr_type_node); |
| 11389 | gimplify_assign (x, v, &ilist); |
| 11390 | nc = NULL_TREE; |
| 11391 | } |
| 11392 | } |
| 11393 | else |
| 11394 | { |
| 11395 | if (DECL_SIZE (ovar) |
| 11396 | && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST) |
| 11397 | { |
| 11398 | tree ovar2 = DECL_VALUE_EXPR (ovar); |
| 11399 | gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF); |
| 11400 | ovar2 = TREE_OPERAND (ovar2, 0); |
| 11401 | gcc_assert (DECL_P (ovar2)); |
| 11402 | ovar = ovar2; |
| 11403 | } |
| 11404 | if (!maybe_lookup_field (ovar, ctx)) |
| 11405 | continue; |
| 11406 | } |
| 11407 | |
| 11408 | unsigned int talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar)); |
| 11409 | if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign) |
| 11410 | talign = DECL_ALIGN_UNIT (ovar); |
| 11411 | if (nc) |
| 11412 | { |
| 11413 | tree var = lookup_decl_in_outer_ctx (ovar, ctx); |
| 11414 | tree x = build_sender_ref (ovar, ctx); |
| 11415 | if (maybe_lookup_oacc_reduction (var, ctx)) |
| 11416 | { |
| 11417 | gcc_checking_assert (offloaded |
| 11418 | && is_gimple_omp_oacc (stmt)); |
| 11419 | gimplify_assign (x, var, &ilist); |
| 11420 | } |
| 11421 | else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP |
| 11422 | && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER |
| 11423 | && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) |
| 11424 | && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE) |
| 11425 | { |
| 11426 | gcc_assert (offloaded); |
| 11427 | tree avar |
| 11428 | = create_tmp_var (TREE_TYPE (TREE_TYPE (x))); |
| 11429 | mark_addressable (avar); |
| 11430 | gimplify_assign (avar, build_fold_addr_expr (var), &ilist); |
| 11431 | talign = DECL_ALIGN_UNIT (avar); |
| 11432 | avar = build_fold_addr_expr (avar); |
| 11433 | gimplify_assign (x, avar, &ilist); |
| 11434 | } |
| 11435 | else if (is_gimple_reg (var)) |
| 11436 | { |
| 11437 | gcc_assert (offloaded); |
| 11438 | tree avar = create_tmp_var (TREE_TYPE (var)); |
| 11439 | mark_addressable (avar); |
| 11440 | enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c); |
| 11441 | if (GOMP_MAP_COPY_TO_P (map_kind) |
| 11442 | || map_kind == GOMP_MAP_POINTER |
| 11443 | || map_kind == GOMP_MAP_TO_PSET |
| 11444 | || map_kind == GOMP_MAP_FORCE_DEVICEPTR) |
| 11445 | gimplify_assign (avar, var, &ilist); |
| 11446 | avar = build_fold_addr_expr (avar); |
| 11447 | gimplify_assign (x, avar, &ilist); |
| 11448 | if ((GOMP_MAP_COPY_FROM_P (map_kind) |
| 11449 | || map_kind == GOMP_MAP_FORCE_DEVICEPTR) |
| 11450 | && !TYPE_READONLY (TREE_TYPE (var))) |
| 11451 | { |
| 11452 | x = build_sender_ref (ovar, ctx); |
| 11453 | x = build_simple_mem_ref (x); |
| 11454 | gimplify_assign (var, x, &olist); |
| 11455 | } |
| 11456 | } |
| 11457 | else |
| 11458 | { |
| 11459 | var = build_fold_addr_expr (var); |
| 11460 | gimplify_assign (x, var, &ilist); |
| 11461 | } |
| 11462 | } |
| 11463 | tree s = OMP_CLAUSE_SIZE (c); |
| 11464 | if (s == NULL_TREE) |
| 11465 | s = TYPE_SIZE_UNIT (TREE_TYPE (ovar)); |
| 11466 | s = fold_convert (size_type_node, s); |
| 11467 | tree purpose = size_int (map_idx++); |
| 11468 | CONSTRUCTOR_APPEND_ELT (vsize, purpose, s); |
| 11469 | if (TREE_CODE (s) != INTEGER_CST) |
| 11470 | TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0; |
| 11471 | |
| 11472 | unsigned HOST_WIDE_INT tkind; |
| 11473 | switch (OMP_CLAUSE_CODE (c)) |
| 11474 | { |
| 11475 | case OMP_CLAUSE_MAP: |
| 11476 | tkind = OMP_CLAUSE_MAP_KIND (c); |
| 11477 | break; |
| 11478 | case OMP_CLAUSE_TO: |
| 11479 | tkind = GOMP_MAP_TO; |
| 11480 | break; |
| 11481 | case OMP_CLAUSE_FROM: |
| 11482 | tkind = GOMP_MAP_FROM; |
| 11483 | break; |
| 11484 | default: |
| 11485 | gcc_unreachable (); |
| 11486 | } |
| 11487 | gcc_checking_assert (tkind |
| 11488 | < (HOST_WIDE_INT_C (1U) << talign_shift)); |
| 11489 | talign = ceil_log2 (talign); |
| 11490 | tkind |= talign << talign_shift; |
| 11491 | gcc_checking_assert (tkind |
| 11492 | <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type))); |
| 11493 | CONSTRUCTOR_APPEND_ELT (vkind, purpose, |
| 11494 | build_int_cstu (tkind_type, tkind)); |
| 11495 | if (nc && nc != c) |
| 11496 | c = nc; |
| 11497 | } |
| 11498 | |
| 11499 | gcc_assert (map_idx == map_cnt); |
| 11500 | |
| 11501 | DECL_INITIAL (TREE_VEC_ELT (t, 1)) |
| 11502 | = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize); |
| 11503 | DECL_INITIAL (TREE_VEC_ELT (t, 2)) |
| 11504 | = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind); |
| 11505 | if (!TREE_STATIC (TREE_VEC_ELT (t, 1))) |
| 11506 | { |
| 11507 | gimple_seq initlist = NULL; |
| 11508 | force_gimple_operand (build1 (DECL_EXPR, void_type_node, |
| 11509 | TREE_VEC_ELT (t, 1)), |
| 11510 | &initlist, true, NULL_TREE); |
| 11511 | gimple_seq_add_seq (&ilist, initlist); |
| 11512 | |
| 11513 | tree clobber = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), |
| 11514 | NULL); |
| 11515 | TREE_THIS_VOLATILE (clobber) = 1; |
| 11516 | gimple_seq_add_stmt (&olist, |
| 11517 | gimple_build_assign (TREE_VEC_ELT (t, 1), |
| 11518 | clobber)); |
| 11519 | } |
| 11520 | |
| 11521 | tree clobber = build_constructor (ctx->record_type, NULL); |
| 11522 | TREE_THIS_VOLATILE (clobber) = 1; |
| 11523 | gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl, |
| 11524 | clobber)); |
| 11525 | } |
| 11526 | |
| 11527 | /* Once all the expansions are done, sequence all the different |
| 11528 | fragments inside gimple_omp_body. */ |
| 11529 | |
| 11530 | new_body = NULL; |
| 11531 | |
| 11532 | if (offloaded |
| 11533 | && ctx->record_type) |
| 11534 | { |
| 11535 | t = build_fold_addr_expr_loc (loc, ctx->sender_decl); |
| 11536 | /* fixup_child_record_type might have changed receiver_decl's type. */ |
| 11537 | t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t); |
| 11538 | gimple_seq_add_stmt (&new_body, |
| 11539 | gimple_build_assign (ctx->receiver_decl, t)); |
| 11540 | } |
| 11541 | |
| 11542 | if (offloaded) |
| 11543 | { |
| 11544 | gimple_seq_add_seq (&new_body, tgt_body); |
| 11545 | new_body = maybe_catch_exception (new_body); |
| 11546 | } |
| 11547 | else if (data_region) |
| 11548 | new_body = tgt_body; |
| 11549 | if (offloaded || data_region) |
| 11550 | { |
| 11551 | gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false)); |
| 11552 | gimple_omp_set_body (stmt, new_body); |
| 11553 | } |
| 11554 | |
| 11555 | bind = gimple_build_bind (NULL, NULL, |
| 11556 | tgt_bind ? gimple_bind_block (tgt_bind) |
| 11557 | : NULL_TREE); |
| 11558 | gsi_replace (gsi_p, bind, true); |
| 11559 | gimple_bind_add_seq (bind, irlist); |
| 11560 | gimple_bind_add_seq (bind, ilist); |
| 11561 | gimple_bind_add_stmt (bind, stmt); |
| 11562 | gimple_bind_add_seq (bind, olist); |
| 11563 | gimple_bind_add_seq (bind, orlist); |
| 11564 | |
| 11565 | pop_gimplify_context (NULL); |
| 11566 | } |
| 11567 | |
| 11568 | /* Expand code for an OpenMP teams directive. */ |
| 11569 | |
| 11570 | static void |
| 11571 | lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
| 11572 | { |
| 11573 | gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p)); |
| 11574 | push_gimplify_context (); |
| 11575 | |
| 11576 | tree block = make_node (BLOCK); |
| 11577 | gbind *bind = gimple_build_bind (NULL, NULL, block); |
| 11578 | gsi_replace (gsi_p, bind, true); |
| 11579 | gimple_seq bind_body = NULL; |
| 11580 | gimple_seq dlist = NULL; |
| 11581 | gimple_seq olist = NULL; |
| 11582 | |
| 11583 | tree num_teams = find_omp_clause (gimple_omp_teams_clauses (teams_stmt), |
| 11584 | OMP_CLAUSE_NUM_TEAMS); |
| 11585 | if (num_teams == NULL_TREE) |
| 11586 | num_teams = build_int_cst (unsigned_type_node, 0); |
| 11587 | else |
| 11588 | { |
| 11589 | num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams); |
| 11590 | num_teams = fold_convert (unsigned_type_node, num_teams); |
| 11591 | gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue); |
| 11592 | } |
| 11593 | tree thread_limit = find_omp_clause (gimple_omp_teams_clauses (teams_stmt), |
| 11594 | OMP_CLAUSE_THREAD_LIMIT); |
| 11595 | if (thread_limit == NULL_TREE) |
| 11596 | thread_limit = build_int_cst (unsigned_type_node, 0); |
| 11597 | else |
| 11598 | { |
| 11599 | thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit); |
| 11600 | thread_limit = fold_convert (unsigned_type_node, thread_limit); |
| 11601 | gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val, |
| 11602 | fb_rvalue); |
| 11603 | } |
| 11604 | |
| 11605 | lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt), |
| 11606 | &bind_body, &dlist, ctx, NULL); |
| 11607 | lower_omp (gimple_omp_body_ptr (teams_stmt), ctx); |
| 11608 | lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist, ctx); |
| 11609 | gimple_seq_add_stmt (&bind_body, teams_stmt); |
| 11610 | |
| 11611 | location_t loc = gimple_location (teams_stmt); |
| 11612 | tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS); |
| 11613 | gimple call = gimple_build_call (decl, 2, num_teams, thread_limit); |
| 11614 | gimple_set_location (call, loc); |
| 11615 | gimple_seq_add_stmt (&bind_body, call); |
| 11616 | |
| 11617 | gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt)); |
| 11618 | gimple_omp_set_body (teams_stmt, NULL); |
| 11619 | gimple_seq_add_seq (&bind_body, olist); |
| 11620 | gimple_seq_add_seq (&bind_body, dlist); |
| 11621 | gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true)); |
| 11622 | gimple_bind_set_body (bind, bind_body); |
| 11623 | |
| 11624 | pop_gimplify_context (bind); |
| 11625 | |
| 11626 | gimple_bind_append_vars (bind, ctx->block_vars); |
| 11627 | BLOCK_VARS (block) = ctx->block_vars; |
| 11628 | if (BLOCK_VARS (block)) |
| 11629 | TREE_USED (block) = 1; |
| 11630 | } |
| 11631 | |
| 11632 | |
| 11633 | /* Callback for lower_omp_1. Return non-NULL if *tp needs to be |
| 11634 | regimplified. If DATA is non-NULL, lower_omp_1 is outside |
| 11635 | of OMP context, but with task_shared_vars set. */ |
| 11636 | |
| 11637 | static tree |
| 11638 | lower_omp_regimplify_p (tree *tp, int *walk_subtrees, |
| 11639 | void *data) |
| 11640 | { |
| 11641 | tree t = *tp; |
| 11642 | |
| 11643 | /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */ |
| 11644 | if (TREE_CODE (t) == VAR_DECL && data == NULL && DECL_HAS_VALUE_EXPR_P (t)) |
| 11645 | return t; |
| 11646 | |
| 11647 | if (task_shared_vars |
| 11648 | && DECL_P (t) |
| 11649 | && bitmap_bit_p (task_shared_vars, DECL_UID (t))) |
| 11650 | return t; |
| 11651 | |
| 11652 | /* If a global variable has been privatized, TREE_CONSTANT on |
| 11653 | ADDR_EXPR might be wrong. */ |
| 11654 | if (data == NULL && TREE_CODE (t) == ADDR_EXPR) |
| 11655 | recompute_tree_invariant_for_addr_expr (t); |
| 11656 | |
| 11657 | *walk_subtrees = !TYPE_P (t) && !DECL_P (t); |
| 11658 | return NULL_TREE; |
| 11659 | } |
| 11660 | |
| 11661 | static void |
| 11662 | lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx) |
| 11663 | { |
| 11664 | gimple stmt = gsi_stmt (*gsi_p); |
| 11665 | struct walk_stmt_info wi; |
| 11666 | gcall *call_stmt; |
| 11667 | |
| 11668 | if (gimple_has_location (stmt)) |
| 11669 | input_location = gimple_location (stmt); |
| 11670 | |
| 11671 | if (task_shared_vars) |
| 11672 | memset (&wi, '\0', sizeof (wi)); |
| 11673 | |
| 11674 | /* If we have issued syntax errors, avoid doing any heavy lifting. |
| 11675 | Just replace the OMP directives with a NOP to avoid |
| 11676 | confusing RTL expansion. */ |
| 11677 | if (seen_error () && is_gimple_omp (stmt)) |
| 11678 | { |
| 11679 | gsi_replace (gsi_p, gimple_build_nop (), true); |
| 11680 | return; |
| 11681 | } |
| 11682 | |
| 11683 | switch (gimple_code (stmt)) |
| 11684 | { |
| 11685 | case GIMPLE_COND: |
| 11686 | { |
| 11687 | gcond *cond_stmt = as_a <gcond *> (stmt); |
| 11688 | if ((ctx || task_shared_vars) |
| 11689 | && (walk_tree (gimple_cond_lhs_ptr (cond_stmt), |
| 11690 | lower_omp_regimplify_p, |
| 11691 | ctx ? NULL : &wi, NULL) |
| 11692 | || walk_tree (gimple_cond_rhs_ptr (cond_stmt), |
| 11693 | lower_omp_regimplify_p, |
| 11694 | ctx ? NULL : &wi, NULL))) |
| 11695 | gimple_regimplify_operands (cond_stmt, gsi_p); |
| 11696 | } |
| 11697 | break; |
| 11698 | case GIMPLE_CATCH: |
| 11699 | lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx); |
| 11700 | break; |
| 11701 | case GIMPLE_EH_FILTER: |
| 11702 | lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx); |
| 11703 | break; |
| 11704 | case GIMPLE_TRY: |
| 11705 | lower_omp (gimple_try_eval_ptr (stmt), ctx); |
| 11706 | lower_omp (gimple_try_cleanup_ptr (stmt), ctx); |
| 11707 | break; |
| 11708 | case GIMPLE_TRANSACTION: |
| 11709 | lower_omp (gimple_transaction_body_ptr ( |
| 11710 | as_a <gtransaction *> (stmt)), |
| 11711 | ctx); |
| 11712 | break; |
| 11713 | case GIMPLE_BIND: |
| 11714 | lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx); |
| 11715 | break; |
| 11716 | case GIMPLE_OMP_PARALLEL: |
| 11717 | case GIMPLE_OMP_TASK: |
| 11718 | ctx = maybe_lookup_ctx (stmt); |
| 11719 | gcc_assert (ctx); |
| 11720 | if (ctx->cancellable) |
| 11721 | ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); |
| 11722 | lower_omp_taskreg (gsi_p, ctx); |
| 11723 | break; |
| 11724 | case GIMPLE_OMP_FOR: |
| 11725 | ctx = maybe_lookup_ctx (stmt); |
| 11726 | gcc_assert (ctx); |
| 11727 | if (ctx->cancellable) |
| 11728 | ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); |
| 11729 | lower_omp_for (gsi_p, ctx); |
| 11730 | break; |
| 11731 | case GIMPLE_OMP_SECTIONS: |
| 11732 | ctx = maybe_lookup_ctx (stmt); |
| 11733 | gcc_assert (ctx); |
| 11734 | if (ctx->cancellable) |
| 11735 | ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); |
| 11736 | lower_omp_sections (gsi_p, ctx); |
| 11737 | break; |
| 11738 | case GIMPLE_OMP_SINGLE: |
| 11739 | ctx = maybe_lookup_ctx (stmt); |
| 11740 | gcc_assert (ctx); |
| 11741 | lower_omp_single (gsi_p, ctx); |
| 11742 | break; |
| 11743 | case GIMPLE_OMP_MASTER: |
| 11744 | ctx = maybe_lookup_ctx (stmt); |
| 11745 | gcc_assert (ctx); |
| 11746 | lower_omp_master (gsi_p, ctx); |
| 11747 | break; |
| 11748 | case GIMPLE_OMP_TASKGROUP: |
| 11749 | ctx = maybe_lookup_ctx (stmt); |
| 11750 | gcc_assert (ctx); |
| 11751 | lower_omp_taskgroup (gsi_p, ctx); |
| 11752 | break; |
| 11753 | case GIMPLE_OMP_ORDERED: |
| 11754 | ctx = maybe_lookup_ctx (stmt); |
| 11755 | gcc_assert (ctx); |
| 11756 | lower_omp_ordered (gsi_p, ctx); |
| 11757 | break; |
| 11758 | case GIMPLE_OMP_CRITICAL: |
| 11759 | ctx = maybe_lookup_ctx (stmt); |
| 11760 | gcc_assert (ctx); |
| 11761 | lower_omp_critical (gsi_p, ctx); |
| 11762 | break; |
| 11763 | case GIMPLE_OMP_ATOMIC_LOAD: |
| 11764 | if ((ctx || task_shared_vars) |
| 11765 | && walk_tree (gimple_omp_atomic_load_rhs_ptr ( |
| 11766 | as_a <gomp_atomic_load *> (stmt)), |
| 11767 | lower_omp_regimplify_p, ctx ? NULL : &wi, NULL)) |
| 11768 | gimple_regimplify_operands (stmt, gsi_p); |
| 11769 | break; |
| 11770 | case GIMPLE_OMP_TARGET: |
| 11771 | ctx = maybe_lookup_ctx (stmt); |
| 11772 | gcc_assert (ctx); |
| 11773 | lower_omp_target (gsi_p, ctx); |
| 11774 | break; |
| 11775 | case GIMPLE_OMP_TEAMS: |
| 11776 | ctx = maybe_lookup_ctx (stmt); |
| 11777 | gcc_assert (ctx); |
| 11778 | lower_omp_teams (gsi_p, ctx); |
| 11779 | break; |
| 11780 | case GIMPLE_CALL: |
| 11781 | tree fndecl; |
| 11782 | call_stmt = as_a <gcall *> (stmt); |
| 11783 | fndecl = gimple_call_fndecl (call_stmt); |
| 11784 | if (fndecl |
| 11785 | && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) |
| 11786 | switch (DECL_FUNCTION_CODE (fndecl)) |
| 11787 | { |
| 11788 | case BUILT_IN_GOMP_BARRIER: |
| 11789 | if (ctx == NULL) |
| 11790 | break; |
| 11791 | /* FALLTHRU */ |
| 11792 | case BUILT_IN_GOMP_CANCEL: |
| 11793 | case BUILT_IN_GOMP_CANCELLATION_POINT: |
| 11794 | omp_context *cctx; |
| 11795 | cctx = ctx; |
| 11796 | if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION) |
| 11797 | cctx = cctx->outer; |
| 11798 | gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE); |
| 11799 | if (!cctx->cancellable) |
| 11800 | { |
| 11801 | if (DECL_FUNCTION_CODE (fndecl) |
| 11802 | == BUILT_IN_GOMP_CANCELLATION_POINT) |
| 11803 | { |
| 11804 | stmt = gimple_build_nop (); |
| 11805 | gsi_replace (gsi_p, stmt, false); |
| 11806 | } |
| 11807 | break; |
| 11808 | } |
| 11809 | if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER) |
| 11810 | { |
| 11811 | fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL); |
| 11812 | gimple_call_set_fndecl (call_stmt, fndecl); |
| 11813 | gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl)); |
| 11814 | } |
| 11815 | tree lhs; |
| 11816 | lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl))); |
| 11817 | gimple_call_set_lhs (call_stmt, lhs); |
| 11818 | tree fallthru_label; |
| 11819 | fallthru_label = create_artificial_label (UNKNOWN_LOCATION); |
| 11820 | gimple g; |
| 11821 | g = gimple_build_label (fallthru_label); |
| 11822 | gsi_insert_after (gsi_p, g, GSI_SAME_STMT); |
| 11823 | g = gimple_build_cond (NE_EXPR, lhs, |
| 11824 | fold_convert (TREE_TYPE (lhs), |
| 11825 | boolean_false_node), |
| 11826 | cctx->cancel_label, fallthru_label); |
| 11827 | gsi_insert_after (gsi_p, g, GSI_SAME_STMT); |
| 11828 | break; |
| 11829 | default: |
| 11830 | break; |
| 11831 | } |
| 11832 | /* FALLTHRU */ |
| 11833 | default: |
| 11834 | if ((ctx || task_shared_vars) |
| 11835 | && walk_gimple_op (stmt, lower_omp_regimplify_p, |
| 11836 | ctx ? NULL : &wi)) |
| 11837 | { |
| 11838 | /* Just remove clobbers, this should happen only if we have |
| 11839 | "privatized" local addressable variables in SIMD regions, |
| 11840 | the clobber isn't needed in that case and gimplifying address |
| 11841 | of the ARRAY_REF into a pointer and creating MEM_REF based |
| 11842 | clobber would create worse code than we get with the clobber |
| 11843 | dropped. */ |
| 11844 | if (gimple_clobber_p (stmt)) |
| 11845 | { |
| 11846 | gsi_replace (gsi_p, gimple_build_nop (), true); |
| 11847 | break; |
| 11848 | } |
| 11849 | gimple_regimplify_operands (stmt, gsi_p); |
| 11850 | } |
| 11851 | break; |
| 11852 | } |
| 11853 | } |
| 11854 | |
| 11855 | static void |
| 11856 | lower_omp (gimple_seq *body, omp_context *ctx) |
| 11857 | { |
| 11858 | location_t saved_location = input_location; |
| 11859 | gimple_stmt_iterator gsi; |
| 11860 | for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi)) |
| 11861 | lower_omp_1 (&gsi, ctx); |
| 11862 | /* During gimplification, we haven't folded statments inside offloading |
| 11863 | regions (gimplify.c:maybe_fold_stmt); do that now. */ |
| 11864 | if (target_nesting_level) |
| 11865 | for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi)) |
| 11866 | fold_stmt (&gsi); |
| 11867 | input_location = saved_location; |
| 11868 | } |
| 11869 | \f |
| 11870 | /* Main entry point. */ |
| 11871 | |
| 11872 | static unsigned int |
| 11873 | execute_lower_omp (void) |
| 11874 | { |
| 11875 | gimple_seq body; |
| 11876 | int i; |
| 11877 | omp_context *ctx; |
| 11878 | |
| 11879 | /* This pass always runs, to provide PROP_gimple_lomp. |
| 11880 | But often, there is nothing to do. */ |
| 11881 | if (flag_cilkplus == 0 && flag_openacc == 0 && flag_openmp == 0 |
| 11882 | && flag_openmp_simd == 0) |
| 11883 | return 0; |
| 11884 | |
| 11885 | all_contexts = splay_tree_new (splay_tree_compare_pointers, 0, |
| 11886 | delete_omp_context); |
| 11887 | |
| 11888 | body = gimple_body (current_function_decl); |
| 11889 | scan_omp (&body, NULL); |
| 11890 | gcc_assert (taskreg_nesting_level == 0); |
| 11891 | FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx) |
| 11892 | finish_taskreg_scan (ctx); |
| 11893 | taskreg_contexts.release (); |
| 11894 | |
| 11895 | if (all_contexts->root) |
| 11896 | { |
| 11897 | if (task_shared_vars) |
| 11898 | push_gimplify_context (); |
| 11899 | lower_omp (&body, NULL); |
| 11900 | if (task_shared_vars) |
| 11901 | pop_gimplify_context (NULL); |
| 11902 | } |
| 11903 | |
| 11904 | if (all_contexts) |
| 11905 | { |
| 11906 | splay_tree_delete (all_contexts); |
| 11907 | all_contexts = NULL; |
| 11908 | } |
| 11909 | BITMAP_FREE (task_shared_vars); |
| 11910 | return 0; |
| 11911 | } |
| 11912 | |
| 11913 | namespace { |
| 11914 | |
| 11915 | const pass_data pass_data_lower_omp = |
| 11916 | { |
| 11917 | GIMPLE_PASS, /* type */ |
| 11918 | "omplower", /* name */ |
| 11919 | OPTGROUP_NONE, /* optinfo_flags */ |
| 11920 | TV_NONE, /* tv_id */ |
| 11921 | PROP_gimple_any, /* properties_required */ |
| 11922 | PROP_gimple_lomp, /* properties_provided */ |
| 11923 | 0, /* properties_destroyed */ |
| 11924 | 0, /* todo_flags_start */ |
| 11925 | 0, /* todo_flags_finish */ |
| 11926 | }; |
| 11927 | |
| 11928 | class pass_lower_omp : public gimple_opt_pass |
| 11929 | { |
| 11930 | public: |
| 11931 | pass_lower_omp (gcc::context *ctxt) |
| 11932 | : gimple_opt_pass (pass_data_lower_omp, ctxt) |
| 11933 | {} |
| 11934 | |
| 11935 | /* opt_pass methods: */ |
| 11936 | virtual unsigned int execute (function *) { return execute_lower_omp (); } |
| 11937 | |
| 11938 | }; // class pass_lower_omp |
| 11939 | |
| 11940 | } // anon namespace |
| 11941 | |
| 11942 | gimple_opt_pass * |
| 11943 | make_pass_lower_omp (gcc::context *ctxt) |
| 11944 | { |
| 11945 | return new pass_lower_omp (ctxt); |
| 11946 | } |
| 11947 | \f |
| 11948 | /* The following is a utility to diagnose structured block violations. |
| 11949 | It is not part of the "omplower" pass, as that's invoked too late. It |
| 11950 | should be invoked by the respective front ends after gimplification. */ |
| 11951 | |
| 11952 | static splay_tree all_labels; |
| 11953 | |
| 11954 | /* Check for mismatched contexts and generate an error if needed. Return |
| 11955 | true if an error is detected. */ |
| 11956 | |
| 11957 | static bool |
| 11958 | diagnose_sb_0 (gimple_stmt_iterator *gsi_p, |
| 11959 | gimple branch_ctx, gimple label_ctx) |
| 11960 | { |
| 11961 | gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx)); |
| 11962 | gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx)); |
| 11963 | |
| 11964 | if (label_ctx == branch_ctx) |
| 11965 | return false; |
| 11966 | |
| 11967 | const char* kind = NULL; |
| 11968 | |
| 11969 | if (flag_cilkplus) |
| 11970 | { |
| 11971 | if ((branch_ctx |
| 11972 | && gimple_code (branch_ctx) == GIMPLE_OMP_FOR |
| 11973 | && gimple_omp_for_kind (branch_ctx) == GF_OMP_FOR_KIND_CILKSIMD) |
| 11974 | || (label_ctx |
| 11975 | && gimple_code (label_ctx) == GIMPLE_OMP_FOR |
| 11976 | && gimple_omp_for_kind (label_ctx) == GF_OMP_FOR_KIND_CILKSIMD)) |
| 11977 | kind = "Cilk Plus"; |
| 11978 | } |
| 11979 | if (flag_openacc) |
| 11980 | { |
| 11981 | if ((branch_ctx && is_gimple_omp_oacc (branch_ctx)) |
| 11982 | || (label_ctx && is_gimple_omp_oacc (label_ctx))) |
| 11983 | { |
| 11984 | gcc_checking_assert (kind == NULL); |
| 11985 | kind = "OpenACC"; |
| 11986 | } |
| 11987 | } |
| 11988 | if (kind == NULL) |
| 11989 | { |
| 11990 | gcc_checking_assert (flag_openmp); |
| 11991 | kind = "OpenMP"; |
| 11992 | } |
| 11993 | |
| 11994 | /* |
| 11995 | Previously we kept track of the label's entire context in diagnose_sb_[12] |
| 11996 | so we could traverse it and issue a correct "exit" or "enter" error |
| 11997 | message upon a structured block violation. |
| 11998 | |
| 11999 | We built the context by building a list with tree_cons'ing, but there is |
| 12000 | no easy counterpart in gimple tuples. It seems like far too much work |
| 12001 | for issuing exit/enter error messages. If someone really misses the |
| 12002 | distinct error message... patches welcome. |
| 12003 | */ |
| 12004 | |
| 12005 | #if 0 |
| 12006 | /* Try to avoid confusing the user by producing and error message |
| 12007 | with correct "exit" or "enter" verbiage. We prefer "exit" |
| 12008 | unless we can show that LABEL_CTX is nested within BRANCH_CTX. */ |
| 12009 | if (branch_ctx == NULL) |
| 12010 | exit_p = false; |
| 12011 | else |
| 12012 | { |
| 12013 | while (label_ctx) |
| 12014 | { |
| 12015 | if (TREE_VALUE (label_ctx) == branch_ctx) |
| 12016 | { |
| 12017 | exit_p = false; |
| 12018 | break; |
| 12019 | } |
| 12020 | label_ctx = TREE_CHAIN (label_ctx); |
| 12021 | } |
| 12022 | } |
| 12023 | |
| 12024 | if (exit_p) |
| 12025 | error ("invalid exit from %s structured block", kind); |
| 12026 | else |
| 12027 | error ("invalid entry to %s structured block", kind); |
| 12028 | #endif |
| 12029 | |
| 12030 | /* If it's obvious we have an invalid entry, be specific about the error. */ |
| 12031 | if (branch_ctx == NULL) |
| 12032 | error ("invalid entry to %s structured block", kind); |
| 12033 | else |
| 12034 | { |
| 12035 | /* Otherwise, be vague and lazy, but efficient. */ |
| 12036 | error ("invalid branch to/from %s structured block", kind); |
| 12037 | } |
| 12038 | |
| 12039 | gsi_replace (gsi_p, gimple_build_nop (), false); |
| 12040 | return true; |
| 12041 | } |
| 12042 | |
| 12043 | /* Pass 1: Create a minimal tree of structured blocks, and record |
| 12044 | where each label is found. */ |
| 12045 | |
| 12046 | static tree |
| 12047 | diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p, |
| 12048 | struct walk_stmt_info *wi) |
| 12049 | { |
| 12050 | gimple context = (gimple) wi->info; |
| 12051 | gimple inner_context; |
| 12052 | gimple stmt = gsi_stmt (*gsi_p); |
| 12053 | |
| 12054 | *handled_ops_p = true; |
| 12055 | |
| 12056 | switch (gimple_code (stmt)) |
| 12057 | { |
| 12058 | WALK_SUBSTMTS; |
| 12059 | |
| 12060 | case GIMPLE_OMP_PARALLEL: |
| 12061 | case GIMPLE_OMP_TASK: |
| 12062 | case GIMPLE_OMP_SECTIONS: |
| 12063 | case GIMPLE_OMP_SINGLE: |
| 12064 | case GIMPLE_OMP_SECTION: |
| 12065 | case GIMPLE_OMP_MASTER: |
| 12066 | case GIMPLE_OMP_ORDERED: |
| 12067 | case GIMPLE_OMP_CRITICAL: |
| 12068 | case GIMPLE_OMP_TARGET: |
| 12069 | case GIMPLE_OMP_TEAMS: |
| 12070 | case GIMPLE_OMP_TASKGROUP: |
| 12071 | /* The minimal context here is just the current OMP construct. */ |
| 12072 | inner_context = stmt; |
| 12073 | wi->info = inner_context; |
| 12074 | walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi); |
| 12075 | wi->info = context; |
| 12076 | break; |
| 12077 | |
| 12078 | case GIMPLE_OMP_FOR: |
| 12079 | inner_context = stmt; |
| 12080 | wi->info = inner_context; |
| 12081 | /* gimple_omp_for_{index,initial,final} are all DECLs; no need to |
| 12082 | walk them. */ |
| 12083 | walk_gimple_seq (gimple_omp_for_pre_body (stmt), |
| 12084 | diagnose_sb_1, NULL, wi); |
| 12085 | walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi); |
| 12086 | wi->info = context; |
| 12087 | break; |
| 12088 | |
| 12089 | case GIMPLE_LABEL: |
| 12090 | splay_tree_insert (all_labels, |
| 12091 | (splay_tree_key) gimple_label_label ( |
| 12092 | as_a <glabel *> (stmt)), |
| 12093 | (splay_tree_value) context); |
| 12094 | break; |
| 12095 | |
| 12096 | default: |
| 12097 | break; |
| 12098 | } |
| 12099 | |
| 12100 | return NULL_TREE; |
| 12101 | } |
| 12102 | |
| 12103 | /* Pass 2: Check each branch and see if its context differs from that of |
| 12104 | the destination label's context. */ |
| 12105 | |
| 12106 | static tree |
| 12107 | diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p, |
| 12108 | struct walk_stmt_info *wi) |
| 12109 | { |
| 12110 | gimple context = (gimple) wi->info; |
| 12111 | splay_tree_node n; |
| 12112 | gimple stmt = gsi_stmt (*gsi_p); |
| 12113 | |
| 12114 | *handled_ops_p = true; |
| 12115 | |
| 12116 | switch (gimple_code (stmt)) |
| 12117 | { |
| 12118 | WALK_SUBSTMTS; |
| 12119 | |
| 12120 | case GIMPLE_OMP_PARALLEL: |
| 12121 | case GIMPLE_OMP_TASK: |
| 12122 | case GIMPLE_OMP_SECTIONS: |
| 12123 | case GIMPLE_OMP_SINGLE: |
| 12124 | case GIMPLE_OMP_SECTION: |
| 12125 | case GIMPLE_OMP_MASTER: |
| 12126 | case GIMPLE_OMP_ORDERED: |
| 12127 | case GIMPLE_OMP_CRITICAL: |
| 12128 | case GIMPLE_OMP_TARGET: |
| 12129 | case GIMPLE_OMP_TEAMS: |
| 12130 | case GIMPLE_OMP_TASKGROUP: |
| 12131 | wi->info = stmt; |
| 12132 | walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi); |
| 12133 | wi->info = context; |
| 12134 | break; |
| 12135 | |
| 12136 | case GIMPLE_OMP_FOR: |
| 12137 | wi->info = stmt; |
| 12138 | /* gimple_omp_for_{index,initial,final} are all DECLs; no need to |
| 12139 | walk them. */ |
| 12140 | walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt), |
| 12141 | diagnose_sb_2, NULL, wi); |
| 12142 | walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi); |
| 12143 | wi->info = context; |
| 12144 | break; |
| 12145 | |
| 12146 | case GIMPLE_COND: |
| 12147 | { |
| 12148 | gcond *cond_stmt = as_a <gcond *> (stmt); |
| 12149 | tree lab = gimple_cond_true_label (cond_stmt); |
| 12150 | if (lab) |
| 12151 | { |
| 12152 | n = splay_tree_lookup (all_labels, |
| 12153 | (splay_tree_key) lab); |
| 12154 | diagnose_sb_0 (gsi_p, context, |
| 12155 | n ? (gimple) n->value : NULL); |
| 12156 | } |
| 12157 | lab = gimple_cond_false_label (cond_stmt); |
| 12158 | if (lab) |
| 12159 | { |
| 12160 | n = splay_tree_lookup (all_labels, |
| 12161 | (splay_tree_key) lab); |
| 12162 | diagnose_sb_0 (gsi_p, context, |
| 12163 | n ? (gimple) n->value : NULL); |
| 12164 | } |
| 12165 | } |
| 12166 | break; |
| 12167 | |
| 12168 | case GIMPLE_GOTO: |
| 12169 | { |
| 12170 | tree lab = gimple_goto_dest (stmt); |
| 12171 | if (TREE_CODE (lab) != LABEL_DECL) |
| 12172 | break; |
| 12173 | |
| 12174 | n = splay_tree_lookup (all_labels, (splay_tree_key) lab); |
| 12175 | diagnose_sb_0 (gsi_p, context, n ? (gimple) n->value : NULL); |
| 12176 | } |
| 12177 | break; |
| 12178 | |
| 12179 | case GIMPLE_SWITCH: |
| 12180 | { |
| 12181 | gswitch *switch_stmt = as_a <gswitch *> (stmt); |
| 12182 | unsigned int i; |
| 12183 | for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i) |
| 12184 | { |
| 12185 | tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i)); |
| 12186 | n = splay_tree_lookup (all_labels, (splay_tree_key) lab); |
| 12187 | if (n && diagnose_sb_0 (gsi_p, context, (gimple) n->value)) |
| 12188 | break; |
| 12189 | } |
| 12190 | } |
| 12191 | break; |
| 12192 | |
| 12193 | case GIMPLE_RETURN: |
| 12194 | diagnose_sb_0 (gsi_p, context, NULL); |
| 12195 | break; |
| 12196 | |
| 12197 | default: |
| 12198 | break; |
| 12199 | } |
| 12200 | |
| 12201 | return NULL_TREE; |
| 12202 | } |
| 12203 | |
| 12204 | /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant |
| 12205 | GIMPLE_* codes. */ |
| 12206 | bool |
| 12207 | make_gimple_omp_edges (basic_block bb, struct omp_region **region, |
| 12208 | int *region_idx) |
| 12209 | { |
| 12210 | gimple last = last_stmt (bb); |
| 12211 | enum gimple_code code = gimple_code (last); |
| 12212 | struct omp_region *cur_region = *region; |
| 12213 | bool fallthru = false; |
| 12214 | |
| 12215 | switch (code) |
| 12216 | { |
| 12217 | case GIMPLE_OMP_PARALLEL: |
| 12218 | case GIMPLE_OMP_TASK: |
| 12219 | case GIMPLE_OMP_FOR: |
| 12220 | case GIMPLE_OMP_SINGLE: |
| 12221 | case GIMPLE_OMP_TEAMS: |
| 12222 | case GIMPLE_OMP_MASTER: |
| 12223 | case GIMPLE_OMP_TASKGROUP: |
| 12224 | case GIMPLE_OMP_ORDERED: |
| 12225 | case GIMPLE_OMP_CRITICAL: |
| 12226 | case GIMPLE_OMP_SECTION: |
| 12227 | cur_region = new_omp_region (bb, code, cur_region); |
| 12228 | fallthru = true; |
| 12229 | break; |
| 12230 | |
| 12231 | case GIMPLE_OMP_TARGET: |
| 12232 | cur_region = new_omp_region (bb, code, cur_region); |
| 12233 | fallthru = true; |
| 12234 | switch (gimple_omp_target_kind (last)) |
| 12235 | { |
| 12236 | case GF_OMP_TARGET_KIND_REGION: |
| 12237 | case GF_OMP_TARGET_KIND_DATA: |
| 12238 | case GF_OMP_TARGET_KIND_OACC_PARALLEL: |
| 12239 | case GF_OMP_TARGET_KIND_OACC_KERNELS: |
| 12240 | case GF_OMP_TARGET_KIND_OACC_DATA: |
| 12241 | break; |
| 12242 | case GF_OMP_TARGET_KIND_UPDATE: |
| 12243 | case GF_OMP_TARGET_KIND_OACC_UPDATE: |
| 12244 | case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: |
| 12245 | cur_region = cur_region->outer; |
| 12246 | break; |
| 12247 | default: |
| 12248 | gcc_unreachable (); |
| 12249 | } |
| 12250 | break; |
| 12251 | |
| 12252 | case GIMPLE_OMP_SECTIONS: |
| 12253 | cur_region = new_omp_region (bb, code, cur_region); |
| 12254 | fallthru = true; |
| 12255 | break; |
| 12256 | |
| 12257 | case GIMPLE_OMP_SECTIONS_SWITCH: |
| 12258 | fallthru = false; |
| 12259 | break; |
| 12260 | |
| 12261 | case GIMPLE_OMP_ATOMIC_LOAD: |
| 12262 | case GIMPLE_OMP_ATOMIC_STORE: |
| 12263 | fallthru = true; |
| 12264 | break; |
| 12265 | |
| 12266 | case GIMPLE_OMP_RETURN: |
| 12267 | /* In the case of a GIMPLE_OMP_SECTION, the edge will go |
| 12268 | somewhere other than the next block. This will be |
| 12269 | created later. */ |
| 12270 | cur_region->exit = bb; |
| 12271 | fallthru = cur_region->type != GIMPLE_OMP_SECTION; |
| 12272 | cur_region = cur_region->outer; |
| 12273 | break; |
| 12274 | |
| 12275 | case GIMPLE_OMP_CONTINUE: |
| 12276 | cur_region->cont = bb; |
| 12277 | switch (cur_region->type) |
| 12278 | { |
| 12279 | case GIMPLE_OMP_FOR: |
| 12280 | /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE |
| 12281 | succs edges as abnormal to prevent splitting |
| 12282 | them. */ |
| 12283 | single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL; |
| 12284 | /* Make the loopback edge. */ |
| 12285 | make_edge (bb, single_succ (cur_region->entry), |
| 12286 | EDGE_ABNORMAL); |
| 12287 | |
| 12288 | /* Create an edge from GIMPLE_OMP_FOR to exit, which |
| 12289 | corresponds to the case that the body of the loop |
| 12290 | is not executed at all. */ |
| 12291 | make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL); |
| 12292 | make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL); |
| 12293 | fallthru = false; |
| 12294 | break; |
| 12295 | |
| 12296 | case GIMPLE_OMP_SECTIONS: |
| 12297 | /* Wire up the edges into and out of the nested sections. */ |
| 12298 | { |
| 12299 | basic_block switch_bb = single_succ (cur_region->entry); |
| 12300 | |
| 12301 | struct omp_region *i; |
| 12302 | for (i = cur_region->inner; i ; i = i->next) |
| 12303 | { |
| 12304 | gcc_assert (i->type == GIMPLE_OMP_SECTION); |
| 12305 | make_edge (switch_bb, i->entry, 0); |
| 12306 | make_edge (i->exit, bb, EDGE_FALLTHRU); |
| 12307 | } |
| 12308 | |
| 12309 | /* Make the loopback edge to the block with |
| 12310 | GIMPLE_OMP_SECTIONS_SWITCH. */ |
| 12311 | make_edge (bb, switch_bb, 0); |
| 12312 | |
| 12313 | /* Make the edge from the switch to exit. */ |
| 12314 | make_edge (switch_bb, bb->next_bb, 0); |
| 12315 | fallthru = false; |
| 12316 | } |
| 12317 | break; |
| 12318 | |
| 12319 | default: |
| 12320 | gcc_unreachable (); |
| 12321 | } |
| 12322 | break; |
| 12323 | |
| 12324 | default: |
| 12325 | gcc_unreachable (); |
| 12326 | } |
| 12327 | |
| 12328 | if (*region != cur_region) |
| 12329 | { |
| 12330 | *region = cur_region; |
| 12331 | if (cur_region) |
| 12332 | *region_idx = cur_region->entry->index; |
| 12333 | else |
| 12334 | *region_idx = 0; |
| 12335 | } |
| 12336 | |
| 12337 | return fallthru; |
| 12338 | } |
| 12339 | |
| 12340 | static unsigned int |
| 12341 | diagnose_omp_structured_block_errors (void) |
| 12342 | { |
| 12343 | struct walk_stmt_info wi; |
| 12344 | gimple_seq body = gimple_body (current_function_decl); |
| 12345 | |
| 12346 | all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0); |
| 12347 | |
| 12348 | memset (&wi, 0, sizeof (wi)); |
| 12349 | walk_gimple_seq (body, diagnose_sb_1, NULL, &wi); |
| 12350 | |
| 12351 | memset (&wi, 0, sizeof (wi)); |
| 12352 | wi.want_locations = true; |
| 12353 | walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi); |
| 12354 | |
| 12355 | gimple_set_body (current_function_decl, body); |
| 12356 | |
| 12357 | splay_tree_delete (all_labels); |
| 12358 | all_labels = NULL; |
| 12359 | |
| 12360 | return 0; |
| 12361 | } |
| 12362 | |
| 12363 | namespace { |
| 12364 | |
| 12365 | const pass_data pass_data_diagnose_omp_blocks = |
| 12366 | { |
| 12367 | GIMPLE_PASS, /* type */ |
| 12368 | "*diagnose_omp_blocks", /* name */ |
| 12369 | OPTGROUP_NONE, /* optinfo_flags */ |
| 12370 | TV_NONE, /* tv_id */ |
| 12371 | PROP_gimple_any, /* properties_required */ |
| 12372 | 0, /* properties_provided */ |
| 12373 | 0, /* properties_destroyed */ |
| 12374 | 0, /* todo_flags_start */ |
| 12375 | 0, /* todo_flags_finish */ |
| 12376 | }; |
| 12377 | |
| 12378 | class pass_diagnose_omp_blocks : public gimple_opt_pass |
| 12379 | { |
| 12380 | public: |
| 12381 | pass_diagnose_omp_blocks (gcc::context *ctxt) |
| 12382 | : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt) |
| 12383 | {} |
| 12384 | |
| 12385 | /* opt_pass methods: */ |
| 12386 | virtual bool gate (function *) |
| 12387 | { |
| 12388 | return flag_cilkplus || flag_openacc || flag_openmp; |
| 12389 | } |
| 12390 | virtual unsigned int execute (function *) |
| 12391 | { |
| 12392 | return diagnose_omp_structured_block_errors (); |
| 12393 | } |
| 12394 | |
| 12395 | }; // class pass_diagnose_omp_blocks |
| 12396 | |
| 12397 | } // anon namespace |
| 12398 | |
| 12399 | gimple_opt_pass * |
| 12400 | make_pass_diagnose_omp_blocks (gcc::context *ctxt) |
| 12401 | { |
| 12402 | return new pass_diagnose_omp_blocks (ctxt); |
| 12403 | } |
| 12404 | \f |
| 12405 | /* SIMD clone supporting code. */ |
| 12406 | |
| 12407 | /* Allocate a fresh `simd_clone' and return it. NARGS is the number |
| 12408 | of arguments to reserve space for. */ |
| 12409 | |
| 12410 | static struct cgraph_simd_clone * |
| 12411 | simd_clone_struct_alloc (int nargs) |
| 12412 | { |
| 12413 | struct cgraph_simd_clone *clone_info; |
| 12414 | size_t len = (sizeof (struct cgraph_simd_clone) |
| 12415 | + nargs * sizeof (struct cgraph_simd_clone_arg)); |
| 12416 | clone_info = (struct cgraph_simd_clone *) |
| 12417 | ggc_internal_cleared_alloc (len); |
| 12418 | return clone_info; |
| 12419 | } |
| 12420 | |
| 12421 | /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */ |
| 12422 | |
| 12423 | static inline void |
| 12424 | simd_clone_struct_copy (struct cgraph_simd_clone *to, |
| 12425 | struct cgraph_simd_clone *from) |
| 12426 | { |
| 12427 | memcpy (to, from, (sizeof (struct cgraph_simd_clone) |
| 12428 | + ((from->nargs - from->inbranch) |
| 12429 | * sizeof (struct cgraph_simd_clone_arg)))); |
| 12430 | } |
| 12431 | |
| 12432 | /* Return vector of parameter types of function FNDECL. This uses |
| 12433 | TYPE_ARG_TYPES if available, otherwise falls back to types of |
| 12434 | DECL_ARGUMENTS types. */ |
| 12435 | |
| 12436 | vec<tree> |
| 12437 | simd_clone_vector_of_formal_parm_types (tree fndecl) |
| 12438 | { |
| 12439 | if (TYPE_ARG_TYPES (TREE_TYPE (fndecl))) |
| 12440 | return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl)); |
| 12441 | vec<tree> args = ipa_get_vector_of_formal_parms (fndecl); |
| 12442 | unsigned int i; |
| 12443 | tree arg; |
| 12444 | FOR_EACH_VEC_ELT (args, i, arg) |
| 12445 | args[i] = TREE_TYPE (args[i]); |
| 12446 | return args; |
| 12447 | } |
| 12448 | |
| 12449 | /* Given a simd function in NODE, extract the simd specific |
| 12450 | information from the OMP clauses passed in CLAUSES, and return |
| 12451 | the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED |
| 12452 | is set to TRUE if the `inbranch' or `notinbranch' clause specified, |
| 12453 | otherwise set to FALSE. */ |
| 12454 | |
| 12455 | static struct cgraph_simd_clone * |
| 12456 | simd_clone_clauses_extract (struct cgraph_node *node, tree clauses, |
| 12457 | bool *inbranch_specified) |
| 12458 | { |
| 12459 | vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl); |
| 12460 | tree t; |
| 12461 | int n; |
| 12462 | *inbranch_specified = false; |
| 12463 | |
| 12464 | n = args.length (); |
| 12465 | if (n > 0 && args.last () == void_type_node) |
| 12466 | n--; |
| 12467 | |
| 12468 | /* To distinguish from an OpenMP simd clone, Cilk Plus functions to |
| 12469 | be cloned have a distinctive artificial label in addition to "omp |
| 12470 | declare simd". */ |
| 12471 | bool cilk_clone |
| 12472 | = (flag_cilkplus |
| 12473 | && lookup_attribute ("cilk simd function", |
| 12474 | DECL_ATTRIBUTES (node->decl))); |
| 12475 | |
| 12476 | /* Allocate one more than needed just in case this is an in-branch |
| 12477 | clone which will require a mask argument. */ |
| 12478 | struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1); |
| 12479 | clone_info->nargs = n; |
| 12480 | clone_info->cilk_elemental = cilk_clone; |
| 12481 | |
| 12482 | if (!clauses) |
| 12483 | { |
| 12484 | args.release (); |
| 12485 | return clone_info; |
| 12486 | } |
| 12487 | clauses = TREE_VALUE (clauses); |
| 12488 | if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE) |
| 12489 | return clone_info; |
| 12490 | |
| 12491 | for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t)) |
| 12492 | { |
| 12493 | switch (OMP_CLAUSE_CODE (t)) |
| 12494 | { |
| 12495 | case OMP_CLAUSE_INBRANCH: |
| 12496 | clone_info->inbranch = 1; |
| 12497 | *inbranch_specified = true; |
| 12498 | break; |
| 12499 | case OMP_CLAUSE_NOTINBRANCH: |
| 12500 | clone_info->inbranch = 0; |
| 12501 | *inbranch_specified = true; |
| 12502 | break; |
| 12503 | case OMP_CLAUSE_SIMDLEN: |
| 12504 | clone_info->simdlen |
| 12505 | = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t)); |
| 12506 | break; |
| 12507 | case OMP_CLAUSE_LINEAR: |
| 12508 | { |
| 12509 | tree decl = OMP_CLAUSE_DECL (t); |
| 12510 | tree step = OMP_CLAUSE_LINEAR_STEP (t); |
| 12511 | int argno = TREE_INT_CST_LOW (decl); |
| 12512 | if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t)) |
| 12513 | { |
| 12514 | clone_info->args[argno].arg_type |
| 12515 | = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP; |
| 12516 | clone_info->args[argno].linear_step = tree_to_shwi (step); |
| 12517 | gcc_assert (clone_info->args[argno].linear_step >= 0 |
| 12518 | && clone_info->args[argno].linear_step < n); |
| 12519 | } |
| 12520 | else |
| 12521 | { |
| 12522 | if (POINTER_TYPE_P (args[argno])) |
| 12523 | step = fold_convert (ssizetype, step); |
| 12524 | if (!tree_fits_shwi_p (step)) |
| 12525 | { |
| 12526 | warning_at (OMP_CLAUSE_LOCATION (t), 0, |
| 12527 | "ignoring large linear step"); |
| 12528 | args.release (); |
| 12529 | return NULL; |
| 12530 | } |
| 12531 | else if (integer_zerop (step)) |
| 12532 | { |
| 12533 | warning_at (OMP_CLAUSE_LOCATION (t), 0, |
| 12534 | "ignoring zero linear step"); |
| 12535 | args.release (); |
| 12536 | return NULL; |
| 12537 | } |
| 12538 | else |
| 12539 | { |
| 12540 | clone_info->args[argno].arg_type |
| 12541 | = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP; |
| 12542 | clone_info->args[argno].linear_step = tree_to_shwi (step); |
| 12543 | } |
| 12544 | } |
| 12545 | break; |
| 12546 | } |
| 12547 | case OMP_CLAUSE_UNIFORM: |
| 12548 | { |
| 12549 | tree decl = OMP_CLAUSE_DECL (t); |
| 12550 | int argno = tree_to_uhwi (decl); |
| 12551 | clone_info->args[argno].arg_type |
| 12552 | = SIMD_CLONE_ARG_TYPE_UNIFORM; |
| 12553 | break; |
| 12554 | } |
| 12555 | case OMP_CLAUSE_ALIGNED: |
| 12556 | { |
| 12557 | tree decl = OMP_CLAUSE_DECL (t); |
| 12558 | int argno = tree_to_uhwi (decl); |
| 12559 | clone_info->args[argno].alignment |
| 12560 | = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t)); |
| 12561 | break; |
| 12562 | } |
| 12563 | default: |
| 12564 | break; |
| 12565 | } |
| 12566 | } |
| 12567 | args.release (); |
| 12568 | return clone_info; |
| 12569 | } |
| 12570 | |
| 12571 | /* Given a SIMD clone in NODE, calculate the characteristic data |
| 12572 | type and return the coresponding type. The characteristic data |
| 12573 | type is computed as described in the Intel Vector ABI. */ |
| 12574 | |
| 12575 | static tree |
| 12576 | simd_clone_compute_base_data_type (struct cgraph_node *node, |
| 12577 | struct cgraph_simd_clone *clone_info) |
| 12578 | { |
| 12579 | tree type = integer_type_node; |
| 12580 | tree fndecl = node->decl; |
| 12581 | |
| 12582 | /* a) For non-void function, the characteristic data type is the |
| 12583 | return type. */ |
| 12584 | if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE) |
| 12585 | type = TREE_TYPE (TREE_TYPE (fndecl)); |
| 12586 | |
| 12587 | /* b) If the function has any non-uniform, non-linear parameters, |
| 12588 | then the characteristic data type is the type of the first |
| 12589 | such parameter. */ |
| 12590 | else |
| 12591 | { |
| 12592 | vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl); |
| 12593 | for (unsigned int i = 0; i < clone_info->nargs; ++i) |
| 12594 | if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR) |
| 12595 | { |
| 12596 | type = map[i]; |
| 12597 | break; |
| 12598 | } |
| 12599 | map.release (); |
| 12600 | } |
| 12601 | |
| 12602 | /* c) If the characteristic data type determined by a) or b) above |
| 12603 | is struct, union, or class type which is pass-by-value (except |
| 12604 | for the type that maps to the built-in complex data type), the |
| 12605 | characteristic data type is int. */ |
| 12606 | if (RECORD_OR_UNION_TYPE_P (type) |
| 12607 | && !aggregate_value_p (type, NULL) |
| 12608 | && TREE_CODE (type) != COMPLEX_TYPE) |
| 12609 | return integer_type_node; |
| 12610 | |
| 12611 | /* d) If none of the above three classes is applicable, the |
| 12612 | characteristic data type is int. */ |
| 12613 | |
| 12614 | return type; |
| 12615 | |
| 12616 | /* e) For Intel Xeon Phi native and offload compilation, if the |
| 12617 | resulting characteristic data type is 8-bit or 16-bit integer |
| 12618 | data type, the characteristic data type is int. */ |
| 12619 | /* Well, we don't handle Xeon Phi yet. */ |
| 12620 | } |
| 12621 | |
| 12622 | static tree |
| 12623 | simd_clone_mangle (struct cgraph_node *node, |
| 12624 | struct cgraph_simd_clone *clone_info) |
| 12625 | { |
| 12626 | char vecsize_mangle = clone_info->vecsize_mangle; |
| 12627 | char mask = clone_info->inbranch ? 'M' : 'N'; |
| 12628 | unsigned int simdlen = clone_info->simdlen; |
| 12629 | unsigned int n; |
| 12630 | pretty_printer pp; |
| 12631 | |
| 12632 | gcc_assert (vecsize_mangle && simdlen); |
| 12633 | |
| 12634 | pp_string (&pp, "_ZGV"); |
| 12635 | pp_character (&pp, vecsize_mangle); |
| 12636 | pp_character (&pp, mask); |
| 12637 | pp_decimal_int (&pp, simdlen); |
| 12638 | |
| 12639 | for (n = 0; n < clone_info->nargs; ++n) |
| 12640 | { |
| 12641 | struct cgraph_simd_clone_arg arg = clone_info->args[n]; |
| 12642 | |
| 12643 | if (arg.arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM) |
| 12644 | pp_character (&pp, 'u'); |
| 12645 | else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP) |
| 12646 | { |
| 12647 | gcc_assert (arg.linear_step != 0); |
| 12648 | pp_character (&pp, 'l'); |
| 12649 | if (arg.linear_step > 1) |
| 12650 | pp_unsigned_wide_integer (&pp, arg.linear_step); |
| 12651 | else if (arg.linear_step < 0) |
| 12652 | { |
| 12653 | pp_character (&pp, 'n'); |
| 12654 | pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT) |
| 12655 | arg.linear_step)); |
| 12656 | } |
| 12657 | } |
| 12658 | else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP) |
| 12659 | { |
| 12660 | pp_character (&pp, 's'); |
| 12661 | pp_unsigned_wide_integer (&pp, arg.linear_step); |
| 12662 | } |
| 12663 | else |
| 12664 | pp_character (&pp, 'v'); |
| 12665 | if (arg.alignment) |
| 12666 | { |
| 12667 | pp_character (&pp, 'a'); |
| 12668 | pp_decimal_int (&pp, arg.alignment); |
| 12669 | } |
| 12670 | } |
| 12671 | |
| 12672 | pp_underscore (&pp); |
| 12673 | const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl)); |
| 12674 | if (*str == '*') |
| 12675 | ++str; |
| 12676 | pp_string (&pp, str); |
| 12677 | str = pp_formatted_text (&pp); |
| 12678 | |
| 12679 | /* If there already is a SIMD clone with the same mangled name, don't |
| 12680 | add another one. This can happen e.g. for |
| 12681 | #pragma omp declare simd |
| 12682 | #pragma omp declare simd simdlen(8) |
| 12683 | int foo (int, int); |
| 12684 | if the simdlen is assumed to be 8 for the first one, etc. */ |
| 12685 | for (struct cgraph_node *clone = node->simd_clones; clone; |
| 12686 | clone = clone->simdclone->next_clone) |
| 12687 | if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)), |
| 12688 | str) == 0) |
| 12689 | return NULL_TREE; |
| 12690 | |
| 12691 | return get_identifier (str); |
| 12692 | } |
| 12693 | |
| 12694 | /* Create a simd clone of OLD_NODE and return it. */ |
| 12695 | |
| 12696 | static struct cgraph_node * |
| 12697 | simd_clone_create (struct cgraph_node *old_node) |
| 12698 | { |
| 12699 | struct cgraph_node *new_node; |
| 12700 | if (old_node->definition) |
| 12701 | { |
| 12702 | if (!old_node->has_gimple_body_p ()) |
| 12703 | return NULL; |
| 12704 | old_node->get_body (); |
| 12705 | new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL, |
| 12706 | false, NULL, NULL, |
| 12707 | "simdclone"); |
| 12708 | } |
| 12709 | else |
| 12710 | { |
| 12711 | tree old_decl = old_node->decl; |
| 12712 | tree new_decl = copy_node (old_node->decl); |
| 12713 | DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone"); |
| 12714 | SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); |
| 12715 | SET_DECL_RTL (new_decl, NULL); |
| 12716 | DECL_STATIC_CONSTRUCTOR (new_decl) = 0; |
| 12717 | DECL_STATIC_DESTRUCTOR (new_decl) = 0; |
| 12718 | new_node = old_node->create_version_clone (new_decl, vNULL, NULL); |
| 12719 | symtab->call_cgraph_insertion_hooks (new_node); |
| 12720 | } |
| 12721 | if (new_node == NULL) |
| 12722 | return new_node; |
| 12723 | |
| 12724 | TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl); |
| 12725 | |
| 12726 | /* The function cgraph_function_versioning () will force the new |
| 12727 | symbol local. Undo this, and inherit external visability from |
| 12728 | the old node. */ |
| 12729 | new_node->local.local = old_node->local.local; |
| 12730 | new_node->externally_visible = old_node->externally_visible; |
| 12731 | |
| 12732 | return new_node; |
| 12733 | } |
| 12734 | |
| 12735 | /* Adjust the return type of the given function to its appropriate |
| 12736 | vector counterpart. Returns a simd array to be used throughout the |
| 12737 | function as a return value. */ |
| 12738 | |
| 12739 | static tree |
| 12740 | simd_clone_adjust_return_type (struct cgraph_node *node) |
| 12741 | { |
| 12742 | tree fndecl = node->decl; |
| 12743 | tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl)); |
| 12744 | unsigned int veclen; |
| 12745 | tree t; |
| 12746 | |
| 12747 | /* Adjust the function return type. */ |
| 12748 | if (orig_rettype == void_type_node) |
| 12749 | return NULL_TREE; |
| 12750 | TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl)); |
| 12751 | t = TREE_TYPE (TREE_TYPE (fndecl)); |
| 12752 | if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t)) |
| 12753 | veclen = node->simdclone->vecsize_int; |
| 12754 | else |
| 12755 | veclen = node->simdclone->vecsize_float; |
| 12756 | veclen /= GET_MODE_BITSIZE (TYPE_MODE (t)); |
| 12757 | if (veclen > node->simdclone->simdlen) |
| 12758 | veclen = node->simdclone->simdlen; |
| 12759 | if (POINTER_TYPE_P (t)) |
| 12760 | t = pointer_sized_int_node; |
| 12761 | if (veclen == node->simdclone->simdlen) |
| 12762 | t = build_vector_type (t, node->simdclone->simdlen); |
| 12763 | else |
| 12764 | { |
| 12765 | t = build_vector_type (t, veclen); |
| 12766 | t = build_array_type_nelts (t, node->simdclone->simdlen / veclen); |
| 12767 | } |
| 12768 | TREE_TYPE (TREE_TYPE (fndecl)) = t; |
| 12769 | if (!node->definition) |
| 12770 | return NULL_TREE; |
| 12771 | |
| 12772 | t = DECL_RESULT (fndecl); |
| 12773 | /* Adjust the DECL_RESULT. */ |
| 12774 | gcc_assert (TREE_TYPE (t) != void_type_node); |
| 12775 | TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl)); |
| 12776 | relayout_decl (t); |
| 12777 | |
| 12778 | tree atype = build_array_type_nelts (orig_rettype, |
| 12779 | node->simdclone->simdlen); |
| 12780 | if (veclen != node->simdclone->simdlen) |
| 12781 | return build1 (VIEW_CONVERT_EXPR, atype, t); |
| 12782 | |
| 12783 | /* Set up a SIMD array to use as the return value. */ |
| 12784 | tree retval = create_tmp_var_raw (atype, "retval"); |
| 12785 | gimple_add_tmp_var (retval); |
| 12786 | return retval; |
| 12787 | } |
| 12788 | |
| 12789 | /* Each vector argument has a corresponding array to be used locally |
| 12790 | as part of the eventual loop. Create such temporary array and |
| 12791 | return it. |
| 12792 | |
| 12793 | PREFIX is the prefix to be used for the temporary. |
| 12794 | |
| 12795 | TYPE is the inner element type. |
| 12796 | |
| 12797 | SIMDLEN is the number of elements. */ |
| 12798 | |
| 12799 | static tree |
| 12800 | create_tmp_simd_array (const char *prefix, tree type, int simdlen) |
| 12801 | { |
| 12802 | tree atype = build_array_type_nelts (type, simdlen); |
| 12803 | tree avar = create_tmp_var_raw (atype, prefix); |
| 12804 | gimple_add_tmp_var (avar); |
| 12805 | return avar; |
| 12806 | } |
| 12807 | |
| 12808 | /* Modify the function argument types to their corresponding vector |
| 12809 | counterparts if appropriate. Also, create one array for each simd |
| 12810 | argument to be used locally when using the function arguments as |
| 12811 | part of the loop. |
| 12812 | |
| 12813 | NODE is the function whose arguments are to be adjusted. |
| 12814 | |
| 12815 | Returns an adjustment vector that will be filled describing how the |
| 12816 | argument types will be adjusted. */ |
| 12817 | |
| 12818 | static ipa_parm_adjustment_vec |
| 12819 | simd_clone_adjust_argument_types (struct cgraph_node *node) |
| 12820 | { |
| 12821 | vec<tree> args; |
| 12822 | ipa_parm_adjustment_vec adjustments; |
| 12823 | |
| 12824 | if (node->definition) |
| 12825 | args = ipa_get_vector_of_formal_parms (node->decl); |
| 12826 | else |
| 12827 | args = simd_clone_vector_of_formal_parm_types (node->decl); |
| 12828 | adjustments.create (args.length ()); |
| 12829 | unsigned i, j, veclen; |
| 12830 | struct ipa_parm_adjustment adj; |
| 12831 | for (i = 0; i < node->simdclone->nargs; ++i) |
| 12832 | { |
| 12833 | memset (&adj, 0, sizeof (adj)); |
| 12834 | tree parm = args[i]; |
| 12835 | tree parm_type = node->definition ? TREE_TYPE (parm) : parm; |
| 12836 | adj.base_index = i; |
| 12837 | adj.base = parm; |
| 12838 | |
| 12839 | node->simdclone->args[i].orig_arg = node->definition ? parm : NULL_TREE; |
| 12840 | node->simdclone->args[i].orig_type = parm_type; |
| 12841 | |
| 12842 | if (node->simdclone->args[i].arg_type != SIMD_CLONE_ARG_TYPE_VECTOR) |
| 12843 | { |
| 12844 | /* No adjustment necessary for scalar arguments. */ |
| 12845 | adj.op = IPA_PARM_OP_COPY; |
| 12846 | } |
| 12847 | else |
| 12848 | { |
| 12849 | if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type)) |
| 12850 | veclen = node->simdclone->vecsize_int; |
| 12851 | else |
| 12852 | veclen = node->simdclone->vecsize_float; |
| 12853 | veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type)); |
| 12854 | if (veclen > node->simdclone->simdlen) |
| 12855 | veclen = node->simdclone->simdlen; |
| 12856 | adj.arg_prefix = "simd"; |
| 12857 | if (POINTER_TYPE_P (parm_type)) |
| 12858 | adj.type = build_vector_type (pointer_sized_int_node, veclen); |
| 12859 | else |
| 12860 | adj.type = build_vector_type (parm_type, veclen); |
| 12861 | node->simdclone->args[i].vector_type = adj.type; |
| 12862 | for (j = veclen; j < node->simdclone->simdlen; j += veclen) |
| 12863 | { |
| 12864 | adjustments.safe_push (adj); |
| 12865 | if (j == veclen) |
| 12866 | { |
| 12867 | memset (&adj, 0, sizeof (adj)); |
| 12868 | adj.op = IPA_PARM_OP_NEW; |
| 12869 | adj.arg_prefix = "simd"; |
| 12870 | adj.base_index = i; |
| 12871 | adj.type = node->simdclone->args[i].vector_type; |
| 12872 | } |
| 12873 | } |
| 12874 | |
| 12875 | if (node->definition) |
| 12876 | node->simdclone->args[i].simd_array |
| 12877 | = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)), |
| 12878 | parm_type, node->simdclone->simdlen); |
| 12879 | } |
| 12880 | adjustments.safe_push (adj); |
| 12881 | } |
| 12882 | |
| 12883 | if (node->simdclone->inbranch) |
| 12884 | { |
| 12885 | tree base_type |
| 12886 | = simd_clone_compute_base_data_type (node->simdclone->origin, |
| 12887 | node->simdclone); |
| 12888 | |
| 12889 | memset (&adj, 0, sizeof (adj)); |
| 12890 | adj.op = IPA_PARM_OP_NEW; |
| 12891 | adj.arg_prefix = "mask"; |
| 12892 | |
| 12893 | adj.base_index = i; |
| 12894 | if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type)) |
| 12895 | veclen = node->simdclone->vecsize_int; |
| 12896 | else |
| 12897 | veclen = node->simdclone->vecsize_float; |
| 12898 | veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type)); |
| 12899 | if (veclen > node->simdclone->simdlen) |
| 12900 | veclen = node->simdclone->simdlen; |
| 12901 | if (POINTER_TYPE_P (base_type)) |
| 12902 | adj.type = build_vector_type (pointer_sized_int_node, veclen); |
| 12903 | else |
| 12904 | adj.type = build_vector_type (base_type, veclen); |
| 12905 | adjustments.safe_push (adj); |
| 12906 | |
| 12907 | for (j = veclen; j < node->simdclone->simdlen; j += veclen) |
| 12908 | adjustments.safe_push (adj); |
| 12909 | |
| 12910 | /* We have previously allocated one extra entry for the mask. Use |
| 12911 | it and fill it. */ |
| 12912 | struct cgraph_simd_clone *sc = node->simdclone; |
| 12913 | sc->nargs++; |
| 12914 | if (node->definition) |
| 12915 | { |
| 12916 | sc->args[i].orig_arg |
| 12917 | = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type); |
| 12918 | sc->args[i].simd_array |
| 12919 | = create_tmp_simd_array ("mask", base_type, sc->simdlen); |
| 12920 | } |
| 12921 | sc->args[i].orig_type = base_type; |
| 12922 | sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK; |
| 12923 | } |
| 12924 | |
| 12925 | if (node->definition) |
| 12926 | ipa_modify_formal_parameters (node->decl, adjustments); |
| 12927 | else |
| 12928 | { |
| 12929 | tree new_arg_types = NULL_TREE, new_reversed; |
| 12930 | bool last_parm_void = false; |
| 12931 | if (args.length () > 0 && args.last () == void_type_node) |
| 12932 | last_parm_void = true; |
| 12933 | |
| 12934 | gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl))); |
| 12935 | j = adjustments.length (); |
| 12936 | for (i = 0; i < j; i++) |
| 12937 | { |
| 12938 | struct ipa_parm_adjustment *adj = &adjustments[i]; |
| 12939 | tree ptype; |
| 12940 | if (adj->op == IPA_PARM_OP_COPY) |
| 12941 | ptype = args[adj->base_index]; |
| 12942 | else |
| 12943 | ptype = adj->type; |
| 12944 | new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types); |
| 12945 | } |
| 12946 | new_reversed = nreverse (new_arg_types); |
| 12947 | if (last_parm_void) |
| 12948 | { |
| 12949 | if (new_reversed) |
| 12950 | TREE_CHAIN (new_arg_types) = void_list_node; |
| 12951 | else |
| 12952 | new_reversed = void_list_node; |
| 12953 | } |
| 12954 | |
| 12955 | tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl)); |
| 12956 | TYPE_ARG_TYPES (new_type) = new_reversed; |
| 12957 | TREE_TYPE (node->decl) = new_type; |
| 12958 | |
| 12959 | adjustments.release (); |
| 12960 | } |
| 12961 | args.release (); |
| 12962 | return adjustments; |
| 12963 | } |
| 12964 | |
| 12965 | /* Initialize and copy the function arguments in NODE to their |
| 12966 | corresponding local simd arrays. Returns a fresh gimple_seq with |
| 12967 | the instruction sequence generated. */ |
| 12968 | |
| 12969 | static gimple_seq |
| 12970 | simd_clone_init_simd_arrays (struct cgraph_node *node, |
| 12971 | ipa_parm_adjustment_vec adjustments) |
| 12972 | { |
| 12973 | gimple_seq seq = NULL; |
| 12974 | unsigned i = 0, j = 0, k; |
| 12975 | |
| 12976 | for (tree arg = DECL_ARGUMENTS (node->decl); |
| 12977 | arg; |
| 12978 | arg = DECL_CHAIN (arg), i++, j++) |
| 12979 | { |
| 12980 | if (adjustments[j].op == IPA_PARM_OP_COPY) |
| 12981 | continue; |
| 12982 | |
| 12983 | node->simdclone->args[i].vector_arg = arg; |
| 12984 | |
| 12985 | tree array = node->simdclone->args[i].simd_array; |
| 12986 | if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen) |
| 12987 | { |
| 12988 | tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array))); |
| 12989 | tree ptr = build_fold_addr_expr (array); |
| 12990 | tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr, |
| 12991 | build_int_cst (ptype, 0)); |
| 12992 | t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); |
| 12993 | gimplify_and_add (t, &seq); |
| 12994 | } |
| 12995 | else |
| 12996 | { |
| 12997 | unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)); |
| 12998 | tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array))); |
| 12999 | for (k = 0; k < node->simdclone->simdlen; k += simdlen) |
| 13000 | { |
| 13001 | tree ptr = build_fold_addr_expr (array); |
| 13002 | int elemsize; |
| 13003 | if (k) |
| 13004 | { |
| 13005 | arg = DECL_CHAIN (arg); |
| 13006 | j++; |
| 13007 | } |
| 13008 | elemsize |
| 13009 | = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg)))); |
| 13010 | tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr, |
| 13011 | build_int_cst (ptype, k * elemsize)); |
| 13012 | t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); |
| 13013 | gimplify_and_add (t, &seq); |
| 13014 | } |
| 13015 | } |
| 13016 | } |
| 13017 | return seq; |
| 13018 | } |
| 13019 | |
| 13020 | /* Callback info for ipa_simd_modify_stmt_ops below. */ |
| 13021 | |
| 13022 | struct modify_stmt_info { |
| 13023 | ipa_parm_adjustment_vec adjustments; |
| 13024 | gimple stmt; |
| 13025 | /* True if the parent statement was modified by |
| 13026 | ipa_simd_modify_stmt_ops. */ |
| 13027 | bool modified; |
| 13028 | }; |
| 13029 | |
| 13030 | /* Callback for walk_gimple_op. |
| 13031 | |
| 13032 | Adjust operands from a given statement as specified in the |
| 13033 | adjustments vector in the callback data. */ |
| 13034 | |
| 13035 | static tree |
| 13036 | ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data) |
| 13037 | { |
| 13038 | struct walk_stmt_info *wi = (struct walk_stmt_info *) data; |
| 13039 | struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info; |
| 13040 | tree *orig_tp = tp; |
| 13041 | if (TREE_CODE (*tp) == ADDR_EXPR) |
| 13042 | tp = &TREE_OPERAND (*tp, 0); |
| 13043 | struct ipa_parm_adjustment *cand = NULL; |
| 13044 | if (TREE_CODE (*tp) == PARM_DECL) |
| 13045 | cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true); |
| 13046 | else |
| 13047 | { |
| 13048 | if (TYPE_P (*tp)) |
| 13049 | *walk_subtrees = 0; |
| 13050 | } |
| 13051 | |
| 13052 | tree repl = NULL_TREE; |
| 13053 | if (cand) |
| 13054 | repl = unshare_expr (cand->new_decl); |
| 13055 | else |
| 13056 | { |
| 13057 | if (tp != orig_tp) |
| 13058 | { |
| 13059 | *walk_subtrees = 0; |
| 13060 | bool modified = info->modified; |
| 13061 | info->modified = false; |
| 13062 | walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset); |
| 13063 | if (!info->modified) |
| 13064 | { |
| 13065 | info->modified = modified; |
| 13066 | return NULL_TREE; |
| 13067 | } |
| 13068 | info->modified = modified; |
| 13069 | repl = *tp; |
| 13070 | } |
| 13071 | else |
| 13072 | return NULL_TREE; |
| 13073 | } |
| 13074 | |
| 13075 | if (tp != orig_tp) |
| 13076 | { |
| 13077 | repl = build_fold_addr_expr (repl); |
| 13078 | gimple stmt; |
| 13079 | if (is_gimple_debug (info->stmt)) |
| 13080 | { |
| 13081 | tree vexpr = make_node (DEBUG_EXPR_DECL); |
| 13082 | stmt = gimple_build_debug_source_bind (vexpr, repl, NULL); |
| 13083 | DECL_ARTIFICIAL (vexpr) = 1; |
| 13084 | TREE_TYPE (vexpr) = TREE_TYPE (repl); |
| 13085 | DECL_MODE (vexpr) = TYPE_MODE (TREE_TYPE (repl)); |
| 13086 | repl = vexpr; |
| 13087 | } |
| 13088 | else |
| 13089 | { |
| 13090 | stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl); |
| 13091 | repl = gimple_assign_lhs (stmt); |
| 13092 | } |
| 13093 | gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt); |
| 13094 | gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); |
| 13095 | *orig_tp = repl; |
| 13096 | } |
| 13097 | else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl))) |
| 13098 | { |
| 13099 | tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl); |
| 13100 | *tp = vce; |
| 13101 | } |
| 13102 | else |
| 13103 | *tp = repl; |
| 13104 | |
| 13105 | info->modified = true; |
| 13106 | return NULL_TREE; |
| 13107 | } |
| 13108 | |
| 13109 | /* Traverse the function body and perform all modifications as |
| 13110 | described in ADJUSTMENTS. At function return, ADJUSTMENTS will be |
| 13111 | modified such that the replacement/reduction value will now be an |
| 13112 | offset into the corresponding simd_array. |
| 13113 | |
| 13114 | This function will replace all function argument uses with their |
| 13115 | corresponding simd array elements, and ajust the return values |
| 13116 | accordingly. */ |
| 13117 | |
| 13118 | static void |
| 13119 | ipa_simd_modify_function_body (struct cgraph_node *node, |
| 13120 | ipa_parm_adjustment_vec adjustments, |
| 13121 | tree retval_array, tree iter) |
| 13122 | { |
| 13123 | basic_block bb; |
| 13124 | unsigned int i, j, l; |
| 13125 | |
| 13126 | /* Re-use the adjustments array, but this time use it to replace |
| 13127 | every function argument use to an offset into the corresponding |
| 13128 | simd_array. */ |
| 13129 | for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j) |
| 13130 | { |
| 13131 | if (!node->simdclone->args[i].vector_arg) |
| 13132 | continue; |
| 13133 | |
| 13134 | tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg); |
| 13135 | tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg); |
| 13136 | adjustments[j].new_decl |
| 13137 | = build4 (ARRAY_REF, |
| 13138 | basetype, |
| 13139 | node->simdclone->args[i].simd_array, |
| 13140 | iter, |
| 13141 | NULL_TREE, NULL_TREE); |
| 13142 | if (adjustments[j].op == IPA_PARM_OP_NONE |
| 13143 | && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen) |
| 13144 | j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1; |
| 13145 | } |
| 13146 | |
| 13147 | l = adjustments.length (); |
| 13148 | for (i = 1; i < num_ssa_names; i++) |
| 13149 | { |
| 13150 | tree name = ssa_name (i); |
| 13151 | if (name |
| 13152 | && SSA_NAME_VAR (name) |
| 13153 | && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL) |
| 13154 | { |
| 13155 | for (j = 0; j < l; j++) |
| 13156 | if (SSA_NAME_VAR (name) == adjustments[j].base |
| 13157 | && adjustments[j].new_decl) |
| 13158 | { |
| 13159 | tree base_var; |
| 13160 | if (adjustments[j].new_ssa_base == NULL_TREE) |
| 13161 | { |
| 13162 | base_var |
| 13163 | = copy_var_decl (adjustments[j].base, |
| 13164 | DECL_NAME (adjustments[j].base), |
| 13165 | TREE_TYPE (adjustments[j].base)); |
| 13166 | adjustments[j].new_ssa_base = base_var; |
| 13167 | } |
| 13168 | else |
| 13169 | base_var = adjustments[j].new_ssa_base; |
| 13170 | if (SSA_NAME_IS_DEFAULT_DEF (name)) |
| 13171 | { |
| 13172 | bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); |
| 13173 | gimple_stmt_iterator gsi = gsi_after_labels (bb); |
| 13174 | tree new_decl = unshare_expr (adjustments[j].new_decl); |
| 13175 | set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE); |
| 13176 | SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var); |
| 13177 | SSA_NAME_IS_DEFAULT_DEF (name) = 0; |
| 13178 | gimple stmt = gimple_build_assign (name, new_decl); |
| 13179 | gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); |
| 13180 | } |
| 13181 | else |
| 13182 | SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var); |
| 13183 | } |
| 13184 | } |
| 13185 | } |
| 13186 | |
| 13187 | struct modify_stmt_info info; |
| 13188 | info.adjustments = adjustments; |
| 13189 | |
| 13190 | FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl)) |
| 13191 | { |
| 13192 | gimple_stmt_iterator gsi; |
| 13193 | |
| 13194 | gsi = gsi_start_bb (bb); |
| 13195 | while (!gsi_end_p (gsi)) |
| 13196 | { |
| 13197 | gimple stmt = gsi_stmt (gsi); |
| 13198 | info.stmt = stmt; |
| 13199 | struct walk_stmt_info wi; |
| 13200 | |
| 13201 | memset (&wi, 0, sizeof (wi)); |
| 13202 | info.modified = false; |
| 13203 | wi.info = &info; |
| 13204 | walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi); |
| 13205 | |
| 13206 | if (greturn *return_stmt = dyn_cast <greturn *> (stmt)) |
| 13207 | { |
| 13208 | tree retval = gimple_return_retval (return_stmt); |
| 13209 | if (!retval) |
| 13210 | { |
| 13211 | gsi_remove (&gsi, true); |
| 13212 | continue; |
| 13213 | } |
| 13214 | |
| 13215 | /* Replace `return foo' with `retval_array[iter] = foo'. */ |
| 13216 | tree ref = build4 (ARRAY_REF, TREE_TYPE (retval), |
| 13217 | retval_array, iter, NULL, NULL); |
| 13218 | stmt = gimple_build_assign (ref, retval); |
| 13219 | gsi_replace (&gsi, stmt, true); |
| 13220 | info.modified = true; |
| 13221 | } |
| 13222 | |
| 13223 | if (info.modified) |
| 13224 | { |
| 13225 | update_stmt (stmt); |
| 13226 | if (maybe_clean_eh_stmt (stmt)) |
| 13227 | gimple_purge_dead_eh_edges (gimple_bb (stmt)); |
| 13228 | } |
| 13229 | gsi_next (&gsi); |
| 13230 | } |
| 13231 | } |
| 13232 | } |
| 13233 | |
| 13234 | /* Adjust the argument types in NODE to their appropriate vector |
| 13235 | counterparts. */ |
| 13236 | |
| 13237 | static void |
| 13238 | simd_clone_adjust (struct cgraph_node *node) |
| 13239 | { |
| 13240 | push_cfun (DECL_STRUCT_FUNCTION (node->decl)); |
| 13241 | |
| 13242 | targetm.simd_clone.adjust (node); |
| 13243 | |
| 13244 | tree retval = simd_clone_adjust_return_type (node); |
| 13245 | ipa_parm_adjustment_vec adjustments |
| 13246 | = simd_clone_adjust_argument_types (node); |
| 13247 | |
| 13248 | push_gimplify_context (); |
| 13249 | |
| 13250 | gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments); |
| 13251 | |
| 13252 | /* Adjust all uses of vector arguments accordingly. Adjust all |
| 13253 | return values accordingly. */ |
| 13254 | tree iter = create_tmp_var (unsigned_type_node, "iter"); |
| 13255 | tree iter1 = make_ssa_name (iter); |
| 13256 | tree iter2 = make_ssa_name (iter); |
| 13257 | ipa_simd_modify_function_body (node, adjustments, retval, iter1); |
| 13258 | |
| 13259 | /* Initialize the iteration variable. */ |
| 13260 | basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); |
| 13261 | basic_block body_bb = split_block_after_labels (entry_bb)->dest; |
| 13262 | gimple_stmt_iterator gsi = gsi_after_labels (entry_bb); |
| 13263 | /* Insert the SIMD array and iv initialization at function |
| 13264 | entry. */ |
| 13265 | gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT); |
| 13266 | |
| 13267 | pop_gimplify_context (NULL); |
| 13268 | |
| 13269 | /* Create a new BB right before the original exit BB, to hold the |
| 13270 | iteration increment and the condition/branch. */ |
| 13271 | basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src; |
| 13272 | basic_block incr_bb = create_empty_bb (orig_exit); |
| 13273 | add_bb_to_loop (incr_bb, body_bb->loop_father); |
| 13274 | /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty |
| 13275 | flag. Set it now to be a FALLTHRU_EDGE. */ |
| 13276 | gcc_assert (EDGE_COUNT (orig_exit->succs) == 1); |
| 13277 | EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU; |
| 13278 | for (unsigned i = 0; |
| 13279 | i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i) |
| 13280 | { |
| 13281 | edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i); |
| 13282 | redirect_edge_succ (e, incr_bb); |
| 13283 | } |
| 13284 | edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); |
| 13285 | e->probability = REG_BR_PROB_BASE; |
| 13286 | gsi = gsi_last_bb (incr_bb); |
| 13287 | gimple g = gimple_build_assign (iter2, PLUS_EXPR, iter1, |
| 13288 | build_int_cst (unsigned_type_node, 1)); |
| 13289 | gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); |
| 13290 | |
| 13291 | /* Mostly annotate the loop for the vectorizer (the rest is done below). */ |
| 13292 | struct loop *loop = alloc_loop (); |
| 13293 | cfun->has_force_vectorize_loops = true; |
| 13294 | loop->safelen = node->simdclone->simdlen; |
| 13295 | loop->force_vectorize = true; |
| 13296 | loop->header = body_bb; |
| 13297 | |
| 13298 | /* Branch around the body if the mask applies. */ |
| 13299 | if (node->simdclone->inbranch) |
| 13300 | { |
| 13301 | gimple_stmt_iterator gsi = gsi_last_bb (loop->header); |
| 13302 | tree mask_array |
| 13303 | = node->simdclone->args[node->simdclone->nargs - 1].simd_array; |
| 13304 | tree mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array))); |
| 13305 | tree aref = build4 (ARRAY_REF, |
| 13306 | TREE_TYPE (TREE_TYPE (mask_array)), |
| 13307 | mask_array, iter1, |
| 13308 | NULL, NULL); |
| 13309 | g = gimple_build_assign (mask, aref); |
| 13310 | gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); |
| 13311 | int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref))); |
| 13312 | if (!INTEGRAL_TYPE_P (TREE_TYPE (aref))) |
| 13313 | { |
| 13314 | aref = build1 (VIEW_CONVERT_EXPR, |
| 13315 | build_nonstandard_integer_type (bitsize, 0), mask); |
| 13316 | mask = make_ssa_name (TREE_TYPE (aref)); |
| 13317 | g = gimple_build_assign (mask, aref); |
| 13318 | gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); |
| 13319 | } |
| 13320 | |
| 13321 | g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)), |
| 13322 | NULL, NULL); |
| 13323 | gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); |
| 13324 | make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE); |
| 13325 | FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE; |
| 13326 | } |
| 13327 | |
| 13328 | /* Generate the condition. */ |
| 13329 | g = gimple_build_cond (LT_EXPR, |
| 13330 | iter2, |
| 13331 | build_int_cst (unsigned_type_node, |
| 13332 | node->simdclone->simdlen), |
| 13333 | NULL, NULL); |
| 13334 | gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); |
| 13335 | e = split_block (incr_bb, gsi_stmt (gsi)); |
| 13336 | basic_block latch_bb = e->dest; |
| 13337 | basic_block new_exit_bb; |
| 13338 | new_exit_bb = split_block (latch_bb, NULL)->dest; |
| 13339 | loop->latch = latch_bb; |
| 13340 | |
| 13341 | redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb); |
| 13342 | |
| 13343 | make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE); |
| 13344 | /* The successor of incr_bb is already pointing to latch_bb; just |
| 13345 | change the flags. |
| 13346 | make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */ |
| 13347 | FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE; |
| 13348 | |
| 13349 | gphi *phi = create_phi_node (iter1, body_bb); |
| 13350 | edge preheader_edge = find_edge (entry_bb, body_bb); |
| 13351 | edge latch_edge = single_succ_edge (latch_bb); |
| 13352 | add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge, |
| 13353 | UNKNOWN_LOCATION); |
| 13354 | add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); |
| 13355 | |
| 13356 | /* Generate the new return. */ |
| 13357 | gsi = gsi_last_bb (new_exit_bb); |
| 13358 | if (retval |
| 13359 | && TREE_CODE (retval) == VIEW_CONVERT_EXPR |
| 13360 | && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL) |
| 13361 | retval = TREE_OPERAND (retval, 0); |
| 13362 | else if (retval) |
| 13363 | { |
| 13364 | retval = build1 (VIEW_CONVERT_EXPR, |
| 13365 | TREE_TYPE (TREE_TYPE (node->decl)), |
| 13366 | retval); |
| 13367 | retval = force_gimple_operand_gsi (&gsi, retval, true, NULL, |
| 13368 | false, GSI_CONTINUE_LINKING); |
| 13369 | } |
| 13370 | g = gimple_build_return (retval); |
| 13371 | gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); |
| 13372 | |
| 13373 | /* Handle aligned clauses by replacing default defs of the aligned |
| 13374 | uniform args with __builtin_assume_aligned (arg_N(D), alignment) |
| 13375 | lhs. Handle linear by adding PHIs. */ |
| 13376 | for (unsigned i = 0; i < node->simdclone->nargs; i++) |
| 13377 | if (node->simdclone->args[i].alignment |
| 13378 | && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM |
| 13379 | && (node->simdclone->args[i].alignment |
| 13380 | & (node->simdclone->args[i].alignment - 1)) == 0 |
| 13381 | && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg)) |
| 13382 | == POINTER_TYPE) |
| 13383 | { |
| 13384 | unsigned int alignment = node->simdclone->args[i].alignment; |
| 13385 | tree orig_arg = node->simdclone->args[i].orig_arg; |
| 13386 | tree def = ssa_default_def (cfun, orig_arg); |
| 13387 | if (def && !has_zero_uses (def)) |
| 13388 | { |
| 13389 | tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED); |
| 13390 | gimple_seq seq = NULL; |
| 13391 | bool need_cvt = false; |
| 13392 | gcall *call |
| 13393 | = gimple_build_call (fn, 2, def, size_int (alignment)); |
| 13394 | g = call; |
| 13395 | if (!useless_type_conversion_p (TREE_TYPE (orig_arg), |
| 13396 | ptr_type_node)) |
| 13397 | need_cvt = true; |
| 13398 | tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg); |
| 13399 | gimple_call_set_lhs (g, t); |
| 13400 | gimple_seq_add_stmt_without_update (&seq, g); |
| 13401 | if (need_cvt) |
| 13402 | { |
| 13403 | t = make_ssa_name (orig_arg); |
| 13404 | g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g)); |
| 13405 | gimple_seq_add_stmt_without_update (&seq, g); |
| 13406 | } |
| 13407 | gsi_insert_seq_on_edge_immediate |
| 13408 | (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq); |
| 13409 | |
| 13410 | entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); |
| 13411 | int freq = compute_call_stmt_bb_frequency (current_function_decl, |
| 13412 | entry_bb); |
| 13413 | node->create_edge (cgraph_node::get_create (fn), |
| 13414 | call, entry_bb->count, freq); |
| 13415 | |
| 13416 | imm_use_iterator iter; |
| 13417 | use_operand_p use_p; |
| 13418 | gimple use_stmt; |
| 13419 | tree repl = gimple_get_lhs (g); |
| 13420 | FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) |
| 13421 | if (is_gimple_debug (use_stmt) || use_stmt == call) |
| 13422 | continue; |
| 13423 | else |
| 13424 | FOR_EACH_IMM_USE_ON_STMT (use_p, iter) |
| 13425 | SET_USE (use_p, repl); |
| 13426 | } |
| 13427 | } |
| 13428 | else if (node->simdclone->args[i].arg_type |
| 13429 | == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP) |
| 13430 | { |
| 13431 | tree orig_arg = node->simdclone->args[i].orig_arg; |
| 13432 | tree def = ssa_default_def (cfun, orig_arg); |
| 13433 | gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) |
| 13434 | || POINTER_TYPE_P (TREE_TYPE (orig_arg))); |
| 13435 | if (def && !has_zero_uses (def)) |
| 13436 | { |
| 13437 | iter1 = make_ssa_name (orig_arg); |
| 13438 | iter2 = make_ssa_name (orig_arg); |
| 13439 | phi = create_phi_node (iter1, body_bb); |
| 13440 | add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION); |
| 13441 | add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); |
| 13442 | enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) |
| 13443 | ? PLUS_EXPR : POINTER_PLUS_EXPR; |
| 13444 | tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) |
| 13445 | ? TREE_TYPE (orig_arg) : sizetype; |
| 13446 | tree addcst |
| 13447 | = build_int_cst (addtype, node->simdclone->args[i].linear_step); |
| 13448 | g = gimple_build_assign (iter2, code, iter1, addcst); |
| 13449 | gsi = gsi_last_bb (incr_bb); |
| 13450 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); |
| 13451 | |
| 13452 | imm_use_iterator iter; |
| 13453 | use_operand_p use_p; |
| 13454 | gimple use_stmt; |
| 13455 | FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) |
| 13456 | if (use_stmt == phi) |
| 13457 | continue; |
| 13458 | else |
| 13459 | FOR_EACH_IMM_USE_ON_STMT (use_p, iter) |
| 13460 | SET_USE (use_p, iter1); |
| 13461 | } |
| 13462 | } |
| 13463 | |
| 13464 | calculate_dominance_info (CDI_DOMINATORS); |
| 13465 | add_loop (loop, loop->header->loop_father); |
| 13466 | update_ssa (TODO_update_ssa); |
| 13467 | |
| 13468 | pop_cfun (); |
| 13469 | } |
| 13470 | |
| 13471 | /* If the function in NODE is tagged as an elemental SIMD function, |
| 13472 | create the appropriate SIMD clones. */ |
| 13473 | |
| 13474 | static void |
| 13475 | expand_simd_clones (struct cgraph_node *node) |
| 13476 | { |
| 13477 | tree attr = lookup_attribute ("omp declare simd", |
| 13478 | DECL_ATTRIBUTES (node->decl)); |
| 13479 | if (attr == NULL_TREE |
| 13480 | || node->global.inlined_to |
| 13481 | || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl))) |
| 13482 | return; |
| 13483 | |
| 13484 | /* Ignore |
| 13485 | #pragma omp declare simd |
| 13486 | extern int foo (); |
| 13487 | in C, there we don't know the argument types at all. */ |
| 13488 | if (!node->definition |
| 13489 | && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE) |
| 13490 | return; |
| 13491 | |
| 13492 | do |
| 13493 | { |
| 13494 | /* Start with parsing the "omp declare simd" attribute(s). */ |
| 13495 | bool inbranch_clause_specified; |
| 13496 | struct cgraph_simd_clone *clone_info |
| 13497 | = simd_clone_clauses_extract (node, TREE_VALUE (attr), |
| 13498 | &inbranch_clause_specified); |
| 13499 | if (clone_info == NULL) |
| 13500 | continue; |
| 13501 | |
| 13502 | int orig_simdlen = clone_info->simdlen; |
| 13503 | tree base_type = simd_clone_compute_base_data_type (node, clone_info); |
| 13504 | /* The target can return 0 (no simd clones should be created), |
| 13505 | 1 (just one ISA of simd clones should be created) or higher |
| 13506 | count of ISA variants. In that case, clone_info is initialized |
| 13507 | for the first ISA variant. */ |
| 13508 | int count |
| 13509 | = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info, |
| 13510 | base_type, 0); |
| 13511 | if (count == 0) |
| 13512 | continue; |
| 13513 | |
| 13514 | /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED, |
| 13515 | also create one inbranch and one !inbranch clone of it. */ |
| 13516 | for (int i = 0; i < count * 2; i++) |
| 13517 | { |
| 13518 | struct cgraph_simd_clone *clone = clone_info; |
| 13519 | if (inbranch_clause_specified && (i & 1) != 0) |
| 13520 | continue; |
| 13521 | |
| 13522 | if (i != 0) |
| 13523 | { |
| 13524 | clone = simd_clone_struct_alloc (clone_info->nargs |
| 13525 | + ((i & 1) != 0)); |
| 13526 | simd_clone_struct_copy (clone, clone_info); |
| 13527 | /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen |
| 13528 | and simd_clone_adjust_argument_types did to the first |
| 13529 | clone's info. */ |
| 13530 | clone->nargs -= clone_info->inbranch; |
| 13531 | clone->simdlen = orig_simdlen; |
| 13532 | /* And call the target hook again to get the right ISA. */ |
| 13533 | targetm.simd_clone.compute_vecsize_and_simdlen (node, clone, |
| 13534 | base_type, |
| 13535 | i / 2); |
| 13536 | if ((i & 1) != 0) |
| 13537 | clone->inbranch = 1; |
| 13538 | } |
| 13539 | |
| 13540 | /* simd_clone_mangle might fail if such a clone has been created |
| 13541 | already. */ |
| 13542 | tree id = simd_clone_mangle (node, clone); |
| 13543 | if (id == NULL_TREE) |
| 13544 | continue; |
| 13545 | |
| 13546 | /* Only when we are sure we want to create the clone actually |
| 13547 | clone the function (or definitions) or create another |
| 13548 | extern FUNCTION_DECL (for prototypes without definitions). */ |
| 13549 | struct cgraph_node *n = simd_clone_create (node); |
| 13550 | if (n == NULL) |
| 13551 | continue; |
| 13552 | |
| 13553 | n->simdclone = clone; |
| 13554 | clone->origin = node; |
| 13555 | clone->next_clone = NULL; |
| 13556 | if (node->simd_clones == NULL) |
| 13557 | { |
| 13558 | clone->prev_clone = n; |
| 13559 | node->simd_clones = n; |
| 13560 | } |
| 13561 | else |
| 13562 | { |
| 13563 | clone->prev_clone = node->simd_clones->simdclone->prev_clone; |
| 13564 | clone->prev_clone->simdclone->next_clone = n; |
| 13565 | node->simd_clones->simdclone->prev_clone = n; |
| 13566 | } |
| 13567 | symtab->change_decl_assembler_name (n->decl, id); |
| 13568 | /* And finally adjust the return type, parameters and for |
| 13569 | definitions also function body. */ |
| 13570 | if (node->definition) |
| 13571 | simd_clone_adjust (n); |
| 13572 | else |
| 13573 | { |
| 13574 | simd_clone_adjust_return_type (n); |
| 13575 | simd_clone_adjust_argument_types (n); |
| 13576 | } |
| 13577 | } |
| 13578 | } |
| 13579 | while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr)))); |
| 13580 | } |
| 13581 | |
| 13582 | /* Entry point for IPA simd clone creation pass. */ |
| 13583 | |
| 13584 | static unsigned int |
| 13585 | ipa_omp_simd_clone (void) |
| 13586 | { |
| 13587 | struct cgraph_node *node; |
| 13588 | FOR_EACH_FUNCTION (node) |
| 13589 | expand_simd_clones (node); |
| 13590 | return 0; |
| 13591 | } |
| 13592 | |
| 13593 | namespace { |
| 13594 | |
| 13595 | const pass_data pass_data_omp_simd_clone = |
| 13596 | { |
| 13597 | SIMPLE_IPA_PASS, /* type */ |
| 13598 | "simdclone", /* name */ |
| 13599 | OPTGROUP_NONE, /* optinfo_flags */ |
| 13600 | TV_NONE, /* tv_id */ |
| 13601 | ( PROP_ssa | PROP_cfg ), /* properties_required */ |
| 13602 | 0, /* properties_provided */ |
| 13603 | 0, /* properties_destroyed */ |
| 13604 | 0, /* todo_flags_start */ |
| 13605 | 0, /* todo_flags_finish */ |
| 13606 | }; |
| 13607 | |
| 13608 | class pass_omp_simd_clone : public simple_ipa_opt_pass |
| 13609 | { |
| 13610 | public: |
| 13611 | pass_omp_simd_clone(gcc::context *ctxt) |
| 13612 | : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt) |
| 13613 | {} |
| 13614 | |
| 13615 | /* opt_pass methods: */ |
| 13616 | virtual bool gate (function *); |
| 13617 | virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); } |
| 13618 | }; |
| 13619 | |
| 13620 | bool |
| 13621 | pass_omp_simd_clone::gate (function *) |
| 13622 | { |
| 13623 | return ((flag_openmp || flag_openmp_simd |
| 13624 | || flag_cilkplus |
| 13625 | || (in_lto_p && !flag_wpa)) |
| 13626 | && (targetm.simd_clone.compute_vecsize_and_simdlen != NULL)); |
| 13627 | } |
| 13628 | |
| 13629 | } // anon namespace |
| 13630 | |
| 13631 | simple_ipa_opt_pass * |
| 13632 | make_pass_omp_simd_clone (gcc::context *ctxt) |
| 13633 | { |
| 13634 | return new pass_omp_simd_clone (ctxt); |
| 13635 | } |
| 13636 | |
| 13637 | /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and |
| 13638 | adds their addresses and sizes to constructor-vector V_CTOR. */ |
| 13639 | static void |
| 13640 | add_decls_addresses_to_decl_constructor (vec<tree, va_gc> *v_decls, |
| 13641 | vec<constructor_elt, va_gc> *v_ctor) |
| 13642 | { |
| 13643 | unsigned len = vec_safe_length (v_decls); |
| 13644 | for (unsigned i = 0; i < len; i++) |
| 13645 | { |
| 13646 | tree it = (*v_decls)[i]; |
| 13647 | bool is_function = TREE_CODE (it) != VAR_DECL; |
| 13648 | |
| 13649 | CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, build_fold_addr_expr (it)); |
| 13650 | if (!is_function) |
| 13651 | CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, |
| 13652 | fold_convert (const_ptr_type_node, |
| 13653 | DECL_SIZE_UNIT (it))); |
| 13654 | } |
| 13655 | } |
| 13656 | |
| 13657 | /* Create new symbols containing (address, size) pairs for global variables, |
| 13658 | marked with "omp declare target" attribute, as well as addresses for the |
| 13659 | functions, which are outlined offloading regions. */ |
| 13660 | void |
| 13661 | omp_finish_file (void) |
| 13662 | { |
| 13663 | unsigned num_funcs = vec_safe_length (offload_funcs); |
| 13664 | unsigned num_vars = vec_safe_length (offload_vars); |
| 13665 | |
| 13666 | if (num_funcs == 0 && num_vars == 0) |
| 13667 | return; |
| 13668 | |
| 13669 | if (targetm_common.have_named_sections) |
| 13670 | { |
| 13671 | vec<constructor_elt, va_gc> *v_f, *v_v; |
| 13672 | vec_alloc (v_f, num_funcs); |
| 13673 | vec_alloc (v_v, num_vars * 2); |
| 13674 | |
| 13675 | add_decls_addresses_to_decl_constructor (offload_funcs, v_f); |
| 13676 | add_decls_addresses_to_decl_constructor (offload_vars, v_v); |
| 13677 | |
| 13678 | tree vars_decl_type = build_array_type_nelts (pointer_sized_int_node, |
| 13679 | num_vars * 2); |
| 13680 | tree funcs_decl_type = build_array_type_nelts (pointer_sized_int_node, |
| 13681 | num_funcs); |
| 13682 | TYPE_ALIGN (vars_decl_type) = TYPE_ALIGN (pointer_sized_int_node); |
| 13683 | TYPE_ALIGN (funcs_decl_type) = TYPE_ALIGN (pointer_sized_int_node); |
| 13684 | tree ctor_v = build_constructor (vars_decl_type, v_v); |
| 13685 | tree ctor_f = build_constructor (funcs_decl_type, v_f); |
| 13686 | TREE_CONSTANT (ctor_v) = TREE_CONSTANT (ctor_f) = 1; |
| 13687 | TREE_STATIC (ctor_v) = TREE_STATIC (ctor_f) = 1; |
| 13688 | tree funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL, |
| 13689 | get_identifier (".offload_func_table"), |
| 13690 | funcs_decl_type); |
| 13691 | tree vars_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL, |
| 13692 | get_identifier (".offload_var_table"), |
| 13693 | vars_decl_type); |
| 13694 | TREE_STATIC (funcs_decl) = TREE_STATIC (vars_decl) = 1; |
| 13695 | /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node), |
| 13696 | otherwise a joint table in a binary will contain padding between |
| 13697 | tables from multiple object files. */ |
| 13698 | DECL_USER_ALIGN (funcs_decl) = DECL_USER_ALIGN (vars_decl) = 1; |
| 13699 | DECL_ALIGN (funcs_decl) = TYPE_ALIGN (funcs_decl_type); |
| 13700 | DECL_ALIGN (vars_decl) = TYPE_ALIGN (vars_decl_type); |
| 13701 | DECL_INITIAL (funcs_decl) = ctor_f; |
| 13702 | DECL_INITIAL (vars_decl) = ctor_v; |
| 13703 | set_decl_section_name (funcs_decl, OFFLOAD_FUNC_TABLE_SECTION_NAME); |
| 13704 | set_decl_section_name (vars_decl, OFFLOAD_VAR_TABLE_SECTION_NAME); |
| 13705 | |
| 13706 | varpool_node::finalize_decl (vars_decl); |
| 13707 | varpool_node::finalize_decl (funcs_decl); |
| 13708 | } |
| 13709 | else |
| 13710 | { |
| 13711 | for (unsigned i = 0; i < num_funcs; i++) |
| 13712 | { |
| 13713 | tree it = (*offload_funcs)[i]; |
| 13714 | targetm.record_offload_symbol (it); |
| 13715 | } |
| 13716 | for (unsigned i = 0; i < num_vars; i++) |
| 13717 | { |
| 13718 | tree it = (*offload_vars)[i]; |
| 13719 | targetm.record_offload_symbol (it); |
| 13720 | } |
| 13721 | } |
| 13722 | } |
| 13723 | |
| 13724 | #include "gt-omp-low.h" |