Lines 4339-4345
Link Here
|
4339 |
inside_cost = add_stmt_cost (target_cost_data, ncopies, vector_stmt, |
4339 |
inside_cost = add_stmt_cost (target_cost_data, ncopies, vector_stmt, |
4340 |
stmt_info, 0, vect_body); |
4340 |
stmt_info, 0, vect_body); |
4341 |
|
4341 |
|
4342 |
/* prologue cost for vec_init and vec_step. */ |
4342 |
/* prologue cost for vec_init and gcc_vec_step. */ |
4343 |
prologue_cost = add_stmt_cost (target_cost_data, 2, scalar_to_vec, |
4343 |
prologue_cost = add_stmt_cost (target_cost_data, 2, scalar_to_vec, |
4344 |
stmt_info, 0, vect_prologue); |
4344 |
stmt_info, 0, vect_prologue); |
4345 |
|
4345 |
|
Lines 4976-4988
Link Here
|
4976 |
|
4976 |
|
4977 |
/* Create a vector of the step value. */ |
4977 |
/* Create a vector of the step value. */ |
4978 |
tree step = build_int_cst (cr_index_scalar_type, nunits_out); |
4978 |
tree step = build_int_cst (cr_index_scalar_type, nunits_out); |
4979 |
tree vec_step = build_vector_from_val (cr_index_vector_type, step); |
4979 |
tree gcc_vec_step = build_vector_from_val (cr_index_vector_type, step); |
4980 |
|
4980 |
|
4981 |
/* Create an induction variable. */ |
4981 |
/* Create an induction variable. */ |
4982 |
gimple_stmt_iterator incr_gsi; |
4982 |
gimple_stmt_iterator incr_gsi; |
4983 |
bool insert_after; |
4983 |
bool insert_after; |
4984 |
standard_iv_increment_position (loop, &incr_gsi, &insert_after); |
4984 |
standard_iv_increment_position (loop, &incr_gsi, &insert_after); |
4985 |
create_iv (series_vect, vec_step, NULL_TREE, loop, &incr_gsi, |
4985 |
create_iv (series_vect, gcc_vec_step, NULL_TREE, loop, &incr_gsi, |
4986 |
insert_after, &indx_before_incr, &indx_after_incr); |
4986 |
insert_after, &indx_before_incr, &indx_after_incr); |
4987 |
|
4987 |
|
4988 |
/* Next create a new phi node vector (NEW_PHI_TREE) which starts |
4988 |
/* Next create a new phi node vector (NEW_PHI_TREE) which starts |
Lines 7650-7656
Link Here
|
7650 |
tree vec_def; |
7650 |
tree vec_def; |
7651 |
edge pe = loop_preheader_edge (loop); |
7651 |
edge pe = loop_preheader_edge (loop); |
7652 |
basic_block new_bb; |
7652 |
basic_block new_bb; |
7653 |
tree new_vec, vec_init, vec_step, t; |
7653 |
tree new_vec, vec_init, gcc_vec_step, t; |
7654 |
tree new_name; |
7654 |
tree new_name; |
7655 |
gimple *new_stmt; |
7655 |
gimple *new_stmt; |
7656 |
gphi *induction_phi; |
7656 |
gphi *induction_phi; |
Lines 7843-7849
Link Here
|
7843 |
new_name = vect_init_vector (phi, new_name, |
7843 |
new_name = vect_init_vector (phi, new_name, |
7844 |
TREE_TYPE (step_expr), NULL); |
7844 |
TREE_TYPE (step_expr), NULL); |
7845 |
new_vec = build_vector_from_val (vectype, new_name); |
7845 |
new_vec = build_vector_from_val (vectype, new_name); |
7846 |
vec_step = vect_init_vector (phi, new_vec, vectype, NULL); |
7846 |
gcc_vec_step = vect_init_vector (phi, new_vec, vectype, NULL); |
7847 |
|
7847 |
|
7848 |
/* Now generate the IVs. */ |
7848 |
/* Now generate the IVs. */ |
7849 |
unsigned group_size = SLP_TREE_SCALAR_STMTS (slp_node).length (); |
7849 |
unsigned group_size = SLP_TREE_SCALAR_STMTS (slp_node).length (); |
Lines 7882-7888
Link Here
|
7882 |
|
7882 |
|
7883 |
/* Create the iv update inside the loop */ |
7883 |
/* Create the iv update inside the loop */ |
7884 |
vec_def = make_ssa_name (vec_dest); |
7884 |
vec_def = make_ssa_name (vec_dest); |
7885 |
new_stmt = gimple_build_assign (vec_def, PLUS_EXPR, induc_def, vec_step); |
7885 |
new_stmt = gimple_build_assign (vec_def, PLUS_EXPR, induc_def, gcc_vec_step); |
7886 |
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT); |
7886 |
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT); |
7887 |
set_vinfo_for_stmt (new_stmt, new_stmt_vec_info (new_stmt, loop_vinfo)); |
7887 |
set_vinfo_for_stmt (new_stmt, new_stmt_vec_info (new_stmt, loop_vinfo)); |
7888 |
|
7888 |
|
Lines 7913-7919
Link Here
|
7913 |
new_name = vect_init_vector (phi, new_name, |
7913 |
new_name = vect_init_vector (phi, new_name, |
7914 |
TREE_TYPE (step_expr), NULL); |
7914 |
TREE_TYPE (step_expr), NULL); |
7915 |
new_vec = build_vector_from_val (vectype, new_name); |
7915 |
new_vec = build_vector_from_val (vectype, new_name); |
7916 |
vec_step = vect_init_vector (phi, new_vec, vectype, NULL); |
7916 |
gcc_vec_step = vect_init_vector (phi, new_vec, vectype, NULL); |
7917 |
for (; ivn < nvects; ++ivn) |
7917 |
for (; ivn < nvects; ++ivn) |
7918 |
{ |
7918 |
{ |
7919 |
gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]; |
7919 |
gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]; |
Lines 7924-7930
Link Here
|
7924 |
def = gimple_assign_lhs (iv); |
7924 |
def = gimple_assign_lhs (iv); |
7925 |
new_stmt = gimple_build_assign (make_ssa_name (vectype), |
7925 |
new_stmt = gimple_build_assign (make_ssa_name (vectype), |
7926 |
PLUS_EXPR, |
7926 |
PLUS_EXPR, |
7927 |
def, vec_step); |
7927 |
def, gcc_vec_step); |
7928 |
if (gimple_code (iv) == GIMPLE_PHI) |
7928 |
if (gimple_code (iv) == GIMPLE_PHI) |
7929 |
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT); |
7929 |
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT); |
7930 |
else |
7930 |
else |
Lines 8023-8034
Link Here
|
8023 |
/* Create the vector that holds the step of the induction. */ |
8023 |
/* Create the vector that holds the step of the induction. */ |
8024 |
if (nested_in_vect_loop) |
8024 |
if (nested_in_vect_loop) |
8025 |
/* iv_loop is nested in the loop to be vectorized. Generate: |
8025 |
/* iv_loop is nested in the loop to be vectorized. Generate: |
8026 |
vec_step = [S, S, S, S] */ |
8026 |
gcc_vec_step = [S, S, S, S] */ |
8027 |
new_name = step_expr; |
8027 |
new_name = step_expr; |
8028 |
else |
8028 |
else |
8029 |
{ |
8029 |
{ |
8030 |
/* iv_loop is the loop to be vectorized. Generate: |
8030 |
/* iv_loop is the loop to be vectorized. Generate: |
8031 |
vec_step = [VF*S, VF*S, VF*S, VF*S] */ |
8031 |
gcc_vec_step = [VF*S, VF*S, VF*S, VF*S] */ |
8032 |
gimple_seq seq = NULL; |
8032 |
gimple_seq seq = NULL; |
8033 |
if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr))) |
8033 |
if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr))) |
8034 |
{ |
8034 |
{ |
Lines 8050-8068
Link Here
|
8050 |
gcc_assert (CONSTANT_CLASS_P (new_name) |
8050 |
gcc_assert (CONSTANT_CLASS_P (new_name) |
8051 |
|| TREE_CODE (new_name) == SSA_NAME); |
8051 |
|| TREE_CODE (new_name) == SSA_NAME); |
8052 |
new_vec = build_vector_from_val (vectype, t); |
8052 |
new_vec = build_vector_from_val (vectype, t); |
8053 |
vec_step = vect_init_vector (phi, new_vec, vectype, NULL); |
8053 |
gcc_vec_step = vect_init_vector (phi, new_vec, vectype, NULL); |
8054 |
|
8054 |
|
8055 |
|
8055 |
|
8056 |
/* Create the following def-use cycle: |
8056 |
/* Create the following def-use cycle: |
8057 |
loop prolog: |
8057 |
loop prolog: |
8058 |
vec_init = ... |
8058 |
vec_init = ... |
8059 |
vec_step = ... |
8059 |
gcc_vec_step = ... |
8060 |
loop: |
8060 |
loop: |
8061 |
vec_iv = PHI <vec_init, vec_loop> |
8061 |
vec_iv = PHI <vec_init, vec_loop> |
8062 |
... |
8062 |
... |
8063 |
STMT |
8063 |
STMT |
8064 |
... |
8064 |
... |
8065 |
vec_loop = vec_iv + vec_step; */ |
8065 |
vec_loop = vec_iv + gcc_vec_step; */ |
8066 |
|
8066 |
|
8067 |
/* Create the induction-phi that defines the induction-operand. */ |
8067 |
/* Create the induction-phi that defines the induction-operand. */ |
8068 |
vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_"); |
8068 |
vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_"); |
Lines 8073-8079
Link Here
|
8073 |
|
8073 |
|
8074 |
/* Create the iv update inside the loop */ |
8074 |
/* Create the iv update inside the loop */ |
8075 |
vec_def = make_ssa_name (vec_dest); |
8075 |
vec_def = make_ssa_name (vec_dest); |
8076 |
new_stmt = gimple_build_assign (vec_def, PLUS_EXPR, induc_def, vec_step); |
8076 |
new_stmt = gimple_build_assign (vec_def, PLUS_EXPR, induc_def, gcc_vec_step); |
8077 |
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT); |
8077 |
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT); |
8078 |
set_vinfo_for_stmt (new_stmt, new_stmt_vec_info (new_stmt, loop_vinfo)); |
8078 |
set_vinfo_for_stmt (new_stmt, new_stmt_vec_info (new_stmt, loop_vinfo)); |
8079 |
|
8079 |
|
Lines 8117-8131
Link Here
|
8117 |
gcc_assert (CONSTANT_CLASS_P (new_name) |
8117 |
gcc_assert (CONSTANT_CLASS_P (new_name) |
8118 |
|| TREE_CODE (new_name) == SSA_NAME); |
8118 |
|| TREE_CODE (new_name) == SSA_NAME); |
8119 |
new_vec = build_vector_from_val (vectype, t); |
8119 |
new_vec = build_vector_from_val (vectype, t); |
8120 |
vec_step = vect_init_vector (phi, new_vec, vectype, NULL); |
8120 |
gcc_vec_step = vect_init_vector (phi, new_vec, vectype, NULL); |
8121 |
|
8121 |
|
8122 |
vec_def = induc_def; |
8122 |
vec_def = induc_def; |
8123 |
prev_stmt_vinfo = vinfo_for_stmt (induction_phi); |
8123 |
prev_stmt_vinfo = vinfo_for_stmt (induction_phi); |
8124 |
for (i = 1; i < ncopies; i++) |
8124 |
for (i = 1; i < ncopies; i++) |
8125 |
{ |
8125 |
{ |
8126 |
/* vec_i = vec_prev + vec_step */ |
8126 |
/* vec_i = vec_prev + gcc_vec_step */ |
8127 |
new_stmt = gimple_build_assign (vec_dest, PLUS_EXPR, |
8127 |
new_stmt = gimple_build_assign (vec_dest, PLUS_EXPR, |
8128 |
vec_def, vec_step); |
8128 |
vec_def, gcc_vec_step); |
8129 |
vec_def = make_ssa_name (vec_dest, new_stmt); |
8129 |
vec_def = make_ssa_name (vec_dest, new_stmt); |
8130 |
gimple_assign_set_lhs (new_stmt, vec_def); |
8130 |
gimple_assign_set_lhs (new_stmt, vec_def); |
8131 |
|
8131 |
|