433-milc Report

**433-milc**
Exec Time (%)	Codelet Name	Error (%)
99.8	control_main_32	99.8
1.1	gauge_stuff_g_measure_345	1.7
22.2	path_product_path_product_49	7.8
1.3	com_vanilla_start_gather_from_temp_581	11.5
96.1	update_update_40	6.2
24.9	gauge_stuff_imp_gauge_force_230	13.4
1.2	path_product_path_product_156	36.4
22.2	path_product_path_product_49	7.8
1.3	com_vanilla_start_gather_from_temp_581	11.5
46.6	quark_stuff_eo_fermion_force_946	6.6
10.7	quark_stuff_u_shift_fermion_1452	7.8
2.0	quark_stuff_u_shift_fermion_1466	4.6
10.1	quark_stuff_u_shift_fermion_1460	19.8
19.9	quark_stuff_add_force_to_mom_1523	8.6
1.3	com_vanilla_start_gather_from_temp_581	11.5
22.2	path_product_path_product_49	7.8
1.3	com_vanilla_start_gather_from_temp_581	11.5
17.8	quark_stuff_load_fatlinks_566	1.7
3.5	quark_stuff_compute_gen_staple_1368	4.9
6.9	quark_stuff_compute_gen_staple_1396	100.0
4.2	quark_stuff_compute_gen_staple_1375	8.3
1.3	quark_stuff_compute_gen_staple_1406	100.0
1.3	com_vanilla_start_gather_from_temp_581	11.5
1.5	update_u_update_u_40	1.5
22.2	path_product_path_product_49	7.8
1.3	com_vanilla_start_gather_from_temp_581	11.5
17.8	quark_stuff_load_fatlinks_566	1.7
3.5	quark_stuff_compute_gen_staple_1368	4.9
6.9	quark_stuff_compute_gen_staple_1396	100.0
4.2	quark_stuff_compute_gen_staple_1375	8.3
1.3	quark_stuff_compute_gen_staple_1406	100.0
1.3	com_vanilla_start_gather_from_temp_581	11.5

Exec Time : 1.330863e+10 cycles

Matching : 97.2%

                /****** quark_stuff.c  -- ******************/
/* MIMD version 6 */
/* quark action stuff for improved action
* D.T. 1/28/98, starting from gauge_stuff.c
* K.O. 3/99 Added optimized fattening for Asq actions
* D.T. 4/99 Combine force calculations for both mass quarks
* K.O. 4/99 Optimized force for Asq action
* S.G. 7/01, modified to use t_longlink and t_fatlink
* C.D. 10/02, consolidated quark_stuff.c and quark_stuff_tmp.c
*
* This code combines quark_stuff.c and quark_stuff_tmp.c
* with DSLASH_TMP_LINKS defined, puts links in field-major variables
* t_longlink and t_fatlink. Otherwise, puts them in the site structure.
*
* In this directory, assume all paths connect even to odd sites, etc.
* Tabulate "backwards" paths (e.g. "XDOWN" is backward path to "XUP")
* as separate parity transforms of the fundamental paths.  They will
* generally need a negative sign in Dslash.  See bottom for a long
* comment on sign conventions.
*/

/**#define FFTIME**/
/**#define LLTIME**/

#include "generic_ks_includes.h"	/* definitions files and prototypes */


#define NULL_FP -1 /* NULL field_offset to be used in the optimized version *
                    * of the load_fatlinks subroutine */

    /* Specify paths in orientation in which they appear in the
       forward part of the x component of dslash().  Rotations and
       reflections will be automatically included. Be careful
       about signs of coefficients.  See long comment at bottom. */
#include <quark_action.h>
    /* Include file specifies the basic paths */

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void printpath( int *path, int length );
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length );
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length );
#ifdef  ASQ_OPTIMIZED_FATTENING
void compute_gen_staple(field_offset staple, int mu, int nu,
                        field_offset link, double coef ) ;
#endif
#ifdef  ASQ_OPTIMIZED_FORCE
#ifndef FN
BOMB THE COMPILE
#endif
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) ;
void add_force_to_mom(su3_vector *back, su3_vector *forw, int dir, double coef);
void side_link_force(int mu, int nu, double coeff, su3_vector *Path,
		     su3_vector *Path_nu, su3_vector *Path_mu, 
		     su3_vector *Path_numu) ;

void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) ;
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) ;
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) ;
#endif

int path_num[MAX_BASIC_PATHS];	/* number of rotations/reflections for each 
					kind */
static double act_path_coeff[MAX_BASIC_PATHS]; /* actual path coefficient     *
                                               * it is equal to path_coeff   *
                                               * if not tadpole improvement  *
                                               * is specified                *
                                               * or path_coeff*u_0^(L-1) when*
                                               * tadpole improvement is      *
                                               * specified                   */

/* Array of structures, for each rotation and reflection of each kind of
	path.  */
struct {
    int dir[MAX_LENGTH];	/* directions in path */
    int length;		/* length of path */
    double coeff;	/* coefficient, including minus sign if backwards */
    double forwback;	/* +1 if in forward Dslash, -1 if in backward */
}   q_paths[MAX_NUM];
int num_q_paths;	/* number of paths in dslash */
int num_basic_paths;	/* number of paths before rotation/reflection */

int is_path_equal( int *path1, int* path2, int length );
int add_basic_path( int *vec, int length, double coeff );

/* Make table of paths in action */
void make_path_table() {

    int i,j;
#ifdef TADPOLE_IMPROVE
    int k;
#endif

    /* table of directions, 1 for each kind of path */
    /**int path_ind[MAX_BASIC_PATHS][MAX_LENGTH];**/
    /* table of coefficients in action, for each path */

    node0_printf("%s\n",quark_action_description);
    num_q_paths = 0;
    num_basic_paths = 0;


    /* add rots. and reflects to table, print out the path coefficients */
    node0_printf("path coefficients: npath  path_coeff  multiplicity\n");
    for(j=0;j<quark_action_npaths;j++) {
	double this_coeff;
	this_coeff = path_coeff[j];
#ifdef TADPOLE_IMPROVE
	for(k=1;k< path_length_in[j];k++)this_coeff /= u0;
#endif
	act_path_coeff[j] = this_coeff ;
	i = add_basic_path( path_ind[j], path_length_in[j],
	    this_coeff );
	node0_printf("                    %d      %e     %d\n",
	    j,this_coeff,i);
    }
}

/* add rotations and reflections of a path to the table.  Return
   multiplicity of paths added */
int add_basic_path( int *basic_vec, int length, double coeff ) {

    int perm[8],pp[8],ir[4];
    int j,path_num;
    int vec[MAX_LENGTH];
    int flag;

    path_num = 0;
    /* now fill the long table with all rotations and reflections
	of the fundamental path.  The path presented to us is for
        the positive x component of dslash, so if the x coordinate
        is reflected it will appear with a negative sign. */
      /* permutations */
      for(perm[0]=0;perm[0]<4;perm[0]++)
      for(perm[1]=0;perm[1]<4;perm[1]++)
      for(perm[2]=0;perm[2]<4;perm[2]++)
      for(perm[3]=0;perm[3]<4;perm[3]++){
	if(perm[0] != perm[1] && perm[0] != perm[2] 
	  && perm[0] != perm[3] && perm[1] != perm[2]
	  && perm[1] != perm[3] && perm[2] != perm[3] ) {
	  /* reflections*/
	  for(ir[0]=0;ir[0]<2;ir[0]++)
	  for(ir[1]=0;ir[1]<2;ir[1]++)
	  for(ir[2]=0;ir[2]<2;ir[2]++)
	  for(ir[3]=0;ir[3]<2;ir[3]++){
	    for(j=0;j<4;j++){
	      pp[j]=perm[j];

	      if(ir[j] == 1) pp[j]=OPP_DIR(pp[j]);
	      pp[OPP_DIR(j)]=OPP_DIR(pp[j]);
	    }
	    /* create new vector*/
	    for(j=0;j<length;j++) vec[j]=pp[basic_vec[j]];
	    for(j=length;j<MAX_LENGTH;j++) vec[j]=NODIR;

            flag=0;
	    /* check if it's a new set: */
	    for(j=0;j<num_q_paths;j++){
	      flag = is_path_equal( vec, q_paths[j].dir, MAX_LENGTH );
	      if(flag==1)break;
	    }
	    if(flag == 0 ){
	      if(num_q_paths>=MAX_NUM){
		node0_printf("OOPS: MAX_NUM too small\n");
		exit(0);
	      }
	      q_paths[num_q_paths].length=length;
	      for(j=0;j<MAX_LENGTH;j++) q_paths[num_q_paths].dir[j]=vec[j];
		/* remember to copy NODIR's, or comparison will get confused */
	      if(ir[0]==0){
		q_paths[num_q_paths].coeff =  coeff;
		q_paths[num_q_paths].forwback =  +1;
	      }
	      else{
		q_paths[num_q_paths].coeff = -coeff;
		q_paths[num_q_paths].forwback = -1;
	      }
	      num_q_paths++;
	      path_num++;
	      /**node0_printf("ADD PATH %d:  rx=%d ",num_q_paths-1,ir[0]);
		 printpath( vec, length );**/
	    }

	  } /* end reflection*/
        } /* end permutation if block */
      } /* end permutation */
    num_basic_paths++;
    return(path_num);
} /* add_basic_path */

/* parallel transport a vector to the current site along a path.
   For example, if the path is "XUP", bring in the vector from
   the +X direction to the current site. If KS phases are in lattice,
   this transport will automatically include them. 
   OK for src and dest to be the same.  OK for length=0.  */
/* NOT OPTIMIZED at the moment - do lots of extra copying.  Use temp.
   vectors rather than stuff in lattice.h */
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    su3_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    su3_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_dest = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_work = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(su3_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_vec( &(s->link[dir[j]]),
		    (su3_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_vec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(su3_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = *(su3_vector *)F_PT(s,src);
    }
  }
} /* path_transport */

/* Path transport a half_wilson_vector */
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    half_wilson_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    half_wilson_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (half_wilson_vector *)calloc(
      sites_on_node, sizeof(half_wilson_vector));
    tmp_dest = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));
    tmp_work = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(half_wilson_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_hwvec( &(s->link[dir[j]]),
		    (half_wilson_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_hwvec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(half_wilson_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) =
	  *(half_wilson_vector *)F_PT(s,src);
    }
  }
} /* path_transport_hwv */


#ifdef EO
/* Stupid dslash routine that follows all the paths.
  Should optimize by precomputing sums of paths to all the displacements
  Use dslash_fn for actions that involve only +X and +X+X+X couplings.
*/
/* D_slash routine - sets dest. on each site equal to sum of
   sources parallel transported to site, with minus sign for transport
   from negative directions.  */
void dslash_eo( field_offset src, field_offset dest, int parity ) {
   register int i;
   register site *s;
   register int ipath,otherparity;
   register double x; /* coefficient of path */

    switch(parity){
	case EVEN:	otherparity=ODD; break;
	case ODD:	otherparity=EVEN; break;
	case EVENANDODD:	otherparity=EVENANDODD; break;
    }

    /* Parallel transport by all the paths in the action.  
       Multiply by coefficient in table
    */
    FORSOMEPARITY(i,s,parity){ clearvec( (su3_vector *)F_PT(s,dest) ); }

    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	    path_transport( src, F_OFFSET(tempvec[0]), parity,
		q_paths[ipath].dir, q_paths[ipath].length );
	    x=q_paths[ipath].coeff;
	    FORSOMEPARITY(i,s,parity){
		scalar_mult_add_su3_vector(  (su3_vector *)F_PT(s,dest),
		    &(s->tempvec[0]), x, (su3_vector *)F_PT(s,dest) );
	    }
    }   /* ipath */
} /* dslash_eo */
#endif /* ifdef EO */

#ifdef FN
/* Sum over paths connecting to nearest neighbor point (fat link) and to third
   nearest neighbor (longlinks) */
/* Doug Toussaint 2/4/98 */
/* modified to use t_longlinks, S. Gottlieb 7/13/01 */
/* long link calculating routine */
/* path_product() follows the path starting at step 0, and
   leaves the answer at the end of the path.  We want the answer
   at the site where the path begins.  So we look for paths with
   the opposite displacement from the displacement of the point
   that we want to transport to this site, and take the adjoint
   of the matrix at the end. clear? */
/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_longlinks() {
  register int i;
  register site *s;
  int ipath,dir;
  int disp[4];
  register su3_matrix *long1;

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_longlinks needs phases in\n");
    terminate(0);
  }
  for (dir=XUP; dir<=TUP; dir++){ /* loop over longlink directions */
    /* set longlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      long1 = &(t_longlink[4*i+dir]);
#else
      long1 = &(s->longlink[dir]);
#endif
      clear_su3mat( long1 );
    }

    /* loop over paths, checking for ones with total displacement 3*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=3,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("ipath = %d, found a path:  ",ipath);
for(j=0;j<q_paths[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  long1 = &(t_longlink[4*i+dir]);
#else
	  long1 = &(s->longlink[dir]);
#endif
          scalar_mult_add_su3_matrix( long1,
	    &(s->staple), -q_paths[ipath].coeff, long1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */

  } /* loop over directions */

  valid_longlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Long):  %e\n",dtime);
#endif
}  /* load_longlinks() */

/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_fatlinks() {
  register int i;
  register site *s;
  int dir;
  register su3_matrix *fat1;
#ifdef ASQ_OPTIMIZED_FATTENING
  int  nu,rho,sig ;
  double one_link ; /*needed to fix the problem with the Lepage term */
#else
  int ipath;
  int disp[4];
#endif

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_fatlinks needs phases in\n");
    terminate(0);
  }

#ifndef  ASQ_OPTIMIZED_FATTENING   /* general case code */
  for (dir=XUP; dir<=TUP; dir++){ /* loop over fatlink directions */
    /* set fatlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+dir]);
#else
      fat1 = &(s->fatlink[dir]);
#endif
      clear_su3mat( fat1 );
    }
    
    /* loop over paths, checking for ones with total displacement 1*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=1,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("dir = %d, found a path:  ",dir);
for(j=0;j<q_paths.[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  fat1 = &(t_fatlink[4*i+dir]);
#else
	  fat1 = &(s->fatlink[dir]);
#endif
          scalar_mult_add_su3_matrix( fat1,
	    &(s->staple), -q_paths[ipath].coeff, fat1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */
  } /* loop over directions */
#else	/* ASQ_OPTIMIZED_FATTENING, for Asq and Asqtad actions */
/*  Optimized fattening code for the Asq and Asqtad actions.           *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
 
 /* to fix up the Lepage term, included by a trick below */
 one_link = (act_path_coeff[0] - 6.0*act_path_coeff[5]) ; 
 
 for (dir=XUP; dir<=TUP; dir++){
   FORALLSITES(i,s) /* Intialize fat links with c_1*U_\mu(x) */
     {
#ifdef DSLASH_TMP_LINKS
       fat1 = &(t_fatlink[4*i+dir]);
#else
       fat1 = &(s->fatlink[dir]);
#endif
       scalar_mult_su3_matrix(&(s->link[dir]), one_link ,
			      fat1 );
     }
   for(nu=XUP; nu<=TUP; nu++) if(nu!=dir)
     {
       compute_gen_staple(F_OFFSET(staple),dir,nu,F_OFFSET(link[dir]),
			  act_path_coeff[2]);
       /* The Lepage term */
       /* Note this also involves modifying c_1 (above) */
       compute_gen_staple(NULL_FP,dir,nu,F_OFFSET(staple),act_path_coeff[5]);
       for(rho=XUP; rho<=TUP; rho++) if((rho!=dir)&&(rho!=nu))
	 {
	   compute_gen_staple(F_OFFSET(tempmat1),dir,rho,F_OFFSET(staple),
			      act_path_coeff[3]);
	   for(sig=XUP; sig<=TUP; sig++)
	     if((sig!=dir)&&(sig!=nu)&&(sig!=rho))
	       {
		 compute_gen_staple(NULL_FP,dir,sig,
				    F_OFFSET(tempmat1),
				    act_path_coeff[4]);
	       } /* sig */
	 } /* rho */
     } /* nu */
 }/* dir */  
#endif

  valid_fatlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Fat):  %e\n",dtime);
#endif
}  /* load_fatlinks() */
#endif /* ifdef FN */


/* compare two paths, return 1 if equal, else zero */
int is_path_equal( int *path1, int* path2, int length ){
   register int i;
   for(i=0;i<length;i++)if(path1[i]!=path2[i])return(0);
   return(1);
}


/* update the  momenta with the fermion force */
/* Assumes that the conjugate gradient has been run, with the answer in
   x_off, and dslash(x_off,x_off,ODD) has been run. (fills in x_off_odd) */
/* SEE LONG COMMENTS AT END */

#ifndef ASQ_OPTIMIZED_FORCE
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon, coeff;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  half_wilson_vector *hw_tmp0,*hw_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  hw_tmp0 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  hw_tmp1 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  /* Use half_wilson_vectors to store x_off transported from ends of
     path.  0 component from forward end, 1 component from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    path_transport( x_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[1] for transport from starting end */
    FORALLSITES(i,s){
      hw_tmp0[i].h[0]=s->tempvec[0];
      hw_tmp0[i].h[1]=*(su3_vector *)F_PT(s,x_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff = ferm_epsilon*q_paths[ipath].coeff;
      if( (ilink%2)==1 )coeff = -coeff;

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "half_wilson_vector" to handle pair of vectors -
	0 component is x_off from forward end, 1 component from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_su3_mat_hwvec( &(s->link[lastdir]),
	      &(hw_tmp0[i]), &(hw_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( hw_tmp1, 2*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     hw_tmp0[i] = *(half_wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( hw_tmp0, 2*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_su3_mat_hwvec( &(s->link[OPP_DIR(lastdir)]),
                (half_wilson_vector *)(gen_pt[0][i]),
		&(hw_tmp1[i]) );
          }
	  tmp_pt = hw_tmp0; hw_tmp0 = hw_tmp1; hw_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( hw_tmp0 ); free( hw_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */


void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off,
  int nflav2, field_offset x2_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/14/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon1, ferm_epsilon2, coeff1, coeff2;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  wilson_vector *w_tmp0,*w_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon1 = 2.0*(nflav1/4.0)*eps;
  ferm_epsilon2 = 2.0*(nflav2/4.0)*eps;
  w_tmp0 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  w_tmp1 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  /* Use wilson_vectors to store x_off transported from ends of
     path.  0 and 1 components from forward end, 2 and 3 components
     from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    /**
    path_transport( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    path_transport( x2_off, F_OFFSET(tempvec[1]),
      EVENANDODD, q_paths[ipath].dir, length );
    **/
/** WARNING!! Assumes xxx1 and xxx2 contiguous **/
if( x2_off-x1_off != sizeof(su3_vector) ){node0_printf("BOTCH\n"); exit(0);}
    path_transport_hwv( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[2] for transport from starting end */
    FORALLSITES(i,s){
      w_tmp0[i].d[0]=s->tempvec[0];
      w_tmp0[i].d[1]=s->tempvec[1];
      w_tmp0[i].d[2]=*(su3_vector *)F_PT(s,x1_off);
      w_tmp0[i].d[3]=*(su3_vector *)F_PT(s,x2_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff1 = ferm_epsilon1*q_paths[ipath].coeff;
      coeff2 = ferm_epsilon2*q_paths[ipath].coeff;
      if( (ilink%2)==1 ){ coeff1 = -coeff1; coeff2 = -coeff2;}

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "wilson_vector" to handle pair of vectors -
	0 component is x_off1 from forward end, 1 component is x_off2
	from forward end,  2 and 3  components are x_off1 and x_off2 
	from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_mat_wilson_vec( &(s->link[lastdir]),
	      &(w_tmp0[i]), &(w_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( w_tmp1, 4*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     w_tmp0[i] = *(wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( w_tmp0, 4*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_mat_wilson_vec( &(s->link[OPP_DIR(lastdir)]),
                (wilson_vector *)(gen_pt[0][i]),
		&(w_tmp1[i]) );
          }
	  tmp_pt = w_tmp0; w_tmp0 = w_tmp1; w_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( w_tmp0 ); free( w_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */



#else /* ASQ_OPTIMIZED_FORCE, for Asq and Asqtad actions */
/* Optimized force code for the Asq and Asqtad actions                 *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
#define Pmu          tempvec[0] 
#define Pnumu        tempvec[1]
#define Prhonumu     tempvec[2]
#define P7           tempvec[3]
#define P7rho        tempvec[4]              
#define P7rhonu      tempvec[5]
#define P5           tempvec[6]
#define P3           tempvec[7]
#define P5nu         tempvec[3]
#define P3mu         tempvec[3]
#define Popmu        tempvec[4]
#define Pmumumu      tempvec[4]
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double ferm_epsilon, coeff;
  double OneLink, Lepage, Naik, FiveSt, ThreeSt, SevenSt ;
  su3_vector *tempvec[8] ;
  su3_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  
  /* Path coefficients times fermion epsilon */
  OneLink = act_path_coeff[0]*ferm_epsilon ; 
  Naik    = act_path_coeff[1]*ferm_epsilon ;
  ThreeSt = act_path_coeff[2]*ferm_epsilon ;
  FiveSt  = act_path_coeff[3]*ferm_epsilon ;
  SevenSt = act_path_coeff[4]*ferm_epsilon ;
  Lepage  = act_path_coeff[5]*ferm_epsilon ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    tempvec[mu] = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

  /*copy x_off to a temporary vector */
  temp_x = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
  FORALLSITES(i,s) temp_x[i] = *(su3_vector *)F_PT(s,x_off) ;

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_force_to_mom(P3, Pmu, sig, -ThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_force_to_mom(P7, Prhonumu, sig, -SevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_fermion(P7, P7rho, rho);
		  side_link_force(rho,sig,SevenSt, Pnumu, P7, Prhonumu, P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff = SevenSt/FiveSt ; 
		  FORALLSITES(i,s)
		    scalar_mult_add_su3_vector(&(P5[i]),&(P7rho[i]),coeff,
					       &(P5[i]));
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_fermion(P5,P5nu, nu);
	      side_link_force(nu,sig,-FiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff = FiveSt/ThreeSt ; 
	      FORALLSITES(i,s)
		scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_fermion(P5,P5nu, mu);
	  side_link_force(mu, sig, -Lepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff = Lepage/ThreeSt ; 
	  FORALLSITES(i,s)
	    scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), -Naik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(tempvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      

#define Pmu          hwvec[0] 
#define Pnumu        hwvec[1]
#define Prhonumu     hwvec[2]
#define P7           hwvec[3]
#define P7rho        hwvec[4]              
#define P7rhonu      hwvec[5]
#define P5           hwvec[6]
#define P3           hwvec[7]
#define P5nu         hwvec[3]
#define P3mu         hwvec[3]
#define Popmu        hwvec[4]
#define Pmumumu      hwvec[4]
void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off, 
			  int nflav2, field_offset x2_off){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/15/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double coeff[2],ferm_epsilon ;
  double OneLink[2], Lepage[2], Naik[2], FiveSt[2], ThreeSt[2], SevenSt[2] ;
  double mNaik[2], mLepage[2], mFiveSt[2], mThreeSt[2], mSevenSt[2] ;
  half_wilson_vector *hwvec[8] ;
  half_wilson_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  
  /* Path coefficients times fermion epsilon */
  ferm_epsilon = 2.0*(nflav1/4.0)*eps;
  OneLink[0] = act_path_coeff[0]*ferm_epsilon ;
  Naik[0]    = act_path_coeff[1]*ferm_epsilon ; mNaik[0]    = -Naik[0] ;
  ThreeSt[0] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[0] = -ThreeSt[0] ;
  FiveSt[0]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[0]  = -FiveSt[0] ;
  SevenSt[0] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[0] = -SevenSt[0] ;
  Lepage[0]  = act_path_coeff[5]*ferm_epsilon ; mLepage[0]  = -Lepage[0] ;

  ferm_epsilon = 2.0*(nflav2/4.0)*eps;
  OneLink[1] = act_path_coeff[0]*ferm_epsilon ;
  Naik[1]    = act_path_coeff[1]*ferm_epsilon ; mNaik[1]    = -Naik[1] ;
  ThreeSt[1] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[1] = -ThreeSt[1] ;
  FiveSt[1]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[1]  = -FiveSt[1] ;
  SevenSt[1] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[1] = -SevenSt[1] ;
  Lepage[1]  = act_path_coeff[5]*ferm_epsilon ; mLepage[1]  = -Lepage[1] ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    hwvec[mu]= 
      (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));

  /*copy x_off to a temporary vector */
  temp_x= 
    (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
  FORALLSITES(i,s)
    {
      temp_x[i].h[0] = *(su3_vector *)F_PT(s,x1_off) ;
      temp_x[i].h[1] = *(su3_vector *)F_PT(s,x2_off) ;
    }

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_hw_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_3f_force_to_mom(P3, Pmu, sig, mThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_hw_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_3f_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_hw_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_hw_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_3f_force_to_mom(P7, Prhonumu, sig, mSevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_hw_fermion(P7, P7rho, rho);
		  side_link_3f_force(rho,sig,SevenSt,Pnumu,P7,Prhonumu,P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff[0] = SevenSt[0]/FiveSt[0] ; 
		  coeff[1] = SevenSt[1]/FiveSt[1] ; 
		  FORALLSITES(i,s)
		    {
		      scalar_mult_add_su3_vector(&(P5[i].h[0]),
						 &(P7rho[i].h[0]),coeff[0],
						 &(P5[i].h[0]));
		      scalar_mult_add_su3_vector(&(P5[i].h[1]),
						 &(P7rho[i].h[1]),coeff[1],
						 &(P5[i].h[1]));
		    }
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_hw_fermion(P5,P5nu, nu);
	      side_link_3f_force(nu,sig,mFiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff[0] = FiveSt[0]/ThreeSt[0] ; 
	      coeff[1] = FiveSt[1]/ThreeSt[1] ; 
	      FORALLSITES(i,s)
		{
		  scalar_mult_add_su3_vector(&(P3[i].h[0]),
					     &(P5nu[i].h[0]), coeff[0],
					     &(P3[i].h[0]));
		  scalar_mult_add_su3_vector(&(P3[i].h[1]),
					     &(P5nu[i].h[1]), coeff[1],
					     &(P3[i].h[1]));
		}
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_3f_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_hw_fermion(P5,P5nu, mu);
	  side_link_3f_force(mu, sig, mLepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff[0] = Lepage[0]/ThreeSt[0] ; 
	  coeff[1] = Lepage[1]/ThreeSt[1] ; 
	  FORALLSITES(i,s)
	    {
	      scalar_mult_add_su3_vector(&(P3[i].h[0]),
					 &(P5nu[i].h[0]),coeff[0],
					 &(P3[i].h[0]));
	      scalar_mult_add_su3_vector(&(P3[i].h[1]),
					 &(P5nu[i].h[1]),coeff[1],
					 &(P3[i].h[1]));
	    }

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_hw_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_3f_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_3f_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_3f_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), mNaik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_hw_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_3f_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_3f_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(hwvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      
#endif /* ASQ_OPTIMIZED_FORCE */


#ifdef  ASQ_OPTIMIZED_FATTENING   /* Asqtad action only, "_fn" executables */
#ifndef FN
BOMB THE COMPILE
#endif
void compute_gen_staple(field_offset staple, int mu, int nu, 
			field_offset link, double coef) {
  su3_matrix tmat1,tmat2;
  msg_tag *mtag0,*mtag1;
  su3_matrix *tempmat ;
  register site *s ;
  register int i ;
  register su3_matrix *fat1;

  /* Computes the staple :
                   mu
                +-------+
          nu	|	|
		|	|
		X	X
    Where the mu link can be any su3_matrix. The result is saved in staple.
    if staple==NULL_FP then the result is not saved.
    It also adds the computed staple to the fatlink[mu] with weight coef.
  */

  /* Upper staple */
  mtag0 = start_gather( link, sizeof(su3_matrix), nu, EVENANDODD, gen_pt[0] );
  mtag1 = start_gather( F_OFFSET(link[nu]), sizeof(su3_matrix), mu, 
			EVENANDODD, gen_pt[1] );
  wait_gather(mtag0);
  wait_gather(mtag1);
  
  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, (su3_matrix *)F_PT(s,staple) );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, &tmat2 );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix(fat1, &tmat2, coef,
				 fat1) ;
    }
  }
  cleanup_gather(mtag0);
  cleanup_gather(mtag1);

  /* lower staple */
  tempmat = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
  mtag0 = start_gather( F_OFFSET(link[nu]),
			sizeof(su3_matrix), mu, EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);
  FORALLSITES(i,s){
    mult_su3_an( &(s->link[nu]),(su3_matrix *)F_PT(s,link), &tmat1 );
    mult_su3_nn( &(tmat1),(su3_matrix *)gen_pt[0][i], &(tempmat[i]) );
  }
  cleanup_gather(mtag0);
  mtag0 = start_gather_from_temp( tempmat, sizeof(su3_matrix),
				  OPP_DIR(nu), EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);

  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      add_su3_matrix( (su3_matrix *)F_PT(s,staple),(su3_matrix *)gen_pt[0][i], 
		      (su3_matrix *)F_PT(s,staple) );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)F_PT(s,staple), coef, 
				 fat1 );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)gen_pt[0][i], coef, 
				 fat1 );
    }
  }

  free(tempmat);
  cleanup_gather(mtag0);
}
#endif  /* ASQ_OPTIMIZED_FATTENING   */


#ifdef  ASQ_OPTIMIZED_FORCE
/*   Covariant shift of the src fermion field in the direction dir  *
 *  by one unit. The result is stored in dest.                       */ 
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) {
  su3_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(su3_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_vec(&(s->link[dir]),(su3_vector *)(gen_pt[0][i]),
			 &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_vec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(su3_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(su3_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/*  Covariant shift of the src half wilson fermion field in the  *
 * direction dir   by one unit. The result is stored in dest.    */
void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) {
  half_wilson_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(half_wilson_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_hwvec(&(s->link[dir]),
			   (half_wilson_vector *)(gen_pt[0][i]), &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = 
	(half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_hwvec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(half_wilson_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(half_wilson_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/* Add in contribution to the force */
/* Put antihermitian traceless part into momentum */
void add_force_to_mom(su3_vector *back,su3_vector *forw,int dir,double coeff) {
  register site *s ;
  register int i ;  
  register double tmp_coeff ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff = -coeff ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD) 
      tmp_coeff = -coeff ;
    else
      tmp_coeff = coeff ;
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i]), &(forw[i]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff, &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/* Add in contribution to the force ( 3flavor case ) */
/* Put antihermitian traceless part into momentum */
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) {
  register site *s ;
  register int i ;  
  double tmp_coeff[2] ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff[0] = -coeff[0] ;
      coeff[1] = -coeff[1] ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD)
      {
	tmp_coeff[0] = -coeff[0] ;
	tmp_coeff[1] = -coeff[1] ;
      }
    else
      {
	tmp_coeff[0] = coeff[0] ;
	tmp_coeff[1] = coeff[1] ;
      }
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i].h[0]), &(forw[i].h[0]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[0], &tmat2 );
    su3_projector(&(back[i].h[1]), &(forw[i].h[1]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[1], &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/*  This routine is needed in order to add the force on the side link *
 * of the paths in the Asq and Asqtad actions. It gets as inputs the  *
 * direction mu of the side link and the direction nu of the Dslash   *
 * term we are dealing with. Then it takes also 4 fermion fields:     *
 * Path: the piece of the path with no hop in the nu or mu direction  *
 * Path_nu: the piece of the path with a hop in nu  but not in mu     *
 * Path_mu: is Path times the link mu                                 *
 * Path_numu: is Path_nu times the link mu                            */
void side_link_force(int mu, int nu, double coeff, 
		     su3_vector *Path   , su3_vector *Path_nu, 
		     su3_vector *Path_mu, su3_vector *Path_numu) {
  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_force_to_mom(Path, Path_numu, OPP_DIR(mu), -coeff );/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_nu, Path_mu, mu, -coeff) ; /* ? extra - */
      else
	add_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
 
/*  The 3 flavor version of side_link_force used *
 * to optimize fermion transports                */
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) {
  double m_coeff[2] ;

  m_coeff[0] = -coeff[0] ;
  m_coeff[1] = -coeff[1] ;

  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_3f_force_to_mom(Path,Path_numu,OPP_DIR(mu),m_coeff);/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_nu, Path_mu, mu, m_coeff) ; /* ? extra - */
      else
	add_3f_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
#endif  /* ASQ_OPTIMIZED_FORCE   */

/* LONG COMMENTS
   Here we have combined "xxx", (offset "x_off")  which is
(M_adjoint M)^{-1} phi, with Dslash times this vector, which goes in the
odd sites of xxx.  Recall that phi is defined only on even sites.  In
computing the fermion force, we are looking at

< X |  d/dt ( Dslash_eo Dslash_oe ) | X >
=
< X | d/dt Dslash_eo | T > + < T | d/dt Dslash_oe | X >
where T = Dslash X.

The subsequent manipulations to get the coefficent of H, the momentum
matrix, in the simulation time derivative above look the same for
the two terms, except for a minus sign at the end, if we simply stick
T, which lives on odd sites, into the odd sites of X

 Each path in the action contributes terms when any link of the path
is the link for which we are computing the force.  We get a minus sign
for odd numbered links in the path, since they connect sites of the
opposite parity from what it would be for an even numbered link.
Minus signs from "going around" plaquette - ie KS phases, are supposed
to be already encoded in the path coefficients.
Minus signs from paths that go backwards are supposed to be already
encoded in the path coefficients.

Here, for example, are comments reproduced from the force routine for
the one-link plus Naik plus single-staple-fat-link action:

 The three link force has three contributions, where the link that
was differentiated is the first, second, or third link in the 3-link
path, respectively.  Diagramatically, where "O" represents the momentum,
the solid line the link corresponding to the momentum, and the dashed
lines the other links:
 

	O______________ x ............ x ...............
+
	x..............O______________x.................
+
	x..............x..............O________________
Think of this as
	< xxx | O | UUUxxx >		(  xxx, UUUX_p3 )
+
	< xxx U | O | UUxxx >		( X_m1U , UUX_p2 )
+
	< xxx U U | O | Uxxx >		( X_m2UU , UX_p1 )
where "U" indicates parallel transport, "X_p3" is xxx displaced
by +3, etc.
Note the second contribution has a relative minus sign
because it effectively contributes to the <odd|even>, or M_adjoint,
part of the force when we work on an even site. i.e., for M on
an even site, this three link path begins on an odd site.

The staple force has six contributions from each plane containing the
link direction:
Call these diagrams A-F:


	x...........x		O____________x
		    .			     .
		    .			     .
		    .			     .
		    .			     .
		    .			     .
	O___________x		x............x
	   (A)			    (B)



	x	    x		O____________x
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	O___________x		x	     x
	   (C)			    (D)



	x...........x		O____________x
	.			.
	.			.
	.			.
	.			.
	.			.
	O___________x		x............x
	   (E)			    (F)

As with the Naik term, diagrams C and D have a relative minus
sign because they connect sites of the other parity.

Also note an overall minus sign in the staple terms relative to the
one link term because, with the KS phase factors included, the fat
link is  "U - w3 * UUU", or the straight link MINUS w3 times the staples.

Finally, diagrams B and E get one more minus sign because the link
we are differentiating is in the opposite direction from the staple
as a whole.  You can think of this as this "U" being a correction to
a "U_adjoint", but the derivative of U is iHU and the derivative
of U_adjoint is -iHU_adjoint.

*/
/* LONG COMMENT on sign conventions
In most of the program, the KS phases and antiperiodic boundary
conditions are absorbed into the link matrices.  This greatly simplfies
multiplying by the fermion matrix.  However, it requires care in
specifying the path coefficients.  Remember that each time you
encircle a plaquette, you pick up a net minus sign from the KS phases.
Thus, when you have more than one path to the same point, you generally
have a relative minus sign for each plaquette in a surface bounded by
this path and the basic path for that displacement.

Examples:
  Fat Link:
    Positive:	X-------X

    Negative     --------
	 	|	|
		|	|
		X	X

  Naik connection, smeared
    Positive:	X-------x-------x-------X

    Negative:	---------
		|	|
		|	|
		X	x-------x-------X

    Positive:	--------x--------
		|		|
		|		|
		X		x-------X

    Negative:	--------x-------x-------x
		|			|
		|			|
		X			X
*/



/* Comment on acceptable actions.
   We construct the backwards part of dslash by reversing all the
   paths in the forwards part.  So, for example, in the p4 action
   the forwards part includes +X+Y+Y

		X
		|
		|
		X
		|
		|
	X---->--X

  so we put -X-Y-Y in the backwards part.  But this isn't the adjoint
  of U_x(0)U_y(+x)U_y(+x+y).  Since much of the code assumes that the
  backwards hop is the adjoint of the forwards (for example, in
  preventing going to 8 flavors), the code only works for actions
  where this is true.  Roughly, this means that the fat link must
  be symmetric about reflection around its midpoint.  Equivalently,
  the paths in the backwards part of Dslash are translations of the
  paths in the forwards part.  In the case of the "P4" or knight's move
  action, this means that we have to have both paths
   +X+Y+Y and +Y+Y+X to the same point, with the same coefficients.
  Alternatively, we could just use the symmetric path +Y+X+Y.
*/

invivoquark_stuff_u_shift_fermion_1452

Real time: 1.434033e+09 -- Predicted time: 1.555542e+09

Callcount: 10160

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
5081	### 1	18.4835594928402	1.745580e+05	1.706760e+05	2.2
747	### 2	1091.68869192986	1.778670e+05	1.457720e+05	18.0
1494	### 3	2060.23463563925	1.782820e+05	1.450760e+05	18.6
112	### 4	1940.04278282046	1.721280e+05	1.445440e+05	16.0
940	### 5	1283.82766135792	0.000000e+00	1.431600e+05	100.0
117	### 6	1528.94390026714	1.738960e+05	1.437440e+05	17.3
485	### 7	1193.99173118011	1.763020e+05	1.441560e+05	18.2
5971	### 8	264.910972320795	1.713880e+05	1.409000e+05	17.7
2244	### 9	131.17634454115	1.659790e+05	1.512040e+05	8.9
344	### 10	647.84990308172	1.749570e+05	1.423880e+05	18.6

                /******************  com_vanilla.c **************************************/
/* Communications routines for the SU3 program
   MIMD version 6.
   This file is communications-scheme dependent.
   Version for single processor machines.
   */
#define NOWHERE -1      /* Not an index in array of fields */

/* Modifications

   1/28/00 combined with Schroedinger functional and
           32 sublattice versions - UMH
   9/2/97  Revised to allow gathers from temporary fields.  neighbor[]
       is now list of indices, add start/restart_gather_from_temp D.T.
   8/05/97 ANSI prototyping for all routines C.D.
   10/5/96 Removed parallel I/O wrappers. Use io_ansi.c now.
   8/30/96 Added restore/save_checkpoint for compatibility
   8/05/96 Added broadcast_bytes and wrappers for system-dependent
    parallel file system calls C.D.
*/

/*
   This is a trivial version, where there is only one node, every site
   is on node ...

   initialize_machine() does any machine dependent setup at the
   very beginning.

   Some routines implement basic functions in a machine independent
   way.
   mynode() returns node number of this node.
   numnodes() returns number of nodes
   g_sync() provides a synchronization point for all nodes.
   g_floatsum() sums a floating point number over all nodes.
   g_doublesum() sums a double over all nodes.
   g_vecdoublesum() sums a vector of doubles over all nodes.
   g_complexsum() sums a single precision complex number over all nodes.
   g_dcomplexsum() sums a double precision complex number over all nodes.
   g_veccomplexsum() sums a vector of single precision complex numbers over all
	nodes.
   g_wvectorsum() sums a single precision wilson vector over all nodes.
   g_xor32() finds global exclusive or of 32-bit word
   g_floatmax() finds maximum of a floating point number over all nodes.
   g_doublemax() finds maximum of a double over all nodes.
   broadcast_float()  broadcasts a single precision number from
	node 0 to all nodes.
   broadcast_double()  broadcasts a double precision number
   broadcast_complex()  broadcasts a single precision complex number
   broadcast_dcomplex()  broadcasts a double precision complex number
   broadcast_bytes()  broadcasts a number of bytes
   send_integer() sends an integer to one other node
   receive_integer() receives an integer
   dclock() returns a double precision time, with arbitrary zero
   terminate() kills the job on all processors

   make_nn_gathers()  makes all necessary lists for communications with
   nodes containing neighbor sites.

   field_pointer_at_coordinates() returns the address of a field in
   the lattice given its coordinates.
   field_pointer_at_direction() returns the address of a field in the
   lattice at a direction from a given site.
   cleanup_field_pointer() frees the buffers that field_pointer...
   allocated.

   start_gather() starts asynchronous sends and receives required
   to gather neighbors.
   start_gather_from_temp() starts asynchronous sends and receives required
   to gather neighbors from temporary array of fields.
   wait_gather()  waits for receives to finish, insuring that the
   data has actually arrived.
   cleanup_gather() frees all the buffers that were allocated, WHICH
   MEANS THAT THE GATHERED DATA MAY SOON DISAPPEAR.
   restart_gather() repeats the internode communications of a previous
   gather.
   restart_gather_from_temp() repeats the internode communications of a
   previous gather of temporary field.

   send_field() sends a field to one other node.
   get_links() receives a field from some other node.
*/

#include "generic_includes.h"
#include <time.h>
#define MAX_GATHERS 32	/* Maximum number of gather tables */

    /* addresses of neighboring sites, NULL if off-node */
    /* neighbor[X][i] is the index of the neighbor of site lattice[i] in
       shuffle number X */
    int ** neighbor;
    /* Number of gathers (mappings) that have been set up */
    int n_gathers;

/**********************************************************************/
/* Machine initialization */
void initialize_machine(int argc, char **argv){}


/**********************************************************************/
/* Set up "comlink" structures needed by gather routines.
   make_lattice() must be called first. */
void make_nn_gathers(){

int i;
void neighbor_coords_special(
 int x,int y,int z,int t, /* coordinates of site */
 int *dirpt,              /* direction (eg XUP) */
 int fb,                  /* "forwards/backwards"  */
 int *x2p,int *y2p,int *z2p,int *t2p);
                          /* pointers to coordinates of neighbor */

    /* Allocate space for lists of pointers to neighbor sites.
       (NULL if neighbor not on this node) */
    neighbor = (int **)calloc(NDIRS, sizeof(int *));
    n_gathers=0;

    for(i=XUP;i<=TUP;i++)
	make_gather(neighbor_coords_special,&i,WANT_INVERSE,
	    ALLOW_EVEN_ODD,SWITCH_PARITY);

    /* Sort into the order we want for nearest neighbor gathers,
	so you can use XUP, XDOWN, etc. as argument in calling them. */
    sort_eight_special((void **) neighbor );
}

/**********************************************************************/
/* sort a list of eight pointers into the order we want for the
  nearest neighbor gathers:  XUP,YUP,ZUP,TUP,TDOWN,ZDOWN,YDOWN,XDOWN,
  starting from the index for the first pointer */

void sort_eight_neighborlists(int index){
  sort_eight_special((void **)&(neighbor[index]) );
}

/**********************************************************************/
/* sort a list of eight pointers into the order we want for the
  nearest neighbor gathers:  XUP,YUP,ZUP,TUP,TDOWN,ZDOWN,YDOWN,XDOWN */
void sort_eight_special(void **pt){
void *tt[8];
register int i;
    for(i=0;i<8;i++)tt[i]=pt[i];
    for(i=XUP;i<=TUP;i++){pt[i]=tt[2*i]; pt[OPP_DIR(i)]=tt[2*i+1];}
}

/**********************************************************************/
/* utility function for finding coordinates of neighbor */
/* This version for use by make_gather for nearest neighbor gathers */
void neighbor_coords_special(
 int x,int y,int z,int t, /* coordinates of site */
 int *dirpt,              /* direction (eg XUP) */
 int fb,                  /* "forwards/backwards"  */
 int *x2p,int *y2p,int *z2p,int *t2p)
                          /* pointers to coordinates of neighbor */
{
int dir;
    dir = (fb==FORWARDS) ? *dirpt : OPP_DIR(*dirpt);
    *x2p = x; *y2p = y; *z2p = z; *t2p = t;
    switch(dir){
	case XUP: *x2p = (x+1)%nx; break;
	case XDOWN: *x2p = (x+nx-1)%nx; break;
	case YUP: *y2p = (y+1)%ny; break;
	case YDOWN: *y2p = (y+ny-1)%ny; break;
	case ZUP: *z2p = (z+1)%nz; break;
	case ZDOWN: *z2p = (z+nz-1)%nz; break;
	case TUP: *t2p = (t+1)%nt; break;
	case TDOWN: *t2p = (t+nt-1)%nt; break;
	default: printf("BOTCH: bad direction\n"); exit(1);
    }
}

/**********************************************************************/
#define RECEIVE 0
#define SEND 1
/* add another gather to the list of tables */
int make_gather(
 void (*func)(int, int, int, int, int *, int, int *, int *, int *, int *),
        		/* function which defines sites to gather from */
 int *args,		/* list of arguments, to be passed to function */
 int inverse,		/* OWN_INVERSE, WANT_INVERSE, or NO_INVERSE */
 int want_even_odd,	/* ALLOW_EVEN_ODD or NO_EVEN_ODD */
 int parity_conserve)	/* {SAME,SWITCH,SCRAMBLE}_PARITY */
			/* SWITCH_PARITY for the 32 sublattice case means,
			   that for all sites in a sublattice, subl, the
			   target sublattice is the same, and different
			   from sublattice subl. */

{
register int i,k;	/* scratch */
register site *s;	/* scratch */
int dir;		/* direction */
int x,y,z,t;		/* coordinates */

#ifdef N_SUBL32
int subl, nsubl, *othersubl;	/* for 32 sublattice case */

    if( want_even_odd==ALLOW_EVEN_ODD && parity_conserve==SCRAMBLE_PARITY ){
	printf("Unsupported combination for make_gather:");
	printf(" ALLOW_EVEN_ODD and SCRAMBLE_PARITY\n");
	terminate(1);
    }
#endif

    /* we will have one or two more gathers */
    if( inverse==WANT_INVERSE ) n_gathers += 2;
    else			n_gathers += 1;
    if(n_gathers > MAX_GATHERS){
	if(this_node==0)printf("Too many gathers! change MAX_GATHERS\n");
	exit(1);
    }
    /* Allocate more space for lists of pointers to neighbor sites.
       (NULL if neighbor not on this node) */
    neighbor = (int **)realloc(neighbor, n_gathers*sizeof(int *));
    if( inverse==WANT_INVERSE) {
        neighbor[n_gathers-2] = (int *)calloc(sites_on_node, sizeof(int));
        if(neighbor[n_gathers-2]==NULL){
            printf("NODE %d: no room for neighbor vector\n",this_node);
            terminate(1);
        }
        neighbor[n_gathers-1] = (int *)calloc(sites_on_node, sizeof(int));
        if(neighbor[n_gathers-1]==NULL){
            printf("NODE %d: no room for neighbor vector\n",this_node);
            terminate(1);
        }
	dir = n_gathers-2;	/* index of gather we are working on */
    }
    else {
        neighbor[n_gathers-1] = (int *)calloc(sites_on_node, sizeof(int));
        if(neighbor[n_gathers-1]==NULL){
            printf("NODE %d: no room for neighbor vector\n",this_node);
            terminate(1);
        }
	dir = n_gathers-1;
    }

#ifndef N_SUBL32
    /* Check to see if mapping has advertised parity and inverse properties */
    /* Also check to see if it returns legal values for coordinates */
    FORALLSITES(i,s){
	/* find coordinates of neighbor who sends us data */
	func( s->x, s->y, s->z, s->t, args, FORWARDS, &x,&y,&z,&t);
	k=(x+y+z+t)%2;

	if( x<0 || y<0 || z<0  || t<0 || x>=nx || y>=ny || z>=nz || t>=nt){
	    printf("DUMMY! Your gather mapping does not stay in lattice\n");
	    printf("It mapped %d %d %d %d to %d %d %d %d\n",
		s->x,s->y,s->z,s->t,x,y,z,t);
	    terminate(1);
	}
	if( ( parity_conserve==SAME_PARITY &&
		((k==0&&s->parity==ODD)||(k==1&&s->parity==EVEN))
	    )
	  ||( parity_conserve==SWITCH_PARITY &&
		((k==0&&s->parity==EVEN)||(k==1&&s->parity==ODD))
#ifdef SCHROED_FUN
	    ) ) if( *args!=TUP && *args!=TDOWN && t!=0 && t!=(nt-1) ) {
#else
	    ) ){
#endif
	    printf("DUMMY! Your gather mapping does not obey claimed parity\n");
	    printf("It mapped %d %d %d %d to %d %d %d %d\n",
		s->x,s->y,s->z,s->t,x,y,z,t);
	    terminate(1);
	}
	if( inverse==OWN_INVERSE ){
	    int x2,y2,z2,t2;
	    func( x, y, z, t, args, FORWARDS, &x2,&y2,&z2,&t2);
	    if( s->x!=x2 || s->y!=y2 || s->z!=z2 || s->t!=t2 ){
		printf(
		    "DUMMY! Your gather mapping is not its own inverse\n");
		printf("It's square mapped %d %d %d %d to %d %d %d %d\n",
		    s->x,s->y,s->z,s->t,x2,y2,z2,t2);
		terminate(1);
	    }
	}
    }

#else	/* N_SUBL32 */

    othersubl = (int *)calloc(N_SUBL32, sizeof(int));
    if(othersubl == NULL){
	printf("NODE %d: no room for othersubl\n",this_node);
	terminate(1);
    }
    for(subl=0;subl<N_SUBL32;subl++) othersubl[subl] = NOWHERE;

    /* Check to see if mapping has advertised parity and inverse properties */
    /* Also check to see if it returns legal values for coordinates */
    FORALLSITES(i,s){
	/* find coordinates of neighbor who sends us data */
	func( s->x, s->y, s->z, s->t, args, FORWARDS, &x,&y,&z,&t);
	subl = s->parity;
	nsubl = (x%2) + 2*(y%2) + 4*(z%2) + 8*(t%2);
	nsubl += 16*((x/2+y/2+z/2+t/2)%2);

	if( want_even_odd==ALLOW_EVEN_ODD ){
	    if( othersubl[subl] == NOWHERE ){
		othersubl[subl] = nsubl;
	    }
	    else if( othersubl[subl] != nsubl ){
		printf("DUMMY! Your gather mixes up sublattices: %d vs %d\n",
		    othersubl[subl], nsubl);
		terminate(1);
	    }
	}

	if( x<0 || y<0 || z<0  || t<0 || x>=nx || y>=ny || z>=nz || t>=nt){
	    printf("DUMMY! Your gather mapping does not stay in lattice\n");
	    printf("It mapped %d %d %d %d to %d %d %d %d\n",
		s->x,s->y,s->z,s->t,x,y,z,t);
	    terminate(1);
	}
	if( parity_conserve==SAME_PARITY && subl!=nsubl ){
	    printf("DUMMY! Your gather mapping does not obey claimed parity");
	    printf(", namely SAME_PARITY\n");
	    printf("It mapped %d %d %d %d with %d to %d %d %d %d with %d\n",
		s->x,s->y,s->z,s->t,subl,x,y,z,t,nsubl);
	    terminate(1);
	}
	if( parity_conserve==SWITCH_PARITY && subl==nsubl ){
	    printf("DUMMY! Your gather mapping does not obey claimed parity");
	    printf(", namely SWITCH_PARITY\n");
	    printf("It mapped %d %d %d %d with %d to %d %d %d %d with %d\n",
		s->x,s->y,s->z,s->t,subl,x,y,z,t,nsubl);
	    terminate(1);
	}
	if( inverse==OWN_INVERSE ){
	    int x2,y2,z2,t2;
            func( x, y, z, t, args, FORWARDS, &x2,&y2,&z2,&t2);
	    if( s->x!=x2 || s->y!=y2 || s->z!=z2 || s->t!=t2 ){
	        printf(
		    "DUMMY! Your gather mapping is not its own inverse\n");
	        printf("It's square mapped %d %d %d %d to %d %d %d %d\n",
		    s->x,s->y,s->z,s->t,x2,y2,z2,t2);
	        terminate(1);
	    }
	}
    }

    free(othersubl);
#endif	/* N_SUBL32 */

    /* RECEIVE LISTS: */
    /* Fill in pointers to sites which are on this node, NOWHERE if
	they are off-node */
    FORALLSITES(i,s){
        /* find coordinates of neighbor who sends us data */
	func( s->x, s->y, s->z, s->t, args, FORWARDS, &x,&y,&z,&t);
	/* neighbor is always on node in vanilla version */
	neighbor[dir][i]= node_index(x,y,z,t);
    }

    if( inverse != WANT_INVERSE) return(dir);

    /* INVERSE GATHER */
    /* Now, if necessary, make inverse gather */
    /* In most cases, we can use the same lists as the gather, in one
	form or another.  Of course, by the time you get to here
	you know that inverse = WANT_INVERSE */
    dir++;	/* inverse gather has direction one more than original */

    /* Always set up pointers to sites on this node */
    /* scan sites in lattice */
    FORALLSITES(i,s){
	/* find coordinates of neighbor who sends us data */
	func( s->x, s->y, s->z, s->t, args, BACKWARDS, &x,&y,&z,&t);

	/* set up pointer */
	neighbor[dir][i]= node_index(x,y,z,t);
    }

    return(dir-1);
}

/**********************************************************************/
/* utility function for finding coordinates of neighbor */
void neighbor_coords(
 int x, int y, int z, int t,  /* coordinates of site */
 int dir,	              /* direction (eg XUP) */
 int *x2p, int *y2p, int *z2p, int *t2p)
                             /* pointers to coordinates of neighbor */
{
    *x2p = x; *y2p = y; *z2p = z; *t2p = t;
    switch(dir){
	case XUP: *x2p = (x+1)%nx; break;
	case XDOWN: *x2p = (x+nx-1)%nx; break;
	case YUP: *y2p = (y+1)%ny; break;
	case YDOWN: *y2p = (y+ny-1)%ny; break;
	case ZUP: *z2p = (z+1)%nz; break;
	case ZDOWN: *z2p = (z+nz-1)%nz; break;
	case TUP: *t2p = (t+1)%nt; break;
	case TDOWN: *t2p = (t+nt-1)%nt; break;
	default: printf("BOTCH: bad direction\n"); exit(1);
    }
}

/**********************************************************************/
/* Set up interrupt handlers to handle field_pointer routines
   make_lattice() must be called first. */
void start_handlers(){
  void fillfieldrequest(int type,int size,int node,int pid);
    /* nothing to do */
}

/**********************************************************************/
/* GATHER ROUTINES */
/* start_gather() returns a pointer to a list of msg_tag's, which will
   be used as input to subsequent wait_gather() and cleanup_gather() calls.
   In the single processor version, the routine will always return NULL.

   usage:  tag = start_gather( source, size, direction, parity, dest )
   example:
	msg_tag *tag;
	tag = start_gather( F_OFFSET(phi), sizeof(su3_vector), XUP,
	    EVEN, gen_pt[0] );
	  ** do other stuff **
	wait_gather(tag);
	  ** gen_pt[0][i] now contains the address of the phi
	   vector (or a copy therof) on the neighbor of site i in the
	   XUP direction for all even sites i.
	   Do whatever you want with it here.
	  **
	cleanup_gather(tag);
	  ** subsequent calls will overwrite the gathered fields. but if you
	   don't clean up, you will eventually run out of space **

    Under certain circumstances it is possible to efficiently gather
    a field that has been previously gathered.  This happens when the
    field being gathered has been modified, but the pointers (the
    destination of start_gather() ) have not been modified.  To use
    restart_gather, the original gather must have been waited for but
    not cleaned up.  The usage is:
	msg_tag *tag;
	tag = start_gather( F_OFFSET(phi), sizeof(su3_vector), XUP,
	    EVEN, gen_pt[0] );
	  ** do other stuff, but don't modify tag or gen_pt[0] **
	wait_gather(tag);
	  ** gen_pt[0][i] now contains the address of the phi
	   vector (or a copy thereof) on the neighbor of site i in the
	   XUP direction for all even sites i.
	   Do whatever you want with it here, but don't modify tag or
	   gen_pt[0].
	   Do modify the source field phi. **
	restart_gather( F_OFFSET(phi), sizeof(su3_vector), XUP,
	    EVEN, gen_pt[0], tag );
	  ** do other stuff **
	wait_gather(tag);
	  ** gen_pt[0][i] now contains the address of the modified phi.
	   The restart-wait may be repeated as often as desired.  **
	cleanup_gather(tag);
	  ** subsequent calls will overwrite the gathered fields. but if you
	   don't clean up, you will eventually run out of space **

   Internal convention for message types is (schematically):
	( f_offset(src) | sending_node | dir ) + GATHER_BASE
*/

/**********************************************************************/
#ifndef N_SUBL32

msg_tag * start_gather(
/* arguments */
 field_offset field,	/* which field? Some member of structure "site" */
 int size,		/* size in bytes of the field (eg sizeof(su3_vector))*/
 int index,		/* direction to gather from. eg XUP - index into
			   neighbor tables */
 int parity,		/* parity of sites whose neighbors we gather.
			   one of EVEN, ODD or EVENANDODD. */
 char ** dest)		/* one of the vectors of pointers */
{
/* local variables */
register int j;		/* scratch */
register site *s;	/* scratch pointer to site */

    /* set pointers in sites whose neighbors are on this node.  (If all
	neighbors are on this node, this is the only thing done.) */
    switch(parity){
	case EVEN:
	    FOREVENSITES(j,s){ 
                dest[j] = ((char *)(lattice+neighbor[index][j]))+field;
	    }
	    break;
	case ODD:
	    FORODDSITES(j,s){ 
                dest[j] = ((char *)(lattice+neighbor[index][j]))+field;
	    }
	    break;
	case EVENANDODD:
	    FORALLSITES(j,s){ 
                dest[j] = ((char *)(lattice+neighbor[index][j]))+field;
	    }
	    break;
    }

    /* return */
    return(NULL);
}

#else	/* N_SUBL32 */

msg_tag * start_gather(
/* arguments */
 field_offset field,	/* which field? Some member of structure "site" */
 int size,		/* size in bytes of the field (eg sizeof(su3_vector))*/
 int index,		/* direction to gather from. eg XUP - index into
			   neighbor tables */
 int subl,		/* subl of sites whose neighbors we gather.
			   It is EVENANDODD, if all sublattices are done. */
 char ** dest)		/* one of the vectors of pointers */
{
/* local variables */
register int j;		/* scratch */
register site *s;	/* scratch pointer to site */

    /* set pointers in sites whose neighbors are on this node.  (If all
	neighbors are on this node, this is the only thing done.) */
    if( subl == EVENANDODD ){
	FORALLSITES(j,s){
	    dest[j] = ((char *)(lattice+neighbor[index][j]))+field;
	}
    }
    else{
	FORSOMESUBLATTICE(j,s,subl){
	    dest[j] = ((char *)(lattice+neighbor[index][j]))+field;
	}
    }

    /* return */
    return(NULL);
}
#endif	/* N_SUBL32 */

/**********************************************************************/
/* Repeat a gather with the same source and destination as a
  previous gather.  The previous gather must have been waited for
  but not cleaned up.  Pointers to sites on the same node are not
  reset, and the same buffers are reused. */
void restart_gather(
/* arguments */
 field_offset field,	/* which field? Some member of structure "site" */
 int size,		/* size in bytes of the field (eg sizeof(su3_vector))*/
 int index,		/* direction to gather from. eg XUP - index into
			   neighbor tables */
 int parity,		/* parity of sites whose neighbors we gather.
			   one of EVEN, ODD or EVENANDODD. */
 char ** dest,		/* one of the vectors of pointers */
 msg_tag *mbuf)          /* previously returned by start_gather */
{
    /* Nothing to do */
}
/**********************************************************************/
#ifndef N_SUBL32

msg_tag * start_gather_from_temp(
/* arguments */
 void * field,		/* which field? Pointer returned by malloc() */
 int size,		/* size in bytes of the field (eg sizeof(su3_vector))*/
 int index,		/* direction to gather from. eg XUP - index into
			   neighbor tables */
 int parity,		/* parity of sites whose neighbors we gather.
			   one of EVEN, ODD or EVENANDODD. */
 char ** dest)		/* one of the vectors of pointers */
{
/* local variables */
register int j;		/* scratch */
register site *s;	/* scratch pointer to site */

    /* set pointers in sites whose neighbors are on this node.  (If all
	neighbors are on this node, this is the only thing done.) */
    switch(parity){
	case EVEN:
	    FOREVENSITES(j,s){ 
                dest[j] = (char *)field + neighbor[index][j]*size;
	    }
	    break;
	case ODD:
	    FORODDSITES(j,s){ 
                dest[j] = (char *)field + neighbor[index][j]*size;
	    }
	    break;
	case EVENANDODD:
	    FORALLSITES(j,s){ 
                dest[j] = (char *)field + neighbor[index][j]*size;
	    }
	    break;
    }

    /* return */
    return(NULL);
}

#else	/* N_SUBL32 */

msg_tag * start_gather_from_temp(
/* arguments */
 void * field,		/* which field? Pointer returned by malloc() */
 int size,		/* size in bytes of the field (eg sizeof(su3_vector))*/
 int index,		/* direction to gather from. eg XUP - index into
			   neighbor tables */
 int subl,		/* subl of sites whose neighbors we gather.
			   It is EVENANDODD, if all sublattices are done. */
 char ** dest)		/* one of the vectors of pointers */
{
/* local variables */
register int j;		/* scratch */
register site *s;	/* scratch pointer to site */

    /* set pointers in sites whose neighbors are on this node.  (If all
	neighbors are on this node, this is the only thing done.) */
    if( subl == EVENANDODD ){
	FORALLSITES(j,s){
	    dest[j] = (char *)field + neighbor[index][j]*size;
	}
    }
    else{
	FORSOMESUBLATTICE(j,s,subl){
	    dest[j] = (char *)field + neighbor[index][j]*size;
	}
    }

    /* return */
    return(NULL);
}
#endif	/* N_SUBL32 */

/**********************************************************************/
/* Repeat a gather with the same source and destination as a
  previous gather.  The previous gather must have been waited for
  but not cleaned up.  Pointers to sites on the same node are not
  reset, and the same buffers are reused. */
void restart_gather_from_temp(
/* arguments */
 void * field,		/* which field? Pointer returned by malloc() */
 int size,		/* size in bytes of the field (eg sizeof(su3_vector))*/
 int index,		/* direction to gather from. eg XUP - index into
			   neighbor tables */
 int parity,		/* parity of sites whose neighbors we gather.
			   one of EVEN, ODD or EVENANDODD. */
 char ** dest,		/* one of the vectors of pointers */
 msg_tag *mbuf)          /* previously returned by start_gather */
{
    /* Nothing to do */
}
/**********************************************************************/
void wait_gather(msg_tag *mbuf) {
    return;
}

/**********************************************************************/
void cleanup_gather(msg_tag *mbuf) {
    return;
}

/**********************************************************************/
/* GENERAL_GATHER ROUTINES */
/* start_general_gather() returns a pointer to a list of msg_tag's, which will
   be used as input to subsequent wait_general_gather() and
   cleanup_general_gather() calls.
   This list contains msg_tags for all receive buffers, followed by
   a msg_tag with msg_id = 0 and msg_buf = NULL, followed by msg_tags
   for all send buffers, followed by a msg_tag with id=0, buf=NULL.
   If no messages at all are required, the routine will return NULL.
   msg_buf=NULL should be a reliable indicator of no message.

   usage:  tag = start_general_gather( source, size, displacement, parity, dest)
   example:
	msg_tag *tag;
	int disp[4]; 
        disp[XUP]=1; disp[YUP]= -1; disp[ZUP] = disp[TUP] = 0;
	tag = start_general_gather( F_OFFSET(phi), sizeof(su3_vector), disp,
	    EVEN, gen_pt[0] );
	  ** do other stuff **
	wait_general_gather(tag);
	  ** gen_pt[0][i] now contains the address of the phi
	   vector (or a copy thereof) on the neighbor of site i in the
	   XUP direction for all even sites i.
	   Do whatever you want with it here.
	  **
	cleanup_general_gather(tag);
	  ** subsequent calls will overwrite the gathered fields. but if you
	   don't clean up, you will eventually run out of space **

*/
struct msg_tmp { int node,count; }; /* temporary structure for keeping
	track of messages to be sent or received */
int g_gather_flag=0;	/* flag to tell if general gather in progress */

#ifndef N_SUBL32

msg_tag * start_general_gather(
/* arguments */
 field_offset field,	/* which field? Some member of structure "site" */
 int size,		/* size in bytes of the field (eg sizeof(su3_vector))*/
 int *displacement,	/* displacement to gather from. four components */
 int parity,		/* parity of sites to which we gather.
			   one of EVEN, ODD or EVENANDODD. */
 char ** dest)		/* one of the vectors of pointers */
{
/* local variables */
register int i;		/* scratch */
register site *s;	/* scratch pointer to site */
int tx,ty,tz,tt;	/* temporary coordinates */


    /* check for gather already in progress */
    if(g_gather_flag!=0){
	fprintf(stderr,"ERROR: node %d, two general_gathers() at once!\n",
	    mynode() );
	exit(1);
    }

    /* set pointers in sites whose neighbors are on this node.  (If all
	neighbors are on this node, this is the only thing done.) Make
	list of nodes from whom we expect messages */
    FORSOMEPARITY(i,s,parity){
	if(displacement[XUP]!=0)tx = (s->x + displacement[XUP] + nx)%nx;
			else    tx = s->x;
	if(displacement[YUP]!=0)ty = (s->y + displacement[YUP] + ny)%ny;
			else    ty = s->y;
	if(displacement[ZUP]!=0)tz = (s->z + displacement[ZUP] + nz)%nz;
			else    tz = s->z;
	if(displacement[TUP]!=0)tt = (s->t + displacement[TUP] + nt)%nt;
			else    tt = s->t;
	dest[i] = F_PT( &lattice[node_index(tx,ty,tz,tt)], field );
    }

    /* mark gather in progress and return */
    g_gather_flag=1;

    return(NULL);
}

#else	/* N_SUBL32 */

msg_tag * start_general_gather(
/* arguments */
 field_offset field,	/* which field? Some member of structure "site" */
 int size,		/* size in bytes of the field (eg sizeof(su3_vector))*/
 int *displacement,	/* displacement to gather from. four components */
 int subl,		/* subl of sites whose neighbors we gather.
			   It is EVENANDODD, if all sublattices are done. */
 char ** dest)		/* one of the vectors of pointers */
{
/* local variables */
register int i;		/* scratch */
register site *s;	/* scratch pointer to site */
int tx,ty,tz,tt;	/* temporary coordinates */


    /* check for gather already in progress */
    if(g_gather_flag!=0){
	fprintf(stderr,"ERROR: node %d, two general_gathers() at once!\n",
	    mynode() );
	exit(1);
    }

    /* set pointers in sites whose neighbors are on this node.  (If all
	neighbors are on this node, this is the only thing done.) Make
	list of nodes from whom we expect messages */
    if( subl == EVENANDODD ){
	FORALLSITES(i,s){
	    if(displacement[XUP]!=0)tx = (s->x + displacement[XUP] + nx)%nx;
			else    tx = s->x;
	    if(displacement[YUP]!=0)ty = (s->y + displacement[YUP] + ny)%ny;
			else    ty = s->y;
	    if(displacement[ZUP]!=0)tz = (s->z + displacement[ZUP] + nz)%nz;
			else    tz = s->z;
	    if(displacement[TUP]!=0)tt = (s->t + displacement[TUP] + nt)%nt;
			else    tt = s->t;
	    dest[i] = (char *)( &lattice[node_index(tx,ty,tz,tt)]) + field;
	}
    }
    else{
	FORSOMESUBLATTICE(i,s,subl){
	    if(displacement[XUP]!=0)tx = (s->x + displacement[XUP] + nx)%nx;
			else    tx = s->x;
	    if(displacement[YUP]!=0)ty = (s->y + displacement[YUP] + ny)%ny;
			else    ty = s->y;
	    if(displacement[ZUP]!=0)tz = (s->z + displacement[ZUP] + nz)%nz;
			else    tz = s->z;
	    if(displacement[TUP]!=0)tt = (s->t + displacement[TUP] + nt)%nt;
			else    tt = s->t;
	    dest[i] = (char *)( &lattice[node_index(tx,ty,tz,tt)]) + field;
	}
    }

    /* mark gather in progress and return */
    g_gather_flag=1;

    return(NULL);
}
#endif	/* N_SUBL32 */

/**********************************************************************/
void wait_general_gather(msg_tag *mbuf) {
    g_gather_flag=0;
    return;
}

/**********************************************************************/
void cleanup_general_gather(msg_tag *mbuf) {
    return;
}


/**********************************************************************/
/* FIELD POINTER ROUTINES */

/* Return a pointer to a field in the lattice at some coordinates.
   If the site is on this node, just return its address.  If it is
   on another node, make a buffer, get the data, and return address of
   buffer.
   Example:  
	su3_matrix *pt;
	pt = (su3_matrix *)field_pointer_at_coordinates(
	    F_OFFSET(xlink), sizeof(su3_matrix), x,y,z,t );
	... do stuff here ...
	cleanup_field_pointer( (char *)pt );
*/
char * field_pointer_at_coordinates(
/* arguments */
 int field,	/* offset of one of the fields in lattice[] */
 int size,	/* size of the field in bytes */
 int x,int y,int z,int t)	/* coordinates of point to get field from */
{
int index;
    index=node_index(x,y,z,t);
    /* return address */
    return( F_PT( &(lattice[index]), field ) );
}

/**********************************************************************/
/* Return a pointer to a field in the lattice at some direction from
   a site on this node. (Usually the "current" site.)
   Works like field_pointer_at_coordinates. */
char * field_pointer_at_direction(
/* arguments */
 field_offset field,	/* offset of one of the fields in lattice[] */
 int size,	/* size of the field in bytes */
 site *s,	/* pointer to a site on this node */
 int direction)	/* direction of site's neighbor to get data from.
		   one of XUP,XDOWN,YUP... */
{
    return( F_PT( neighbor[direction][s-lattice], field ) );
}

/**********************************************************************/
/* free any buffers allocated by above routines.  Argument is the
address returned by field_pointer...() */
void cleanup_field_pointer(char * buf) {
}

/**********************************************************************/
/* SEND AND RECEIVE FIELD */
void send_field(char *buf, int size, int tonode) {
    printf("BOTCH: this never happens\n");
}
/**********************************************************************/
void get_field(char *buf, int size){
    printf("BOTCH: this never happens\n");
}

/**********************************************************************/
/* BASIC COMMUNICATIONS FUNCTIONS */

/* Tell what kind of machine we are on */
static char name[]="Scalar processor";
char * machine_type(){
    return(name);
}

/* Return my node number */
int mynode(){ return(0); }

/* Return number of nodes */
int numnodes(){ return(1); }

/* Synchronize all nodes */
void g_sync(){
}

/* Sum double over all nodes */
void g_floatsum( float *fpt) {
}

/* Sum double over all nodes */
void g_doublesum( double *dpt) {
}

/* Sum a vector of doubles over all nodes */
void g_vecdoublesum( double *dpt, int ndoubles) {
}

/* Sum complex over all nodes */
void g_complexsum( complex *cpt) {
}

/* Sum double_complex over all nodes */
void g_dcomplexsum( double_complex *cpt) {
}

/* Sum a vector of complex over all nodes */
void g_veccomplexsum( complex *cpt, int ncomplex){
}

/* Sum wilson_vector over all nodes */
void g_wvectorsum( wilson_vector *wvpt) {
}

/* 32 bit exclusive or over all nodes */
void g_xor32( u_int32type *pt ) {
}

/* Find maximum of double over all nodes */
void g_floatmax( float *fpt) {
}

/* Find maximum of double over all nodes */
void g_doublemax( double *dpt) {
}

/* Broadcast floating point number from node zero */
void broadcast_float(float *fpt) {
}

/* Broadcast double precision doubleing point number from node zero */
void broadcast_double(double *dpt) {
}

/* Broadcast single precision complex number from node zero */
void broadcast_complex(complex *cpt) {
}

/* Broadcast double precision complex number from node zero */
void broadcast_dcomplex(double_complex *cpt) {
}

/* Broadcast bytes from node zero */
void broadcast_bytes(char *buf,int size)  {
}

/* Send an integer to one other node */
/* This is to be called only by the node doing the sending */
void send_integer(int tonode, int *address) {
    printf("BOTCH: this never happens\n");
}

/* Receive an integer from another node */
/* Note we do not check if this was really meant for us */
void receive_integer(int *address) {
    printf("BOTCH: this never happens\n");
}

/* Double precision time */
/* This one wraps around after 36 minutes!! It gives the cpu time,
   not the wall clock time */
double dclock(){
long fine;
  /*CPU2006    fine = clock(); */
    fine = 0.;
    return( ((double)fine)/1000000.0 );
}

/* Print time stamp */
void time_stamp(char *msg){
  time_t time_stamp;
  
  if(mynode()==0){
    time(&time_stamp);
    printf("%s: %s\n",msg,ctime(&time_stamp));
  }
}

/* version of exit for multinode processes -- kill all nodes */
void terminate(int status) {
  time_stamp("termination");
  printf("Termination: status = %d\n",status);
  fflush(stdout);fflush(stderr);
  exit(status);
}

/* version of normal exit for scalar processes */
void normal_exit(int status) {
  /*CPU2006  time_stamp("exit");*/
  exit(status);
}

invivocom_vanilla_start_gather_from_temp_581

Real time: 1.784641e+08 -- Predicted time: 2.017281e+08

Callcount: 26472

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
76	### 1	24477.6146953405	7.600000e+03	6.696000e+03	11.8
65	### 2	2022.97321937322	7.760000e+03	7.020000e+03	9.5

                /******************  m_mat_an.c  (in su3.a) *****************************
*									*
* void mult_su3_an( su3_matrix *a,*b,*c )				*
* matrix multiply, first matrix is adjoint 				*
* C <-  A_adjoint*B							*
*/
#include "config.h"
#include "complex.h"
#include "su3.h"

#ifndef FAST
void mult_su3_an( su3_matrix *a, su3_matrix *b, su3_matrix *c ){
register int i,j,k;
register complex x,y;
    for(i=0;i<3;i++)for(j=0;j<3;j++){
	x.real=x.imag=0.0;
	for(k=0;k<3;k++){
	    CMULJ_( a->e[k][i] , b->e[k][j], y );
	    CSUM( x , y );
	}
	c->e[i][j] = x;
    }
}

/* "Hand coded" routines, clearer coding is up above */
#else

void mult_su3_an( su3_matrix *a, su3_matrix *b, su3_matrix *c ){
  int j;

#ifdef NATIVEDOUBLE
  register double a0r,a0i,a1r,a1i,a2r,a2i;
  register double b0r,b0i,b1r,b1i,b2r,b2i;
#else
  register double a0r,a0i,a1r,a1i,a2r,a2i;
  register double b0r,b0i,b1r,b1i,b2r,b2i;
#endif

  for(j=0;j<3;j++){
    
    a0r=a->e[0][0].real; a0i=a->e[0][0].imag;
    b0r=b->e[0][j].real; b0i=b->e[0][j].imag;
    a1r=a->e[1][0].real; a1i=a->e[1][0].imag;
    b1r=b->e[1][j].real; b1i=b->e[1][j].imag;
    a2r=a->e[2][0].real; a2i=a->e[2][0].imag;
    b2r=b->e[2][j].real; b2i=b->e[2][j].imag;
    
    c->e[0][j].real = a0r*b0r + a0i*b0i + a1r*b1r + a1i*b1i + a2r*b2r + a2i*b2i;
    c->e[0][j].imag = a0r*b0i - a0i*b0r + a1r*b1i - a1i*b1r + a2r*b2i - a2i*b2r;
  
    a0r=a->e[0][1].real; a0i=a->e[0][1].imag;
    b0r=b->e[0][j].real; b0i=b->e[0][j].imag;
    a1r=a->e[1][1].real; a1i=a->e[1][1].imag;
    b1r=b->e[1][j].real; b1i=b->e[1][j].imag;
    a2r=a->e[2][1].real; a2i=a->e[2][1].imag;
    b2r=b->e[2][j].real; b2i=b->e[2][j].imag;
    
    c->e[1][j].real = a0r*b0r + a0i*b0i + a1r*b1r + a1i*b1i + a2r*b2r + a2i*b2i;
    c->e[1][j].imag = a0r*b0i - a0i*b0r + a1r*b1i - a1i*b1r + a2r*b2i - a2i*b2r;
  
    a0r=a->e[0][2].real; a0i=a->e[0][2].imag;
    b0r=b->e[0][j].real; b0i=b->e[0][j].imag;
    a1r=a->e[1][2].real; a1i=a->e[1][2].imag;
    b1r=b->e[1][j].real; b1i=b->e[1][j].imag;
    a2r=a->e[2][2].real; a2i=a->e[2][2].imag;
    b2r=b->e[2][j].real; b2i=b->e[2][j].imag;
    
    c->e[2][j].real = a0r*b0r + a0i*b0i + a1r*b1r + a1i*b1i + a2r*b2r + a2i*b2i;
    c->e[2][j].imag = a0r*b0i - a0i*b0r + a1r*b1i - a1i*b1r + a2r*b2i - a2i*b2r;
  
    }
}

#endif	/* End of "#ifdef FAST" */

invivom_mat_an_mult_su3_an_39

Real time: 5.050794e+08 -- Predicted time: 2.255115e+09

Callcount: 3216000

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
632253	### 1	919253.191630562	5.300000e+02	1.520000e+02	71.3
2304026	### 2	815546.539081672	5.390000e+02	1.560000e+02	71.0
3084728	### 3	485037.946393254	8.370000e+02	1.480000e+02	82.3
2337338	### 4	451413.801902248	1.032000e+03	1.600000e+02	84.4
2637716	### 5	203548.460903411	9.600000e+02	1.640000e+02	82.9
3157735	### 6	162916.985807615	6.000000e+02	1.680000e+02	72.0
1859244	### 7	172645.810865939	8.780000e+02	1.440000e+02	83.5
979226	### 8	11449.7997346746	8.860000e+02	3.240000e+02	63.4
2708617	### 9	2806.6130459105	5.740000e+02	2.240000e+02	60.9

                /************************* control.c *******************************/
/* MIMD version 6 */
/* Main procedure for SU3 with dynamical fermions 			*/
/* Naik plus fat link fermions, general gauge action */

/* This version combines code for the PHI algorithm (approriate for 4
   flavors) and the R algorithm for "epsilon squared" updating of 
   1 to 4 flavors.  Compilation should occur with PHI_ALGORITHM defined
   for the former and not defined for the latter.  It also contains code
   for the hybrid Monte Carlo algorithm, for which HMC_ALGORITHM and
   PHI_ALGORITHM should be defined.  (Actually, the
   changes to control.c are minimal and the real differences will appear
   in update.c */

#define CONTROL
#include "ks_imp_includes.h"	/* definitions files and prototypes */

EXTERN  gauge_header start_lat_hdr;     /* Input gauge field header */

int main( int argc, char **argv ){
    int meascount,traj_done;
    int prompt;
    int s_iters,avs_iters,avspect_iters,
        avbcorr_iters;
    double dtime, dclock();

    initialize_machine(argc,argv);
    g_sync();
    /* set up */
    prompt = setup();
    /* loop over input sets */
    while( readin(prompt) == 0){

	/* perform warmup trajectories */
	dtime = -dclock();
	for( traj_done=0; traj_done < warms; traj_done++ ){
            update();
	}
	node0_printf("WARMUPS COMPLETED\n"); fflush(stdout);

	/* perform measuring trajectories, reunitarizing and measuring 	*/
	meascount=0;		/* number of measurements 		*/
	avspect_iters = avs_iters = avbcorr_iters = 0;
	for( traj_done=0; traj_done < trajecs; traj_done++ ){ 

	    /* do the trajectories */
	    s_iters=update();

	    /* measure every "propinterval" trajectories */
	    if( (traj_done%propinterval)==(propinterval-1) ){

	        /* call gauge_variable fermion_variable measuring routines */
		/* results are printed in output file */
	        rephase(OFF);
		g_measure( );
		rephase(ON);
		f_meas_imp(F_OFFSET(phi),F_OFFSET(xxx),mass);
#ifdef SPECTRUM 
                rephase( OFF );
                gaugefix(TUP,(double)1.8,500,(double)GAUGE_FIX_TOL,
          	   F_OFFSET(tempmat1),F_OFFSET(tempvec[0]),
			 0,NULL,NULL,0,NULL,NULL);
                rephase( ON );
#ifdef FN
		valid_fatlinks = valid_longlinks = 0;
#endif
		avspect_iters += spectrum2(mass,F_OFFSET(phi),F_OFFSET(xxx));

		/* Fix TUP Coulomb gauge - gauge links only*/
		rephase( OFF );
		gaugefix(TUP,(double)1.8,500,(double)GAUGE_FIX_TOL,
			 F_OFFSET(tempmat1),F_OFFSET(staple),
			 0,NULL,NULL,0,NULL,NULL);
		rephase( ON );
		valid_fatlinks = valid_longlinks = 0;

		/* Comment these spectrum calls out as you wish */
		/* There are no side-effects */
                avspect_iters += nl_spectrum(mass,F_OFFSET(phi),F_OFFSET(xxx),
					F_OFFSET(tempmat1),F_OFFSET(staple));
		avspect_iters += spectrum_mom(mass,mass,F_OFFSET(phi),5e-3);
		avspect_iters += spectrum_multimom(mass,mass,mass/4.,4,5e-3);
		avspect_iters += spectrum_nlpi2(mass,mass,F_OFFSET(phi),5e-3);
#ifdef FN
		if( !valid_fatlinks )load_fatlinks();
#endif
		rephase( OFF );
		hvy_pot( F_OFFSET(link[XUP]) );
		rephase( ON );
#endif
		avs_iters += s_iters;
	        ++meascount;
		fflush(stdout);
	    }
	}	/* end loop over trajectories */

	node0_printf("RUNNING COMPLETED\n"); fflush(stdout);
	if(meascount>0)  {
	    node0_printf("average cg iters for step= %e\n",
		(double)avs_iters/meascount);
#ifdef SPECTRUM
	    node0_printf("average cg iters for spectrum = %e\n",
		(double)avspect_iters/meascount);
#endif
	}

	dtime += dclock();
	if(this_node==0){
	  /*CPU2006  printf("Time = %e seconds\n",dtime);*/
	    printf("total_iters = %d\n",total_iters);
	}
	fflush(stdout);

	/* save lattice if requested */
        if( saveflag != FORGET ){
          rephase( OFF );
          save_lattice( saveflag, savefile );
          rephase( ON );
        }
    }
    return 0;
}

invivocontrol_main_32

Real time: 1.329345e+10 -- Predicted time: 2.289945e+07

Callcount: 1

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
1	### 1	1	2.289945e+07	1.330103e+10	99.8

                /****** quark_stuff.c  -- ******************/
/* MIMD version 6 */
/* quark action stuff for improved action
* D.T. 1/28/98, starting from gauge_stuff.c
* K.O. 3/99 Added optimized fattening for Asq actions
* D.T. 4/99 Combine force calculations for both mass quarks
* K.O. 4/99 Optimized force for Asq action
* S.G. 7/01, modified to use t_longlink and t_fatlink
* C.D. 10/02, consolidated quark_stuff.c and quark_stuff_tmp.c
*
* This code combines quark_stuff.c and quark_stuff_tmp.c
* with DSLASH_TMP_LINKS defined, puts links in field-major variables
* t_longlink and t_fatlink. Otherwise, puts them in the site structure.
*
* In this directory, assume all paths connect even to odd sites, etc.
* Tabulate "backwards" paths (e.g. "XDOWN" is backward path to "XUP")
* as separate parity transforms of the fundamental paths.  They will
* generally need a negative sign in Dslash.  See bottom for a long
* comment on sign conventions.
*/

/**#define FFTIME**/
/**#define LLTIME**/

#include "generic_ks_includes.h"	/* definitions files and prototypes */


#define NULL_FP -1 /* NULL field_offset to be used in the optimized version *
                    * of the load_fatlinks subroutine */

    /* Specify paths in orientation in which they appear in the
       forward part of the x component of dslash().  Rotations and
       reflections will be automatically included. Be careful
       about signs of coefficients.  See long comment at bottom. */
#include <quark_action.h>
    /* Include file specifies the basic paths */

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void printpath( int *path, int length );
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length );
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length );
#ifdef  ASQ_OPTIMIZED_FATTENING
void compute_gen_staple(field_offset staple, int mu, int nu,
                        field_offset link, double coef ) ;
#endif
#ifdef  ASQ_OPTIMIZED_FORCE
#ifndef FN
BOMB THE COMPILE
#endif
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) ;
void add_force_to_mom(su3_vector *back, su3_vector *forw, int dir, double coef);
void side_link_force(int mu, int nu, double coeff, su3_vector *Path,
		     su3_vector *Path_nu, su3_vector *Path_mu, 
		     su3_vector *Path_numu) ;

void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) ;
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) ;
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) ;
#endif

int path_num[MAX_BASIC_PATHS];	/* number of rotations/reflections for each 
					kind */
static double act_path_coeff[MAX_BASIC_PATHS]; /* actual path coefficient     *
                                               * it is equal to path_coeff   *
                                               * if not tadpole improvement  *
                                               * is specified                *
                                               * or path_coeff*u_0^(L-1) when*
                                               * tadpole improvement is      *
                                               * specified                   */

/* Array of structures, for each rotation and reflection of each kind of
	path.  */
struct {
    int dir[MAX_LENGTH];	/* directions in path */
    int length;		/* length of path */
    double coeff;	/* coefficient, including minus sign if backwards */
    double forwback;	/* +1 if in forward Dslash, -1 if in backward */
}   q_paths[MAX_NUM];
int num_q_paths;	/* number of paths in dslash */
int num_basic_paths;	/* number of paths before rotation/reflection */

int is_path_equal( int *path1, int* path2, int length );
int add_basic_path( int *vec, int length, double coeff );

/* Make table of paths in action */
void make_path_table() {

    int i,j;
#ifdef TADPOLE_IMPROVE
    int k;
#endif

    /* table of directions, 1 for each kind of path */
    /**int path_ind[MAX_BASIC_PATHS][MAX_LENGTH];**/
    /* table of coefficients in action, for each path */

    node0_printf("%s\n",quark_action_description);
    num_q_paths = 0;
    num_basic_paths = 0;


    /* add rots. and reflects to table, print out the path coefficients */
    node0_printf("path coefficients: npath  path_coeff  multiplicity\n");
    for(j=0;j<quark_action_npaths;j++) {
	double this_coeff;
	this_coeff = path_coeff[j];
#ifdef TADPOLE_IMPROVE
	for(k=1;k< path_length_in[j];k++)this_coeff /= u0;
#endif
	act_path_coeff[j] = this_coeff ;
	i = add_basic_path( path_ind[j], path_length_in[j],
	    this_coeff );
	node0_printf("                    %d      %e     %d\n",
	    j,this_coeff,i);
    }
}

/* add rotations and reflections of a path to the table.  Return
   multiplicity of paths added */
int add_basic_path( int *basic_vec, int length, double coeff ) {

    int perm[8],pp[8],ir[4];
    int j,path_num;
    int vec[MAX_LENGTH];
    int flag;

    path_num = 0;
    /* now fill the long table with all rotations and reflections
	of the fundamental path.  The path presented to us is for
        the positive x component of dslash, so if the x coordinate
        is reflected it will appear with a negative sign. */
      /* permutations */
      for(perm[0]=0;perm[0]<4;perm[0]++)
      for(perm[1]=0;perm[1]<4;perm[1]++)
      for(perm[2]=0;perm[2]<4;perm[2]++)
      for(perm[3]=0;perm[3]<4;perm[3]++){
	if(perm[0] != perm[1] && perm[0] != perm[2] 
	  && perm[0] != perm[3] && perm[1] != perm[2]
	  && perm[1] != perm[3] && perm[2] != perm[3] ) {
	  /* reflections*/
	  for(ir[0]=0;ir[0]<2;ir[0]++)
	  for(ir[1]=0;ir[1]<2;ir[1]++)
	  for(ir[2]=0;ir[2]<2;ir[2]++)
	  for(ir[3]=0;ir[3]<2;ir[3]++){
	    for(j=0;j<4;j++){
	      pp[j]=perm[j];

	      if(ir[j] == 1) pp[j]=OPP_DIR(pp[j]);
	      pp[OPP_DIR(j)]=OPP_DIR(pp[j]);
	    }
	    /* create new vector*/
	    for(j=0;j<length;j++) vec[j]=pp[basic_vec[j]];
	    for(j=length;j<MAX_LENGTH;j++) vec[j]=NODIR;

            flag=0;
	    /* check if it's a new set: */
	    for(j=0;j<num_q_paths;j++){
	      flag = is_path_equal( vec, q_paths[j].dir, MAX_LENGTH );
	      if(flag==1)break;
	    }
	    if(flag == 0 ){
	      if(num_q_paths>=MAX_NUM){
		node0_printf("OOPS: MAX_NUM too small\n");
		exit(0);
	      }
	      q_paths[num_q_paths].length=length;
	      for(j=0;j<MAX_LENGTH;j++) q_paths[num_q_paths].dir[j]=vec[j];
		/* remember to copy NODIR's, or comparison will get confused */
	      if(ir[0]==0){
		q_paths[num_q_paths].coeff =  coeff;
		q_paths[num_q_paths].forwback =  +1;
	      }
	      else{
		q_paths[num_q_paths].coeff = -coeff;
		q_paths[num_q_paths].forwback = -1;
	      }
	      num_q_paths++;
	      path_num++;
	      /**node0_printf("ADD PATH %d:  rx=%d ",num_q_paths-1,ir[0]);
		 printpath( vec, length );**/
	    }

	  } /* end reflection*/
        } /* end permutation if block */
      } /* end permutation */
    num_basic_paths++;
    return(path_num);
} /* add_basic_path */

/* parallel transport a vector to the current site along a path.
   For example, if the path is "XUP", bring in the vector from
   the +X direction to the current site. If KS phases are in lattice,
   this transport will automatically include them. 
   OK for src and dest to be the same.  OK for length=0.  */
/* NOT OPTIMIZED at the moment - do lots of extra copying.  Use temp.
   vectors rather than stuff in lattice.h */
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    su3_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    su3_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_dest = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_work = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(su3_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_vec( &(s->link[dir[j]]),
		    (su3_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_vec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(su3_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = *(su3_vector *)F_PT(s,src);
    }
  }
} /* path_transport */

/* Path transport a half_wilson_vector */
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    half_wilson_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    half_wilson_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (half_wilson_vector *)calloc(
      sites_on_node, sizeof(half_wilson_vector));
    tmp_dest = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));
    tmp_work = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(half_wilson_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_hwvec( &(s->link[dir[j]]),
		    (half_wilson_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_hwvec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(half_wilson_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) =
	  *(half_wilson_vector *)F_PT(s,src);
    }
  }
} /* path_transport_hwv */


#ifdef EO
/* Stupid dslash routine that follows all the paths.
  Should optimize by precomputing sums of paths to all the displacements
  Use dslash_fn for actions that involve only +X and +X+X+X couplings.
*/
/* D_slash routine - sets dest. on each site equal to sum of
   sources parallel transported to site, with minus sign for transport
   from negative directions.  */
void dslash_eo( field_offset src, field_offset dest, int parity ) {
   register int i;
   register site *s;
   register int ipath,otherparity;
   register double x; /* coefficient of path */

    switch(parity){
	case EVEN:	otherparity=ODD; break;
	case ODD:	otherparity=EVEN; break;
	case EVENANDODD:	otherparity=EVENANDODD; break;
    }

    /* Parallel transport by all the paths in the action.  
       Multiply by coefficient in table
    */
    FORSOMEPARITY(i,s,parity){ clearvec( (su3_vector *)F_PT(s,dest) ); }

    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	    path_transport( src, F_OFFSET(tempvec[0]), parity,
		q_paths[ipath].dir, q_paths[ipath].length );
	    x=q_paths[ipath].coeff;
	    FORSOMEPARITY(i,s,parity){
		scalar_mult_add_su3_vector(  (su3_vector *)F_PT(s,dest),
		    &(s->tempvec[0]), x, (su3_vector *)F_PT(s,dest) );
	    }
    }   /* ipath */
} /* dslash_eo */
#endif /* ifdef EO */

#ifdef FN
/* Sum over paths connecting to nearest neighbor point (fat link) and to third
   nearest neighbor (longlinks) */
/* Doug Toussaint 2/4/98 */
/* modified to use t_longlinks, S. Gottlieb 7/13/01 */
/* long link calculating routine */
/* path_product() follows the path starting at step 0, and
   leaves the answer at the end of the path.  We want the answer
   at the site where the path begins.  So we look for paths with
   the opposite displacement from the displacement of the point
   that we want to transport to this site, and take the adjoint
   of the matrix at the end. clear? */
/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_longlinks() {
  register int i;
  register site *s;
  int ipath,dir;
  int disp[4];
  register su3_matrix *long1;

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_longlinks needs phases in\n");
    terminate(0);
  }
  for (dir=XUP; dir<=TUP; dir++){ /* loop over longlink directions */
    /* set longlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      long1 = &(t_longlink[4*i+dir]);
#else
      long1 = &(s->longlink[dir]);
#endif
      clear_su3mat( long1 );
    }

    /* loop over paths, checking for ones with total displacement 3*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=3,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("ipath = %d, found a path:  ",ipath);
for(j=0;j<q_paths[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  long1 = &(t_longlink[4*i+dir]);
#else
	  long1 = &(s->longlink[dir]);
#endif
          scalar_mult_add_su3_matrix( long1,
	    &(s->staple), -q_paths[ipath].coeff, long1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */

  } /* loop over directions */

  valid_longlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Long):  %e\n",dtime);
#endif
}  /* load_longlinks() */

/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_fatlinks() {
  register int i;
  register site *s;
  int dir;
  register su3_matrix *fat1;
#ifdef ASQ_OPTIMIZED_FATTENING
  int  nu,rho,sig ;
  double one_link ; /*needed to fix the problem with the Lepage term */
#else
  int ipath;
  int disp[4];
#endif

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_fatlinks needs phases in\n");
    terminate(0);
  }

#ifndef  ASQ_OPTIMIZED_FATTENING   /* general case code */
  for (dir=XUP; dir<=TUP; dir++){ /* loop over fatlink directions */
    /* set fatlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+dir]);
#else
      fat1 = &(s->fatlink[dir]);
#endif
      clear_su3mat( fat1 );
    }
    
    /* loop over paths, checking for ones with total displacement 1*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=1,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("dir = %d, found a path:  ",dir);
for(j=0;j<q_paths.[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  fat1 = &(t_fatlink[4*i+dir]);
#else
	  fat1 = &(s->fatlink[dir]);
#endif
          scalar_mult_add_su3_matrix( fat1,
	    &(s->staple), -q_paths[ipath].coeff, fat1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */
  } /* loop over directions */
#else	/* ASQ_OPTIMIZED_FATTENING, for Asq and Asqtad actions */
/*  Optimized fattening code for the Asq and Asqtad actions.           *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
 
 /* to fix up the Lepage term, included by a trick below */
 one_link = (act_path_coeff[0] - 6.0*act_path_coeff[5]) ; 
 
 for (dir=XUP; dir<=TUP; dir++){
   FORALLSITES(i,s) /* Intialize fat links with c_1*U_\mu(x) */
     {
#ifdef DSLASH_TMP_LINKS
       fat1 = &(t_fatlink[4*i+dir]);
#else
       fat1 = &(s->fatlink[dir]);
#endif
       scalar_mult_su3_matrix(&(s->link[dir]), one_link ,
			      fat1 );
     }
   for(nu=XUP; nu<=TUP; nu++) if(nu!=dir)
     {
       compute_gen_staple(F_OFFSET(staple),dir,nu,F_OFFSET(link[dir]),
			  act_path_coeff[2]);
       /* The Lepage term */
       /* Note this also involves modifying c_1 (above) */
       compute_gen_staple(NULL_FP,dir,nu,F_OFFSET(staple),act_path_coeff[5]);
       for(rho=XUP; rho<=TUP; rho++) if((rho!=dir)&&(rho!=nu))
	 {
	   compute_gen_staple(F_OFFSET(tempmat1),dir,rho,F_OFFSET(staple),
			      act_path_coeff[3]);
	   for(sig=XUP; sig<=TUP; sig++)
	     if((sig!=dir)&&(sig!=nu)&&(sig!=rho))
	       {
		 compute_gen_staple(NULL_FP,dir,sig,
				    F_OFFSET(tempmat1),
				    act_path_coeff[4]);
	       } /* sig */
	 } /* rho */
     } /* nu */
 }/* dir */  
#endif

  valid_fatlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Fat):  %e\n",dtime);
#endif
}  /* load_fatlinks() */
#endif /* ifdef FN */


/* compare two paths, return 1 if equal, else zero */
int is_path_equal( int *path1, int* path2, int length ){
   register int i;
   for(i=0;i<length;i++)if(path1[i]!=path2[i])return(0);
   return(1);
}


/* update the  momenta with the fermion force */
/* Assumes that the conjugate gradient has been run, with the answer in
   x_off, and dslash(x_off,x_off,ODD) has been run. (fills in x_off_odd) */
/* SEE LONG COMMENTS AT END */

#ifndef ASQ_OPTIMIZED_FORCE
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon, coeff;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  half_wilson_vector *hw_tmp0,*hw_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  hw_tmp0 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  hw_tmp1 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  /* Use half_wilson_vectors to store x_off transported from ends of
     path.  0 component from forward end, 1 component from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    path_transport( x_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[1] for transport from starting end */
    FORALLSITES(i,s){
      hw_tmp0[i].h[0]=s->tempvec[0];
      hw_tmp0[i].h[1]=*(su3_vector *)F_PT(s,x_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff = ferm_epsilon*q_paths[ipath].coeff;
      if( (ilink%2)==1 )coeff = -coeff;

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "half_wilson_vector" to handle pair of vectors -
	0 component is x_off from forward end, 1 component from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_su3_mat_hwvec( &(s->link[lastdir]),
	      &(hw_tmp0[i]), &(hw_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( hw_tmp1, 2*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     hw_tmp0[i] = *(half_wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( hw_tmp0, 2*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_su3_mat_hwvec( &(s->link[OPP_DIR(lastdir)]),
                (half_wilson_vector *)(gen_pt[0][i]),
		&(hw_tmp1[i]) );
          }
	  tmp_pt = hw_tmp0; hw_tmp0 = hw_tmp1; hw_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( hw_tmp0 ); free( hw_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */


void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off,
  int nflav2, field_offset x2_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/14/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon1, ferm_epsilon2, coeff1, coeff2;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  wilson_vector *w_tmp0,*w_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon1 = 2.0*(nflav1/4.0)*eps;
  ferm_epsilon2 = 2.0*(nflav2/4.0)*eps;
  w_tmp0 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  w_tmp1 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  /* Use wilson_vectors to store x_off transported from ends of
     path.  0 and 1 components from forward end, 2 and 3 components
     from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    /**
    path_transport( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    path_transport( x2_off, F_OFFSET(tempvec[1]),
      EVENANDODD, q_paths[ipath].dir, length );
    **/
/** WARNING!! Assumes xxx1 and xxx2 contiguous **/
if( x2_off-x1_off != sizeof(su3_vector) ){node0_printf("BOTCH\n"); exit(0);}
    path_transport_hwv( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[2] for transport from starting end */
    FORALLSITES(i,s){
      w_tmp0[i].d[0]=s->tempvec[0];
      w_tmp0[i].d[1]=s->tempvec[1];
      w_tmp0[i].d[2]=*(su3_vector *)F_PT(s,x1_off);
      w_tmp0[i].d[3]=*(su3_vector *)F_PT(s,x2_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff1 = ferm_epsilon1*q_paths[ipath].coeff;
      coeff2 = ferm_epsilon2*q_paths[ipath].coeff;
      if( (ilink%2)==1 ){ coeff1 = -coeff1; coeff2 = -coeff2;}

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "wilson_vector" to handle pair of vectors -
	0 component is x_off1 from forward end, 1 component is x_off2
	from forward end,  2 and 3  components are x_off1 and x_off2 
	from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_mat_wilson_vec( &(s->link[lastdir]),
	      &(w_tmp0[i]), &(w_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( w_tmp1, 4*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     w_tmp0[i] = *(wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( w_tmp0, 4*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_mat_wilson_vec( &(s->link[OPP_DIR(lastdir)]),
                (wilson_vector *)(gen_pt[0][i]),
		&(w_tmp1[i]) );
          }
	  tmp_pt = w_tmp0; w_tmp0 = w_tmp1; w_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( w_tmp0 ); free( w_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */



#else /* ASQ_OPTIMIZED_FORCE, for Asq and Asqtad actions */
/* Optimized force code for the Asq and Asqtad actions                 *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
#define Pmu          tempvec[0] 
#define Pnumu        tempvec[1]
#define Prhonumu     tempvec[2]
#define P7           tempvec[3]
#define P7rho        tempvec[4]              
#define P7rhonu      tempvec[5]
#define P5           tempvec[6]
#define P3           tempvec[7]
#define P5nu         tempvec[3]
#define P3mu         tempvec[3]
#define Popmu        tempvec[4]
#define Pmumumu      tempvec[4]
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double ferm_epsilon, coeff;
  double OneLink, Lepage, Naik, FiveSt, ThreeSt, SevenSt ;
  su3_vector *tempvec[8] ;
  su3_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  
  /* Path coefficients times fermion epsilon */
  OneLink = act_path_coeff[0]*ferm_epsilon ; 
  Naik    = act_path_coeff[1]*ferm_epsilon ;
  ThreeSt = act_path_coeff[2]*ferm_epsilon ;
  FiveSt  = act_path_coeff[3]*ferm_epsilon ;
  SevenSt = act_path_coeff[4]*ferm_epsilon ;
  Lepage  = act_path_coeff[5]*ferm_epsilon ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    tempvec[mu] = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

  /*copy x_off to a temporary vector */
  temp_x = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
  FORALLSITES(i,s) temp_x[i] = *(su3_vector *)F_PT(s,x_off) ;

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_force_to_mom(P3, Pmu, sig, -ThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_force_to_mom(P7, Prhonumu, sig, -SevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_fermion(P7, P7rho, rho);
		  side_link_force(rho,sig,SevenSt, Pnumu, P7, Prhonumu, P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff = SevenSt/FiveSt ; 
		  FORALLSITES(i,s)
		    scalar_mult_add_su3_vector(&(P5[i]),&(P7rho[i]),coeff,
					       &(P5[i]));
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_fermion(P5,P5nu, nu);
	      side_link_force(nu,sig,-FiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff = FiveSt/ThreeSt ; 
	      FORALLSITES(i,s)
		scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_fermion(P5,P5nu, mu);
	  side_link_force(mu, sig, -Lepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff = Lepage/ThreeSt ; 
	  FORALLSITES(i,s)
	    scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), -Naik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(tempvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      

#define Pmu          hwvec[0] 
#define Pnumu        hwvec[1]
#define Prhonumu     hwvec[2]
#define P7           hwvec[3]
#define P7rho        hwvec[4]              
#define P7rhonu      hwvec[5]
#define P5           hwvec[6]
#define P3           hwvec[7]
#define P5nu         hwvec[3]
#define P3mu         hwvec[3]
#define Popmu        hwvec[4]
#define Pmumumu      hwvec[4]
void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off, 
			  int nflav2, field_offset x2_off){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/15/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double coeff[2],ferm_epsilon ;
  double OneLink[2], Lepage[2], Naik[2], FiveSt[2], ThreeSt[2], SevenSt[2] ;
  double mNaik[2], mLepage[2], mFiveSt[2], mThreeSt[2], mSevenSt[2] ;
  half_wilson_vector *hwvec[8] ;
  half_wilson_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  
  /* Path coefficients times fermion epsilon */
  ferm_epsilon = 2.0*(nflav1/4.0)*eps;
  OneLink[0] = act_path_coeff[0]*ferm_epsilon ;
  Naik[0]    = act_path_coeff[1]*ferm_epsilon ; mNaik[0]    = -Naik[0] ;
  ThreeSt[0] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[0] = -ThreeSt[0] ;
  FiveSt[0]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[0]  = -FiveSt[0] ;
  SevenSt[0] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[0] = -SevenSt[0] ;
  Lepage[0]  = act_path_coeff[5]*ferm_epsilon ; mLepage[0]  = -Lepage[0] ;

  ferm_epsilon = 2.0*(nflav2/4.0)*eps;
  OneLink[1] = act_path_coeff[0]*ferm_epsilon ;
  Naik[1]    = act_path_coeff[1]*ferm_epsilon ; mNaik[1]    = -Naik[1] ;
  ThreeSt[1] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[1] = -ThreeSt[1] ;
  FiveSt[1]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[1]  = -FiveSt[1] ;
  SevenSt[1] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[1] = -SevenSt[1] ;
  Lepage[1]  = act_path_coeff[5]*ferm_epsilon ; mLepage[1]  = -Lepage[1] ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    hwvec[mu]= 
      (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));

  /*copy x_off to a temporary vector */
  temp_x= 
    (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
  FORALLSITES(i,s)
    {
      temp_x[i].h[0] = *(su3_vector *)F_PT(s,x1_off) ;
      temp_x[i].h[1] = *(su3_vector *)F_PT(s,x2_off) ;
    }

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_hw_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_3f_force_to_mom(P3, Pmu, sig, mThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_hw_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_3f_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_hw_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_hw_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_3f_force_to_mom(P7, Prhonumu, sig, mSevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_hw_fermion(P7, P7rho, rho);
		  side_link_3f_force(rho,sig,SevenSt,Pnumu,P7,Prhonumu,P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff[0] = SevenSt[0]/FiveSt[0] ; 
		  coeff[1] = SevenSt[1]/FiveSt[1] ; 
		  FORALLSITES(i,s)
		    {
		      scalar_mult_add_su3_vector(&(P5[i].h[0]),
						 &(P7rho[i].h[0]),coeff[0],
						 &(P5[i].h[0]));
		      scalar_mult_add_su3_vector(&(P5[i].h[1]),
						 &(P7rho[i].h[1]),coeff[1],
						 &(P5[i].h[1]));
		    }
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_hw_fermion(P5,P5nu, nu);
	      side_link_3f_force(nu,sig,mFiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff[0] = FiveSt[0]/ThreeSt[0] ; 
	      coeff[1] = FiveSt[1]/ThreeSt[1] ; 
	      FORALLSITES(i,s)
		{
		  scalar_mult_add_su3_vector(&(P3[i].h[0]),
					     &(P5nu[i].h[0]), coeff[0],
					     &(P3[i].h[0]));
		  scalar_mult_add_su3_vector(&(P3[i].h[1]),
					     &(P5nu[i].h[1]), coeff[1],
					     &(P3[i].h[1]));
		}
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_3f_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_hw_fermion(P5,P5nu, mu);
	  side_link_3f_force(mu, sig, mLepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff[0] = Lepage[0]/ThreeSt[0] ; 
	  coeff[1] = Lepage[1]/ThreeSt[1] ; 
	  FORALLSITES(i,s)
	    {
	      scalar_mult_add_su3_vector(&(P3[i].h[0]),
					 &(P5nu[i].h[0]),coeff[0],
					 &(P3[i].h[0]));
	      scalar_mult_add_su3_vector(&(P3[i].h[1]),
					 &(P5nu[i].h[1]),coeff[1],
					 &(P3[i].h[1]));
	    }

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_hw_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_3f_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_3f_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_3f_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), mNaik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_hw_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_3f_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_3f_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(hwvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      
#endif /* ASQ_OPTIMIZED_FORCE */


#ifdef  ASQ_OPTIMIZED_FATTENING   /* Asqtad action only, "_fn" executables */
#ifndef FN
BOMB THE COMPILE
#endif
void compute_gen_staple(field_offset staple, int mu, int nu, 
			field_offset link, double coef) {
  su3_matrix tmat1,tmat2;
  msg_tag *mtag0,*mtag1;
  su3_matrix *tempmat ;
  register site *s ;
  register int i ;
  register su3_matrix *fat1;

  /* Computes the staple :
                   mu
                +-------+
          nu	|	|
		|	|
		X	X
    Where the mu link can be any su3_matrix. The result is saved in staple.
    if staple==NULL_FP then the result is not saved.
    It also adds the computed staple to the fatlink[mu] with weight coef.
  */

  /* Upper staple */
  mtag0 = start_gather( link, sizeof(su3_matrix), nu, EVENANDODD, gen_pt[0] );
  mtag1 = start_gather( F_OFFSET(link[nu]), sizeof(su3_matrix), mu, 
			EVENANDODD, gen_pt[1] );
  wait_gather(mtag0);
  wait_gather(mtag1);
  
  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, (su3_matrix *)F_PT(s,staple) );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, &tmat2 );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix(fat1, &tmat2, coef,
				 fat1) ;
    }
  }
  cleanup_gather(mtag0);
  cleanup_gather(mtag1);

  /* lower staple */
  tempmat = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
  mtag0 = start_gather( F_OFFSET(link[nu]),
			sizeof(su3_matrix), mu, EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);
  FORALLSITES(i,s){
    mult_su3_an( &(s->link[nu]),(su3_matrix *)F_PT(s,link), &tmat1 );
    mult_su3_nn( &(tmat1),(su3_matrix *)gen_pt[0][i], &(tempmat[i]) );
  }
  cleanup_gather(mtag0);
  mtag0 = start_gather_from_temp( tempmat, sizeof(su3_matrix),
				  OPP_DIR(nu), EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);

  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      add_su3_matrix( (su3_matrix *)F_PT(s,staple),(su3_matrix *)gen_pt[0][i], 
		      (su3_matrix *)F_PT(s,staple) );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)F_PT(s,staple), coef, 
				 fat1 );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)gen_pt[0][i], coef, 
				 fat1 );
    }
  }

  free(tempmat);
  cleanup_gather(mtag0);
}
#endif  /* ASQ_OPTIMIZED_FATTENING   */


#ifdef  ASQ_OPTIMIZED_FORCE
/*   Covariant shift of the src fermion field in the direction dir  *
 *  by one unit. The result is stored in dest.                       */ 
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) {
  su3_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(su3_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_vec(&(s->link[dir]),(su3_vector *)(gen_pt[0][i]),
			 &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_vec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(su3_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(su3_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/*  Covariant shift of the src half wilson fermion field in the  *
 * direction dir   by one unit. The result is stored in dest.    */
void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) {
  half_wilson_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(half_wilson_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_hwvec(&(s->link[dir]),
			   (half_wilson_vector *)(gen_pt[0][i]), &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = 
	(half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_hwvec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(half_wilson_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(half_wilson_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/* Add in contribution to the force */
/* Put antihermitian traceless part into momentum */
void add_force_to_mom(su3_vector *back,su3_vector *forw,int dir,double coeff) {
  register site *s ;
  register int i ;  
  register double tmp_coeff ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff = -coeff ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD) 
      tmp_coeff = -coeff ;
    else
      tmp_coeff = coeff ;
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i]), &(forw[i]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff, &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/* Add in contribution to the force ( 3flavor case ) */
/* Put antihermitian traceless part into momentum */
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) {
  register site *s ;
  register int i ;  
  double tmp_coeff[2] ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff[0] = -coeff[0] ;
      coeff[1] = -coeff[1] ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD)
      {
	tmp_coeff[0] = -coeff[0] ;
	tmp_coeff[1] = -coeff[1] ;
      }
    else
      {
	tmp_coeff[0] = coeff[0] ;
	tmp_coeff[1] = coeff[1] ;
      }
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i].h[0]), &(forw[i].h[0]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[0], &tmat2 );
    su3_projector(&(back[i].h[1]), &(forw[i].h[1]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[1], &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/*  This routine is needed in order to add the force on the side link *
 * of the paths in the Asq and Asqtad actions. It gets as inputs the  *
 * direction mu of the side link and the direction nu of the Dslash   *
 * term we are dealing with. Then it takes also 4 fermion fields:     *
 * Path: the piece of the path with no hop in the nu or mu direction  *
 * Path_nu: the piece of the path with a hop in nu  but not in mu     *
 * Path_mu: is Path times the link mu                                 *
 * Path_numu: is Path_nu times the link mu                            */
void side_link_force(int mu, int nu, double coeff, 
		     su3_vector *Path   , su3_vector *Path_nu, 
		     su3_vector *Path_mu, su3_vector *Path_numu) {
  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_force_to_mom(Path, Path_numu, OPP_DIR(mu), -coeff );/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_nu, Path_mu, mu, -coeff) ; /* ? extra - */
      else
	add_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
 
/*  The 3 flavor version of side_link_force used *
 * to optimize fermion transports                */
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) {
  double m_coeff[2] ;

  m_coeff[0] = -coeff[0] ;
  m_coeff[1] = -coeff[1] ;

  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_3f_force_to_mom(Path,Path_numu,OPP_DIR(mu),m_coeff);/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_nu, Path_mu, mu, m_coeff) ; /* ? extra - */
      else
	add_3f_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
#endif  /* ASQ_OPTIMIZED_FORCE   */

/* LONG COMMENTS
   Here we have combined "xxx", (offset "x_off")  which is
(M_adjoint M)^{-1} phi, with Dslash times this vector, which goes in the
odd sites of xxx.  Recall that phi is defined only on even sites.  In
computing the fermion force, we are looking at

< X |  d/dt ( Dslash_eo Dslash_oe ) | X >
=
< X | d/dt Dslash_eo | T > + < T | d/dt Dslash_oe | X >
where T = Dslash X.

The subsequent manipulations to get the coefficent of H, the momentum
matrix, in the simulation time derivative above look the same for
the two terms, except for a minus sign at the end, if we simply stick
T, which lives on odd sites, into the odd sites of X

 Each path in the action contributes terms when any link of the path
is the link for which we are computing the force.  We get a minus sign
for odd numbered links in the path, since they connect sites of the
opposite parity from what it would be for an even numbered link.
Minus signs from "going around" plaquette - ie KS phases, are supposed
to be already encoded in the path coefficients.
Minus signs from paths that go backwards are supposed to be already
encoded in the path coefficients.

Here, for example, are comments reproduced from the force routine for
the one-link plus Naik plus single-staple-fat-link action:

 The three link force has three contributions, where the link that
was differentiated is the first, second, or third link in the 3-link
path, respectively.  Diagramatically, where "O" represents the momentum,
the solid line the link corresponding to the momentum, and the dashed
lines the other links:
 

	O______________ x ............ x ...............
+
	x..............O______________x.................
+
	x..............x..............O________________
Think of this as
	< xxx | O | UUUxxx >		(  xxx, UUUX_p3 )
+
	< xxx U | O | UUxxx >		( X_m1U , UUX_p2 )
+
	< xxx U U | O | Uxxx >		( X_m2UU , UX_p1 )
where "U" indicates parallel transport, "X_p3" is xxx displaced
by +3, etc.
Note the second contribution has a relative minus sign
because it effectively contributes to the <odd|even>, or M_adjoint,
part of the force when we work on an even site. i.e., for M on
an even site, this three link path begins on an odd site.

The staple force has six contributions from each plane containing the
link direction:
Call these diagrams A-F:


	x...........x		O____________x
		    .			     .
		    .			     .
		    .			     .
		    .			     .
		    .			     .
	O___________x		x............x
	   (A)			    (B)



	x	    x		O____________x
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	O___________x		x	     x
	   (C)			    (D)



	x...........x		O____________x
	.			.
	.			.
	.			.
	.			.
	.			.
	O___________x		x............x
	   (E)			    (F)

As with the Naik term, diagrams C and D have a relative minus
sign because they connect sites of the other parity.

Also note an overall minus sign in the staple terms relative to the
one link term because, with the KS phase factors included, the fat
link is  "U - w3 * UUU", or the straight link MINUS w3 times the staples.

Finally, diagrams B and E get one more minus sign because the link
we are differentiating is in the opposite direction from the staple
as a whole.  You can think of this as this "U" being a correction to
a "U_adjoint", but the derivative of U is iHU and the derivative
of U_adjoint is -iHU_adjoint.

*/
/* LONG COMMENT on sign conventions
In most of the program, the KS phases and antiperiodic boundary
conditions are absorbed into the link matrices.  This greatly simplfies
multiplying by the fermion matrix.  However, it requires care in
specifying the path coefficients.  Remember that each time you
encircle a plaquette, you pick up a net minus sign from the KS phases.
Thus, when you have more than one path to the same point, you generally
have a relative minus sign for each plaquette in a surface bounded by
this path and the basic path for that displacement.

Examples:
  Fat Link:
    Positive:	X-------X

    Negative     --------
	 	|	|
		|	|
		X	X

  Naik connection, smeared
    Positive:	X-------x-------x-------X

    Negative:	---------
		|	|
		|	|
		X	x-------x-------X

    Positive:	--------x--------
		|		|
		|		|
		X		x-------X

    Negative:	--------x-------x-------x
		|			|
		|			|
		X			X
*/



/* Comment on acceptable actions.
   We construct the backwards part of dslash by reversing all the
   paths in the forwards part.  So, for example, in the p4 action
   the forwards part includes +X+Y+Y

		X
		|
		|
		X
		|
		|
	X---->--X

  so we put -X-Y-Y in the backwards part.  But this isn't the adjoint
  of U_x(0)U_y(+x)U_y(+x+y).  Since much of the code assumes that the
  backwards hop is the adjoint of the forwards (for example, in
  preventing going to 8 flavors), the code only works for actions
  where this is true.  Roughly, this means that the fat link must
  be symmetric about reflection around its midpoint.  Equivalently,
  the paths in the backwards part of Dslash are translations of the
  paths in the forwards part.  In the case of the "P4" or knight's move
  action, this means that we have to have both paths
   +X+Y+Y and +Y+Y+X to the same point, with the same coefficients.
  Alternatively, we could just use the symmetric path +Y+X+Y.
*/

invivoquark_stuff_load_fatlinks_566

Real time: 2.371161e+09 -- Predicted time: 2.412550e+09

Callcount: 22

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
14	### 1	10.0038326362174	1.107993e+08	1.072275e+08	3.2
12	### 2	12.0000792662136	1.086769e+08	1.064262e+08	2.0

                /********** update.c ****************************************************/
/* MIMD version 6 */

/*
 Update lattice.
 Improved method for 1-4 flavors:
	update U by (epsilon/2)*(1-Nf/4)
	compute PHI
	update U to epsilon/2
	compute X
	update H, full step
	update U to next time needed

 This routine does not refresh the antihermitian momenta.
 This routine begins at "integral" time, with H and U evaluated
 at same time.
*/
#include "ks_imp_includes.h"	/* definitions files and prototypes */

int update()  {
int step, iters=0;
double final_rsq;
void predict_next_xxx(double *oldtime,double *newtime,double *nexttime);
double cg_time,old_cg_time,next_cg_time;	/* simulation time for last two CG's */
#ifdef HMC_ALGORITHM
double startaction,endaction,d_action();
double xrandom;
#endif

    /* refresh the momenta */
    ranmom();

/*DEBUG*/
/**grsource_imp(F_OFFSET(phi), mass, EVENANDODD);
ks_congrad(F_OFFSET(phi),F_OFFSET(xxx),mass,niter,rsqmin,EVENANDODD,&final_rsq);
checkmul();**/
/*ENDDEBUG*/

    /* do "steps" microcanonical steps  */
    for(step=1; step <= steps; step++){
 
#ifdef PHI_ALGORITHM
        /* generate a pseudofermion configuration only at start*/
     	if(step==1){grsource_imp(F_OFFSET(phi), mass, EVEN); old_cg_time = cg_time = -1.0e6;}

#ifdef HMC_ALGORITHM
        /* find action */
        /* do conjugate gradient to get (Madj M)inverse * phi */
        if(step==1){
            /* do conjugate gradient to get (Madj M)inverse * phi */
	    iters += ks_congrad( F_OFFSET(phi), F_OFFSET(xxx), mass, 
				 niter, rsqmin, EVEN, &final_rsq );
	    cg_time = 0.0;

     	    startaction=d_action();
            /* copy link field to old_link */
	    gauge_field_copy( F_OFFSET(link[0]), F_OFFSET(old_link[0]));
        }
#endif

	/* update U's to middle of interval */
     	update_u(0.5*epsilon);

	/* save conjugate gradient solution, predict next one */
	next_cg_time = ((double)step-0.5)*epsilon;
	predict_next_xxx(&old_cg_time,&cg_time,&next_cg_time);

#else /* "R" algorithm */
       	/* first update the U's to special time interval */
       	update_u(epsilon*(0.5-nflavors/8.0));

        /* generate a pseudofermion configuration */
     	grsource_imp(F_OFFSET(phi), mass, EVEN); cg_time = -1.0e6;

	/* update U's to middle of interval */
     	update_u(epsilon*nflavors/8.0);
#endif

        /* do conjugate gradient to get (Madj M)inverse * phi */
     	iters += ks_congrad( F_OFFSET(phi), F_OFFSET(xxx), mass, 
			     niter, rsqmin, EVEN, &final_rsq );
	cg_time = ((double)step - 0.5)*epsilon;
	/* now update H by full time interval */
    	update_h(epsilon);

    	/* update U's by half time step to get to even time */
    	update_u(epsilon*0.5);

        /* reunitarize the gauge field */
	rephase( OFF );
        reunitarize();
	rephase( ON );

    }	/* end loop over microcanonical steps */

#ifdef HMC_ALGORITHM
    /* find action */
    /* do conjugate gradient to get (Madj M)inverse * phi */
    next_cg_time = steps*epsilon;
    predict_next_xxx(&old_cg_time,&cg_time,&next_cg_time);
    iters += ks_congrad( F_OFFSET(phi), F_OFFSET(xxx), mass,
			 niter, rsqmin, EVEN, &final_rsq );
    cg_time = steps*epsilon;
    endaction=d_action();
    /* decide whether to accept, if not, copy old link field back */
    /* careful - must generate only one random number for whole lattice */
    if(this_node==0)xrandom = myrand(&node_prn);
    broadcast_double(&xrandom);
    if( exp( (double)(startaction-endaction) ) < xrandom ){
	if(steps > 0)
	    gauge_field_copy( F_OFFSET(old_link[0]), F_OFFSET(link[0]) );
#ifdef FN
	valid_longlinks=valid_fatlinks=0;
#endif
	node0_printf("REJECT: delta S = %e\n", (double)(endaction-startaction));
    }
    else {
	node0_printf("ACCEPT: delta S = %e\n", (double)(endaction-startaction));
    }
#endif

    if(steps > 0)return (iters/steps);
    else return(-99);
}

#ifdef PHI_ALGORITHM
/* use linear extrapolation to predict next conjugate gradient solution */
/* only need even sites */
void predict_next_xxx(double *oldtime,double *newtime,double *nexttime)
{
register int i;
register site *s;
register double x;
su3_vector tvec;
    if( *newtime != *oldtime ) x = (*nexttime-*newtime)/(*newtime-*oldtime);
    else x = 0.0;
    if( *oldtime < 0.0 ){
        FOREVENSITES(i,s){
	    s->old_xxx = s->xxx;
        }
    }
    else  {
        FOREVENSITES(i,s){
            sub_su3_vector( &(s->xxx), &(s->old_xxx), &tvec);
	    s->old_xxx = s->xxx;
            scalar_mult_add_su3_vector( &(s->xxx), &tvec,x, &(s->xxx) );
        }
    }
    *oldtime = *newtime;
    *newtime = *nexttime;
}
#endif

invivoupdate_update_40

Real time: 1.279301e+10 -- Predicted time: 1.363889e+10

Callcount: 2

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
1	### 1	1	6.607406e+09	6.371209e+09	3.5
2	### 2	1	7.031489e+09	6.446610e+09	8.3

                /****** quark_stuff.c  -- ******************/
/* MIMD version 6 */
/* quark action stuff for improved action
* D.T. 1/28/98, starting from gauge_stuff.c
* K.O. 3/99 Added optimized fattening for Asq actions
* D.T. 4/99 Combine force calculations for both mass quarks
* K.O. 4/99 Optimized force for Asq action
* S.G. 7/01, modified to use t_longlink and t_fatlink
* C.D. 10/02, consolidated quark_stuff.c and quark_stuff_tmp.c
*
* This code combines quark_stuff.c and quark_stuff_tmp.c
* with DSLASH_TMP_LINKS defined, puts links in field-major variables
* t_longlink and t_fatlink. Otherwise, puts them in the site structure.
*
* In this directory, assume all paths connect even to odd sites, etc.
* Tabulate "backwards" paths (e.g. "XDOWN" is backward path to "XUP")
* as separate parity transforms of the fundamental paths.  They will
* generally need a negative sign in Dslash.  See bottom for a long
* comment on sign conventions.
*/

/**#define FFTIME**/
/**#define LLTIME**/

#include "generic_ks_includes.h"	/* definitions files and prototypes */


#define NULL_FP -1 /* NULL field_offset to be used in the optimized version *
                    * of the load_fatlinks subroutine */

    /* Specify paths in orientation in which they appear in the
       forward part of the x component of dslash().  Rotations and
       reflections will be automatically included. Be careful
       about signs of coefficients.  See long comment at bottom. */
#include <quark_action.h>
    /* Include file specifies the basic paths */

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void printpath( int *path, int length );
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length );
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length );
#ifdef  ASQ_OPTIMIZED_FATTENING
void compute_gen_staple(field_offset staple, int mu, int nu,
                        field_offset link, double coef ) ;
#endif
#ifdef  ASQ_OPTIMIZED_FORCE
#ifndef FN
BOMB THE COMPILE
#endif
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) ;
void add_force_to_mom(su3_vector *back, su3_vector *forw, int dir, double coef);
void side_link_force(int mu, int nu, double coeff, su3_vector *Path,
		     su3_vector *Path_nu, su3_vector *Path_mu, 
		     su3_vector *Path_numu) ;

void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) ;
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) ;
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) ;
#endif

int path_num[MAX_BASIC_PATHS];	/* number of rotations/reflections for each 
					kind */
static double act_path_coeff[MAX_BASIC_PATHS]; /* actual path coefficient     *
                                               * it is equal to path_coeff   *
                                               * if not tadpole improvement  *
                                               * is specified                *
                                               * or path_coeff*u_0^(L-1) when*
                                               * tadpole improvement is      *
                                               * specified                   */

/* Array of structures, for each rotation and reflection of each kind of
	path.  */
struct {
    int dir[MAX_LENGTH];	/* directions in path */
    int length;		/* length of path */
    double coeff;	/* coefficient, including minus sign if backwards */
    double forwback;	/* +1 if in forward Dslash, -1 if in backward */
}   q_paths[MAX_NUM];
int num_q_paths;	/* number of paths in dslash */
int num_basic_paths;	/* number of paths before rotation/reflection */

int is_path_equal( int *path1, int* path2, int length );
int add_basic_path( int *vec, int length, double coeff );

/* Make table of paths in action */
void make_path_table() {

    int i,j;
#ifdef TADPOLE_IMPROVE
    int k;
#endif

    /* table of directions, 1 for each kind of path */
    /**int path_ind[MAX_BASIC_PATHS][MAX_LENGTH];**/
    /* table of coefficients in action, for each path */

    node0_printf("%s\n",quark_action_description);
    num_q_paths = 0;
    num_basic_paths = 0;


    /* add rots. and reflects to table, print out the path coefficients */
    node0_printf("path coefficients: npath  path_coeff  multiplicity\n");
    for(j=0;j<quark_action_npaths;j++) {
	double this_coeff;
	this_coeff = path_coeff[j];
#ifdef TADPOLE_IMPROVE
	for(k=1;k< path_length_in[j];k++)this_coeff /= u0;
#endif
	act_path_coeff[j] = this_coeff ;
	i = add_basic_path( path_ind[j], path_length_in[j],
	    this_coeff );
	node0_printf("                    %d      %e     %d\n",
	    j,this_coeff,i);
    }
}

/* add rotations and reflections of a path to the table.  Return
   multiplicity of paths added */
int add_basic_path( int *basic_vec, int length, double coeff ) {

    int perm[8],pp[8],ir[4];
    int j,path_num;
    int vec[MAX_LENGTH];
    int flag;

    path_num = 0;
    /* now fill the long table with all rotations and reflections
	of the fundamental path.  The path presented to us is for
        the positive x component of dslash, so if the x coordinate
        is reflected it will appear with a negative sign. */
      /* permutations */
      for(perm[0]=0;perm[0]<4;perm[0]++)
      for(perm[1]=0;perm[1]<4;perm[1]++)
      for(perm[2]=0;perm[2]<4;perm[2]++)
      for(perm[3]=0;perm[3]<4;perm[3]++){
	if(perm[0] != perm[1] && perm[0] != perm[2] 
	  && perm[0] != perm[3] && perm[1] != perm[2]
	  && perm[1] != perm[3] && perm[2] != perm[3] ) {
	  /* reflections*/
	  for(ir[0]=0;ir[0]<2;ir[0]++)
	  for(ir[1]=0;ir[1]<2;ir[1]++)
	  for(ir[2]=0;ir[2]<2;ir[2]++)
	  for(ir[3]=0;ir[3]<2;ir[3]++){
	    for(j=0;j<4;j++){
	      pp[j]=perm[j];

	      if(ir[j] == 1) pp[j]=OPP_DIR(pp[j]);
	      pp[OPP_DIR(j)]=OPP_DIR(pp[j]);
	    }
	    /* create new vector*/
	    for(j=0;j<length;j++) vec[j]=pp[basic_vec[j]];
	    for(j=length;j<MAX_LENGTH;j++) vec[j]=NODIR;

            flag=0;
	    /* check if it's a new set: */
	    for(j=0;j<num_q_paths;j++){
	      flag = is_path_equal( vec, q_paths[j].dir, MAX_LENGTH );
	      if(flag==1)break;
	    }
	    if(flag == 0 ){
	      if(num_q_paths>=MAX_NUM){
		node0_printf("OOPS: MAX_NUM too small\n");
		exit(0);
	      }
	      q_paths[num_q_paths].length=length;
	      for(j=0;j<MAX_LENGTH;j++) q_paths[num_q_paths].dir[j]=vec[j];
		/* remember to copy NODIR's, or comparison will get confused */
	      if(ir[0]==0){
		q_paths[num_q_paths].coeff =  coeff;
		q_paths[num_q_paths].forwback =  +1;
	      }
	      else{
		q_paths[num_q_paths].coeff = -coeff;
		q_paths[num_q_paths].forwback = -1;
	      }
	      num_q_paths++;
	      path_num++;
	      /**node0_printf("ADD PATH %d:  rx=%d ",num_q_paths-1,ir[0]);
		 printpath( vec, length );**/
	    }

	  } /* end reflection*/
        } /* end permutation if block */
      } /* end permutation */
    num_basic_paths++;
    return(path_num);
} /* add_basic_path */

/* parallel transport a vector to the current site along a path.
   For example, if the path is "XUP", bring in the vector from
   the +X direction to the current site. If KS phases are in lattice,
   this transport will automatically include them. 
   OK for src and dest to be the same.  OK for length=0.  */
/* NOT OPTIMIZED at the moment - do lots of extra copying.  Use temp.
   vectors rather than stuff in lattice.h */
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    su3_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    su3_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_dest = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_work = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(su3_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_vec( &(s->link[dir[j]]),
		    (su3_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_vec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(su3_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = *(su3_vector *)F_PT(s,src);
    }
  }
} /* path_transport */

/* Path transport a half_wilson_vector */
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    half_wilson_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    half_wilson_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (half_wilson_vector *)calloc(
      sites_on_node, sizeof(half_wilson_vector));
    tmp_dest = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));
    tmp_work = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(half_wilson_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_hwvec( &(s->link[dir[j]]),
		    (half_wilson_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_hwvec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(half_wilson_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) =
	  *(half_wilson_vector *)F_PT(s,src);
    }
  }
} /* path_transport_hwv */


#ifdef EO
/* Stupid dslash routine that follows all the paths.
  Should optimize by precomputing sums of paths to all the displacements
  Use dslash_fn for actions that involve only +X and +X+X+X couplings.
*/
/* D_slash routine - sets dest. on each site equal to sum of
   sources parallel transported to site, with minus sign for transport
   from negative directions.  */
void dslash_eo( field_offset src, field_offset dest, int parity ) {
   register int i;
   register site *s;
   register int ipath,otherparity;
   register double x; /* coefficient of path */

    switch(parity){
	case EVEN:	otherparity=ODD; break;
	case ODD:	otherparity=EVEN; break;
	case EVENANDODD:	otherparity=EVENANDODD; break;
    }

    /* Parallel transport by all the paths in the action.  
       Multiply by coefficient in table
    */
    FORSOMEPARITY(i,s,parity){ clearvec( (su3_vector *)F_PT(s,dest) ); }

    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	    path_transport( src, F_OFFSET(tempvec[0]), parity,
		q_paths[ipath].dir, q_paths[ipath].length );
	    x=q_paths[ipath].coeff;
	    FORSOMEPARITY(i,s,parity){
		scalar_mult_add_su3_vector(  (su3_vector *)F_PT(s,dest),
		    &(s->tempvec[0]), x, (su3_vector *)F_PT(s,dest) );
	    }
    }   /* ipath */
} /* dslash_eo */
#endif /* ifdef EO */

#ifdef FN
/* Sum over paths connecting to nearest neighbor point (fat link) and to third
   nearest neighbor (longlinks) */
/* Doug Toussaint 2/4/98 */
/* modified to use t_longlinks, S. Gottlieb 7/13/01 */
/* long link calculating routine */
/* path_product() follows the path starting at step 0, and
   leaves the answer at the end of the path.  We want the answer
   at the site where the path begins.  So we look for paths with
   the opposite displacement from the displacement of the point
   that we want to transport to this site, and take the adjoint
   of the matrix at the end. clear? */
/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_longlinks() {
  register int i;
  register site *s;
  int ipath,dir;
  int disp[4];
  register su3_matrix *long1;

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_longlinks needs phases in\n");
    terminate(0);
  }
  for (dir=XUP; dir<=TUP; dir++){ /* loop over longlink directions */
    /* set longlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      long1 = &(t_longlink[4*i+dir]);
#else
      long1 = &(s->longlink[dir]);
#endif
      clear_su3mat( long1 );
    }

    /* loop over paths, checking for ones with total displacement 3*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=3,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("ipath = %d, found a path:  ",ipath);
for(j=0;j<q_paths[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  long1 = &(t_longlink[4*i+dir]);
#else
	  long1 = &(s->longlink[dir]);
#endif
          scalar_mult_add_su3_matrix( long1,
	    &(s->staple), -q_paths[ipath].coeff, long1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */

  } /* loop over directions */

  valid_longlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Long):  %e\n",dtime);
#endif
}  /* load_longlinks() */

/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_fatlinks() {
  register int i;
  register site *s;
  int dir;
  register su3_matrix *fat1;
#ifdef ASQ_OPTIMIZED_FATTENING
  int  nu,rho,sig ;
  double one_link ; /*needed to fix the problem with the Lepage term */
#else
  int ipath;
  int disp[4];
#endif

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_fatlinks needs phases in\n");
    terminate(0);
  }

#ifndef  ASQ_OPTIMIZED_FATTENING   /* general case code */
  for (dir=XUP; dir<=TUP; dir++){ /* loop over fatlink directions */
    /* set fatlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+dir]);
#else
      fat1 = &(s->fatlink[dir]);
#endif
      clear_su3mat( fat1 );
    }
    
    /* loop over paths, checking for ones with total displacement 1*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=1,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("dir = %d, found a path:  ",dir);
for(j=0;j<q_paths.[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  fat1 = &(t_fatlink[4*i+dir]);
#else
	  fat1 = &(s->fatlink[dir]);
#endif
          scalar_mult_add_su3_matrix( fat1,
	    &(s->staple), -q_paths[ipath].coeff, fat1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */
  } /* loop over directions */
#else	/* ASQ_OPTIMIZED_FATTENING, for Asq and Asqtad actions */
/*  Optimized fattening code for the Asq and Asqtad actions.           *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
 
 /* to fix up the Lepage term, included by a trick below */
 one_link = (act_path_coeff[0] - 6.0*act_path_coeff[5]) ; 
 
 for (dir=XUP; dir<=TUP; dir++){
   FORALLSITES(i,s) /* Intialize fat links with c_1*U_\mu(x) */
     {
#ifdef DSLASH_TMP_LINKS
       fat1 = &(t_fatlink[4*i+dir]);
#else
       fat1 = &(s->fatlink[dir]);
#endif
       scalar_mult_su3_matrix(&(s->link[dir]), one_link ,
			      fat1 );
     }
   for(nu=XUP; nu<=TUP; nu++) if(nu!=dir)
     {
       compute_gen_staple(F_OFFSET(staple),dir,nu,F_OFFSET(link[dir]),
			  act_path_coeff[2]);
       /* The Lepage term */
       /* Note this also involves modifying c_1 (above) */
       compute_gen_staple(NULL_FP,dir,nu,F_OFFSET(staple),act_path_coeff[5]);
       for(rho=XUP; rho<=TUP; rho++) if((rho!=dir)&&(rho!=nu))
	 {
	   compute_gen_staple(F_OFFSET(tempmat1),dir,rho,F_OFFSET(staple),
			      act_path_coeff[3]);
	   for(sig=XUP; sig<=TUP; sig++)
	     if((sig!=dir)&&(sig!=nu)&&(sig!=rho))
	       {
		 compute_gen_staple(NULL_FP,dir,sig,
				    F_OFFSET(tempmat1),
				    act_path_coeff[4]);
	       } /* sig */
	 } /* rho */
     } /* nu */
 }/* dir */  
#endif

  valid_fatlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Fat):  %e\n",dtime);
#endif
}  /* load_fatlinks() */
#endif /* ifdef FN */


/* compare two paths, return 1 if equal, else zero */
int is_path_equal( int *path1, int* path2, int length ){
   register int i;
   for(i=0;i<length;i++)if(path1[i]!=path2[i])return(0);
   return(1);
}


/* update the  momenta with the fermion force */
/* Assumes that the conjugate gradient has been run, with the answer in
   x_off, and dslash(x_off,x_off,ODD) has been run. (fills in x_off_odd) */
/* SEE LONG COMMENTS AT END */

#ifndef ASQ_OPTIMIZED_FORCE
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon, coeff;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  half_wilson_vector *hw_tmp0,*hw_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  hw_tmp0 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  hw_tmp1 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  /* Use half_wilson_vectors to store x_off transported from ends of
     path.  0 component from forward end, 1 component from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    path_transport( x_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[1] for transport from starting end */
    FORALLSITES(i,s){
      hw_tmp0[i].h[0]=s->tempvec[0];
      hw_tmp0[i].h[1]=*(su3_vector *)F_PT(s,x_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff = ferm_epsilon*q_paths[ipath].coeff;
      if( (ilink%2)==1 )coeff = -coeff;

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "half_wilson_vector" to handle pair of vectors -
	0 component is x_off from forward end, 1 component from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_su3_mat_hwvec( &(s->link[lastdir]),
	      &(hw_tmp0[i]), &(hw_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( hw_tmp1, 2*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     hw_tmp0[i] = *(half_wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( hw_tmp0, 2*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_su3_mat_hwvec( &(s->link[OPP_DIR(lastdir)]),
                (half_wilson_vector *)(gen_pt[0][i]),
		&(hw_tmp1[i]) );
          }
	  tmp_pt = hw_tmp0; hw_tmp0 = hw_tmp1; hw_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( hw_tmp0 ); free( hw_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */


void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off,
  int nflav2, field_offset x2_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/14/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon1, ferm_epsilon2, coeff1, coeff2;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  wilson_vector *w_tmp0,*w_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon1 = 2.0*(nflav1/4.0)*eps;
  ferm_epsilon2 = 2.0*(nflav2/4.0)*eps;
  w_tmp0 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  w_tmp1 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  /* Use wilson_vectors to store x_off transported from ends of
     path.  0 and 1 components from forward end, 2 and 3 components
     from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    /**
    path_transport( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    path_transport( x2_off, F_OFFSET(tempvec[1]),
      EVENANDODD, q_paths[ipath].dir, length );
    **/
/** WARNING!! Assumes xxx1 and xxx2 contiguous **/
if( x2_off-x1_off != sizeof(su3_vector) ){node0_printf("BOTCH\n"); exit(0);}
    path_transport_hwv( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[2] for transport from starting end */
    FORALLSITES(i,s){
      w_tmp0[i].d[0]=s->tempvec[0];
      w_tmp0[i].d[1]=s->tempvec[1];
      w_tmp0[i].d[2]=*(su3_vector *)F_PT(s,x1_off);
      w_tmp0[i].d[3]=*(su3_vector *)F_PT(s,x2_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff1 = ferm_epsilon1*q_paths[ipath].coeff;
      coeff2 = ferm_epsilon2*q_paths[ipath].coeff;
      if( (ilink%2)==1 ){ coeff1 = -coeff1; coeff2 = -coeff2;}

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "wilson_vector" to handle pair of vectors -
	0 component is x_off1 from forward end, 1 component is x_off2
	from forward end,  2 and 3  components are x_off1 and x_off2 
	from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_mat_wilson_vec( &(s->link[lastdir]),
	      &(w_tmp0[i]), &(w_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( w_tmp1, 4*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     w_tmp0[i] = *(wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( w_tmp0, 4*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_mat_wilson_vec( &(s->link[OPP_DIR(lastdir)]),
                (wilson_vector *)(gen_pt[0][i]),
		&(w_tmp1[i]) );
          }
	  tmp_pt = w_tmp0; w_tmp0 = w_tmp1; w_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( w_tmp0 ); free( w_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */



#else /* ASQ_OPTIMIZED_FORCE, for Asq and Asqtad actions */
/* Optimized force code for the Asq and Asqtad actions                 *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
#define Pmu          tempvec[0] 
#define Pnumu        tempvec[1]
#define Prhonumu     tempvec[2]
#define P7           tempvec[3]
#define P7rho        tempvec[4]              
#define P7rhonu      tempvec[5]
#define P5           tempvec[6]
#define P3           tempvec[7]
#define P5nu         tempvec[3]
#define P3mu         tempvec[3]
#define Popmu        tempvec[4]
#define Pmumumu      tempvec[4]
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double ferm_epsilon, coeff;
  double OneLink, Lepage, Naik, FiveSt, ThreeSt, SevenSt ;
  su3_vector *tempvec[8] ;
  su3_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  
  /* Path coefficients times fermion epsilon */
  OneLink = act_path_coeff[0]*ferm_epsilon ; 
  Naik    = act_path_coeff[1]*ferm_epsilon ;
  ThreeSt = act_path_coeff[2]*ferm_epsilon ;
  FiveSt  = act_path_coeff[3]*ferm_epsilon ;
  SevenSt = act_path_coeff[4]*ferm_epsilon ;
  Lepage  = act_path_coeff[5]*ferm_epsilon ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    tempvec[mu] = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

  /*copy x_off to a temporary vector */
  temp_x = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
  FORALLSITES(i,s) temp_x[i] = *(su3_vector *)F_PT(s,x_off) ;

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_force_to_mom(P3, Pmu, sig, -ThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_force_to_mom(P7, Prhonumu, sig, -SevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_fermion(P7, P7rho, rho);
		  side_link_force(rho,sig,SevenSt, Pnumu, P7, Prhonumu, P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff = SevenSt/FiveSt ; 
		  FORALLSITES(i,s)
		    scalar_mult_add_su3_vector(&(P5[i]),&(P7rho[i]),coeff,
					       &(P5[i]));
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_fermion(P5,P5nu, nu);
	      side_link_force(nu,sig,-FiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff = FiveSt/ThreeSt ; 
	      FORALLSITES(i,s)
		scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_fermion(P5,P5nu, mu);
	  side_link_force(mu, sig, -Lepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff = Lepage/ThreeSt ; 
	  FORALLSITES(i,s)
	    scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), -Naik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(tempvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      

#define Pmu          hwvec[0] 
#define Pnumu        hwvec[1]
#define Prhonumu     hwvec[2]
#define P7           hwvec[3]
#define P7rho        hwvec[4]              
#define P7rhonu      hwvec[5]
#define P5           hwvec[6]
#define P3           hwvec[7]
#define P5nu         hwvec[3]
#define P3mu         hwvec[3]
#define Popmu        hwvec[4]
#define Pmumumu      hwvec[4]
void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off, 
			  int nflav2, field_offset x2_off){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/15/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double coeff[2],ferm_epsilon ;
  double OneLink[2], Lepage[2], Naik[2], FiveSt[2], ThreeSt[2], SevenSt[2] ;
  double mNaik[2], mLepage[2], mFiveSt[2], mThreeSt[2], mSevenSt[2] ;
  half_wilson_vector *hwvec[8] ;
  half_wilson_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  
  /* Path coefficients times fermion epsilon */
  ferm_epsilon = 2.0*(nflav1/4.0)*eps;
  OneLink[0] = act_path_coeff[0]*ferm_epsilon ;
  Naik[0]    = act_path_coeff[1]*ferm_epsilon ; mNaik[0]    = -Naik[0] ;
  ThreeSt[0] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[0] = -ThreeSt[0] ;
  FiveSt[0]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[0]  = -FiveSt[0] ;
  SevenSt[0] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[0] = -SevenSt[0] ;
  Lepage[0]  = act_path_coeff[5]*ferm_epsilon ; mLepage[0]  = -Lepage[0] ;

  ferm_epsilon = 2.0*(nflav2/4.0)*eps;
  OneLink[1] = act_path_coeff[0]*ferm_epsilon ;
  Naik[1]    = act_path_coeff[1]*ferm_epsilon ; mNaik[1]    = -Naik[1] ;
  ThreeSt[1] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[1] = -ThreeSt[1] ;
  FiveSt[1]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[1]  = -FiveSt[1] ;
  SevenSt[1] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[1] = -SevenSt[1] ;
  Lepage[1]  = act_path_coeff[5]*ferm_epsilon ; mLepage[1]  = -Lepage[1] ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    hwvec[mu]= 
      (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));

  /*copy x_off to a temporary vector */
  temp_x= 
    (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
  FORALLSITES(i,s)
    {
      temp_x[i].h[0] = *(su3_vector *)F_PT(s,x1_off) ;
      temp_x[i].h[1] = *(su3_vector *)F_PT(s,x2_off) ;
    }

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_hw_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_3f_force_to_mom(P3, Pmu, sig, mThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_hw_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_3f_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_hw_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_hw_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_3f_force_to_mom(P7, Prhonumu, sig, mSevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_hw_fermion(P7, P7rho, rho);
		  side_link_3f_force(rho,sig,SevenSt,Pnumu,P7,Prhonumu,P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff[0] = SevenSt[0]/FiveSt[0] ; 
		  coeff[1] = SevenSt[1]/FiveSt[1] ; 
		  FORALLSITES(i,s)
		    {
		      scalar_mult_add_su3_vector(&(P5[i].h[0]),
						 &(P7rho[i].h[0]),coeff[0],
						 &(P5[i].h[0]));
		      scalar_mult_add_su3_vector(&(P5[i].h[1]),
						 &(P7rho[i].h[1]),coeff[1],
						 &(P5[i].h[1]));
		    }
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_hw_fermion(P5,P5nu, nu);
	      side_link_3f_force(nu,sig,mFiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff[0] = FiveSt[0]/ThreeSt[0] ; 
	      coeff[1] = FiveSt[1]/ThreeSt[1] ; 
	      FORALLSITES(i,s)
		{
		  scalar_mult_add_su3_vector(&(P3[i].h[0]),
					     &(P5nu[i].h[0]), coeff[0],
					     &(P3[i].h[0]));
		  scalar_mult_add_su3_vector(&(P3[i].h[1]),
					     &(P5nu[i].h[1]), coeff[1],
					     &(P3[i].h[1]));
		}
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_3f_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_hw_fermion(P5,P5nu, mu);
	  side_link_3f_force(mu, sig, mLepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff[0] = Lepage[0]/ThreeSt[0] ; 
	  coeff[1] = Lepage[1]/ThreeSt[1] ; 
	  FORALLSITES(i,s)
	    {
	      scalar_mult_add_su3_vector(&(P3[i].h[0]),
					 &(P5nu[i].h[0]),coeff[0],
					 &(P3[i].h[0]));
	      scalar_mult_add_su3_vector(&(P3[i].h[1]),
					 &(P5nu[i].h[1]),coeff[1],
					 &(P3[i].h[1]));
	    }

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_hw_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_3f_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_3f_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_3f_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), mNaik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_hw_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_3f_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_3f_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(hwvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      
#endif /* ASQ_OPTIMIZED_FORCE */


#ifdef  ASQ_OPTIMIZED_FATTENING   /* Asqtad action only, "_fn" executables */
#ifndef FN
BOMB THE COMPILE
#endif
void compute_gen_staple(field_offset staple, int mu, int nu, 
			field_offset link, double coef) {
  su3_matrix tmat1,tmat2;
  msg_tag *mtag0,*mtag1;
  su3_matrix *tempmat ;
  register site *s ;
  register int i ;
  register su3_matrix *fat1;

  /* Computes the staple :
                   mu
                +-------+
          nu	|	|
		|	|
		X	X
    Where the mu link can be any su3_matrix. The result is saved in staple.
    if staple==NULL_FP then the result is not saved.
    It also adds the computed staple to the fatlink[mu] with weight coef.
  */

  /* Upper staple */
  mtag0 = start_gather( link, sizeof(su3_matrix), nu, EVENANDODD, gen_pt[0] );
  mtag1 = start_gather( F_OFFSET(link[nu]), sizeof(su3_matrix), mu, 
			EVENANDODD, gen_pt[1] );
  wait_gather(mtag0);
  wait_gather(mtag1);
  
  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, (su3_matrix *)F_PT(s,staple) );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, &tmat2 );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix(fat1, &tmat2, coef,
				 fat1) ;
    }
  }
  cleanup_gather(mtag0);
  cleanup_gather(mtag1);

  /* lower staple */
  tempmat = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
  mtag0 = start_gather( F_OFFSET(link[nu]),
			sizeof(su3_matrix), mu, EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);
  FORALLSITES(i,s){
    mult_su3_an( &(s->link[nu]),(su3_matrix *)F_PT(s,link), &tmat1 );
    mult_su3_nn( &(tmat1),(su3_matrix *)gen_pt[0][i], &(tempmat[i]) );
  }
  cleanup_gather(mtag0);
  mtag0 = start_gather_from_temp( tempmat, sizeof(su3_matrix),
				  OPP_DIR(nu), EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);

  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      add_su3_matrix( (su3_matrix *)F_PT(s,staple),(su3_matrix *)gen_pt[0][i], 
		      (su3_matrix *)F_PT(s,staple) );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)F_PT(s,staple), coef, 
				 fat1 );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)gen_pt[0][i], coef, 
				 fat1 );
    }
  }

  free(tempmat);
  cleanup_gather(mtag0);
}
#endif  /* ASQ_OPTIMIZED_FATTENING   */


#ifdef  ASQ_OPTIMIZED_FORCE
/*   Covariant shift of the src fermion field in the direction dir  *
 *  by one unit. The result is stored in dest.                       */ 
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) {
  su3_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(su3_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_vec(&(s->link[dir]),(su3_vector *)(gen_pt[0][i]),
			 &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_vec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(su3_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(su3_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/*  Covariant shift of the src half wilson fermion field in the  *
 * direction dir   by one unit. The result is stored in dest.    */
void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) {
  half_wilson_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(half_wilson_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_hwvec(&(s->link[dir]),
			   (half_wilson_vector *)(gen_pt[0][i]), &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = 
	(half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_hwvec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(half_wilson_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(half_wilson_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/* Add in contribution to the force */
/* Put antihermitian traceless part into momentum */
void add_force_to_mom(su3_vector *back,su3_vector *forw,int dir,double coeff) {
  register site *s ;
  register int i ;  
  register double tmp_coeff ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff = -coeff ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD) 
      tmp_coeff = -coeff ;
    else
      tmp_coeff = coeff ;
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i]), &(forw[i]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff, &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/* Add in contribution to the force ( 3flavor case ) */
/* Put antihermitian traceless part into momentum */
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) {
  register site *s ;
  register int i ;  
  double tmp_coeff[2] ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff[0] = -coeff[0] ;
      coeff[1] = -coeff[1] ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD)
      {
	tmp_coeff[0] = -coeff[0] ;
	tmp_coeff[1] = -coeff[1] ;
      }
    else
      {
	tmp_coeff[0] = coeff[0] ;
	tmp_coeff[1] = coeff[1] ;
      }
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i].h[0]), &(forw[i].h[0]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[0], &tmat2 );
    su3_projector(&(back[i].h[1]), &(forw[i].h[1]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[1], &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/*  This routine is needed in order to add the force on the side link *
 * of the paths in the Asq and Asqtad actions. It gets as inputs the  *
 * direction mu of the side link and the direction nu of the Dslash   *
 * term we are dealing with. Then it takes also 4 fermion fields:     *
 * Path: the piece of the path with no hop in the nu or mu direction  *
 * Path_nu: the piece of the path with a hop in nu  but not in mu     *
 * Path_mu: is Path times the link mu                                 *
 * Path_numu: is Path_nu times the link mu                            */
void side_link_force(int mu, int nu, double coeff, 
		     su3_vector *Path   , su3_vector *Path_nu, 
		     su3_vector *Path_mu, su3_vector *Path_numu) {
  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_force_to_mom(Path, Path_numu, OPP_DIR(mu), -coeff );/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_nu, Path_mu, mu, -coeff) ; /* ? extra - */
      else
	add_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
 
/*  The 3 flavor version of side_link_force used *
 * to optimize fermion transports                */
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) {
  double m_coeff[2] ;

  m_coeff[0] = -coeff[0] ;
  m_coeff[1] = -coeff[1] ;

  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_3f_force_to_mom(Path,Path_numu,OPP_DIR(mu),m_coeff);/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_nu, Path_mu, mu, m_coeff) ; /* ? extra - */
      else
	add_3f_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
#endif  /* ASQ_OPTIMIZED_FORCE   */

/* LONG COMMENTS
   Here we have combined "xxx", (offset "x_off")  which is
(M_adjoint M)^{-1} phi, with Dslash times this vector, which goes in the
odd sites of xxx.  Recall that phi is defined only on even sites.  In
computing the fermion force, we are looking at

< X |  d/dt ( Dslash_eo Dslash_oe ) | X >
=
< X | d/dt Dslash_eo | T > + < T | d/dt Dslash_oe | X >
where T = Dslash X.

The subsequent manipulations to get the coefficent of H, the momentum
matrix, in the simulation time derivative above look the same for
the two terms, except for a minus sign at the end, if we simply stick
T, which lives on odd sites, into the odd sites of X

 Each path in the action contributes terms when any link of the path
is the link for which we are computing the force.  We get a minus sign
for odd numbered links in the path, since they connect sites of the
opposite parity from what it would be for an even numbered link.
Minus signs from "going around" plaquette - ie KS phases, are supposed
to be already encoded in the path coefficients.
Minus signs from paths that go backwards are supposed to be already
encoded in the path coefficients.

Here, for example, are comments reproduced from the force routine for
the one-link plus Naik plus single-staple-fat-link action:

 The three link force has three contributions, where the link that
was differentiated is the first, second, or third link in the 3-link
path, respectively.  Diagramatically, where "O" represents the momentum,
the solid line the link corresponding to the momentum, and the dashed
lines the other links:
 

	O______________ x ............ x ...............
+
	x..............O______________x.................
+
	x..............x..............O________________
Think of this as
	< xxx | O | UUUxxx >		(  xxx, UUUX_p3 )
+
	< xxx U | O | UUxxx >		( X_m1U , UUX_p2 )
+
	< xxx U U | O | Uxxx >		( X_m2UU , UX_p1 )
where "U" indicates parallel transport, "X_p3" is xxx displaced
by +3, etc.
Note the second contribution has a relative minus sign
because it effectively contributes to the <odd|even>, or M_adjoint,
part of the force when we work on an even site. i.e., for M on
an even site, this three link path begins on an odd site.

The staple force has six contributions from each plane containing the
link direction:
Call these diagrams A-F:


	x...........x		O____________x
		    .			     .
		    .			     .
		    .			     .
		    .			     .
		    .			     .
	O___________x		x............x
	   (A)			    (B)



	x	    x		O____________x
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	O___________x		x	     x
	   (C)			    (D)



	x...........x		O____________x
	.			.
	.			.
	.			.
	.			.
	.			.
	O___________x		x............x
	   (E)			    (F)

As with the Naik term, diagrams C and D have a relative minus
sign because they connect sites of the other parity.

Also note an overall minus sign in the staple terms relative to the
one link term because, with the KS phase factors included, the fat
link is  "U - w3 * UUU", or the straight link MINUS w3 times the staples.

Finally, diagrams B and E get one more minus sign because the link
we are differentiating is in the opposite direction from the staple
as a whole.  You can think of this as this "U" being a correction to
a "U_adjoint", but the derivative of U is iHU and the derivative
of U_adjoint is -iHU_adjoint.

*/
/* LONG COMMENT on sign conventions
In most of the program, the KS phases and antiperiodic boundary
conditions are absorbed into the link matrices.  This greatly simplfies
multiplying by the fermion matrix.  However, it requires care in
specifying the path coefficients.  Remember that each time you
encircle a plaquette, you pick up a net minus sign from the KS phases.
Thus, when you have more than one path to the same point, you generally
have a relative minus sign for each plaquette in a surface bounded by
this path and the basic path for that displacement.

Examples:
  Fat Link:
    Positive:	X-------X

    Negative     --------
	 	|	|
		|	|
		X	X

  Naik connection, smeared
    Positive:	X-------x-------x-------X

    Negative:	---------
		|	|
		|	|
		X	x-------x-------X

    Positive:	--------x--------
		|		|
		|		|
		X		x-------X

    Negative:	--------x-------x-------x
		|			|
		|			|
		X			X
*/



/* Comment on acceptable actions.
   We construct the backwards part of dslash by reversing all the
   paths in the forwards part.  So, for example, in the p4 action
   the forwards part includes +X+Y+Y

		X
		|
		|
		X
		|
		|
	X---->--X

  so we put -X-Y-Y in the backwards part.  But this isn't the adjoint
  of U_x(0)U_y(+x)U_y(+x+y).  Since much of the code assumes that the
  backwards hop is the adjoint of the forwards (for example, in
  preventing going to 8 flavors), the code only works for actions
  where this is true.  Roughly, this means that the fat link must
  be symmetric about reflection around its midpoint.  Equivalently,
  the paths in the backwards part of Dslash are translations of the
  paths in the forwards part.  In the case of the "P4" or knight's move
  action, this means that we have to have both paths
   +X+Y+Y and +Y+Y+X to the same point, with the same coefficients.
  Alternatively, we could just use the symmetric path +Y+X+Y.
*/

invivoquark_stuff_compute_gen_staple_1406

Real time: 1.792900e+08 -- Predicted time: 0.000000e+00

Callcount: 792

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
145	### 1	66.4787883366067	0.000000e+00	2.647600e+05	100.0
294	### 2	729.493444598313	0.000000e+00	2.224120e+05	100.0

                /*****************  su3mat_copy.c  (in su3.a) ***************************
*									*
* void su3mat_copy( su3_matrix *a, su3_matrix *b )			*
* Copy an su3 matrix:  B <- A   						*
*/
#include "config.h"
#include "complex.h"
#include "su3.h"

/* Copy a su3 matrix:  b <- a   */
void su3mat_copy( su3_matrix *a, su3_matrix *b ){
register int i,j;
    for(i=0;i<3;i++)for(j=0;j<3;j++){
	b->e[i][j].real = a->e[i][j].real;
	b->e[i][j].imag = a->e[i][j].imag;
    }
}

invivosu3mat_copy_su3mat_copy_13

Real time: 4.508012e+08 -- Predicted time: 1.863417e+09

Callcount: 7984000

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
1569622	### 1	5991501.89311295	2.110000e+02	2.800000e+01	86.7
6493442	### 2	1841489.02266925	3.160000e+02	2.400000e+01	92.4
73682	### 3	77231.2005381121	2.240000e+02	3.064000e+03	92.6

                /*******************  m_mat_nn.c  (in su3.a) ****************************
*									*
* void mult_su3_nn( su3_matrix *a,*b,*c )				*
* matrix multiply, no adjoints 						*
* C  <-  A*B								*
*/
#include "config.h"
#include "complex.h"
#include "su3.h"

#ifndef FAST
void mult_su3_nn( su3_matrix *a, su3_matrix *b, su3_matrix *c ){
register int i,j,k;
register complex x,y;
    for(i=0;i<3;i++)for(j=0;j<3;j++){
	x.real=x.imag=0.0;
	for(k=0;k<3;k++){
	    CMUL( a->e[i][k] , b->e[k][j] , y );
	    CSUM( x , y );
	}
	c->e[i][j] = x;
    }
}

/* "Hand coded" routines, clearer coding is up above */
#else
#ifdef NATIVEDOUBLE   /* RS6000 version */

void mult_su3_nn( su3_matrix *a, su3_matrix *b, su3_matrix *c ){
  int j;
  register double a0r,a0i,a1r,a1i,a2r,a2i;
  register double b0r,b0i,b1r,b1i,b2r,b2i;
  
  for(j=0;j<3;j++){
    
    a0r=a->e[0][0].real; a0i=a->e[0][0].imag;
    b0r=b->e[0][j].real; b0i=b->e[0][j].imag;
    a1r=a->e[0][1].real; a1i=a->e[0][1].imag;
    b1r=b->e[1][j].real; b1i=b->e[1][j].imag;
    a2r=a->e[0][2].real; a2i=a->e[0][2].imag;
    b2r=b->e[2][j].real; b2i=b->e[2][j].imag;
    
    c->e[0][j].real = a0r*b0r - a0i*b0i + a1r*b1r - a1i*b1i + a2r*b2r - a2i*b2i;
    c->e[0][j].imag = a0r*b0i + a0i*b0r + a1r*b1i + a1i*b1r + a2r*b2i + a2i*b2r;
    
    a0r=a->e[1][0].real; a0i=a->e[1][0].imag;
    b0r=b->e[0][j].real; b0i=b->e[0][j].imag;
    a1r=a->e[1][1].real; a1i=a->e[1][1].imag;
    b1r=b->e[1][j].real; b1i=b->e[1][j].imag;
    a2r=a->e[1][2].real; a2i=a->e[1][2].imag;
    b2r=b->e[2][j].real; b2i=b->e[2][j].imag;
    
    c->e[1][j].real = a0r*b0r - a0i*b0i + a1r*b1r - a1i*b1i + a2r*b2r - a2i*b2i;
    c->e[1][j].imag = a0r*b0i + a0i*b0r + a1r*b1i + a1i*b1r + a2r*b2i + a2i*b2r;
    
    a0r=a->e[2][0].real; a0i=a->e[2][0].imag;
    b0r=b->e[0][j].real; b0i=b->e[0][j].imag;
    a1r=a->e[2][1].real; a1i=a->e[2][1].imag;
    b1r=b->e[1][j].real; b1i=b->e[1][j].imag;
    a2r=a->e[2][2].real; a2i=a->e[2][2].imag;
    b2r=b->e[2][j].real; b2i=b->e[2][j].imag;
    
    c->e[2][j].real = a0r*b0r - a0i*b0i + a1r*b1r - a1i*b1i + a2r*b2r - a2i*b2i;
    c->e[2][j].imag = a0r*b0i + a0i*b0r + a1r*b1i + a1i*b1r + a2r*b2i + a2i*b2r;
    
  }
}
#else

void mult_su3_nn( su3_matrix *a, su3_matrix *b, su3_matrix *c ){
  int i,j;
  register double t,ar,ai,br,bi,cr,ci;
    for(i=0;i<3;i++)for(j=0;j<3;j++){

	ar=a->e[i][0].real; ai=a->e[i][0].imag;
	br=b->e[0][j].real; bi=b->e[0][j].imag;
	cr=ar*br; t=ai*bi; cr -= t;
	ci=ar*bi; t=ai*br; ci += t;

	ar=a->e[i][1].real; ai=a->e[i][1].imag;
	br=b->e[1][j].real; bi=b->e[1][j].imag;
	t=ar*br; cr += t; t=ai*bi; cr -= t;
	t=ar*bi; ci += t; t=ai*br; ci += t;

	ar=a->e[i][2].real; ai=a->e[i][2].imag;
	br=b->e[2][j].real; bi=b->e[2][j].imag;
	t=ar*br; cr += t; t=ai*bi; cr -= t;
	t=ar*bi; ci += t; t=ai*br; ci += t;

	c->e[i][j].real=cr;
	c->e[i][j].imag=ci;
    }
}
#endif	/* End of "#ifdef NATIVEDOUBLE" */
#endif	/* End of "#ifdef FAST" */

invivom_mat_nn_mult_su3_nn_73

Real time: 2.073895e+09 -- Predicted time: 8.405498e+09

Callcount: 13114000

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
2578159	### 1	1832880.04935445	4.210000e+02	1.600000e+02	61.9
1592566	### 2	1416879.74220797	5.270000e+02	1.280000e+02	75.7
5128590	### 3	3619222.89221726	4.560000e+02	1.480000e+02	67.5
10665711	### 4	3169029.29840069	1.017000e+03	1.520000e+02	85.0
10755934	### 5	2369272.6246912	6.760000e+02	1.560000e+02	76.9
5623132	### 6	521517.931062955	6.800000e+02	1.720000e+02	74.7
11026139	### 7	84218.1332932698	3.780000e+02	3.240000e+02	14.2
6242758	### 8	28570.1958509917	8.020000e+02	3.148000e+03	74.5
12366531	### 9	3581.16211734922	8.050000e+02	4.900000e+03	83.5

                /****** gauge_stuff.c  -- ******************/
/* MIMD version 6 */
/* gauge action stuff for improved action
* T.D. and A.H. general gauge action updating code
* D.T. modified  5/97
* D.T. modified 12/97, optimized gauge_force a little
* D.T. modified 3/99, gauge action in include file */

/**#define GFTIME**/ /* For timing gauge force calculation */
#include "generic_includes.h"	/* definitions files and prototypes */

#ifdef LOOPEND
#undef FORALLSITES
#define FORALLSITES(i,s) \
{ register int loopend; loopend=sites_on_node; \
for( i=0,  s=lattice ; i<loopend; i++,s++ )
#define END_LOOP }
#else
#define END_LOOP        /* define it to be nothing */
#endif

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)
void printpath( int *path, int length );

#define GAUGE_ACTION_PART1
/* defines NREPS NLOOP MAX_LENGTH MAX_NUM */
#include <gauge_action.h>
#undef GAUGE_ACTION_PART1

char gauge_action_description[128];
int  gauge_action_nloops=NLOOP;
int  gauge_action_nreps=NREPS;
int loop_length[NLOOP];	/* lengths of various kinds of loops */
int loop_num[NLOOP];	/* number of rotations/reflections  for each kind */

    /* table of directions, 1 for each kind of loop */
int loop_ind[NLOOP][MAX_LENGTH];
    /* table of directions, for each rotation and reflection of each kind of
	loop.  tabulated with "canonical" starting point and direction. */
int loop_table[NLOOP][MAX_NUM][MAX_LENGTH];
    /* table of coefficients in action, for various "representations" (actually,
	powers of the trace) */
double loop_coeff[NLOOP][NREPS];
    /* for each rotation/reflection, an integer distinct for each starting
	point, or each cyclic permutation of the links */
int loop_char[MAX_NUM];
    /* for each kind of loop for each rotation/reflection, the expectation
	value of the loop */
double loop_expect[NLOOP][NREPS][MAX_NUM];


/* Make table of loops in action */
void make_loop_table() {

    int perm[8],pp[8],ir[4];
    int length,iloop,i,j,chr;
    int vec[MAX_LENGTH];
    int count,flag;
    void char_num( int *dig, int *chr, int length);

#define GAUGE_ACTION_PART2
/* defines all loops and their coefficients */
#include <gauge_action.h>
#undef GAUGE_ACTION_PART2

    for(iloop=0;iloop<NLOOP;iloop++){
	length=loop_length[iloop];
	count=0;
	/* permutations */
	for(perm[0]=0;perm[0]<4;perm[0]++)
	for(perm[1]=0;perm[1]<4;perm[1]++)
	for(perm[2]=0;perm[2]<4;perm[2]++)
	for(perm[3]=0;perm[3]<4;perm[3]++){
	    if(perm[0] != perm[1] && perm[0] != perm[2] 
		&& perm[0] != perm[3] && perm[1] != perm[2]
	    	&& perm[1] != perm[3] && perm[2] != perm[3] ) {
	        /* reflections*/
	 	for(ir[0]=0;ir[0]<2;ir[0]++)
		for(ir[1]=0;ir[1]<2;ir[1]++)
		for(ir[2]=0;ir[2]<2;ir[2]++)
		for(ir[3]=0;ir[3]<2;ir[3]++){
		    for(j=0;j<4;j++){
			pp[j]=perm[j];

			if(ir[j] == 1) pp[j]=7-pp[j];
			pp[7-j]=7-pp[j];
		    }
		    /* create new vector*/
		    for(j=0;j<length;j++) vec[j]=pp[loop_ind[iloop][j]];

         	    char_num(vec,&chr,length);
         	    flag=0;
		    /* check if it's a new set: */
		    for(j=0;j<count;j++) if(chr == loop_char[j])flag=1;
		    if(flag == 0 ){
			loop_char[count]=chr;
			for(j=0;j<length;j++)
			    loop_table[iloop][count][j]=vec[j];
			count++;
/**node0_printf("ADD LOOP: "); printpath( vec, length );**/
		    }
		    if(count>MAX_NUM){
			node0_printf("OOPS: MAX_NUM too small\n");
			exit(0);
		    }
		    loop_num[iloop]=count;

		} /* end reflection*/
	    } /* end permutation if block */
	} /* end permutation */
    } /* end iloop */

    /* print out the loop coefficients */
    node0_printf("loop coefficients: nloop rep loop_coeff  multiplicity\n");
    for(i=0;i<NREPS;i++) for(j=0;j<NLOOP;j++) {
	node0_printf("                    %d %d      %e     %d\n",
	    j,i,loop_coeff[j][i],loop_num[j]);
    }

} /* make_loop_table */


/* find a number uniquely identifying the cyclic permutation of a path,
   or the starting point on the path.  Backwards paths are considered
   equivalent here, so scan those too. */
void char_num( int *dig, int *chr, int length){
    int j;
    int bdig[MAX_LENGTH],tenl,newv,old;
    /* "dig" is array of directions.  "bdig" is array of directions for
	backwards path. */

  tenl=1;
  for(j=0;j<length-1;j++) tenl=tenl*10;

  *chr=dig[length-1];
  for(j=length-2;j>=0;j--) *chr= *chr*10+dig[j];

  /* forward*/
  old=*chr;
  for(j=length-1;j>=1;j--){
       newv=old-tenl*dig[j];
       newv=newv*10+dig[j];
       if(newv < *chr) *chr=newv;
       old=newv;           }

   /* backward*/
   for(j=0;j<length;j++)bdig[j]=7-dig[length-j-1];
   old=bdig[length-1];
   for(j=length-2;j>=0;j--) old=old*10+bdig[j];
   if(old < *chr ) *chr=old;
   for(j=length-1;j>=1;j--){
       newv=old-tenl*bdig[j];
       newv=newv*10+bdig[j];
       if(newv < *chr) *chr=newv;
       old=newv;           }

} /* char_num */

double imp_gauge_action() {
    register int i;
    int rep;
    register site *s;
    complex trace;
    double g_action;
    double action,act2,total_action;
    int length;

    /* these are for loop_table  */
    int ln,iloop;

    g_action=0.0;

    /* gauge action */
    for(iloop=0;iloop<NLOOP;iloop++){
	length=loop_length[iloop];
	/* loop over rotations and reflections */
	for(ln=0;ln<loop_num[iloop];ln++){

	    path_product( loop_table[iloop][ln] , length );

	    FORALLSITES(i,s){
		trace=trace_su3( &s->tempmat1 );
		action =  3.0 - (double)trace.real;
		/* need the "3 -" for higher characters */
        	total_action= (double)loop_coeff[iloop][0]*action;
        	act2=action;
		for(rep=1;rep<NREPS;rep++){
		    act2 *= action;
		    total_action += (double)loop_coeff[iloop][rep]*act2;
		}

        	g_action  += total_action;

	    } END_LOOP /* sites */
	} /* ln */
    } /* iloop */

    g_doublesum( &g_action );
    return( g_action );
} /* imp_gauge_action */


/* update the momenta with the gauge force */
void imp_gauge_force( double eps, field_offset mom_off ){
    register int i,dir;
    register site *st;
    su3_matrix tmat1,tmat2;
    register double eb3;
    register anti_hermitmat* momentum;
#ifdef GFTIME
    double dtime;
#endif

    int j,k;
    int dirs[MAX_LENGTH],length;
    int path_dir[MAX_LENGTH],path_length;

    int ln,iloop;
    double action,act2,new_term;

    int ncount;

#ifdef GFTIME
dtime=-dclock();
#endif
    eb3 = eps*beta/3.0;

    /* Loop over directions, update mom[dir] */
    for(dir=XUP; dir<=TUP; dir++){

	FORALLSITES(i,st)for(j=0;j<3;j++)for(k=0;k<3;k++){
			st->staple.e[j][k]=cmplx(0.0,0.0);
	} END_LOOP

	ncount=0;
	for(iloop=0;iloop<NLOOP;iloop++){
	    length=loop_length[iloop];
	    for(ln=0;ln<loop_num[iloop];ln++){
/**printf("UPD:  "); printpath( loop_table[iloop][ln], length );**/
		/* set up dirs.  we are looking at loop starting in "XUP"
		   direction, rotate so it starts in "dir" direction. */
		for(k=0;k<length;k++){
                    if( GOES_FORWARDS(loop_table[iloop][ln][k]) ){
                	dirs[k]=(dir+loop_table[iloop][ln][k] )% 4;
		    }
            	    else {
                        dirs[k]=OPP_DIR(
			    (dir+OPP_DIR(loop_table[iloop][ln][k]))%4 );
		    }
		}

		path_length= length-1;  /* generalized "staple" */

		/* check for links in direction of momentum to be
		   updated, each such link gives a contribution. Note
		   the direction of the path - opposite the link. */
		for(k=0;k<length;k++)if( dirs[k]==dir||dirs[k]==OPP_DIR(dir)) {
		    if( GOES_FORWARDS(dirs[k]) ) for(j=0;j<path_length;j++) {
			path_dir[j] = dirs[(k+j+1)%length];
		    }
		    if( GOES_BACKWARDS(dirs[k]) ) for(j=0;j<path_length;j++) {
			path_dir[path_length-1-j] =
			    OPP_DIR(dirs[(k+j+1)%length]);
		    }
/**if(dir==XUP)printf("X_UPDATE PATH: "); printpath( path_dir, path_length );**/
		    path_product(path_dir,path_length);

		    /* We took the path in the other direction from our
			old convention in order to get it to end up
			"at our site", so now take adjoint */
		    /* then compute "single_action" contribution to
			staple */
		    FORALLSITES(i,st){
			su3_adjoint( &(st->tempmat1), &tmat1 );
			/* first we compute the fundamental term */
			new_term = loop_coeff[iloop][0];

			/* now we add in the higher representations */
			if(NREPS > 1){
node0_printf("WARNING: THIS CODE IS NOT TESTED\n"); exit(0);
			    act2=1.0;
			    action = 3.0 - realtrace_su3(&(st->link[dir]),
				&tmat1 ); 

			    for(j=1;j<NREPS;j++){
				act2 *= action;
				new_term +=
				    loop_coeff[iloop][j]*act2*(double)(j+1);
			    }
			}  /* end if NREPS > 1 */

			scalar_mult_add_su3_matrix( &(st->staple), &tmat1,
				new_term, &(st->staple) );

		    } END_LOOP

		    ncount++;

		} /* k (location in path) */
	    } /* ln */
	} /* iloop */

	/* Now multiply the staple sum by the link, then update momentum */
	FORALLSITES(i,st){
	    mult_su3_na( &(st->link[dir]), &(st->staple), &tmat1 );
	    momentum = (anti_hermitmat *)F_PT(st,mom_off);
	    uncompress_anti_hermitian( &momentum[dir], &tmat2 );
	    scalar_mult_sub_su3_matrix( &tmat2, &tmat1,
		eb3, &(st->staple) );
	    make_anti_hermitian( &(st->staple), &momentum[dir] );
	} END_LOOP
    } /* dir loop */
#ifdef GFTIME
dtime+=dclock();
node0_printf("GFTIME:  %e\n",dtime);
#endif
} /* imp_gauge_force.c */

/* Measure gauge observables:
    Loops in action (time and space directions treated differently)
    Polyakov loop

*/
void g_measure( ){
    double ss_plaquette, st_plaquette;
    complex p_loop;
    register int i;
    register site *s;
    complex trace;
    double average[NREPS],action,act2,total_action;
    int length;
    /* these are for loop_table  */
    int ln,iloop,rep;

    /* KS and BC minus signs should be out for this routine */
    d_plaquette( &ss_plaquette, &st_plaquette );
    if(this_node==0)printf("PLAQ:\t%f\t%f\n", ss_plaquette, st_plaquette );

    p_loop = ploop();
    if(this_node==0)printf("P_LOOP:\t%e\t%e\n", p_loop.real, p_loop.imag );

    /* gauge action, all loops that contribute */
    total_action=0.0;
    for(iloop=0;iloop<NLOOP;iloop++){
	length=loop_length[iloop];
	/* loop over rotations and reflections */
	for(ln=0;ln<loop_num[iloop];ln++){

	    path_product( loop_table[iloop][ln] , length );

	    for(rep=0;rep<NREPS;rep++)average[rep] = 0.0;
	    FORALLSITES(i,s){
		trace=trace_su3( &s->tempmat1 );
		average[0] += (double)trace.real;
		action =  3.0 - (double)trace.real;
		total_action += (double)loop_coeff[iloop][0]*action;
		/* need the "3 -" for higher characters */
        	act2=action;
		for(rep=1;rep<NREPS;rep++){
		    act2 *= action;
		    average[rep] += act2;
		    total_action += (double)loop_coeff[iloop][rep]*act2;
		} /* reps */
	    } END_LOOP /* sites */
	    g_vecdoublesum( average, NREPS );
	    /* dump the loop */
	    node0_printf("G_LOOP:  %d  %d  %d   ",iloop,ln,length);
	    for(rep=0;rep<NREPS;rep++)node0_printf("\t%e",average[rep]/volume);
	    node0_printf("\t( ");
	    for(i=0;i<length;i++)node0_printf("%d ",loop_table[iloop][ln][i]);
	    node0_printf(" )\n");
	} /* ln */
    } /* iloop */
    g_doublesum( &total_action );
    node0_printf("GACTION: %e\n",total_action/volume);
    /**node0_printf("CHECK:   %e   %e\n",total_action,imp_gauge_action() );**/

    if(this_node==0)fflush(stdout);

}

void printpath( int *path, int length ){
    register int i;
    node0_printf("\t( ");
    for(i=0;i<length;i++)node0_printf("%d ",path[i]);
    node0_printf(",  L = %d )\n", length );
}

#ifdef N_SUBL32
/*** code from symanzik_sl32/dsdu_qhb.c  -- compute the staple ***/
/* This is a version for extended actions where 32 sublattices are
   needed to make the links independent. */
/* U.M. Heller August 1997 */

/* Modifications:
   2/17/98  ANSI prototyping U.M.H.
   */
#include <assert.h>
void dsdu_qhb_subl(int dir, int subl)
{
register site *st;
register int i;
int iloop, ln, k, j;
int dirs[MAX_LENGTH], length;
int path_dir[MAX_LENGTH], path_length;
su3_matrix tmat1;
int fsubl;

 assert(NREPS==1);   /* This procedure designed only for NREPS = 1 */

    FORSOMESUBLATTICE(i,st,subl) {
	clear_su3mat(&(st->staple));
    }

    for(iloop=0;iloop<NLOOP;iloop++){
	length=loop_length[iloop];
	for(ln=0;ln<loop_num[iloop];ln++){
	    /* set up dirs.  we are looking at loop starting in "XUP"
	       direction, rotate so it starts in "dir" direction. */
	    for(k=0;k<length;k++){
		if( GOES_FORWARDS(loop_table[iloop][ln][k]) ){
		    dirs[k]=(dir+loop_table[iloop][ln][k] )% 4;
		}
		else {
		    dirs[k]=OPP_DIR(
			(dir+OPP_DIR(loop_table[iloop][ln][k]))%4 );
		}
	    }

	    path_length = length-1;	/* generalized "staple" */
	    /* The path starts at the forward end of the link */
	    fsubl = neighsubl[subl][dir];

	    /* check for links in direction of link to be updated.
	       Note the direction of the path - opposite the link. */
	    for(k=0;k<length;k++)if( dirs[k]==dir||dirs[k]==OPP_DIR(dir)) {
		if( GOES_FORWARDS(dirs[k]) ) for(j=0;j<path_length;j++) {
		    path_dir[j] = dirs[(k+j+1)%length];
		}
		if( GOES_BACKWARDS(dirs[k]) ) for(j=0;j<path_length;j++) {
		    path_dir[path_length-1-j] =
			OPP_DIR(dirs[(k+j+1)%length]);
		}
		path_prod_subl(path_dir, path_length, fsubl);

		/* We took the path in the other direction from our old
		   convention in order to get it to end up "at our site".
		   So now take adjoint */
		FORSOMESUBLATTICE(i,st,subl) {
		    su3_adjoint(&(st->tempmat1), &tmat1 );
		    scalar_mult_add_su3_matrix(&(st->staple), &tmat1,
			loop_coeff[iloop][0], &(st->staple) );
		}
	    } /* k (location in path) */
	} /* ln */
    } /* iloop */

    g_sync();

} /* dsdu_qhb */

#endif /* N_SUBL32 */

invivogauge_stuff_imp_gauge_force_230

Real time: 3.316111e+09 -- Predicted time: 3.830944e+09

Callcount: 10

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
1	### 1	1	3.921395e+08	3.885231e+08	0.9
2	### 2	1	3.775596e+08	3.880036e+08	2.6
3	### 3	1	4.021845e+08	3.879635e+08	3.5
4	### 4	1	3.778699e+08	3.883666e+08	2.7
5	### 5	1	3.776415e+08	3.878079e+08	2.6
6	### 6	1	3.800310e+08	3.900781e+08	2.5
7	### 7	1	3.820524e+08	3.900300e+08	2.0
8	### 8	1	3.820947e+08	3.888123e+08	1.7
9	### 9	1	3.822644e+08	3.877292e+08	1.4
10	### 10	1	3.771071e+08	3.877194e+08	2.7

                /********************** path_product.c ***************************/
/* MIMD version 6 */
/* Compute product of links along a specified path    */

#include "generic_includes.h"	/* definitions files and prototypes */
#define FETCH_UP 1

/* LOOPEND is required now -CD */
#undef FORALLSITES
#define FORALLSITES(i,s) \
{ register int loopend; loopend=sites_on_node; \
for( i=0,  s=lattice ; i<loopend; i++,s++ )
#define END_LOOP }

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void path_product( int *dir, int length) {
    register int i;
    register site *s;
    msg_tag *mtag0;
    su3_matrix *tempmat2t, *tempmat3t;
    int j;
    /* a forward step leaves the answer in gen_pt[0], which points into
	link, tempmat1 or tempmat2, and backwards step in tempmat1 or tempmat2,
	After a forwards step, need to wait and clean a gather.
	  STEP	leaves answer in
	  even # forward	gen_pt[0]->tempmat1  (gen_pt[0]->link for step 0
	  even # backward	tempmat1
	  odd  # forward	gen_pt[0]->tempmat2
	  odd  # backward	tempmat2
	*/

    /* allocate temporary space */
    tempmat3t = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
    tempmat2t = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));

    /* j=0 */
    if( GOES_FORWARDS(dir[0]) )  {
	mtag0 = start_gather( F_OFFSET(link[dir[0]]), sizeof(su3_matrix),
	    OPP_DIR(dir[0]), EVENANDODD, gen_pt[0] );
    }
    else{  /* if GOES_BACKWARDS(dir[0]) */
	FORALLSITES(i,s){
	    su3_adjoint(&(s->link[OPP_DIR(dir[0])]),&(s->tempmat1) );
	} END_LOOP
    }

    for(j=1;j<length;j++) {
	if( j%2==1 ){
	    if( GOES_FORWARDS(dir[j]) ) {
	      if( GOES_FORWARDS(dir[j-1]) ){
	        wait_gather(mtag0);
	        FORALLSITES(i,s){
		  mult_su3_nn( (su3_matrix *)(gen_pt[0][i]), &(s->link[dir[j]]),
		    &(tempmat2t[i]) );
	        } END_LOOP
	        cleanup_gather(mtag0);
	      }
	      else{ /* last link was backwards */
	        FORALLSITES(i,s){
		  mult_su3_nn( &(s->tempmat1),&(s->link[dir[j]]),
		    &(tempmat2t[i]) );
	        } END_LOOP
	      }
	      mtag0 = start_gather_from_temp( tempmat2t, sizeof(su3_matrix),
		OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	    }  /* for GOES_FORWARDS */

	    else{ /* GOES_BACKWARDS(dir[j]), which is an odd numbered step */
	      if( GOES_FORWARDS(dir[j-1]) ){
	        wait_gather(mtag0);
	        FORALLSITES(i,s){
	          su3mat_copy((su3_matrix *)(gen_pt[0][i]),&(tempmat3t[i]) );
	        } END_LOOP
	        cleanup_gather(mtag0);
	        mtag0 = start_gather_from_temp( tempmat3t, sizeof(su3_matrix),
		  OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	      }
	      else{ /*last step was backwards */
	        mtag0 = start_gather( F_OFFSET(tempmat1), sizeof(su3_matrix),
		  OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	      }
	      wait_gather(mtag0);
	      FORALLSITES(i,s){
		  mult_su3_na((su3_matrix *)(gen_pt[0][i]),
		    &(s->link[OPP_DIR(dir[j])]), &(tempmat2t[i]) );
	      } END_LOOP
	      cleanup_gather(mtag0);
	    } /* end for GOES_BACKWARDS */
	} /* end for j=odd */

	else{	/* j=even */
	  if( GOES_FORWARDS(dir[j]) ) {
	    if( GOES_FORWARDS(dir[j-1]) ){
	      wait_gather(mtag0);
	      FORALLSITES(i,s){
		mult_su3_nn( (su3_matrix *)(gen_pt[0][i]), &(s->link[dir[j]]),
		    &(s->tempmat1) );
	      } END_LOOP
	      cleanup_gather(mtag0);
	    }
	    else{ /* last link goes backwards */
	      FORALLSITES(i,s){
		mult_su3_nn( &(tempmat2t[i]),&(s->link[dir[j]]),
		    &(s->tempmat1) );
	      } END_LOOP
	    }
	    mtag0 = start_gather( F_OFFSET(tempmat1), sizeof(su3_matrix),
		OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	  }  /* for GOES_FORWARDS */

	  else{ /* GOES_BACKWARDS(dir[j]) */
	    if( GOES_FORWARDS(dir[j-1]) ){
	      wait_gather(mtag0);
	      FORALLSITES(i,s){
	        su3mat_copy((su3_matrix *)(gen_pt[0][i]),&(tempmat3t[i]) ); 
	      } END_LOOP
	      cleanup_gather(mtag0);
	      mtag0 = start_gather_from_temp( tempmat3t, sizeof(su3_matrix),
		OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	    }
	    else{ /* last step was backwards */
	      mtag0 = start_gather_from_temp( tempmat2t, sizeof(su3_matrix),
		OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	    }
	    wait_gather(mtag0);
	    FORALLSITES(i,s){
	      mult_su3_na((su3_matrix *)(gen_pt[0][i]),
		    &(s->link[OPP_DIR(dir[j])]), &(s->tempmat1) );
	    } END_LOOP
	    cleanup_gather(mtag0);
	  } /* for GOES_BACKWARDS */
	} /* for j=even */

    }  /* j=link in loop */

    /* Want to end in tempmat1 */
    if( length%2==0 ){  /* last step was odd */
      if( GOES_FORWARDS(dir[length-1]) ){
	wait_gather(mtag0);
	  FORALLSITES(i,s){
	    su3mat_copy((su3_matrix *)(gen_pt[0][i]),&(s->tempmat1) ); 
	} END_LOOP
	cleanup_gather(mtag0);
      }
      else{
	FORALLSITES(i,s){
	  su3mat_copy(&(tempmat2t[i]),&(s->tempmat1) );
	} END_LOOP
      }
    }
    else{ /* odd length path */
      if( GOES_FORWARDS(dir[length-1]) ){
	wait_gather(mtag0);
	FORALLSITES(i,s){
	  su3mat_copy( (su3_matrix *)(gen_pt[0][i]), &(tempmat3t[i]) );
	} END_LOOP
	cleanup_gather(mtag0);
	FORALLSITES(i,s){
	  su3mat_copy( &(tempmat3t[i]), &(s->tempmat1) );
	} END_LOOP
      }
      else{
      }
    }
    free(tempmat3t);
    free(tempmat2t);
} /* path */

#ifdef N_SUBL32
/* code from symanzik_sl32/dsdu_qhb.c ****************************/
/* U.M. Heller August 1997 */

/* This is a modification of "path_product" from gauge_stuff.c
   which works only on one sublattice. */
void path_prod_subl(int *dir, int length, int subl)
{
register int i;
register site *s;
msg_tag *mtag0;
su3_matrix *tempmat2t, *tempmat3t;
int j, nsubl;

    /* A forward step leaves the answer in gen_pt[0], which points into
       link, tempmat1 or tempmat2, and backwards step in tempmat1 or tempmat2.
       After a forwards step, need to wait and clean a gather.
	STEP			leaves answer in
	even # forward		gen_pt[0]->tempmat1 (gen_pt[0]->link for step 0)
	even # backward		tempmat1
	odd  # forward		gen_pt[0]->tempmat2
	odd  # backward		tempmat2
    */

    /* allocate temporary space */
    tempmat3t = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
    tempmat2t = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));

    /* j=0 */
    if( GOES_FORWARDS(dir[0]) ) {
	nsubl = neighsubl[subl][dir[0]];
	mtag0 = start_gather( F_OFFSET(link[dir[0]]), sizeof(su3_matrix),
		OPP_DIR(dir[0]), nsubl, gen_pt[0] );
    }
    else{  /* if GOES_BACKWARDS(dir[0]) */
	nsubl = neighsubl[subl][dir[0]];
	FORSOMESUBLATTICE(i,s,nsubl){
	    su3_adjoint(&(s->link[OPP_DIR(dir[0])]), &(s->tempmat1) );
	}
    }

    for(j=1;j<length;j++) {
	if( j%2==1 ){
	    if( GOES_FORWARDS(dir[j]) ) {
	      if( GOES_FORWARDS(dir[j-1]) ){
		wait_gather(mtag0);
		FORSOMESUBLATTICE(i,s,nsubl){
		  mult_su3_nn( (su3_matrix *)(gen_pt[0][i]),
		    &(s->link[dir[j]]), &(tempmat2t[i]) );
		}
		cleanup_gather(mtag0);
	      }
	      else{ /* last link was backwards */
		FORSOMESUBLATTICE(i,s,nsubl){
		  mult_su3_nn( &(s->tempmat1),
		    &(s->link[dir[j]]), &(tempmat2t[i]) );
		}
	      }
	      nsubl = neighsubl[nsubl][dir[j]];
	      mtag0 = start_gather_from_temp( tempmat2t, sizeof(su3_matrix),
		      OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	    }  /* for GOES_FORWARDS */

	    else{ /* GOES_BACKWARDS(dir[j]), which is an odd numbered step */
	      if( GOES_FORWARDS(dir[j-1]) ){
		wait_gather(mtag0);
		FORSOMESUBLATTICE(i,s,nsubl){
		  su3mat_copy((su3_matrix *)(gen_pt[0][i]), &(tempmat3t[i]) );
		}
		cleanup_gather(mtag0);
		nsubl = neighsubl[nsubl][dir[j]];
		mtag0 = start_gather_from_temp( tempmat3t, sizeof(su3_matrix),
			OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	      }
	      else{ /*last step was backwards */
		nsubl = neighsubl[nsubl][dir[j]];
		mtag0 = start_gather( F_OFFSET(tempmat1), sizeof(su3_matrix),
			OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	      }
	      wait_gather(mtag0);
	      FORSOMESUBLATTICE(i,s,nsubl){
		mult_su3_na((su3_matrix *)(gen_pt[0][i]),
		  &(s->link[OPP_DIR(dir[j])]), &(tempmat2t[i]) );
	      }
	      cleanup_gather(mtag0);
	    } /* end for GOES_BACKWARDS */
	} /* end for j=odd */

	else{   /* j=even */
	    if( GOES_FORWARDS(dir[j]) ) {
	      if( GOES_FORWARDS(dir[j-1]) ){
		wait_gather(mtag0);
		FORSOMESUBLATTICE(i,s,nsubl){
		  mult_su3_nn( (su3_matrix *)(gen_pt[0][i]),
		    &(s->link[dir[j]]), &(s->tempmat1) );
		}
		cleanup_gather(mtag0);
	      }
	      else{ /* last link was backwards */
		FORSOMESUBLATTICE(i,s,nsubl){
		  mult_su3_nn( &(tempmat2t[i]),
		    &(s->link[dir[j]]), &(s->tempmat1) );
		}
	      }
	      nsubl = neighsubl[nsubl][dir[j]];
	      mtag0 = start_gather( F_OFFSET(tempmat1), sizeof(su3_matrix),
		      OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	    }  /* for GOES_FORWARDS */

	    else{ /* GOES_BACKWARDS(dir[j]), which is an even numbered step */
	      if( GOES_FORWARDS(dir[j-1]) ){
		wait_gather(mtag0);
		FORSOMESUBLATTICE(i,s,nsubl){
		  su3mat_copy((su3_matrix *)(gen_pt[0][i]), &(tempmat3t[i]) );
		}
		cleanup_gather(mtag0);
		nsubl = neighsubl[nsubl][dir[j]];
		mtag0 = start_gather_from_temp( tempmat3t, sizeof(su3_matrix),
			OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	      }
	      else{ /*last step was backwards */
		nsubl = neighsubl[nsubl][dir[j]];
		mtag0 = start_gather_from_temp( tempmat2t, sizeof(su3_matrix),
			OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	      }
	      wait_gather(mtag0);
	      FORSOMESUBLATTICE(i,s,nsubl){
		mult_su3_na((su3_matrix *)(gen_pt[0][i]),
		  &(s->link[OPP_DIR(dir[j])]), &(s->tempmat1) );
	      }
	      cleanup_gather(mtag0);
	    } /* end for GOES_BACKWARDS */
	} /* end for j=even */

    }  /* j=link in loop */

    /* Want to end in tempmat1 */
    if( length%2==0 ){  /* last step was odd */
	if( GOES_FORWARDS(dir[length-1]) ){
	    wait_gather(mtag0);
	    FORSOMESUBLATTICE(i,s,nsubl){
	      su3mat_copy((su3_matrix *)(gen_pt[0][i]), &(s->tempmat1) );
	    }
	    cleanup_gather(mtag0);
	}
	else{
	    FORSOMESUBLATTICE(i,s,nsubl){
	      su3mat_copy(&(tempmat2t[i]), &(s->tempmat1) );
	    }
	}
    }
    else{ /* odd length path: last step was even */
	if( GOES_FORWARDS(dir[length-1]) ){
	    wait_gather(mtag0);
	    FORSOMESUBLATTICE(i,s,nsubl){
	      su3mat_copy((su3_matrix *)(gen_pt[0][i]), &(tempmat3t[i]) );
	    }
	    cleanup_gather(mtag0);
	    FORSOMESUBLATTICE(i,s,nsubl){
	      su3mat_copy(&(tempmat3t[i]), &(s->tempmat1) );
	    }
	}
    }

    free(tempmat3t);
    free(tempmat2t);

} /* path_prod_subl */

#endif /* N_SUBL32 */

invivopath_product_path_product_156

Real time: 1.662913e+08 -- Predicted time: 1.056162e+08

Callcount: 1080

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
2	### 1	21.9936511967494	1.001080e+05	1.260080e+05	20.5
281	### 2	1062.12077150066	9.736600e+04	9.353200e+04	3.9

                /****************  s_m_a_mat.c  (in su3.a) ******************************
*									*
* void scalar_mult_add_su3_matrix( su3_matrix *a, su3_matrix *b,	*
*	double s, su3_matrix *c)						*
* C <- A + s*B								*
*/
#include "config.h"
#include "complex.h"
#include "su3.h"

/* c <- a + s*b, matrices */
void scalar_mult_add_su3_matrix(su3_matrix *a,su3_matrix *b,double s,
	su3_matrix *c){

#ifndef NATIVEDOUBLE
register int i,j;
    for(i=0;i<3;i++)for(j=0;j<3;j++){
	c->e[i][j].real = a->e[i][j].real + s*b->e[i][j].real;
	c->e[i][j].imag = a->e[i][j].imag + s*b->e[i][j].imag;
    }

#else /* RS6000 version */

  register double ss;

  ss = s;

  c->e[0][0].real = a->e[0][0].real + ss*b->e[0][0].real;
  c->e[0][0].imag = a->e[0][0].imag + ss*b->e[0][0].imag;
  c->e[0][1].real = a->e[0][1].real + ss*b->e[0][1].real;
  c->e[0][1].imag = a->e[0][1].imag + ss*b->e[0][1].imag;
  c->e[0][2].real = a->e[0][2].real + ss*b->e[0][2].real;
  c->e[0][2].imag = a->e[0][2].imag + ss*b->e[0][2].imag;

  c->e[1][0].real = a->e[1][0].real + ss*b->e[1][0].real;
  c->e[1][0].imag = a->e[1][0].imag + ss*b->e[1][0].imag;
  c->e[1][1].real = a->e[1][1].real + ss*b->e[1][1].real;
  c->e[1][1].imag = a->e[1][1].imag + ss*b->e[1][1].imag;
  c->e[1][2].real = a->e[1][2].real + ss*b->e[1][2].real;
  c->e[1][2].imag = a->e[1][2].imag + ss*b->e[1][2].imag;

  c->e[2][0].real = a->e[2][0].real + ss*b->e[2][0].real;
  c->e[2][0].imag = a->e[2][0].imag + ss*b->e[2][0].imag;
  c->e[2][1].real = a->e[2][1].real + ss*b->e[2][1].real;
  c->e[2][1].imag = a->e[2][1].imag + ss*b->e[2][1].imag;
  c->e[2][2].real = a->e[2][2].real + ss*b->e[2][2].real;
  c->e[2][2].imag = a->e[2][2].imag + ss*b->e[2][2].imag;

#endif
}

invivos_m_a_mat_scalar_mult_add_su3_matrix_17

Real time: 1.332137e+09 -- Predicted time: 6.356343e+09

Callcount: 30688000

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
6033136	### 1	4895929.64907193	4.100000e+02	4.000000e+01	90.2
21985681	### 2	22535405.1663995	1.460000e+02	4.400000e+01	69.8
24958772	### 3	2966429.64559201	3.420000e+02	4.800000e+01	85.9
522379	### 4	335782.994752487	1.320000e+02	5.200000e+01	60.6

                /****** gauge_stuff.c  -- ******************/
/* MIMD version 6 */
/* gauge action stuff for improved action
* T.D. and A.H. general gauge action updating code
* D.T. modified  5/97
* D.T. modified 12/97, optimized gauge_force a little
* D.T. modified 3/99, gauge action in include file */

/**#define GFTIME**/ /* For timing gauge force calculation */
#include "generic_includes.h"	/* definitions files and prototypes */

#ifdef LOOPEND
#undef FORALLSITES
#define FORALLSITES(i,s) \
{ register int loopend; loopend=sites_on_node; \
for( i=0,  s=lattice ; i<loopend; i++,s++ )
#define END_LOOP }
#else
#define END_LOOP        /* define it to be nothing */
#endif

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)
void printpath( int *path, int length );

#define GAUGE_ACTION_PART1
/* defines NREPS NLOOP MAX_LENGTH MAX_NUM */
#include <gauge_action.h>
#undef GAUGE_ACTION_PART1

char gauge_action_description[128];
int  gauge_action_nloops=NLOOP;
int  gauge_action_nreps=NREPS;
int loop_length[NLOOP];	/* lengths of various kinds of loops */
int loop_num[NLOOP];	/* number of rotations/reflections  for each kind */

    /* table of directions, 1 for each kind of loop */
int loop_ind[NLOOP][MAX_LENGTH];
    /* table of directions, for each rotation and reflection of each kind of
	loop.  tabulated with "canonical" starting point and direction. */
int loop_table[NLOOP][MAX_NUM][MAX_LENGTH];
    /* table of coefficients in action, for various "representations" (actually,
	powers of the trace) */
double loop_coeff[NLOOP][NREPS];
    /* for each rotation/reflection, an integer distinct for each starting
	point, or each cyclic permutation of the links */
int loop_char[MAX_NUM];
    /* for each kind of loop for each rotation/reflection, the expectation
	value of the loop */
double loop_expect[NLOOP][NREPS][MAX_NUM];


/* Make table of loops in action */
void make_loop_table() {

    int perm[8],pp[8],ir[4];
    int length,iloop,i,j,chr;
    int vec[MAX_LENGTH];
    int count,flag;
    void char_num( int *dig, int *chr, int length);

#define GAUGE_ACTION_PART2
/* defines all loops and their coefficients */
#include <gauge_action.h>
#undef GAUGE_ACTION_PART2

    for(iloop=0;iloop<NLOOP;iloop++){
	length=loop_length[iloop];
	count=0;
	/* permutations */
	for(perm[0]=0;perm[0]<4;perm[0]++)
	for(perm[1]=0;perm[1]<4;perm[1]++)
	for(perm[2]=0;perm[2]<4;perm[2]++)
	for(perm[3]=0;perm[3]<4;perm[3]++){
	    if(perm[0] != perm[1] && perm[0] != perm[2] 
		&& perm[0] != perm[3] && perm[1] != perm[2]
	    	&& perm[1] != perm[3] && perm[2] != perm[3] ) {
	        /* reflections*/
	 	for(ir[0]=0;ir[0]<2;ir[0]++)
		for(ir[1]=0;ir[1]<2;ir[1]++)
		for(ir[2]=0;ir[2]<2;ir[2]++)
		for(ir[3]=0;ir[3]<2;ir[3]++){
		    for(j=0;j<4;j++){
			pp[j]=perm[j];

			if(ir[j] == 1) pp[j]=7-pp[j];
			pp[7-j]=7-pp[j];
		    }
		    /* create new vector*/
		    for(j=0;j<length;j++) vec[j]=pp[loop_ind[iloop][j]];

         	    char_num(vec,&chr,length);
         	    flag=0;
		    /* check if it's a new set: */
		    for(j=0;j<count;j++) if(chr == loop_char[j])flag=1;
		    if(flag == 0 ){
			loop_char[count]=chr;
			for(j=0;j<length;j++)
			    loop_table[iloop][count][j]=vec[j];
			count++;
/**node0_printf("ADD LOOP: "); printpath( vec, length );**/
		    }
		    if(count>MAX_NUM){
			node0_printf("OOPS: MAX_NUM too small\n");
			exit(0);
		    }
		    loop_num[iloop]=count;

		} /* end reflection*/
	    } /* end permutation if block */
	} /* end permutation */
    } /* end iloop */

    /* print out the loop coefficients */
    node0_printf("loop coefficients: nloop rep loop_coeff  multiplicity\n");
    for(i=0;i<NREPS;i++) for(j=0;j<NLOOP;j++) {
	node0_printf("                    %d %d      %e     %d\n",
	    j,i,loop_coeff[j][i],loop_num[j]);
    }

} /* make_loop_table */


/* find a number uniquely identifying the cyclic permutation of a path,
   or the starting point on the path.  Backwards paths are considered
   equivalent here, so scan those too. */
void char_num( int *dig, int *chr, int length){
    int j;
    int bdig[MAX_LENGTH],tenl,newv,old;
    /* "dig" is array of directions.  "bdig" is array of directions for
	backwards path. */

  tenl=1;
  for(j=0;j<length-1;j++) tenl=tenl*10;

  *chr=dig[length-1];
  for(j=length-2;j>=0;j--) *chr= *chr*10+dig[j];

  /* forward*/
  old=*chr;
  for(j=length-1;j>=1;j--){
       newv=old-tenl*dig[j];
       newv=newv*10+dig[j];
       if(newv < *chr) *chr=newv;
       old=newv;           }

   /* backward*/
   for(j=0;j<length;j++)bdig[j]=7-dig[length-j-1];
   old=bdig[length-1];
   for(j=length-2;j>=0;j--) old=old*10+bdig[j];
   if(old < *chr ) *chr=old;
   for(j=length-1;j>=1;j--){
       newv=old-tenl*bdig[j];
       newv=newv*10+bdig[j];
       if(newv < *chr) *chr=newv;
       old=newv;           }

} /* char_num */

double imp_gauge_action() {
    register int i;
    int rep;
    register site *s;
    complex trace;
    double g_action;
    double action,act2,total_action;
    int length;

    /* these are for loop_table  */
    int ln,iloop;

    g_action=0.0;

    /* gauge action */
    for(iloop=0;iloop<NLOOP;iloop++){
	length=loop_length[iloop];
	/* loop over rotations and reflections */
	for(ln=0;ln<loop_num[iloop];ln++){

	    path_product( loop_table[iloop][ln] , length );

	    FORALLSITES(i,s){
		trace=trace_su3( &s->tempmat1 );
		action =  3.0 - (double)trace.real;
		/* need the "3 -" for higher characters */
        	total_action= (double)loop_coeff[iloop][0]*action;
        	act2=action;
		for(rep=1;rep<NREPS;rep++){
		    act2 *= action;
		    total_action += (double)loop_coeff[iloop][rep]*act2;
		}

        	g_action  += total_action;

	    } END_LOOP /* sites */
	} /* ln */
    } /* iloop */

    g_doublesum( &g_action );
    return( g_action );
} /* imp_gauge_action */


/* update the momenta with the gauge force */
void imp_gauge_force( double eps, field_offset mom_off ){
    register int i,dir;
    register site *st;
    su3_matrix tmat1,tmat2;
    register double eb3;
    register anti_hermitmat* momentum;
#ifdef GFTIME
    double dtime;
#endif

    int j,k;
    int dirs[MAX_LENGTH],length;
    int path_dir[MAX_LENGTH],path_length;

    int ln,iloop;
    double action,act2,new_term;

    int ncount;

#ifdef GFTIME
dtime=-dclock();
#endif
    eb3 = eps*beta/3.0;

    /* Loop over directions, update mom[dir] */
    for(dir=XUP; dir<=TUP; dir++){

	FORALLSITES(i,st)for(j=0;j<3;j++)for(k=0;k<3;k++){
			st->staple.e[j][k]=cmplx(0.0,0.0);
	} END_LOOP

	ncount=0;
	for(iloop=0;iloop<NLOOP;iloop++){
	    length=loop_length[iloop];
	    for(ln=0;ln<loop_num[iloop];ln++){
/**printf("UPD:  "); printpath( loop_table[iloop][ln], length );**/
		/* set up dirs.  we are looking at loop starting in "XUP"
		   direction, rotate so it starts in "dir" direction. */
		for(k=0;k<length;k++){
                    if( GOES_FORWARDS(loop_table[iloop][ln][k]) ){
                	dirs[k]=(dir+loop_table[iloop][ln][k] )% 4;
		    }
            	    else {
                        dirs[k]=OPP_DIR(
			    (dir+OPP_DIR(loop_table[iloop][ln][k]))%4 );
		    }
		}

		path_length= length-1;  /* generalized "staple" */

		/* check for links in direction of momentum to be
		   updated, each such link gives a contribution. Note
		   the direction of the path - opposite the link. */
		for(k=0;k<length;k++)if( dirs[k]==dir||dirs[k]==OPP_DIR(dir)) {
		    if( GOES_FORWARDS(dirs[k]) ) for(j=0;j<path_length;j++) {
			path_dir[j] = dirs[(k+j+1)%length];
		    }
		    if( GOES_BACKWARDS(dirs[k]) ) for(j=0;j<path_length;j++) {
			path_dir[path_length-1-j] =
			    OPP_DIR(dirs[(k+j+1)%length]);
		    }
/**if(dir==XUP)printf("X_UPDATE PATH: "); printpath( path_dir, path_length );**/
		    path_product(path_dir,path_length);

		    /* We took the path in the other direction from our
			old convention in order to get it to end up
			"at our site", so now take adjoint */
		    /* then compute "single_action" contribution to
			staple */
		    FORALLSITES(i,st){
			su3_adjoint( &(st->tempmat1), &tmat1 );
			/* first we compute the fundamental term */
			new_term = loop_coeff[iloop][0];

			/* now we add in the higher representations */
			if(NREPS > 1){
node0_printf("WARNING: THIS CODE IS NOT TESTED\n"); exit(0);
			    act2=1.0;
			    action = 3.0 - realtrace_su3(&(st->link[dir]),
				&tmat1 ); 

			    for(j=1;j<NREPS;j++){
				act2 *= action;
				new_term +=
				    loop_coeff[iloop][j]*act2*(double)(j+1);
			    }
			}  /* end if NREPS > 1 */

			scalar_mult_add_su3_matrix( &(st->staple), &tmat1,
				new_term, &(st->staple) );

		    } END_LOOP

		    ncount++;

		} /* k (location in path) */
	    } /* ln */
	} /* iloop */

	/* Now multiply the staple sum by the link, then update momentum */
	FORALLSITES(i,st){
	    mult_su3_na( &(st->link[dir]), &(st->staple), &tmat1 );
	    momentum = (anti_hermitmat *)F_PT(st,mom_off);
	    uncompress_anti_hermitian( &momentum[dir], &tmat2 );
	    scalar_mult_sub_su3_matrix( &tmat2, &tmat1,
		eb3, &(st->staple) );
	    make_anti_hermitian( &(st->staple), &momentum[dir] );
	} END_LOOP
    } /* dir loop */
#ifdef GFTIME
dtime+=dclock();
node0_printf("GFTIME:  %e\n",dtime);
#endif
} /* imp_gauge_force.c */

/* Measure gauge observables:
    Loops in action (time and space directions treated differently)
    Polyakov loop

*/
void g_measure( ){
    double ss_plaquette, st_plaquette;
    complex p_loop;
    register int i;
    register site *s;
    complex trace;
    double average[NREPS],action,act2,total_action;
    int length;
    /* these are for loop_table  */
    int ln,iloop,rep;

    /* KS and BC minus signs should be out for this routine */
    d_plaquette( &ss_plaquette, &st_plaquette );
    if(this_node==0)printf("PLAQ:\t%f\t%f\n", ss_plaquette, st_plaquette );

    p_loop = ploop();
    if(this_node==0)printf("P_LOOP:\t%e\t%e\n", p_loop.real, p_loop.imag );

    /* gauge action, all loops that contribute */
    total_action=0.0;
    for(iloop=0;iloop<NLOOP;iloop++){
	length=loop_length[iloop];
	/* loop over rotations and reflections */
	for(ln=0;ln<loop_num[iloop];ln++){

	    path_product( loop_table[iloop][ln] , length );

	    for(rep=0;rep<NREPS;rep++)average[rep] = 0.0;
	    FORALLSITES(i,s){
		trace=trace_su3( &s->tempmat1 );
		average[0] += (double)trace.real;
		action =  3.0 - (double)trace.real;
		total_action += (double)loop_coeff[iloop][0]*action;
		/* need the "3 -" for higher characters */
        	act2=action;
		for(rep=1;rep<NREPS;rep++){
		    act2 *= action;
		    average[rep] += act2;
		    total_action += (double)loop_coeff[iloop][rep]*act2;
		} /* reps */
	    } END_LOOP /* sites */
	    g_vecdoublesum( average, NREPS );
	    /* dump the loop */
	    node0_printf("G_LOOP:  %d  %d  %d   ",iloop,ln,length);
	    for(rep=0;rep<NREPS;rep++)node0_printf("\t%e",average[rep]/volume);
	    node0_printf("\t( ");
	    for(i=0;i<length;i++)node0_printf("%d ",loop_table[iloop][ln][i]);
	    node0_printf(" )\n");
	} /* ln */
    } /* iloop */
    g_doublesum( &total_action );
    node0_printf("GACTION: %e\n",total_action/volume);
    /**node0_printf("CHECK:   %e   %e\n",total_action,imp_gauge_action() );**/

    if(this_node==0)fflush(stdout);

}

void printpath( int *path, int length ){
    register int i;
    node0_printf("\t( ");
    for(i=0;i<length;i++)node0_printf("%d ",path[i]);
    node0_printf(",  L = %d )\n", length );
}

#ifdef N_SUBL32
/*** code from symanzik_sl32/dsdu_qhb.c  -- compute the staple ***/
/* This is a version for extended actions where 32 sublattices are
   needed to make the links independent. */
/* U.M. Heller August 1997 */

/* Modifications:
   2/17/98  ANSI prototyping U.M.H.
   */
#include <assert.h>
void dsdu_qhb_subl(int dir, int subl)
{
register site *st;
register int i;
int iloop, ln, k, j;
int dirs[MAX_LENGTH], length;
int path_dir[MAX_LENGTH], path_length;
su3_matrix tmat1;
int fsubl;

 assert(NREPS==1);   /* This procedure designed only for NREPS = 1 */

    FORSOMESUBLATTICE(i,st,subl) {
	clear_su3mat(&(st->staple));
    }

    for(iloop=0;iloop<NLOOP;iloop++){
	length=loop_length[iloop];
	for(ln=0;ln<loop_num[iloop];ln++){
	    /* set up dirs.  we are looking at loop starting in "XUP"
	       direction, rotate so it starts in "dir" direction. */
	    for(k=0;k<length;k++){
		if( GOES_FORWARDS(loop_table[iloop][ln][k]) ){
		    dirs[k]=(dir+loop_table[iloop][ln][k] )% 4;
		}
		else {
		    dirs[k]=OPP_DIR(
			(dir+OPP_DIR(loop_table[iloop][ln][k]))%4 );
		}
	    }

	    path_length = length-1;	/* generalized "staple" */
	    /* The path starts at the forward end of the link */
	    fsubl = neighsubl[subl][dir];

	    /* check for links in direction of link to be updated.
	       Note the direction of the path - opposite the link. */
	    for(k=0;k<length;k++)if( dirs[k]==dir||dirs[k]==OPP_DIR(dir)) {
		if( GOES_FORWARDS(dirs[k]) ) for(j=0;j<path_length;j++) {
		    path_dir[j] = dirs[(k+j+1)%length];
		}
		if( GOES_BACKWARDS(dirs[k]) ) for(j=0;j<path_length;j++) {
		    path_dir[path_length-1-j] =
			OPP_DIR(dirs[(k+j+1)%length]);
		}
		path_prod_subl(path_dir, path_length, fsubl);

		/* We took the path in the other direction from our old
		   convention in order to get it to end up "at our site".
		   So now take adjoint */
		FORSOMESUBLATTICE(i,st,subl) {
		    su3_adjoint(&(st->tempmat1), &tmat1 );
		    scalar_mult_add_su3_matrix(&(st->staple), &tmat1,
			loop_coeff[iloop][0], &(st->staple) );
		}
	    } /* k (location in path) */
	} /* ln */
    } /* iloop */

    g_sync();

} /* dsdu_qhb */

#endif /* N_SUBL32 */

invivogauge_stuff_g_measure_345

Real time: 1.484978e+08 -- Predicted time: 1.511139e+08

Callcount: 2

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
1	### 1	1	7.758141e+07	7.348109e+07	5.2
2	### 2	1	7.353247e+07	7.343599e+07	0.1

                /*****************  m_amatvec.c  (in su3.a) *****************************
*									*
*  void mult_adj_su3_mat_vec( su3_matrix *a, su3_vector *b,*c )		*
*  C  <-  A_adjoint * B							*
*/
#include "config.h"
#include "complex.h"
#include "su3.h"

#ifndef FAST
/* adjoint matrix times vector multiply */
void mult_adj_su3_mat_vec( su3_matrix *a, su3_vector *b, su3_vector *c ){
register int i,j;
register complex x,y,z;
    for(i=0;i<3;i++){
	x.real=x.imag=0.0;
	for(j=0;j<3;j++){
	    CONJG( a->e[j][i], z );
	    CMUL( z , b->c[j], y )
	    CSUM( x , y );
	}
	c->c[i] = x;
    }
}

#else
#ifdef NATIVEDOUBLE /* IBM RS6000 version */
void mult_adj_su3_mat_vec( su3_matrix *a, su3_vector *b, su3_vector *c ){

  register double c0r,c0i,c1r,c1i,c2r,c2i;
  register double br,bi,a0,a1,a2;
  
  br=b->c[0].real;    bi=b->c[0].imag;
  a0=a->e[0][0].real;
  a1=a->e[0][1].real;
  a2=a->e[0][2].real;

  c0r = a0*br;
  c1r = a1*br;
  c2r = a2*br;
  c0i = a0*bi;
  c1i = a1*bi;
  c2i = a2*bi;

  a0=a->e[0][0].imag;
  a1=a->e[0][1].imag;
  a2=a->e[0][2].imag;

  c0r += a0*bi;
  c1r += a1*bi;
  c2r += a2*bi;
  c0i -= a0*br;
  c1i -= a1*br;
  c2i -= a2*br;

  br=b->c[1].real;    bi=b->c[1].imag;
  a0=a->e[1][0].real;
  a1=a->e[1][1].real;
  a2=a->e[1][2].real;

  c0r += a0*br;
  c1r += a1*br;
  c2r += a2*br;
  c0i += a0*bi;
  c1i += a1*bi;
  c2i += a2*bi;

  a0=a->e[1][0].imag;
  a1=a->e[1][1].imag;
  a2=a->e[1][2].imag;

  c0r += a0*bi;
  c1r += a1*bi;
  c2r += a2*bi;
  c0i -= a0*br;
  c1i -= a1*br;
  c2i -= a2*br;

  br=b->c[2].real;    bi=b->c[2].imag;
  a0=a->e[2][0].real;
  a1=a->e[2][1].real;
  a2=a->e[2][2].real;

  c0r += a0*br;
  c1r += a1*br;
  c2r += a2*br;
  c0i += a0*bi;
  c1i += a1*bi;
  c2i += a2*bi;

  a0=a->e[2][0].imag;
  a1=a->e[2][1].imag;
  a2=a->e[2][2].imag;

  c0r += a0*bi;
  c1r += a1*bi;
  c2r += a2*bi;
  c0i -= a0*br;
  c1i -= a1*br;
  c2i -= a2*br;

  c->c[0].real = c0r;
  c->c[0].imag = c0i;
  c->c[1].real = c1r;
  c->c[1].imag = c1i;
  c->c[2].real = c2r;
  c->c[2].imag = c2i;
  
}
#else
void mult_adj_su3_mat_vec( su3_matrix *a, su3_vector *b, su3_vector *c ){
int i;
register double t,ar,ai,br,bi,cr,ci;
    for(i=0;i<3;i++){

	ar=a->e[0][i].real; ai=a->e[0][i].imag;
	br=b->c[0].real; bi=b->c[0].imag;
	cr=ar*br; t=ai*bi; cr += t;
	ci=ar*bi; t=ai*br; ci -= t;

	ar=a->e[1][i].real; ai=a->e[1][i].imag;
	br=b->c[1].real; bi=b->c[1].imag;
	t=ar*br; cr += t; t=ai*bi; cr += t;
	t=ar*bi; ci += t; t=ai*br; ci -= t;

	ar=a->e[2][i].real; ai=a->e[2][i].imag;
	br=b->c[2].real; bi=b->c[2].imag;
	t=ar*br; cr += t; t=ai*bi; cr += t;
	t=ar*bi; ci += t; t=ai*br; ci -= t;

	c->c[i].real=cr;
	c->c[i].imag=ci;
    }
}
#endif	/* End of "#ifdef NATIVEDOUBLE" */
#endif	/* End of "#ifndef FAST" */

invivom_amatvec_mult_adj_su3_mat_vec_114

Real time: 1.575511e+09 -- Predicted time: 8.569147e+09

Callcount: 19760000

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
3884736	### 1	7182121.13988284	5.430000e+02	7.600000e+01	86.0
14156578	### 2	3157675.64503272	2.760000e+02	7.200000e+01	73.9
7727691	### 3	2712077.1249976	5.320000e+02	8.400000e+01	84.2
16206898	### 4	4575616.9252766	2.890000e+02	8.000000e+01	72.3
2813735	### 5	2244691.43641616	4.600000e+02	8.800000e+01	80.8

                /****** quark_stuff.c  -- ******************/
/* MIMD version 6 */
/* quark action stuff for improved action
* D.T. 1/28/98, starting from gauge_stuff.c
* K.O. 3/99 Added optimized fattening for Asq actions
* D.T. 4/99 Combine force calculations for both mass quarks
* K.O. 4/99 Optimized force for Asq action
* S.G. 7/01, modified to use t_longlink and t_fatlink
* C.D. 10/02, consolidated quark_stuff.c and quark_stuff_tmp.c
*
* This code combines quark_stuff.c and quark_stuff_tmp.c
* with DSLASH_TMP_LINKS defined, puts links in field-major variables
* t_longlink and t_fatlink. Otherwise, puts them in the site structure.
*
* In this directory, assume all paths connect even to odd sites, etc.
* Tabulate "backwards" paths (e.g. "XDOWN" is backward path to "XUP")
* as separate parity transforms of the fundamental paths.  They will
* generally need a negative sign in Dslash.  See bottom for a long
* comment on sign conventions.
*/

/**#define FFTIME**/
/**#define LLTIME**/

#include "generic_ks_includes.h"	/* definitions files and prototypes */


#define NULL_FP -1 /* NULL field_offset to be used in the optimized version *
                    * of the load_fatlinks subroutine */

    /* Specify paths in orientation in which they appear in the
       forward part of the x component of dslash().  Rotations and
       reflections will be automatically included. Be careful
       about signs of coefficients.  See long comment at bottom. */
#include <quark_action.h>
    /* Include file specifies the basic paths */

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void printpath( int *path, int length );
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length );
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length );
#ifdef  ASQ_OPTIMIZED_FATTENING
void compute_gen_staple(field_offset staple, int mu, int nu,
                        field_offset link, double coef ) ;
#endif
#ifdef  ASQ_OPTIMIZED_FORCE
#ifndef FN
BOMB THE COMPILE
#endif
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) ;
void add_force_to_mom(su3_vector *back, su3_vector *forw, int dir, double coef);
void side_link_force(int mu, int nu, double coeff, su3_vector *Path,
		     su3_vector *Path_nu, su3_vector *Path_mu, 
		     su3_vector *Path_numu) ;

void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) ;
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) ;
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) ;
#endif

int path_num[MAX_BASIC_PATHS];	/* number of rotations/reflections for each 
					kind */
static double act_path_coeff[MAX_BASIC_PATHS]; /* actual path coefficient     *
                                               * it is equal to path_coeff   *
                                               * if not tadpole improvement  *
                                               * is specified                *
                                               * or path_coeff*u_0^(L-1) when*
                                               * tadpole improvement is      *
                                               * specified                   */

/* Array of structures, for each rotation and reflection of each kind of
	path.  */
struct {
    int dir[MAX_LENGTH];	/* directions in path */
    int length;		/* length of path */
    double coeff;	/* coefficient, including minus sign if backwards */
    double forwback;	/* +1 if in forward Dslash, -1 if in backward */
}   q_paths[MAX_NUM];
int num_q_paths;	/* number of paths in dslash */
int num_basic_paths;	/* number of paths before rotation/reflection */

int is_path_equal( int *path1, int* path2, int length );
int add_basic_path( int *vec, int length, double coeff );

/* Make table of paths in action */
void make_path_table() {

    int i,j;
#ifdef TADPOLE_IMPROVE
    int k;
#endif

    /* table of directions, 1 for each kind of path */
    /**int path_ind[MAX_BASIC_PATHS][MAX_LENGTH];**/
    /* table of coefficients in action, for each path */

    node0_printf("%s\n",quark_action_description);
    num_q_paths = 0;
    num_basic_paths = 0;


    /* add rots. and reflects to table, print out the path coefficients */
    node0_printf("path coefficients: npath  path_coeff  multiplicity\n");
    for(j=0;j<quark_action_npaths;j++) {
	double this_coeff;
	this_coeff = path_coeff[j];
#ifdef TADPOLE_IMPROVE
	for(k=1;k< path_length_in[j];k++)this_coeff /= u0;
#endif
	act_path_coeff[j] = this_coeff ;
	i = add_basic_path( path_ind[j], path_length_in[j],
	    this_coeff );
	node0_printf("                    %d      %e     %d\n",
	    j,this_coeff,i);
    }
}

/* add rotations and reflections of a path to the table.  Return
   multiplicity of paths added */
int add_basic_path( int *basic_vec, int length, double coeff ) {

    int perm[8],pp[8],ir[4];
    int j,path_num;
    int vec[MAX_LENGTH];
    int flag;

    path_num = 0;
    /* now fill the long table with all rotations and reflections
	of the fundamental path.  The path presented to us is for
        the positive x component of dslash, so if the x coordinate
        is reflected it will appear with a negative sign. */
      /* permutations */
      for(perm[0]=0;perm[0]<4;perm[0]++)
      for(perm[1]=0;perm[1]<4;perm[1]++)
      for(perm[2]=0;perm[2]<4;perm[2]++)
      for(perm[3]=0;perm[3]<4;perm[3]++){
	if(perm[0] != perm[1] && perm[0] != perm[2] 
	  && perm[0] != perm[3] && perm[1] != perm[2]
	  && perm[1] != perm[3] && perm[2] != perm[3] ) {
	  /* reflections*/
	  for(ir[0]=0;ir[0]<2;ir[0]++)
	  for(ir[1]=0;ir[1]<2;ir[1]++)
	  for(ir[2]=0;ir[2]<2;ir[2]++)
	  for(ir[3]=0;ir[3]<2;ir[3]++){
	    for(j=0;j<4;j++){
	      pp[j]=perm[j];

	      if(ir[j] == 1) pp[j]=OPP_DIR(pp[j]);
	      pp[OPP_DIR(j)]=OPP_DIR(pp[j]);
	    }
	    /* create new vector*/
	    for(j=0;j<length;j++) vec[j]=pp[basic_vec[j]];
	    for(j=length;j<MAX_LENGTH;j++) vec[j]=NODIR;

            flag=0;
	    /* check if it's a new set: */
	    for(j=0;j<num_q_paths;j++){
	      flag = is_path_equal( vec, q_paths[j].dir, MAX_LENGTH );
	      if(flag==1)break;
	    }
	    if(flag == 0 ){
	      if(num_q_paths>=MAX_NUM){
		node0_printf("OOPS: MAX_NUM too small\n");
		exit(0);
	      }
	      q_paths[num_q_paths].length=length;
	      for(j=0;j<MAX_LENGTH;j++) q_paths[num_q_paths].dir[j]=vec[j];
		/* remember to copy NODIR's, or comparison will get confused */
	      if(ir[0]==0){
		q_paths[num_q_paths].coeff =  coeff;
		q_paths[num_q_paths].forwback =  +1;
	      }
	      else{
		q_paths[num_q_paths].coeff = -coeff;
		q_paths[num_q_paths].forwback = -1;
	      }
	      num_q_paths++;
	      path_num++;
	      /**node0_printf("ADD PATH %d:  rx=%d ",num_q_paths-1,ir[0]);
		 printpath( vec, length );**/
	    }

	  } /* end reflection*/
        } /* end permutation if block */
      } /* end permutation */
    num_basic_paths++;
    return(path_num);
} /* add_basic_path */

/* parallel transport a vector to the current site along a path.
   For example, if the path is "XUP", bring in the vector from
   the +X direction to the current site. If KS phases are in lattice,
   this transport will automatically include them. 
   OK for src and dest to be the same.  OK for length=0.  */
/* NOT OPTIMIZED at the moment - do lots of extra copying.  Use temp.
   vectors rather than stuff in lattice.h */
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    su3_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    su3_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_dest = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_work = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(su3_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_vec( &(s->link[dir[j]]),
		    (su3_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_vec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(su3_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = *(su3_vector *)F_PT(s,src);
    }
  }
} /* path_transport */

/* Path transport a half_wilson_vector */
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    half_wilson_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    half_wilson_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (half_wilson_vector *)calloc(
      sites_on_node, sizeof(half_wilson_vector));
    tmp_dest = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));
    tmp_work = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(half_wilson_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_hwvec( &(s->link[dir[j]]),
		    (half_wilson_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_hwvec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(half_wilson_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) =
	  *(half_wilson_vector *)F_PT(s,src);
    }
  }
} /* path_transport_hwv */


#ifdef EO
/* Stupid dslash routine that follows all the paths.
  Should optimize by precomputing sums of paths to all the displacements
  Use dslash_fn for actions that involve only +X and +X+X+X couplings.
*/
/* D_slash routine - sets dest. on each site equal to sum of
   sources parallel transported to site, with minus sign for transport
   from negative directions.  */
void dslash_eo( field_offset src, field_offset dest, int parity ) {
   register int i;
   register site *s;
   register int ipath,otherparity;
   register double x; /* coefficient of path */

    switch(parity){
	case EVEN:	otherparity=ODD; break;
	case ODD:	otherparity=EVEN; break;
	case EVENANDODD:	otherparity=EVENANDODD; break;
    }

    /* Parallel transport by all the paths in the action.  
       Multiply by coefficient in table
    */
    FORSOMEPARITY(i,s,parity){ clearvec( (su3_vector *)F_PT(s,dest) ); }

    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	    path_transport( src, F_OFFSET(tempvec[0]), parity,
		q_paths[ipath].dir, q_paths[ipath].length );
	    x=q_paths[ipath].coeff;
	    FORSOMEPARITY(i,s,parity){
		scalar_mult_add_su3_vector(  (su3_vector *)F_PT(s,dest),
		    &(s->tempvec[0]), x, (su3_vector *)F_PT(s,dest) );
	    }
    }   /* ipath */
} /* dslash_eo */
#endif /* ifdef EO */

#ifdef FN
/* Sum over paths connecting to nearest neighbor point (fat link) and to third
   nearest neighbor (longlinks) */
/* Doug Toussaint 2/4/98 */
/* modified to use t_longlinks, S. Gottlieb 7/13/01 */
/* long link calculating routine */
/* path_product() follows the path starting at step 0, and
   leaves the answer at the end of the path.  We want the answer
   at the site where the path begins.  So we look for paths with
   the opposite displacement from the displacement of the point
   that we want to transport to this site, and take the adjoint
   of the matrix at the end. clear? */
/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_longlinks() {
  register int i;
  register site *s;
  int ipath,dir;
  int disp[4];
  register su3_matrix *long1;

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_longlinks needs phases in\n");
    terminate(0);
  }
  for (dir=XUP; dir<=TUP; dir++){ /* loop over longlink directions */
    /* set longlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      long1 = &(t_longlink[4*i+dir]);
#else
      long1 = &(s->longlink[dir]);
#endif
      clear_su3mat( long1 );
    }

    /* loop over paths, checking for ones with total displacement 3*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=3,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("ipath = %d, found a path:  ",ipath);
for(j=0;j<q_paths[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  long1 = &(t_longlink[4*i+dir]);
#else
	  long1 = &(s->longlink[dir]);
#endif
          scalar_mult_add_su3_matrix( long1,
	    &(s->staple), -q_paths[ipath].coeff, long1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */

  } /* loop over directions */

  valid_longlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Long):  %e\n",dtime);
#endif
}  /* load_longlinks() */

/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_fatlinks() {
  register int i;
  register site *s;
  int dir;
  register su3_matrix *fat1;
#ifdef ASQ_OPTIMIZED_FATTENING
  int  nu,rho,sig ;
  double one_link ; /*needed to fix the problem with the Lepage term */
#else
  int ipath;
  int disp[4];
#endif

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_fatlinks needs phases in\n");
    terminate(0);
  }

#ifndef  ASQ_OPTIMIZED_FATTENING   /* general case code */
  for (dir=XUP; dir<=TUP; dir++){ /* loop over fatlink directions */
    /* set fatlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+dir]);
#else
      fat1 = &(s->fatlink[dir]);
#endif
      clear_su3mat( fat1 );
    }
    
    /* loop over paths, checking for ones with total displacement 1*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=1,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("dir = %d, found a path:  ",dir);
for(j=0;j<q_paths.[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  fat1 = &(t_fatlink[4*i+dir]);
#else
	  fat1 = &(s->fatlink[dir]);
#endif
          scalar_mult_add_su3_matrix( fat1,
	    &(s->staple), -q_paths[ipath].coeff, fat1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */
  } /* loop over directions */
#else	/* ASQ_OPTIMIZED_FATTENING, for Asq and Asqtad actions */
/*  Optimized fattening code for the Asq and Asqtad actions.           *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
 
 /* to fix up the Lepage term, included by a trick below */
 one_link = (act_path_coeff[0] - 6.0*act_path_coeff[5]) ; 
 
 for (dir=XUP; dir<=TUP; dir++){
   FORALLSITES(i,s) /* Intialize fat links with c_1*U_\mu(x) */
     {
#ifdef DSLASH_TMP_LINKS
       fat1 = &(t_fatlink[4*i+dir]);
#else
       fat1 = &(s->fatlink[dir]);
#endif
       scalar_mult_su3_matrix(&(s->link[dir]), one_link ,
			      fat1 );
     }
   for(nu=XUP; nu<=TUP; nu++) if(nu!=dir)
     {
       compute_gen_staple(F_OFFSET(staple),dir,nu,F_OFFSET(link[dir]),
			  act_path_coeff[2]);
       /* The Lepage term */
       /* Note this also involves modifying c_1 (above) */
       compute_gen_staple(NULL_FP,dir,nu,F_OFFSET(staple),act_path_coeff[5]);
       for(rho=XUP; rho<=TUP; rho++) if((rho!=dir)&&(rho!=nu))
	 {
	   compute_gen_staple(F_OFFSET(tempmat1),dir,rho,F_OFFSET(staple),
			      act_path_coeff[3]);
	   for(sig=XUP; sig<=TUP; sig++)
	     if((sig!=dir)&&(sig!=nu)&&(sig!=rho))
	       {
		 compute_gen_staple(NULL_FP,dir,sig,
				    F_OFFSET(tempmat1),
				    act_path_coeff[4]);
	       } /* sig */
	 } /* rho */
     } /* nu */
 }/* dir */  
#endif

  valid_fatlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Fat):  %e\n",dtime);
#endif
}  /* load_fatlinks() */
#endif /* ifdef FN */


/* compare two paths, return 1 if equal, else zero */
int is_path_equal( int *path1, int* path2, int length ){
   register int i;
   for(i=0;i<length;i++)if(path1[i]!=path2[i])return(0);
   return(1);
}


/* update the  momenta with the fermion force */
/* Assumes that the conjugate gradient has been run, with the answer in
   x_off, and dslash(x_off,x_off,ODD) has been run. (fills in x_off_odd) */
/* SEE LONG COMMENTS AT END */

#ifndef ASQ_OPTIMIZED_FORCE
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon, coeff;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  half_wilson_vector *hw_tmp0,*hw_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  hw_tmp0 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  hw_tmp1 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  /* Use half_wilson_vectors to store x_off transported from ends of
     path.  0 component from forward end, 1 component from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    path_transport( x_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[1] for transport from starting end */
    FORALLSITES(i,s){
      hw_tmp0[i].h[0]=s->tempvec[0];
      hw_tmp0[i].h[1]=*(su3_vector *)F_PT(s,x_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff = ferm_epsilon*q_paths[ipath].coeff;
      if( (ilink%2)==1 )coeff = -coeff;

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "half_wilson_vector" to handle pair of vectors -
	0 component is x_off from forward end, 1 component from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_su3_mat_hwvec( &(s->link[lastdir]),
	      &(hw_tmp0[i]), &(hw_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( hw_tmp1, 2*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     hw_tmp0[i] = *(half_wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( hw_tmp0, 2*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_su3_mat_hwvec( &(s->link[OPP_DIR(lastdir)]),
                (half_wilson_vector *)(gen_pt[0][i]),
		&(hw_tmp1[i]) );
          }
	  tmp_pt = hw_tmp0; hw_tmp0 = hw_tmp1; hw_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( hw_tmp0 ); free( hw_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */


void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off,
  int nflav2, field_offset x2_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/14/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon1, ferm_epsilon2, coeff1, coeff2;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  wilson_vector *w_tmp0,*w_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon1 = 2.0*(nflav1/4.0)*eps;
  ferm_epsilon2 = 2.0*(nflav2/4.0)*eps;
  w_tmp0 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  w_tmp1 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  /* Use wilson_vectors to store x_off transported from ends of
     path.  0 and 1 components from forward end, 2 and 3 components
     from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    /**
    path_transport( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    path_transport( x2_off, F_OFFSET(tempvec[1]),
      EVENANDODD, q_paths[ipath].dir, length );
    **/
/** WARNING!! Assumes xxx1 and xxx2 contiguous **/
if( x2_off-x1_off != sizeof(su3_vector) ){node0_printf("BOTCH\n"); exit(0);}
    path_transport_hwv( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[2] for transport from starting end */
    FORALLSITES(i,s){
      w_tmp0[i].d[0]=s->tempvec[0];
      w_tmp0[i].d[1]=s->tempvec[1];
      w_tmp0[i].d[2]=*(su3_vector *)F_PT(s,x1_off);
      w_tmp0[i].d[3]=*(su3_vector *)F_PT(s,x2_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff1 = ferm_epsilon1*q_paths[ipath].coeff;
      coeff2 = ferm_epsilon2*q_paths[ipath].coeff;
      if( (ilink%2)==1 ){ coeff1 = -coeff1; coeff2 = -coeff2;}

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "wilson_vector" to handle pair of vectors -
	0 component is x_off1 from forward end, 1 component is x_off2
	from forward end,  2 and 3  components are x_off1 and x_off2 
	from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_mat_wilson_vec( &(s->link[lastdir]),
	      &(w_tmp0[i]), &(w_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( w_tmp1, 4*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     w_tmp0[i] = *(wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( w_tmp0, 4*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_mat_wilson_vec( &(s->link[OPP_DIR(lastdir)]),
                (wilson_vector *)(gen_pt[0][i]),
		&(w_tmp1[i]) );
          }
	  tmp_pt = w_tmp0; w_tmp0 = w_tmp1; w_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( w_tmp0 ); free( w_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */



#else /* ASQ_OPTIMIZED_FORCE, for Asq and Asqtad actions */
/* Optimized force code for the Asq and Asqtad actions                 *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
#define Pmu          tempvec[0] 
#define Pnumu        tempvec[1]
#define Prhonumu     tempvec[2]
#define P7           tempvec[3]
#define P7rho        tempvec[4]              
#define P7rhonu      tempvec[5]
#define P5           tempvec[6]
#define P3           tempvec[7]
#define P5nu         tempvec[3]
#define P3mu         tempvec[3]
#define Popmu        tempvec[4]
#define Pmumumu      tempvec[4]
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double ferm_epsilon, coeff;
  double OneLink, Lepage, Naik, FiveSt, ThreeSt, SevenSt ;
  su3_vector *tempvec[8] ;
  su3_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  
  /* Path coefficients times fermion epsilon */
  OneLink = act_path_coeff[0]*ferm_epsilon ; 
  Naik    = act_path_coeff[1]*ferm_epsilon ;
  ThreeSt = act_path_coeff[2]*ferm_epsilon ;
  FiveSt  = act_path_coeff[3]*ferm_epsilon ;
  SevenSt = act_path_coeff[4]*ferm_epsilon ;
  Lepage  = act_path_coeff[5]*ferm_epsilon ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    tempvec[mu] = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

  /*copy x_off to a temporary vector */
  temp_x = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
  FORALLSITES(i,s) temp_x[i] = *(su3_vector *)F_PT(s,x_off) ;

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_force_to_mom(P3, Pmu, sig, -ThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_force_to_mom(P7, Prhonumu, sig, -SevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_fermion(P7, P7rho, rho);
		  side_link_force(rho,sig,SevenSt, Pnumu, P7, Prhonumu, P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff = SevenSt/FiveSt ; 
		  FORALLSITES(i,s)
		    scalar_mult_add_su3_vector(&(P5[i]),&(P7rho[i]),coeff,
					       &(P5[i]));
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_fermion(P5,P5nu, nu);
	      side_link_force(nu,sig,-FiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff = FiveSt/ThreeSt ; 
	      FORALLSITES(i,s)
		scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_fermion(P5,P5nu, mu);
	  side_link_force(mu, sig, -Lepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff = Lepage/ThreeSt ; 
	  FORALLSITES(i,s)
	    scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), -Naik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(tempvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      

#define Pmu          hwvec[0] 
#define Pnumu        hwvec[1]
#define Prhonumu     hwvec[2]
#define P7           hwvec[3]
#define P7rho        hwvec[4]              
#define P7rhonu      hwvec[5]
#define P5           hwvec[6]
#define P3           hwvec[7]
#define P5nu         hwvec[3]
#define P3mu         hwvec[3]
#define Popmu        hwvec[4]
#define Pmumumu      hwvec[4]
void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off, 
			  int nflav2, field_offset x2_off){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/15/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double coeff[2],ferm_epsilon ;
  double OneLink[2], Lepage[2], Naik[2], FiveSt[2], ThreeSt[2], SevenSt[2] ;
  double mNaik[2], mLepage[2], mFiveSt[2], mThreeSt[2], mSevenSt[2] ;
  half_wilson_vector *hwvec[8] ;
  half_wilson_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  
  /* Path coefficients times fermion epsilon */
  ferm_epsilon = 2.0*(nflav1/4.0)*eps;
  OneLink[0] = act_path_coeff[0]*ferm_epsilon ;
  Naik[0]    = act_path_coeff[1]*ferm_epsilon ; mNaik[0]    = -Naik[0] ;
  ThreeSt[0] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[0] = -ThreeSt[0] ;
  FiveSt[0]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[0]  = -FiveSt[0] ;
  SevenSt[0] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[0] = -SevenSt[0] ;
  Lepage[0]  = act_path_coeff[5]*ferm_epsilon ; mLepage[0]  = -Lepage[0] ;

  ferm_epsilon = 2.0*(nflav2/4.0)*eps;
  OneLink[1] = act_path_coeff[0]*ferm_epsilon ;
  Naik[1]    = act_path_coeff[1]*ferm_epsilon ; mNaik[1]    = -Naik[1] ;
  ThreeSt[1] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[1] = -ThreeSt[1] ;
  FiveSt[1]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[1]  = -FiveSt[1] ;
  SevenSt[1] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[1] = -SevenSt[1] ;
  Lepage[1]  = act_path_coeff[5]*ferm_epsilon ; mLepage[1]  = -Lepage[1] ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    hwvec[mu]= 
      (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));

  /*copy x_off to a temporary vector */
  temp_x= 
    (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
  FORALLSITES(i,s)
    {
      temp_x[i].h[0] = *(su3_vector *)F_PT(s,x1_off) ;
      temp_x[i].h[1] = *(su3_vector *)F_PT(s,x2_off) ;
    }

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_hw_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_3f_force_to_mom(P3, Pmu, sig, mThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_hw_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_3f_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_hw_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_hw_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_3f_force_to_mom(P7, Prhonumu, sig, mSevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_hw_fermion(P7, P7rho, rho);
		  side_link_3f_force(rho,sig,SevenSt,Pnumu,P7,Prhonumu,P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff[0] = SevenSt[0]/FiveSt[0] ; 
		  coeff[1] = SevenSt[1]/FiveSt[1] ; 
		  FORALLSITES(i,s)
		    {
		      scalar_mult_add_su3_vector(&(P5[i].h[0]),
						 &(P7rho[i].h[0]),coeff[0],
						 &(P5[i].h[0]));
		      scalar_mult_add_su3_vector(&(P5[i].h[1]),
						 &(P7rho[i].h[1]),coeff[1],
						 &(P5[i].h[1]));
		    }
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_hw_fermion(P5,P5nu, nu);
	      side_link_3f_force(nu,sig,mFiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff[0] = FiveSt[0]/ThreeSt[0] ; 
	      coeff[1] = FiveSt[1]/ThreeSt[1] ; 
	      FORALLSITES(i,s)
		{
		  scalar_mult_add_su3_vector(&(P3[i].h[0]),
					     &(P5nu[i].h[0]), coeff[0],
					     &(P3[i].h[0]));
		  scalar_mult_add_su3_vector(&(P3[i].h[1]),
					     &(P5nu[i].h[1]), coeff[1],
					     &(P3[i].h[1]));
		}
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_3f_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_hw_fermion(P5,P5nu, mu);
	  side_link_3f_force(mu, sig, mLepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff[0] = Lepage[0]/ThreeSt[0] ; 
	  coeff[1] = Lepage[1]/ThreeSt[1] ; 
	  FORALLSITES(i,s)
	    {
	      scalar_mult_add_su3_vector(&(P3[i].h[0]),
					 &(P5nu[i].h[0]),coeff[0],
					 &(P3[i].h[0]));
	      scalar_mult_add_su3_vector(&(P3[i].h[1]),
					 &(P5nu[i].h[1]),coeff[1],
					 &(P3[i].h[1]));
	    }

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_hw_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_3f_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_3f_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_3f_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), mNaik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_hw_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_3f_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_3f_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(hwvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      
#endif /* ASQ_OPTIMIZED_FORCE */


#ifdef  ASQ_OPTIMIZED_FATTENING   /* Asqtad action only, "_fn" executables */
#ifndef FN
BOMB THE COMPILE
#endif
void compute_gen_staple(field_offset staple, int mu, int nu, 
			field_offset link, double coef) {
  su3_matrix tmat1,tmat2;
  msg_tag *mtag0,*mtag1;
  su3_matrix *tempmat ;
  register site *s ;
  register int i ;
  register su3_matrix *fat1;

  /* Computes the staple :
                   mu
                +-------+
          nu	|	|
		|	|
		X	X
    Where the mu link can be any su3_matrix. The result is saved in staple.
    if staple==NULL_FP then the result is not saved.
    It also adds the computed staple to the fatlink[mu] with weight coef.
  */

  /* Upper staple */
  mtag0 = start_gather( link, sizeof(su3_matrix), nu, EVENANDODD, gen_pt[0] );
  mtag1 = start_gather( F_OFFSET(link[nu]), sizeof(su3_matrix), mu, 
			EVENANDODD, gen_pt[1] );
  wait_gather(mtag0);
  wait_gather(mtag1);
  
  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, (su3_matrix *)F_PT(s,staple) );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, &tmat2 );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix(fat1, &tmat2, coef,
				 fat1) ;
    }
  }
  cleanup_gather(mtag0);
  cleanup_gather(mtag1);

  /* lower staple */
  tempmat = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
  mtag0 = start_gather( F_OFFSET(link[nu]),
			sizeof(su3_matrix), mu, EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);
  FORALLSITES(i,s){
    mult_su3_an( &(s->link[nu]),(su3_matrix *)F_PT(s,link), &tmat1 );
    mult_su3_nn( &(tmat1),(su3_matrix *)gen_pt[0][i], &(tempmat[i]) );
  }
  cleanup_gather(mtag0);
  mtag0 = start_gather_from_temp( tempmat, sizeof(su3_matrix),
				  OPP_DIR(nu), EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);

  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      add_su3_matrix( (su3_matrix *)F_PT(s,staple),(su3_matrix *)gen_pt[0][i], 
		      (su3_matrix *)F_PT(s,staple) );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)F_PT(s,staple), coef, 
				 fat1 );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)gen_pt[0][i], coef, 
				 fat1 );
    }
  }

  free(tempmat);
  cleanup_gather(mtag0);
}
#endif  /* ASQ_OPTIMIZED_FATTENING   */


#ifdef  ASQ_OPTIMIZED_FORCE
/*   Covariant shift of the src fermion field in the direction dir  *
 *  by one unit. The result is stored in dest.                       */ 
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) {
  su3_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(su3_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_vec(&(s->link[dir]),(su3_vector *)(gen_pt[0][i]),
			 &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_vec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(su3_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(su3_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/*  Covariant shift of the src half wilson fermion field in the  *
 * direction dir   by one unit. The result is stored in dest.    */
void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) {
  half_wilson_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(half_wilson_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_hwvec(&(s->link[dir]),
			   (half_wilson_vector *)(gen_pt[0][i]), &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = 
	(half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_hwvec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(half_wilson_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(half_wilson_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/* Add in contribution to the force */
/* Put antihermitian traceless part into momentum */
void add_force_to_mom(su3_vector *back,su3_vector *forw,int dir,double coeff) {
  register site *s ;
  register int i ;  
  register double tmp_coeff ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff = -coeff ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD) 
      tmp_coeff = -coeff ;
    else
      tmp_coeff = coeff ;
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i]), &(forw[i]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff, &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/* Add in contribution to the force ( 3flavor case ) */
/* Put antihermitian traceless part into momentum */
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) {
  register site *s ;
  register int i ;  
  double tmp_coeff[2] ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff[0] = -coeff[0] ;
      coeff[1] = -coeff[1] ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD)
      {
	tmp_coeff[0] = -coeff[0] ;
	tmp_coeff[1] = -coeff[1] ;
      }
    else
      {
	tmp_coeff[0] = coeff[0] ;
	tmp_coeff[1] = coeff[1] ;
      }
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i].h[0]), &(forw[i].h[0]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[0], &tmat2 );
    su3_projector(&(back[i].h[1]), &(forw[i].h[1]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[1], &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/*  This routine is needed in order to add the force on the side link *
 * of the paths in the Asq and Asqtad actions. It gets as inputs the  *
 * direction mu of the side link and the direction nu of the Dslash   *
 * term we are dealing with. Then it takes also 4 fermion fields:     *
 * Path: the piece of the path with no hop in the nu or mu direction  *
 * Path_nu: the piece of the path with a hop in nu  but not in mu     *
 * Path_mu: is Path times the link mu                                 *
 * Path_numu: is Path_nu times the link mu                            */
void side_link_force(int mu, int nu, double coeff, 
		     su3_vector *Path   , su3_vector *Path_nu, 
		     su3_vector *Path_mu, su3_vector *Path_numu) {
  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_force_to_mom(Path, Path_numu, OPP_DIR(mu), -coeff );/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_nu, Path_mu, mu, -coeff) ; /* ? extra - */
      else
	add_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
 
/*  The 3 flavor version of side_link_force used *
 * to optimize fermion transports                */
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) {
  double m_coeff[2] ;

  m_coeff[0] = -coeff[0] ;
  m_coeff[1] = -coeff[1] ;

  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_3f_force_to_mom(Path,Path_numu,OPP_DIR(mu),m_coeff);/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_nu, Path_mu, mu, m_coeff) ; /* ? extra - */
      else
	add_3f_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
#endif  /* ASQ_OPTIMIZED_FORCE   */

/* LONG COMMENTS
   Here we have combined "xxx", (offset "x_off")  which is
(M_adjoint M)^{-1} phi, with Dslash times this vector, which goes in the
odd sites of xxx.  Recall that phi is defined only on even sites.  In
computing the fermion force, we are looking at

< X |  d/dt ( Dslash_eo Dslash_oe ) | X >
=
< X | d/dt Dslash_eo | T > + < T | d/dt Dslash_oe | X >
where T = Dslash X.

The subsequent manipulations to get the coefficent of H, the momentum
matrix, in the simulation time derivative above look the same for
the two terms, except for a minus sign at the end, if we simply stick
T, which lives on odd sites, into the odd sites of X

 Each path in the action contributes terms when any link of the path
is the link for which we are computing the force.  We get a minus sign
for odd numbered links in the path, since they connect sites of the
opposite parity from what it would be for an even numbered link.
Minus signs from "going around" plaquette - ie KS phases, are supposed
to be already encoded in the path coefficients.
Minus signs from paths that go backwards are supposed to be already
encoded in the path coefficients.

Here, for example, are comments reproduced from the force routine for
the one-link plus Naik plus single-staple-fat-link action:

 The three link force has three contributions, where the link that
was differentiated is the first, second, or third link in the 3-link
path, respectively.  Diagramatically, where "O" represents the momentum,
the solid line the link corresponding to the momentum, and the dashed
lines the other links:
 

	O______________ x ............ x ...............
+
	x..............O______________x.................
+
	x..............x..............O________________
Think of this as
	< xxx | O | UUUxxx >		(  xxx, UUUX_p3 )
+
	< xxx U | O | UUxxx >		( X_m1U , UUX_p2 )
+
	< xxx U U | O | Uxxx >		( X_m2UU , UX_p1 )
where "U" indicates parallel transport, "X_p3" is xxx displaced
by +3, etc.
Note the second contribution has a relative minus sign
because it effectively contributes to the <odd|even>, or M_adjoint,
part of the force when we work on an even site. i.e., for M on
an even site, this three link path begins on an odd site.

The staple force has six contributions from each plane containing the
link direction:
Call these diagrams A-F:


	x...........x		O____________x
		    .			     .
		    .			     .
		    .			     .
		    .			     .
		    .			     .
	O___________x		x............x
	   (A)			    (B)



	x	    x		O____________x
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	O___________x		x	     x
	   (C)			    (D)



	x...........x		O____________x
	.			.
	.			.
	.			.
	.			.
	.			.
	O___________x		x............x
	   (E)			    (F)

As with the Naik term, diagrams C and D have a relative minus
sign because they connect sites of the other parity.

Also note an overall minus sign in the staple terms relative to the
one link term because, with the KS phase factors included, the fat
link is  "U - w3 * UUU", or the straight link MINUS w3 times the staples.

Finally, diagrams B and E get one more minus sign because the link
we are differentiating is in the opposite direction from the staple
as a whole.  You can think of this as this "U" being a correction to
a "U_adjoint", but the derivative of U is iHU and the derivative
of U_adjoint is -iHU_adjoint.

*/
/* LONG COMMENT on sign conventions
In most of the program, the KS phases and antiperiodic boundary
conditions are absorbed into the link matrices.  This greatly simplfies
multiplying by the fermion matrix.  However, it requires care in
specifying the path coefficients.  Remember that each time you
encircle a plaquette, you pick up a net minus sign from the KS phases.
Thus, when you have more than one path to the same point, you generally
have a relative minus sign for each plaquette in a surface bounded by
this path and the basic path for that displacement.

Examples:
  Fat Link:
    Positive:	X-------X

    Negative     --------
	 	|	|
		|	|
		X	X

  Naik connection, smeared
    Positive:	X-------x-------x-------X

    Negative:	---------
		|	|
		|	|
		X	x-------x-------X

    Positive:	--------x--------
		|		|
		|		|
		X		x-------X

    Negative:	--------x-------x-------x
		|			|
		|			|
		X			X
*/



/* Comment on acceptable actions.
   We construct the backwards part of dslash by reversing all the
   paths in the forwards part.  So, for example, in the p4 action
   the forwards part includes +X+Y+Y

		X
		|
		|
		X
		|
		|
	X---->--X

  so we put -X-Y-Y in the backwards part.  But this isn't the adjoint
  of U_x(0)U_y(+x)U_y(+x+y).  Since much of the code assumes that the
  backwards hop is the adjoint of the forwards (for example, in
  preventing going to 8 flavors), the code only works for actions
  where this is true.  Roughly, this means that the fat link must
  be symmetric about reflection around its midpoint.  Equivalently,
  the paths in the backwards part of Dslash are translations of the
  paths in the forwards part.  In the case of the "P4" or knight's move
  action, this means that we have to have both paths
   +X+Y+Y and +Y+Y+X to the same point, with the same coefficients.
  Alternatively, we could just use the symmetric path +Y+X+Y.
*/

invivoquark_stuff_eo_fermion_force_946

Real time: 6.211676e+09 -- Predicted time: 6.656005e+09

Callcount: 10

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
1	### 1	1	6.685353e+08	6.135942e+08	8.2
2	### 2	1	6.616265e+08	6.132722e+08	7.3
3	### 3	1	6.439233e+08	6.129853e+08	4.8
4	### 4	1	6.769530e+08	6.130985e+08	9.4
5	### 5	1	7.131595e+08	6.131065e+08	14.0
6	### 6	1	6.431018e+08	6.133213e+08	4.6
7	### 7	1	6.635900e+08	6.133025e+08	7.5
8	### 8	1	6.866288e+08	6.130592e+08	10.7
9	### 9	1	6.545162e+08	6.130764e+08	6.3
10	### 10	1	6.439703e+08	6.130254e+08	4.8

                /****** quark_stuff.c  -- ******************/
/* MIMD version 6 */
/* quark action stuff for improved action
* D.T. 1/28/98, starting from gauge_stuff.c
* K.O. 3/99 Added optimized fattening for Asq actions
* D.T. 4/99 Combine force calculations for both mass quarks
* K.O. 4/99 Optimized force for Asq action
* S.G. 7/01, modified to use t_longlink and t_fatlink
* C.D. 10/02, consolidated quark_stuff.c and quark_stuff_tmp.c
*
* This code combines quark_stuff.c and quark_stuff_tmp.c
* with DSLASH_TMP_LINKS defined, puts links in field-major variables
* t_longlink and t_fatlink. Otherwise, puts them in the site structure.
*
* In this directory, assume all paths connect even to odd sites, etc.
* Tabulate "backwards" paths (e.g. "XDOWN" is backward path to "XUP")
* as separate parity transforms of the fundamental paths.  They will
* generally need a negative sign in Dslash.  See bottom for a long
* comment on sign conventions.
*/

/**#define FFTIME**/
/**#define LLTIME**/

#include "generic_ks_includes.h"	/* definitions files and prototypes */


#define NULL_FP -1 /* NULL field_offset to be used in the optimized version *
                    * of the load_fatlinks subroutine */

    /* Specify paths in orientation in which they appear in the
       forward part of the x component of dslash().  Rotations and
       reflections will be automatically included. Be careful
       about signs of coefficients.  See long comment at bottom. */
#include <quark_action.h>
    /* Include file specifies the basic paths */

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void printpath( int *path, int length );
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length );
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length );
#ifdef  ASQ_OPTIMIZED_FATTENING
void compute_gen_staple(field_offset staple, int mu, int nu,
                        field_offset link, double coef ) ;
#endif
#ifdef  ASQ_OPTIMIZED_FORCE
#ifndef FN
BOMB THE COMPILE
#endif
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) ;
void add_force_to_mom(su3_vector *back, su3_vector *forw, int dir, double coef);
void side_link_force(int mu, int nu, double coeff, su3_vector *Path,
		     su3_vector *Path_nu, su3_vector *Path_mu, 
		     su3_vector *Path_numu) ;

void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) ;
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) ;
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) ;
#endif

int path_num[MAX_BASIC_PATHS];	/* number of rotations/reflections for each 
					kind */
static double act_path_coeff[MAX_BASIC_PATHS]; /* actual path coefficient     *
                                               * it is equal to path_coeff   *
                                               * if not tadpole improvement  *
                                               * is specified                *
                                               * or path_coeff*u_0^(L-1) when*
                                               * tadpole improvement is      *
                                               * specified                   */

/* Array of structures, for each rotation and reflection of each kind of
	path.  */
struct {
    int dir[MAX_LENGTH];	/* directions in path */
    int length;		/* length of path */
    double coeff;	/* coefficient, including minus sign if backwards */
    double forwback;	/* +1 if in forward Dslash, -1 if in backward */
}   q_paths[MAX_NUM];
int num_q_paths;	/* number of paths in dslash */
int num_basic_paths;	/* number of paths before rotation/reflection */

int is_path_equal( int *path1, int* path2, int length );
int add_basic_path( int *vec, int length, double coeff );

/* Make table of paths in action */
void make_path_table() {

    int i,j;
#ifdef TADPOLE_IMPROVE
    int k;
#endif

    /* table of directions, 1 for each kind of path */
    /**int path_ind[MAX_BASIC_PATHS][MAX_LENGTH];**/
    /* table of coefficients in action, for each path */

    node0_printf("%s\n",quark_action_description);
    num_q_paths = 0;
    num_basic_paths = 0;


    /* add rots. and reflects to table, print out the path coefficients */
    node0_printf("path coefficients: npath  path_coeff  multiplicity\n");
    for(j=0;j<quark_action_npaths;j++) {
	double this_coeff;
	this_coeff = path_coeff[j];
#ifdef TADPOLE_IMPROVE
	for(k=1;k< path_length_in[j];k++)this_coeff /= u0;
#endif
	act_path_coeff[j] = this_coeff ;
	i = add_basic_path( path_ind[j], path_length_in[j],
	    this_coeff );
	node0_printf("                    %d      %e     %d\n",
	    j,this_coeff,i);
    }
}

/* add rotations and reflections of a path to the table.  Return
   multiplicity of paths added */
int add_basic_path( int *basic_vec, int length, double coeff ) {

    int perm[8],pp[8],ir[4];
    int j,path_num;
    int vec[MAX_LENGTH];
    int flag;

    path_num = 0;
    /* now fill the long table with all rotations and reflections
	of the fundamental path.  The path presented to us is for
        the positive x component of dslash, so if the x coordinate
        is reflected it will appear with a negative sign. */
      /* permutations */
      for(perm[0]=0;perm[0]<4;perm[0]++)
      for(perm[1]=0;perm[1]<4;perm[1]++)
      for(perm[2]=0;perm[2]<4;perm[2]++)
      for(perm[3]=0;perm[3]<4;perm[3]++){
	if(perm[0] != perm[1] && perm[0] != perm[2] 
	  && perm[0] != perm[3] && perm[1] != perm[2]
	  && perm[1] != perm[3] && perm[2] != perm[3] ) {
	  /* reflections*/
	  for(ir[0]=0;ir[0]<2;ir[0]++)
	  for(ir[1]=0;ir[1]<2;ir[1]++)
	  for(ir[2]=0;ir[2]<2;ir[2]++)
	  for(ir[3]=0;ir[3]<2;ir[3]++){
	    for(j=0;j<4;j++){
	      pp[j]=perm[j];

	      if(ir[j] == 1) pp[j]=OPP_DIR(pp[j]);
	      pp[OPP_DIR(j)]=OPP_DIR(pp[j]);
	    }
	    /* create new vector*/
	    for(j=0;j<length;j++) vec[j]=pp[basic_vec[j]];
	    for(j=length;j<MAX_LENGTH;j++) vec[j]=NODIR;

            flag=0;
	    /* check if it's a new set: */
	    for(j=0;j<num_q_paths;j++){
	      flag = is_path_equal( vec, q_paths[j].dir, MAX_LENGTH );
	      if(flag==1)break;
	    }
	    if(flag == 0 ){
	      if(num_q_paths>=MAX_NUM){
		node0_printf("OOPS: MAX_NUM too small\n");
		exit(0);
	      }
	      q_paths[num_q_paths].length=length;
	      for(j=0;j<MAX_LENGTH;j++) q_paths[num_q_paths].dir[j]=vec[j];
		/* remember to copy NODIR's, or comparison will get confused */
	      if(ir[0]==0){
		q_paths[num_q_paths].coeff =  coeff;
		q_paths[num_q_paths].forwback =  +1;
	      }
	      else{
		q_paths[num_q_paths].coeff = -coeff;
		q_paths[num_q_paths].forwback = -1;
	      }
	      num_q_paths++;
	      path_num++;
	      /**node0_printf("ADD PATH %d:  rx=%d ",num_q_paths-1,ir[0]);
		 printpath( vec, length );**/
	    }

	  } /* end reflection*/
        } /* end permutation if block */
      } /* end permutation */
    num_basic_paths++;
    return(path_num);
} /* add_basic_path */

/* parallel transport a vector to the current site along a path.
   For example, if the path is "XUP", bring in the vector from
   the +X direction to the current site. If KS phases are in lattice,
   this transport will automatically include them. 
   OK for src and dest to be the same.  OK for length=0.  */
/* NOT OPTIMIZED at the moment - do lots of extra copying.  Use temp.
   vectors rather than stuff in lattice.h */
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    su3_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    su3_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_dest = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_work = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(su3_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_vec( &(s->link[dir[j]]),
		    (su3_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_vec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(su3_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = *(su3_vector *)F_PT(s,src);
    }
  }
} /* path_transport */

/* Path transport a half_wilson_vector */
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    half_wilson_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    half_wilson_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (half_wilson_vector *)calloc(
      sites_on_node, sizeof(half_wilson_vector));
    tmp_dest = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));
    tmp_work = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(half_wilson_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_hwvec( &(s->link[dir[j]]),
		    (half_wilson_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_hwvec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(half_wilson_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) =
	  *(half_wilson_vector *)F_PT(s,src);
    }
  }
} /* path_transport_hwv */


#ifdef EO
/* Stupid dslash routine that follows all the paths.
  Should optimize by precomputing sums of paths to all the displacements
  Use dslash_fn for actions that involve only +X and +X+X+X couplings.
*/
/* D_slash routine - sets dest. on each site equal to sum of
   sources parallel transported to site, with minus sign for transport
   from negative directions.  */
void dslash_eo( field_offset src, field_offset dest, int parity ) {
   register int i;
   register site *s;
   register int ipath,otherparity;
   register double x; /* coefficient of path */

    switch(parity){
	case EVEN:	otherparity=ODD; break;
	case ODD:	otherparity=EVEN; break;
	case EVENANDODD:	otherparity=EVENANDODD; break;
    }

    /* Parallel transport by all the paths in the action.  
       Multiply by coefficient in table
    */
    FORSOMEPARITY(i,s,parity){ clearvec( (su3_vector *)F_PT(s,dest) ); }

    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	    path_transport( src, F_OFFSET(tempvec[0]), parity,
		q_paths[ipath].dir, q_paths[ipath].length );
	    x=q_paths[ipath].coeff;
	    FORSOMEPARITY(i,s,parity){
		scalar_mult_add_su3_vector(  (su3_vector *)F_PT(s,dest),
		    &(s->tempvec[0]), x, (su3_vector *)F_PT(s,dest) );
	    }
    }   /* ipath */
} /* dslash_eo */
#endif /* ifdef EO */

#ifdef FN
/* Sum over paths connecting to nearest neighbor point (fat link) and to third
   nearest neighbor (longlinks) */
/* Doug Toussaint 2/4/98 */
/* modified to use t_longlinks, S. Gottlieb 7/13/01 */
/* long link calculating routine */
/* path_product() follows the path starting at step 0, and
   leaves the answer at the end of the path.  We want the answer
   at the site where the path begins.  So we look for paths with
   the opposite displacement from the displacement of the point
   that we want to transport to this site, and take the adjoint
   of the matrix at the end. clear? */
/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_longlinks() {
  register int i;
  register site *s;
  int ipath,dir;
  int disp[4];
  register su3_matrix *long1;

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_longlinks needs phases in\n");
    terminate(0);
  }
  for (dir=XUP; dir<=TUP; dir++){ /* loop over longlink directions */
    /* set longlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      long1 = &(t_longlink[4*i+dir]);
#else
      long1 = &(s->longlink[dir]);
#endif
      clear_su3mat( long1 );
    }

    /* loop over paths, checking for ones with total displacement 3*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=3,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("ipath = %d, found a path:  ",ipath);
for(j=0;j<q_paths[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  long1 = &(t_longlink[4*i+dir]);
#else
	  long1 = &(s->longlink[dir]);
#endif
          scalar_mult_add_su3_matrix( long1,
	    &(s->staple), -q_paths[ipath].coeff, long1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */

  } /* loop over directions */

  valid_longlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Long):  %e\n",dtime);
#endif
}  /* load_longlinks() */

/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_fatlinks() {
  register int i;
  register site *s;
  int dir;
  register su3_matrix *fat1;
#ifdef ASQ_OPTIMIZED_FATTENING
  int  nu,rho,sig ;
  double one_link ; /*needed to fix the problem with the Lepage term */
#else
  int ipath;
  int disp[4];
#endif

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_fatlinks needs phases in\n");
    terminate(0);
  }

#ifndef  ASQ_OPTIMIZED_FATTENING   /* general case code */
  for (dir=XUP; dir<=TUP; dir++){ /* loop over fatlink directions */
    /* set fatlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+dir]);
#else
      fat1 = &(s->fatlink[dir]);
#endif
      clear_su3mat( fat1 );
    }
    
    /* loop over paths, checking for ones with total displacement 1*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=1,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("dir = %d, found a path:  ",dir);
for(j=0;j<q_paths.[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  fat1 = &(t_fatlink[4*i+dir]);
#else
	  fat1 = &(s->fatlink[dir]);
#endif
          scalar_mult_add_su3_matrix( fat1,
	    &(s->staple), -q_paths[ipath].coeff, fat1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */
  } /* loop over directions */
#else	/* ASQ_OPTIMIZED_FATTENING, for Asq and Asqtad actions */
/*  Optimized fattening code for the Asq and Asqtad actions.           *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
 
 /* to fix up the Lepage term, included by a trick below */
 one_link = (act_path_coeff[0] - 6.0*act_path_coeff[5]) ; 
 
 for (dir=XUP; dir<=TUP; dir++){
   FORALLSITES(i,s) /* Intialize fat links with c_1*U_\mu(x) */
     {
#ifdef DSLASH_TMP_LINKS
       fat1 = &(t_fatlink[4*i+dir]);
#else
       fat1 = &(s->fatlink[dir]);
#endif
       scalar_mult_su3_matrix(&(s->link[dir]), one_link ,
			      fat1 );
     }
   for(nu=XUP; nu<=TUP; nu++) if(nu!=dir)
     {
       compute_gen_staple(F_OFFSET(staple),dir,nu,F_OFFSET(link[dir]),
			  act_path_coeff[2]);
       /* The Lepage term */
       /* Note this also involves modifying c_1 (above) */
       compute_gen_staple(NULL_FP,dir,nu,F_OFFSET(staple),act_path_coeff[5]);
       for(rho=XUP; rho<=TUP; rho++) if((rho!=dir)&&(rho!=nu))
	 {
	   compute_gen_staple(F_OFFSET(tempmat1),dir,rho,F_OFFSET(staple),
			      act_path_coeff[3]);
	   for(sig=XUP; sig<=TUP; sig++)
	     if((sig!=dir)&&(sig!=nu)&&(sig!=rho))
	       {
		 compute_gen_staple(NULL_FP,dir,sig,
				    F_OFFSET(tempmat1),
				    act_path_coeff[4]);
	       } /* sig */
	 } /* rho */
     } /* nu */
 }/* dir */  
#endif

  valid_fatlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Fat):  %e\n",dtime);
#endif
}  /* load_fatlinks() */
#endif /* ifdef FN */


/* compare two paths, return 1 if equal, else zero */
int is_path_equal( int *path1, int* path2, int length ){
   register int i;
   for(i=0;i<length;i++)if(path1[i]!=path2[i])return(0);
   return(1);
}


/* update the  momenta with the fermion force */
/* Assumes that the conjugate gradient has been run, with the answer in
   x_off, and dslash(x_off,x_off,ODD) has been run. (fills in x_off_odd) */
/* SEE LONG COMMENTS AT END */

#ifndef ASQ_OPTIMIZED_FORCE
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon, coeff;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  half_wilson_vector *hw_tmp0,*hw_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  hw_tmp0 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  hw_tmp1 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  /* Use half_wilson_vectors to store x_off transported from ends of
     path.  0 component from forward end, 1 component from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    path_transport( x_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[1] for transport from starting end */
    FORALLSITES(i,s){
      hw_tmp0[i].h[0]=s->tempvec[0];
      hw_tmp0[i].h[1]=*(su3_vector *)F_PT(s,x_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff = ferm_epsilon*q_paths[ipath].coeff;
      if( (ilink%2)==1 )coeff = -coeff;

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "half_wilson_vector" to handle pair of vectors -
	0 component is x_off from forward end, 1 component from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_su3_mat_hwvec( &(s->link[lastdir]),
	      &(hw_tmp0[i]), &(hw_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( hw_tmp1, 2*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     hw_tmp0[i] = *(half_wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( hw_tmp0, 2*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_su3_mat_hwvec( &(s->link[OPP_DIR(lastdir)]),
                (half_wilson_vector *)(gen_pt[0][i]),
		&(hw_tmp1[i]) );
          }
	  tmp_pt = hw_tmp0; hw_tmp0 = hw_tmp1; hw_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( hw_tmp0 ); free( hw_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */


void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off,
  int nflav2, field_offset x2_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/14/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon1, ferm_epsilon2, coeff1, coeff2;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  wilson_vector *w_tmp0,*w_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon1 = 2.0*(nflav1/4.0)*eps;
  ferm_epsilon2 = 2.0*(nflav2/4.0)*eps;
  w_tmp0 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  w_tmp1 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  /* Use wilson_vectors to store x_off transported from ends of
     path.  0 and 1 components from forward end, 2 and 3 components
     from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    /**
    path_transport( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    path_transport( x2_off, F_OFFSET(tempvec[1]),
      EVENANDODD, q_paths[ipath].dir, length );
    **/
/** WARNING!! Assumes xxx1 and xxx2 contiguous **/
if( x2_off-x1_off != sizeof(su3_vector) ){node0_printf("BOTCH\n"); exit(0);}
    path_transport_hwv( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[2] for transport from starting end */
    FORALLSITES(i,s){
      w_tmp0[i].d[0]=s->tempvec[0];
      w_tmp0[i].d[1]=s->tempvec[1];
      w_tmp0[i].d[2]=*(su3_vector *)F_PT(s,x1_off);
      w_tmp0[i].d[3]=*(su3_vector *)F_PT(s,x2_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff1 = ferm_epsilon1*q_paths[ipath].coeff;
      coeff2 = ferm_epsilon2*q_paths[ipath].coeff;
      if( (ilink%2)==1 ){ coeff1 = -coeff1; coeff2 = -coeff2;}

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "wilson_vector" to handle pair of vectors -
	0 component is x_off1 from forward end, 1 component is x_off2
	from forward end,  2 and 3  components are x_off1 and x_off2 
	from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_mat_wilson_vec( &(s->link[lastdir]),
	      &(w_tmp0[i]), &(w_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( w_tmp1, 4*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     w_tmp0[i] = *(wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( w_tmp0, 4*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_mat_wilson_vec( &(s->link[OPP_DIR(lastdir)]),
                (wilson_vector *)(gen_pt[0][i]),
		&(w_tmp1[i]) );
          }
	  tmp_pt = w_tmp0; w_tmp0 = w_tmp1; w_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( w_tmp0 ); free( w_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */



#else /* ASQ_OPTIMIZED_FORCE, for Asq and Asqtad actions */
/* Optimized force code for the Asq and Asqtad actions                 *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
#define Pmu          tempvec[0] 
#define Pnumu        tempvec[1]
#define Prhonumu     tempvec[2]
#define P7           tempvec[3]
#define P7rho        tempvec[4]              
#define P7rhonu      tempvec[5]
#define P5           tempvec[6]
#define P3           tempvec[7]
#define P5nu         tempvec[3]
#define P3mu         tempvec[3]
#define Popmu        tempvec[4]
#define Pmumumu      tempvec[4]
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double ferm_epsilon, coeff;
  double OneLink, Lepage, Naik, FiveSt, ThreeSt, SevenSt ;
  su3_vector *tempvec[8] ;
  su3_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  
  /* Path coefficients times fermion epsilon */
  OneLink = act_path_coeff[0]*ferm_epsilon ; 
  Naik    = act_path_coeff[1]*ferm_epsilon ;
  ThreeSt = act_path_coeff[2]*ferm_epsilon ;
  FiveSt  = act_path_coeff[3]*ferm_epsilon ;
  SevenSt = act_path_coeff[4]*ferm_epsilon ;
  Lepage  = act_path_coeff[5]*ferm_epsilon ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    tempvec[mu] = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

  /*copy x_off to a temporary vector */
  temp_x = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
  FORALLSITES(i,s) temp_x[i] = *(su3_vector *)F_PT(s,x_off) ;

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_force_to_mom(P3, Pmu, sig, -ThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_force_to_mom(P7, Prhonumu, sig, -SevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_fermion(P7, P7rho, rho);
		  side_link_force(rho,sig,SevenSt, Pnumu, P7, Prhonumu, P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff = SevenSt/FiveSt ; 
		  FORALLSITES(i,s)
		    scalar_mult_add_su3_vector(&(P5[i]),&(P7rho[i]),coeff,
					       &(P5[i]));
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_fermion(P5,P5nu, nu);
	      side_link_force(nu,sig,-FiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff = FiveSt/ThreeSt ; 
	      FORALLSITES(i,s)
		scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_fermion(P5,P5nu, mu);
	  side_link_force(mu, sig, -Lepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff = Lepage/ThreeSt ; 
	  FORALLSITES(i,s)
	    scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), -Naik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(tempvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      

#define Pmu          hwvec[0] 
#define Pnumu        hwvec[1]
#define Prhonumu     hwvec[2]
#define P7           hwvec[3]
#define P7rho        hwvec[4]              
#define P7rhonu      hwvec[5]
#define P5           hwvec[6]
#define P3           hwvec[7]
#define P5nu         hwvec[3]
#define P3mu         hwvec[3]
#define Popmu        hwvec[4]
#define Pmumumu      hwvec[4]
void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off, 
			  int nflav2, field_offset x2_off){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/15/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double coeff[2],ferm_epsilon ;
  double OneLink[2], Lepage[2], Naik[2], FiveSt[2], ThreeSt[2], SevenSt[2] ;
  double mNaik[2], mLepage[2], mFiveSt[2], mThreeSt[2], mSevenSt[2] ;
  half_wilson_vector *hwvec[8] ;
  half_wilson_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  
  /* Path coefficients times fermion epsilon */
  ferm_epsilon = 2.0*(nflav1/4.0)*eps;
  OneLink[0] = act_path_coeff[0]*ferm_epsilon ;
  Naik[0]    = act_path_coeff[1]*ferm_epsilon ; mNaik[0]    = -Naik[0] ;
  ThreeSt[0] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[0] = -ThreeSt[0] ;
  FiveSt[0]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[0]  = -FiveSt[0] ;
  SevenSt[0] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[0] = -SevenSt[0] ;
  Lepage[0]  = act_path_coeff[5]*ferm_epsilon ; mLepage[0]  = -Lepage[0] ;

  ferm_epsilon = 2.0*(nflav2/4.0)*eps;
  OneLink[1] = act_path_coeff[0]*ferm_epsilon ;
  Naik[1]    = act_path_coeff[1]*ferm_epsilon ; mNaik[1]    = -Naik[1] ;
  ThreeSt[1] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[1] = -ThreeSt[1] ;
  FiveSt[1]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[1]  = -FiveSt[1] ;
  SevenSt[1] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[1] = -SevenSt[1] ;
  Lepage[1]  = act_path_coeff[5]*ferm_epsilon ; mLepage[1]  = -Lepage[1] ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    hwvec[mu]= 
      (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));

  /*copy x_off to a temporary vector */
  temp_x= 
    (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
  FORALLSITES(i,s)
    {
      temp_x[i].h[0] = *(su3_vector *)F_PT(s,x1_off) ;
      temp_x[i].h[1] = *(su3_vector *)F_PT(s,x2_off) ;
    }

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_hw_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_3f_force_to_mom(P3, Pmu, sig, mThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_hw_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_3f_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_hw_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_hw_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_3f_force_to_mom(P7, Prhonumu, sig, mSevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_hw_fermion(P7, P7rho, rho);
		  side_link_3f_force(rho,sig,SevenSt,Pnumu,P7,Prhonumu,P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff[0] = SevenSt[0]/FiveSt[0] ; 
		  coeff[1] = SevenSt[1]/FiveSt[1] ; 
		  FORALLSITES(i,s)
		    {
		      scalar_mult_add_su3_vector(&(P5[i].h[0]),
						 &(P7rho[i].h[0]),coeff[0],
						 &(P5[i].h[0]));
		      scalar_mult_add_su3_vector(&(P5[i].h[1]),
						 &(P7rho[i].h[1]),coeff[1],
						 &(P5[i].h[1]));
		    }
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_hw_fermion(P5,P5nu, nu);
	      side_link_3f_force(nu,sig,mFiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff[0] = FiveSt[0]/ThreeSt[0] ; 
	      coeff[1] = FiveSt[1]/ThreeSt[1] ; 
	      FORALLSITES(i,s)
		{
		  scalar_mult_add_su3_vector(&(P3[i].h[0]),
					     &(P5nu[i].h[0]), coeff[0],
					     &(P3[i].h[0]));
		  scalar_mult_add_su3_vector(&(P3[i].h[1]),
					     &(P5nu[i].h[1]), coeff[1],
					     &(P3[i].h[1]));
		}
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_3f_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_hw_fermion(P5,P5nu, mu);
	  side_link_3f_force(mu, sig, mLepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff[0] = Lepage[0]/ThreeSt[0] ; 
	  coeff[1] = Lepage[1]/ThreeSt[1] ; 
	  FORALLSITES(i,s)
	    {
	      scalar_mult_add_su3_vector(&(P3[i].h[0]),
					 &(P5nu[i].h[0]),coeff[0],
					 &(P3[i].h[0]));
	      scalar_mult_add_su3_vector(&(P3[i].h[1]),
					 &(P5nu[i].h[1]),coeff[1],
					 &(P3[i].h[1]));
	    }

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_hw_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_3f_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_3f_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_3f_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), mNaik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_hw_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_3f_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_3f_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(hwvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      
#endif /* ASQ_OPTIMIZED_FORCE */


#ifdef  ASQ_OPTIMIZED_FATTENING   /* Asqtad action only, "_fn" executables */
#ifndef FN
BOMB THE COMPILE
#endif
void compute_gen_staple(field_offset staple, int mu, int nu, 
			field_offset link, double coef) {
  su3_matrix tmat1,tmat2;
  msg_tag *mtag0,*mtag1;
  su3_matrix *tempmat ;
  register site *s ;
  register int i ;
  register su3_matrix *fat1;

  /* Computes the staple :
                   mu
                +-------+
          nu	|	|
		|	|
		X	X
    Where the mu link can be any su3_matrix. The result is saved in staple.
    if staple==NULL_FP then the result is not saved.
    It also adds the computed staple to the fatlink[mu] with weight coef.
  */

  /* Upper staple */
  mtag0 = start_gather( link, sizeof(su3_matrix), nu, EVENANDODD, gen_pt[0] );
  mtag1 = start_gather( F_OFFSET(link[nu]), sizeof(su3_matrix), mu, 
			EVENANDODD, gen_pt[1] );
  wait_gather(mtag0);
  wait_gather(mtag1);
  
  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, (su3_matrix *)F_PT(s,staple) );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, &tmat2 );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix(fat1, &tmat2, coef,
				 fat1) ;
    }
  }
  cleanup_gather(mtag0);
  cleanup_gather(mtag1);

  /* lower staple */
  tempmat = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
  mtag0 = start_gather( F_OFFSET(link[nu]),
			sizeof(su3_matrix), mu, EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);
  FORALLSITES(i,s){
    mult_su3_an( &(s->link[nu]),(su3_matrix *)F_PT(s,link), &tmat1 );
    mult_su3_nn( &(tmat1),(su3_matrix *)gen_pt[0][i], &(tempmat[i]) );
  }
  cleanup_gather(mtag0);
  mtag0 = start_gather_from_temp( tempmat, sizeof(su3_matrix),
				  OPP_DIR(nu), EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);

  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      add_su3_matrix( (su3_matrix *)F_PT(s,staple),(su3_matrix *)gen_pt[0][i], 
		      (su3_matrix *)F_PT(s,staple) );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)F_PT(s,staple), coef, 
				 fat1 );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)gen_pt[0][i], coef, 
				 fat1 );
    }
  }

  free(tempmat);
  cleanup_gather(mtag0);
}
#endif  /* ASQ_OPTIMIZED_FATTENING   */


#ifdef  ASQ_OPTIMIZED_FORCE
/*   Covariant shift of the src fermion field in the direction dir  *
 *  by one unit. The result is stored in dest.                       */ 
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) {
  su3_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(su3_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_vec(&(s->link[dir]),(su3_vector *)(gen_pt[0][i]),
			 &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_vec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(su3_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(su3_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/*  Covariant shift of the src half wilson fermion field in the  *
 * direction dir   by one unit. The result is stored in dest.    */
void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) {
  half_wilson_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(half_wilson_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_hwvec(&(s->link[dir]),
			   (half_wilson_vector *)(gen_pt[0][i]), &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = 
	(half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_hwvec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(half_wilson_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(half_wilson_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/* Add in contribution to the force */
/* Put antihermitian traceless part into momentum */
void add_force_to_mom(su3_vector *back,su3_vector *forw,int dir,double coeff) {
  register site *s ;
  register int i ;  
  register double tmp_coeff ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff = -coeff ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD) 
      tmp_coeff = -coeff ;
    else
      tmp_coeff = coeff ;
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i]), &(forw[i]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff, &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/* Add in contribution to the force ( 3flavor case ) */
/* Put antihermitian traceless part into momentum */
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) {
  register site *s ;
  register int i ;  
  double tmp_coeff[2] ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff[0] = -coeff[0] ;
      coeff[1] = -coeff[1] ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD)
      {
	tmp_coeff[0] = -coeff[0] ;
	tmp_coeff[1] = -coeff[1] ;
      }
    else
      {
	tmp_coeff[0] = coeff[0] ;
	tmp_coeff[1] = coeff[1] ;
      }
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i].h[0]), &(forw[i].h[0]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[0], &tmat2 );
    su3_projector(&(back[i].h[1]), &(forw[i].h[1]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[1], &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/*  This routine is needed in order to add the force on the side link *
 * of the paths in the Asq and Asqtad actions. It gets as inputs the  *
 * direction mu of the side link and the direction nu of the Dslash   *
 * term we are dealing with. Then it takes also 4 fermion fields:     *
 * Path: the piece of the path with no hop in the nu or mu direction  *
 * Path_nu: the piece of the path with a hop in nu  but not in mu     *
 * Path_mu: is Path times the link mu                                 *
 * Path_numu: is Path_nu times the link mu                            */
void side_link_force(int mu, int nu, double coeff, 
		     su3_vector *Path   , su3_vector *Path_nu, 
		     su3_vector *Path_mu, su3_vector *Path_numu) {
  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_force_to_mom(Path, Path_numu, OPP_DIR(mu), -coeff );/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_nu, Path_mu, mu, -coeff) ; /* ? extra - */
      else
	add_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
 
/*  The 3 flavor version of side_link_force used *
 * to optimize fermion transports                */
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) {
  double m_coeff[2] ;

  m_coeff[0] = -coeff[0] ;
  m_coeff[1] = -coeff[1] ;

  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_3f_force_to_mom(Path,Path_numu,OPP_DIR(mu),m_coeff);/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_nu, Path_mu, mu, m_coeff) ; /* ? extra - */
      else
	add_3f_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
#endif  /* ASQ_OPTIMIZED_FORCE   */

/* LONG COMMENTS
   Here we have combined "xxx", (offset "x_off")  which is
(M_adjoint M)^{-1} phi, with Dslash times this vector, which goes in the
odd sites of xxx.  Recall that phi is defined only on even sites.  In
computing the fermion force, we are looking at

< X |  d/dt ( Dslash_eo Dslash_oe ) | X >
=
< X | d/dt Dslash_eo | T > + < T | d/dt Dslash_oe | X >
where T = Dslash X.

The subsequent manipulations to get the coefficent of H, the momentum
matrix, in the simulation time derivative above look the same for
the two terms, except for a minus sign at the end, if we simply stick
T, which lives on odd sites, into the odd sites of X

 Each path in the action contributes terms when any link of the path
is the link for which we are computing the force.  We get a minus sign
for odd numbered links in the path, since they connect sites of the
opposite parity from what it would be for an even numbered link.
Minus signs from "going around" plaquette - ie KS phases, are supposed
to be already encoded in the path coefficients.
Minus signs from paths that go backwards are supposed to be already
encoded in the path coefficients.

Here, for example, are comments reproduced from the force routine for
the one-link plus Naik plus single-staple-fat-link action:

 The three link force has three contributions, where the link that
was differentiated is the first, second, or third link in the 3-link
path, respectively.  Diagramatically, where "O" represents the momentum,
the solid line the link corresponding to the momentum, and the dashed
lines the other links:
 

	O______________ x ............ x ...............
+
	x..............O______________x.................
+
	x..............x..............O________________
Think of this as
	< xxx | O | UUUxxx >		(  xxx, UUUX_p3 )
+
	< xxx U | O | UUxxx >		( X_m1U , UUX_p2 )
+
	< xxx U U | O | Uxxx >		( X_m2UU , UX_p1 )
where "U" indicates parallel transport, "X_p3" is xxx displaced
by +3, etc.
Note the second contribution has a relative minus sign
because it effectively contributes to the <odd|even>, or M_adjoint,
part of the force when we work on an even site. i.e., for M on
an even site, this three link path begins on an odd site.

The staple force has six contributions from each plane containing the
link direction:
Call these diagrams A-F:


	x...........x		O____________x
		    .			     .
		    .			     .
		    .			     .
		    .			     .
		    .			     .
	O___________x		x............x
	   (A)			    (B)



	x	    x		O____________x
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	O___________x		x	     x
	   (C)			    (D)



	x...........x		O____________x
	.			.
	.			.
	.			.
	.			.
	.			.
	O___________x		x............x
	   (E)			    (F)

As with the Naik term, diagrams C and D have a relative minus
sign because they connect sites of the other parity.

Also note an overall minus sign in the staple terms relative to the
one link term because, with the KS phase factors included, the fat
link is  "U - w3 * UUU", or the straight link MINUS w3 times the staples.

Finally, diagrams B and E get one more minus sign because the link
we are differentiating is in the opposite direction from the staple
as a whole.  You can think of this as this "U" being a correction to
a "U_adjoint", but the derivative of U is iHU and the derivative
of U_adjoint is -iHU_adjoint.

*/
/* LONG COMMENT on sign conventions
In most of the program, the KS phases and antiperiodic boundary
conditions are absorbed into the link matrices.  This greatly simplfies
multiplying by the fermion matrix.  However, it requires care in
specifying the path coefficients.  Remember that each time you
encircle a plaquette, you pick up a net minus sign from the KS phases.
Thus, when you have more than one path to the same point, you generally
have a relative minus sign for each plaquette in a surface bounded by
this path and the basic path for that displacement.

Examples:
  Fat Link:
    Positive:	X-------X

    Negative     --------
	 	|	|
		|	|
		X	X

  Naik connection, smeared
    Positive:	X-------x-------x-------X

    Negative:	---------
		|	|
		|	|
		X	x-------x-------X

    Positive:	--------x--------
		|		|
		|		|
		X		x-------X

    Negative:	--------x-------x-------x
		|			|
		|			|
		X			X
*/



/* Comment on acceptable actions.
   We construct the backwards part of dslash by reversing all the
   paths in the forwards part.  So, for example, in the p4 action
   the forwards part includes +X+Y+Y

		X
		|
		|
		X
		|
		|
	X---->--X

  so we put -X-Y-Y in the backwards part.  But this isn't the adjoint
  of U_x(0)U_y(+x)U_y(+x+y).  Since much of the code assumes that the
  backwards hop is the adjoint of the forwards (for example, in
  preventing going to 8 flavors), the code only works for actions
  where this is true.  Roughly, this means that the fat link must
  be symmetric about reflection around its midpoint.  Equivalently,
  the paths in the backwards part of Dslash are translations of the
  paths in the forwards part.  In the case of the "P4" or knight's move
  action, this means that we have to have both paths
   +X+Y+Y and +Y+Y+X to the same point, with the same coefficients.
  Alternatively, we could just use the symmetric path +Y+X+Y.
*/

invivoquark_stuff_compute_gen_staple_1368

Real time: 4.745753e+08 -- Predicted time: 4.991175e+08

Callcount: 792

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
436	### 1	474.038814895352	6.213180e+05	5.886400e+05	5.2
73	### 2	43.0380669370757	5.663150e+05	5.532360e+05	2.3
310	### 3	276.129391947683	6.526490e+05	6.055400e+05	7.2

                /*************** update_u.c ************************************/
/* MIMD version 6 */

/* update the link matrices					*
*  								*
*  Go to sixth order in the exponential of the momentum		*
*  matrices, since unitarity seems important.			*
*  Evaluation is done as:					*
*	exp(H) * U = ( 1 + H + H^2/2 + H^3/3 ...)*U		*
*	= U + H*( U + (H/2)*( U + (H/3)*( ... )))		*
*								*
*/

#include "ks_imp_includes.h"	/* definitions files and prototypes */

void update_u( double eps ){

register int i,dir;
register site *s;
su3_matrix *link,temp1,temp2,htemp;
register double t2,t3,t4,t5,t6;
/**TEMP**
double gf_x,gf_av,gf_max;
int gf_i,gf_j;
**END TEMP **/

/**double dtime,dtime2,dclock();**/
/**dtime = -dclock();**/

/* Temporary by-hand optimization until pgcc compiler bug is fixed */
t2 = eps/2.0;
t3 = eps/3.0;
t4 = eps/4.0;
t5 = eps/5.0;
t6 = eps/6.0;

/** TEMP **
gf_av=gf_max=0.0;
**END TEMP**/
    FORALLSITES(i,s){
	for(dir=XUP; dir <=TUP; dir++){
	    uncompress_anti_hermitian( &(s->mom[dir]) , &htemp );
	    link = &(s->link[dir]);
	    mult_su3_nn(&htemp,link,&temp1);
            /**scalar_mult_add_su3_matrix(link,&temp1,eps/6.0,&temp2);**/
scalar_mult_add_su3_matrix(link,&temp1,t6,&temp2);
	    mult_su3_nn(&htemp,&temp2,&temp1);
            /**scalar_mult_add_su3_matrix(link,&temp1,eps/5.0,&temp2);**/
scalar_mult_add_su3_matrix(link,&temp1,t5,&temp2);
	    mult_su3_nn(&htemp,&temp2,&temp1);
            /**scalar_mult_add_su3_matrix(link,&temp1,eps/4.0,&temp2);**/
scalar_mult_add_su3_matrix(link,&temp1,t4,&temp2);
	    mult_su3_nn(&htemp,&temp2,&temp1);
	    /**scalar_mult_add_su3_matrix(link,&temp1,eps/3.0,&temp2);**/
scalar_mult_add_su3_matrix(link,&temp1,t3,&temp2);
	    mult_su3_nn(&htemp,&temp2,&temp1);
	    /**scalar_mult_add_su3_matrix(link,&temp1,eps/2.0,&temp2);**/
scalar_mult_add_su3_matrix(link,&temp1,t2,&temp2);
	    mult_su3_nn(&htemp,&temp2,&temp1);
	    scalar_mult_add_su3_matrix(link,&temp1,eps    ,&temp2); 
	    su3mat_copy(&temp2,link);
	}
    }
#ifdef FN
    valid_longlinks=0;
    valid_fatlinks=0;
#endif
/**dtime += dclock();
node0_printf("LINK_UPDATE: time = %e  mflops = %e\n",
dtime, (double)(5616.0*volume/(1.0e6*dtime*numnodes())) );**/
} /* update_u */

invivoupdate_u_update_u_40

Real time: 2.115407e+08 -- Predicted time: 2.083390e+08

Callcount: 30

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
19	### 1	19.9996801940081	6.941721e+06	7.079292e+06	1.9
14	### 2	10.0016893795609	6.949504e+06	7.133980e+06	2.5

                /********************** path_product.c ***************************/
/* MIMD version 6 */
/* Compute product of links along a specified path    */

#include "generic_includes.h"	/* definitions files and prototypes */
#define FETCH_UP 1

/* LOOPEND is required now -CD */
#undef FORALLSITES
#define FORALLSITES(i,s) \
{ register int loopend; loopend=sites_on_node; \
for( i=0,  s=lattice ; i<loopend; i++,s++ )
#define END_LOOP }

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void path_product( int *dir, int length) {
    register int i;
    register site *s;
    msg_tag *mtag0;
    su3_matrix *tempmat2t, *tempmat3t;
    int j;
    /* a forward step leaves the answer in gen_pt[0], which points into
	link, tempmat1 or tempmat2, and backwards step in tempmat1 or tempmat2,
	After a forwards step, need to wait and clean a gather.
	  STEP	leaves answer in
	  even # forward	gen_pt[0]->tempmat1  (gen_pt[0]->link for step 0
	  even # backward	tempmat1
	  odd  # forward	gen_pt[0]->tempmat2
	  odd  # backward	tempmat2
	*/

    /* allocate temporary space */
    tempmat3t = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
    tempmat2t = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));

    /* j=0 */
    if( GOES_FORWARDS(dir[0]) )  {
	mtag0 = start_gather( F_OFFSET(link[dir[0]]), sizeof(su3_matrix),
	    OPP_DIR(dir[0]), EVENANDODD, gen_pt[0] );
    }
    else{  /* if GOES_BACKWARDS(dir[0]) */
	FORALLSITES(i,s){
	    su3_adjoint(&(s->link[OPP_DIR(dir[0])]),&(s->tempmat1) );
	} END_LOOP
    }

    for(j=1;j<length;j++) {
	if( j%2==1 ){
	    if( GOES_FORWARDS(dir[j]) ) {
	      if( GOES_FORWARDS(dir[j-1]) ){
	        wait_gather(mtag0);
	        FORALLSITES(i,s){
		  mult_su3_nn( (su3_matrix *)(gen_pt[0][i]), &(s->link[dir[j]]),
		    &(tempmat2t[i]) );
	        } END_LOOP
	        cleanup_gather(mtag0);
	      }
	      else{ /* last link was backwards */
	        FORALLSITES(i,s){
		  mult_su3_nn( &(s->tempmat1),&(s->link[dir[j]]),
		    &(tempmat2t[i]) );
	        } END_LOOP
	      }
	      mtag0 = start_gather_from_temp( tempmat2t, sizeof(su3_matrix),
		OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	    }  /* for GOES_FORWARDS */

	    else{ /* GOES_BACKWARDS(dir[j]), which is an odd numbered step */
	      if( GOES_FORWARDS(dir[j-1]) ){
	        wait_gather(mtag0);
	        FORALLSITES(i,s){
	          su3mat_copy((su3_matrix *)(gen_pt[0][i]),&(tempmat3t[i]) );
	        } END_LOOP
	        cleanup_gather(mtag0);
	        mtag0 = start_gather_from_temp( tempmat3t, sizeof(su3_matrix),
		  OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	      }
	      else{ /*last step was backwards */
	        mtag0 = start_gather( F_OFFSET(tempmat1), sizeof(su3_matrix),
		  OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	      }
	      wait_gather(mtag0);
	      FORALLSITES(i,s){
		  mult_su3_na((su3_matrix *)(gen_pt[0][i]),
		    &(s->link[OPP_DIR(dir[j])]), &(tempmat2t[i]) );
	      } END_LOOP
	      cleanup_gather(mtag0);
	    } /* end for GOES_BACKWARDS */
	} /* end for j=odd */

	else{	/* j=even */
	  if( GOES_FORWARDS(dir[j]) ) {
	    if( GOES_FORWARDS(dir[j-1]) ){
	      wait_gather(mtag0);
	      FORALLSITES(i,s){
		mult_su3_nn( (su3_matrix *)(gen_pt[0][i]), &(s->link[dir[j]]),
		    &(s->tempmat1) );
	      } END_LOOP
	      cleanup_gather(mtag0);
	    }
	    else{ /* last link goes backwards */
	      FORALLSITES(i,s){
		mult_su3_nn( &(tempmat2t[i]),&(s->link[dir[j]]),
		    &(s->tempmat1) );
	      } END_LOOP
	    }
	    mtag0 = start_gather( F_OFFSET(tempmat1), sizeof(su3_matrix),
		OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	  }  /* for GOES_FORWARDS */

	  else{ /* GOES_BACKWARDS(dir[j]) */
	    if( GOES_FORWARDS(dir[j-1]) ){
	      wait_gather(mtag0);
	      FORALLSITES(i,s){
	        su3mat_copy((su3_matrix *)(gen_pt[0][i]),&(tempmat3t[i]) ); 
	      } END_LOOP
	      cleanup_gather(mtag0);
	      mtag0 = start_gather_from_temp( tempmat3t, sizeof(su3_matrix),
		OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	    }
	    else{ /* last step was backwards */
	      mtag0 = start_gather_from_temp( tempmat2t, sizeof(su3_matrix),
		OPP_DIR(dir[j]), EVENANDODD, gen_pt[0] );
	    }
	    wait_gather(mtag0);
	    FORALLSITES(i,s){
	      mult_su3_na((su3_matrix *)(gen_pt[0][i]),
		    &(s->link[OPP_DIR(dir[j])]), &(s->tempmat1) );
	    } END_LOOP
	    cleanup_gather(mtag0);
	  } /* for GOES_BACKWARDS */
	} /* for j=even */

    }  /* j=link in loop */

    /* Want to end in tempmat1 */
    if( length%2==0 ){  /* last step was odd */
      if( GOES_FORWARDS(dir[length-1]) ){
	wait_gather(mtag0);
	  FORALLSITES(i,s){
	    su3mat_copy((su3_matrix *)(gen_pt[0][i]),&(s->tempmat1) ); 
	} END_LOOP
	cleanup_gather(mtag0);
      }
      else{
	FORALLSITES(i,s){
	  su3mat_copy(&(tempmat2t[i]),&(s->tempmat1) );
	} END_LOOP
      }
    }
    else{ /* odd length path */
      if( GOES_FORWARDS(dir[length-1]) ){
	wait_gather(mtag0);
	FORALLSITES(i,s){
	  su3mat_copy( (su3_matrix *)(gen_pt[0][i]), &(tempmat3t[i]) );
	} END_LOOP
	cleanup_gather(mtag0);
	FORALLSITES(i,s){
	  su3mat_copy( &(tempmat3t[i]), &(s->tempmat1) );
	} END_LOOP
      }
      else{
      }
    }
    free(tempmat3t);
    free(tempmat2t);
} /* path */

#ifdef N_SUBL32
/* code from symanzik_sl32/dsdu_qhb.c ****************************/
/* U.M. Heller August 1997 */

/* This is a modification of "path_product" from gauge_stuff.c
   which works only on one sublattice. */
void path_prod_subl(int *dir, int length, int subl)
{
register int i;
register site *s;
msg_tag *mtag0;
su3_matrix *tempmat2t, *tempmat3t;
int j, nsubl;

    /* A forward step leaves the answer in gen_pt[0], which points into
       link, tempmat1 or tempmat2, and backwards step in tempmat1 or tempmat2.
       After a forwards step, need to wait and clean a gather.
	STEP			leaves answer in
	even # forward		gen_pt[0]->tempmat1 (gen_pt[0]->link for step 0)
	even # backward		tempmat1
	odd  # forward		gen_pt[0]->tempmat2
	odd  # backward		tempmat2
    */

    /* allocate temporary space */
    tempmat3t = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
    tempmat2t = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));

    /* j=0 */
    if( GOES_FORWARDS(dir[0]) ) {
	nsubl = neighsubl[subl][dir[0]];
	mtag0 = start_gather( F_OFFSET(link[dir[0]]), sizeof(su3_matrix),
		OPP_DIR(dir[0]), nsubl, gen_pt[0] );
    }
    else{  /* if GOES_BACKWARDS(dir[0]) */
	nsubl = neighsubl[subl][dir[0]];
	FORSOMESUBLATTICE(i,s,nsubl){
	    su3_adjoint(&(s->link[OPP_DIR(dir[0])]), &(s->tempmat1) );
	}
    }

    for(j=1;j<length;j++) {
	if( j%2==1 ){
	    if( GOES_FORWARDS(dir[j]) ) {
	      if( GOES_FORWARDS(dir[j-1]) ){
		wait_gather(mtag0);
		FORSOMESUBLATTICE(i,s,nsubl){
		  mult_su3_nn( (su3_matrix *)(gen_pt[0][i]),
		    &(s->link[dir[j]]), &(tempmat2t[i]) );
		}
		cleanup_gather(mtag0);
	      }
	      else{ /* last link was backwards */
		FORSOMESUBLATTICE(i,s,nsubl){
		  mult_su3_nn( &(s->tempmat1),
		    &(s->link[dir[j]]), &(tempmat2t[i]) );
		}
	      }
	      nsubl = neighsubl[nsubl][dir[j]];
	      mtag0 = start_gather_from_temp( tempmat2t, sizeof(su3_matrix),
		      OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	    }  /* for GOES_FORWARDS */

	    else{ /* GOES_BACKWARDS(dir[j]), which is an odd numbered step */
	      if( GOES_FORWARDS(dir[j-1]) ){
		wait_gather(mtag0);
		FORSOMESUBLATTICE(i,s,nsubl){
		  su3mat_copy((su3_matrix *)(gen_pt[0][i]), &(tempmat3t[i]) );
		}
		cleanup_gather(mtag0);
		nsubl = neighsubl[nsubl][dir[j]];
		mtag0 = start_gather_from_temp( tempmat3t, sizeof(su3_matrix),
			OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	      }
	      else{ /*last step was backwards */
		nsubl = neighsubl[nsubl][dir[j]];
		mtag0 = start_gather( F_OFFSET(tempmat1), sizeof(su3_matrix),
			OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	      }
	      wait_gather(mtag0);
	      FORSOMESUBLATTICE(i,s,nsubl){
		mult_su3_na((su3_matrix *)(gen_pt[0][i]),
		  &(s->link[OPP_DIR(dir[j])]), &(tempmat2t[i]) );
	      }
	      cleanup_gather(mtag0);
	    } /* end for GOES_BACKWARDS */
	} /* end for j=odd */

	else{   /* j=even */
	    if( GOES_FORWARDS(dir[j]) ) {
	      if( GOES_FORWARDS(dir[j-1]) ){
		wait_gather(mtag0);
		FORSOMESUBLATTICE(i,s,nsubl){
		  mult_su3_nn( (su3_matrix *)(gen_pt[0][i]),
		    &(s->link[dir[j]]), &(s->tempmat1) );
		}
		cleanup_gather(mtag0);
	      }
	      else{ /* last link was backwards */
		FORSOMESUBLATTICE(i,s,nsubl){
		  mult_su3_nn( &(tempmat2t[i]),
		    &(s->link[dir[j]]), &(s->tempmat1) );
		}
	      }
	      nsubl = neighsubl[nsubl][dir[j]];
	      mtag0 = start_gather( F_OFFSET(tempmat1), sizeof(su3_matrix),
		      OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	    }  /* for GOES_FORWARDS */

	    else{ /* GOES_BACKWARDS(dir[j]), which is an even numbered step */
	      if( GOES_FORWARDS(dir[j-1]) ){
		wait_gather(mtag0);
		FORSOMESUBLATTICE(i,s,nsubl){
		  su3mat_copy((su3_matrix *)(gen_pt[0][i]), &(tempmat3t[i]) );
		}
		cleanup_gather(mtag0);
		nsubl = neighsubl[nsubl][dir[j]];
		mtag0 = start_gather_from_temp( tempmat3t, sizeof(su3_matrix),
			OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	      }
	      else{ /*last step was backwards */
		nsubl = neighsubl[nsubl][dir[j]];
		mtag0 = start_gather_from_temp( tempmat2t, sizeof(su3_matrix),
			OPP_DIR(dir[j]), nsubl, gen_pt[0] );
	      }
	      wait_gather(mtag0);
	      FORSOMESUBLATTICE(i,s,nsubl){
		mult_su3_na((su3_matrix *)(gen_pt[0][i]),
		  &(s->link[OPP_DIR(dir[j])]), &(s->tempmat1) );
	      }
	      cleanup_gather(mtag0);
	    } /* end for GOES_BACKWARDS */
	} /* end for j=even */

    }  /* j=link in loop */

    /* Want to end in tempmat1 */
    if( length%2==0 ){  /* last step was odd */
	if( GOES_FORWARDS(dir[length-1]) ){
	    wait_gather(mtag0);
	    FORSOMESUBLATTICE(i,s,nsubl){
	      su3mat_copy((su3_matrix *)(gen_pt[0][i]), &(s->tempmat1) );
	    }
	    cleanup_gather(mtag0);
	}
	else{
	    FORSOMESUBLATTICE(i,s,nsubl){
	      su3mat_copy(&(tempmat2t[i]), &(s->tempmat1) );
	    }
	}
    }
    else{ /* odd length path: last step was even */
	if( GOES_FORWARDS(dir[length-1]) ){
	    wait_gather(mtag0);
	    FORSOMESUBLATTICE(i,s,nsubl){
	      su3mat_copy((su3_matrix *)(gen_pt[0][i]), &(tempmat3t[i]) );
	    }
	    cleanup_gather(mtag0);
	    FORSOMESUBLATTICE(i,s,nsubl){
	      su3mat_copy(&(tempmat3t[i]), &(s->tempmat1) );
	    }
	}
    }

    free(tempmat3t);
    free(tempmat2t);

} /* path_prod_subl */

#endif /* N_SUBL32 */

invivopath_product_path_product_49

Real time: 2.967069e+09 -- Predicted time: 2.734133e+09

Callcount: 2076

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
1248	### 1	207.370378720777	6.585400e+05	7.451400e+05	11.6
1659	### 2	491.403956528125	1.313398e+06	1.361800e+06	3.5
1393	### 3	119.697885753495	7.589820e+05	9.327200e+05	18.6
270	### 4	1275.32089447187	1.459487e+06	1.562844e+06	6.6

                /****************  m_matvec.c  (in su3.a) *******************************
*									*
* void mult_su3_mat_vec( su3_matrix *a, su3_vector *b,*c )		*
* matrix times vector multiply, no adjoints 				*
*  C  <-  A*B								*
*/
#include "config.h"
#include "complex.h"
#include "su3.h"

#ifndef FAST
void mult_su3_mat_vec( su3_matrix *a, su3_vector *b, su3_vector *c  ){
register int i,j;
register complex x,y;
    for(i=0;i<3;i++){
	x.real=x.imag=0.0;
	for(j=0;j<3;j++){
	    CMUL( a->e[i][j] , b->c[j] , y )
	    CSUM( x , y );
	}
	c->c[i] = x;
    }
}
#else
#ifdef NATIVEDOUBLE   /* RS6000 version */
void mult_su3_mat_vec( su3_matrix *a, su3_vector *b, su3_vector *c ){

  register double a0r,a0i,a1r,a1i,a2r,a2i;
  register double b0r,b0i,b1r,b1i,b2r,b2i;
  
  a0r=a->e[0][0].real; a0i=a->e[0][0].imag;
  b0r=b->c[0].real;    b0i=b->c[0].imag;
  a1r=a->e[0][1].real; a1i=a->e[0][1].imag;
  b1r=b->c[1].real;    b1i=b->c[1].imag;
  a2r=a->e[0][2].real; a2i=a->e[0][2].imag;
  b2r=b->c[2].real;    b2i=b->c[2].imag;

  c->c[0].real = a0r*b0r - a0i*b0i + a1r*b1r - a1i*b1i + a2r*b2r - a2i*b2i;
  c->c[0].imag = a0r*b0i + a0i*b0r + a1r*b1i + a1i*b1r + a2r*b2i + a2i*b2r;
  
  a0r=a->e[1][0].real; a0i=a->e[1][0].imag;
  b0r=b->c[0].real;    b0i=b->c[0].imag;
  a1r=a->e[1][1].real; a1i=a->e[1][1].imag;
  b1r=b->c[1].real;    b1i=b->c[1].imag;
  a2r=a->e[1][2].real; a2i=a->e[1][2].imag;
  b2r=b->c[2].real;    b2i=b->c[2].imag;

  c->c[1].real = a0r*b0r - a0i*b0i + a1r*b1r - a1i*b1i + a2r*b2r - a2i*b2i;
  c->c[1].imag = a0r*b0i + a0i*b0r + a1r*b1i + a1i*b1r + a2r*b2i + a2i*b2r;

  a0r=a->e[2][0].real; a0i=a->e[2][0].imag;
  b0r=b->c[0].real;    b0i=b->c[0].imag;
  a1r=a->e[2][1].real; a1i=a->e[2][1].imag;
  b1r=b->c[1].real;    b1i=b->c[1].imag;
  a2r=a->e[2][2].real; a2i=a->e[2][2].imag;
  b2r=b->c[2].real;    b2i=b->c[2].imag;

  c->c[2].real = a0r*b0r - a0i*b0i + a1r*b1r - a1i*b1i + a2r*b2r - a2i*b2i;
  c->c[2].imag = a0r*b0i + a0i*b0r + a1r*b1i + a1i*b1r + a2r*b2i + a2i*b2r;

}

#else
void mult_su3_mat_vec( su3_matrix *a, su3_vector *b, su3_vector *c ){
int i;
register double t,ar,ai,br,bi,cr,ci;
    for(i=0;i<3;i++){

	ar=a->e[i][0].real; ai=a->e[i][0].imag;
	br=b->c[0].real; bi=b->c[0].imag;
	cr=ar*br; t=ai*bi; cr -= t;
	ci=ar*bi; t=ai*br; ci += t;

	ar=a->e[i][1].real; ai=a->e[i][1].imag;
	br=b->c[1].real; bi=b->c[1].imag;
	t=ar*br; cr += t; t=ai*bi; cr -= t;
	t=ar*bi; ci += t; t=ai*br; ci += t;

	ar=a->e[i][2].real; ai=a->e[i][2].imag;
	br=b->c[2].real; bi=b->c[2].imag;
	t=ar*br; cr += t; t=ai*bi; cr -= t;
	t=ar*bi; ci += t; t=ai*br; ci += t;

	c->c[i].real=cr;
	c->c[i].imag=ci;
    }
}
#endif	/* End of "#ifdef NATIVEDOUBLE" */
#endif	/* End of "#infdef FAST" */

invivom_matvec_mult_su3_mat_vec_67

Real time: 1.642143e+09 -- Predicted time: 1.018734e+10

Callcount: 20320000

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
3994830	### 1	1939563.89809527	3.460000e+02	7.200000e+01	79.1
14557777	### 2	3278246.83628237	4.290000e+02	8.400000e+01	80.4
7357581	### 3	465430.542701201	4.720000e+02	9.200000e+01	80.5
7946695	### 4	6476833.81886882	4.220000e+02	8.000000e+01	81.0
16526403	### 5	1969142.81202431	4.940000e+02	8.800000e+01	82.1
14768284	### 6	6039923.90701616	6.880000e+02	7.600000e+01	88.9
10486626	### 7	179707.825597154	1.600000e+02	2.680000e+02	40.2

                ERROR UNICODE -> FILE:m_mat_na.c

invivom_mat_na_mult_su3_na_31

Real time: 2.244397e+09 -- Predicted time: 8.754871e+09

Callcount: 13344000

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
2623377	### 1	3332971.73600894	5.500000e+02	1.560000e+02	71.6
9559989	### 2	2671388.57471862	4.770000e+02	1.600000e+02	66.4
5218538	### 3	2763349.16724286	9.040000e+02	1.520000e+02	83.1
12799343	### 4	1601980.17701645	6.390000e+02	1.640000e+02	74.3
13102299	### 5	1943138.48642326	5.100000e+02	1.480000e+02	70.9
12957977	### 6	908099.744825106	1.150000e+03	1.680000e+02	85.3
8257160	### 7	77300.2082087806	8.410000e+02	3.360000e+02	60.0
7453860	### 8	45769.3603601021	5.130000e+02	3.144000e+03	83.6
761689	### 9	3154.7340342963	6.170000e+02	5.028000e+03	87.7

                /****** quark_stuff.c  -- ******************/
/* MIMD version 6 */
/* quark action stuff for improved action
* D.T. 1/28/98, starting from gauge_stuff.c
* K.O. 3/99 Added optimized fattening for Asq actions
* D.T. 4/99 Combine force calculations for both mass quarks
* K.O. 4/99 Optimized force for Asq action
* S.G. 7/01, modified to use t_longlink and t_fatlink
* C.D. 10/02, consolidated quark_stuff.c and quark_stuff_tmp.c
*
* This code combines quark_stuff.c and quark_stuff_tmp.c
* with DSLASH_TMP_LINKS defined, puts links in field-major variables
* t_longlink and t_fatlink. Otherwise, puts them in the site structure.
*
* In this directory, assume all paths connect even to odd sites, etc.
* Tabulate "backwards" paths (e.g. "XDOWN" is backward path to "XUP")
* as separate parity transforms of the fundamental paths.  They will
* generally need a negative sign in Dslash.  See bottom for a long
* comment on sign conventions.
*/

/**#define FFTIME**/
/**#define LLTIME**/

#include "generic_ks_includes.h"	/* definitions files and prototypes */


#define NULL_FP -1 /* NULL field_offset to be used in the optimized version *
                    * of the load_fatlinks subroutine */

    /* Specify paths in orientation in which they appear in the
       forward part of the x component of dslash().  Rotations and
       reflections will be automatically included. Be careful
       about signs of coefficients.  See long comment at bottom. */
#include <quark_action.h>
    /* Include file specifies the basic paths */

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void printpath( int *path, int length );
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length );
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length );
#ifdef  ASQ_OPTIMIZED_FATTENING
void compute_gen_staple(field_offset staple, int mu, int nu,
                        field_offset link, double coef ) ;
#endif
#ifdef  ASQ_OPTIMIZED_FORCE
#ifndef FN
BOMB THE COMPILE
#endif
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) ;
void add_force_to_mom(su3_vector *back, su3_vector *forw, int dir, double coef);
void side_link_force(int mu, int nu, double coeff, su3_vector *Path,
		     su3_vector *Path_nu, su3_vector *Path_mu, 
		     su3_vector *Path_numu) ;

void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) ;
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) ;
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) ;
#endif

int path_num[MAX_BASIC_PATHS];	/* number of rotations/reflections for each 
					kind */
static double act_path_coeff[MAX_BASIC_PATHS]; /* actual path coefficient     *
                                               * it is equal to path_coeff   *
                                               * if not tadpole improvement  *
                                               * is specified                *
                                               * or path_coeff*u_0^(L-1) when*
                                               * tadpole improvement is      *
                                               * specified                   */

/* Array of structures, for each rotation and reflection of each kind of
	path.  */
struct {
    int dir[MAX_LENGTH];	/* directions in path */
    int length;		/* length of path */
    double coeff;	/* coefficient, including minus sign if backwards */
    double forwback;	/* +1 if in forward Dslash, -1 if in backward */
}   q_paths[MAX_NUM];
int num_q_paths;	/* number of paths in dslash */
int num_basic_paths;	/* number of paths before rotation/reflection */

int is_path_equal( int *path1, int* path2, int length );
int add_basic_path( int *vec, int length, double coeff );

/* Make table of paths in action */
void make_path_table() {

    int i,j;
#ifdef TADPOLE_IMPROVE
    int k;
#endif

    /* table of directions, 1 for each kind of path */
    /**int path_ind[MAX_BASIC_PATHS][MAX_LENGTH];**/
    /* table of coefficients in action, for each path */

    node0_printf("%s\n",quark_action_description);
    num_q_paths = 0;
    num_basic_paths = 0;


    /* add rots. and reflects to table, print out the path coefficients */
    node0_printf("path coefficients: npath  path_coeff  multiplicity\n");
    for(j=0;j<quark_action_npaths;j++) {
	double this_coeff;
	this_coeff = path_coeff[j];
#ifdef TADPOLE_IMPROVE
	for(k=1;k< path_length_in[j];k++)this_coeff /= u0;
#endif
	act_path_coeff[j] = this_coeff ;
	i = add_basic_path( path_ind[j], path_length_in[j],
	    this_coeff );
	node0_printf("                    %d      %e     %d\n",
	    j,this_coeff,i);
    }
}

/* add rotations and reflections of a path to the table.  Return
   multiplicity of paths added */
int add_basic_path( int *basic_vec, int length, double coeff ) {

    int perm[8],pp[8],ir[4];
    int j,path_num;
    int vec[MAX_LENGTH];
    int flag;

    path_num = 0;
    /* now fill the long table with all rotations and reflections
	of the fundamental path.  The path presented to us is for
        the positive x component of dslash, so if the x coordinate
        is reflected it will appear with a negative sign. */
      /* permutations */
      for(perm[0]=0;perm[0]<4;perm[0]++)
      for(perm[1]=0;perm[1]<4;perm[1]++)
      for(perm[2]=0;perm[2]<4;perm[2]++)
      for(perm[3]=0;perm[3]<4;perm[3]++){
	if(perm[0] != perm[1] && perm[0] != perm[2] 
	  && perm[0] != perm[3] && perm[1] != perm[2]
	  && perm[1] != perm[3] && perm[2] != perm[3] ) {
	  /* reflections*/
	  for(ir[0]=0;ir[0]<2;ir[0]++)
	  for(ir[1]=0;ir[1]<2;ir[1]++)
	  for(ir[2]=0;ir[2]<2;ir[2]++)
	  for(ir[3]=0;ir[3]<2;ir[3]++){
	    for(j=0;j<4;j++){
	      pp[j]=perm[j];

	      if(ir[j] == 1) pp[j]=OPP_DIR(pp[j]);
	      pp[OPP_DIR(j)]=OPP_DIR(pp[j]);
	    }
	    /* create new vector*/
	    for(j=0;j<length;j++) vec[j]=pp[basic_vec[j]];
	    for(j=length;j<MAX_LENGTH;j++) vec[j]=NODIR;

            flag=0;
	    /* check if it's a new set: */
	    for(j=0;j<num_q_paths;j++){
	      flag = is_path_equal( vec, q_paths[j].dir, MAX_LENGTH );
	      if(flag==1)break;
	    }
	    if(flag == 0 ){
	      if(num_q_paths>=MAX_NUM){
		node0_printf("OOPS: MAX_NUM too small\n");
		exit(0);
	      }
	      q_paths[num_q_paths].length=length;
	      for(j=0;j<MAX_LENGTH;j++) q_paths[num_q_paths].dir[j]=vec[j];
		/* remember to copy NODIR's, or comparison will get confused */
	      if(ir[0]==0){
		q_paths[num_q_paths].coeff =  coeff;
		q_paths[num_q_paths].forwback =  +1;
	      }
	      else{
		q_paths[num_q_paths].coeff = -coeff;
		q_paths[num_q_paths].forwback = -1;
	      }
	      num_q_paths++;
	      path_num++;
	      /**node0_printf("ADD PATH %d:  rx=%d ",num_q_paths-1,ir[0]);
		 printpath( vec, length );**/
	    }

	  } /* end reflection*/
        } /* end permutation if block */
      } /* end permutation */
    num_basic_paths++;
    return(path_num);
} /* add_basic_path */

/* parallel transport a vector to the current site along a path.
   For example, if the path is "XUP", bring in the vector from
   the +X direction to the current site. If KS phases are in lattice,
   this transport will automatically include them. 
   OK for src and dest to be the same.  OK for length=0.  */
/* NOT OPTIMIZED at the moment - do lots of extra copying.  Use temp.
   vectors rather than stuff in lattice.h */
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    su3_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    su3_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_dest = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_work = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(su3_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_vec( &(s->link[dir[j]]),
		    (su3_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_vec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(su3_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = *(su3_vector *)F_PT(s,src);
    }
  }
} /* path_transport */

/* Path transport a half_wilson_vector */
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    half_wilson_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    half_wilson_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (half_wilson_vector *)calloc(
      sites_on_node, sizeof(half_wilson_vector));
    tmp_dest = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));
    tmp_work = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(half_wilson_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_hwvec( &(s->link[dir[j]]),
		    (half_wilson_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_hwvec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(half_wilson_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) =
	  *(half_wilson_vector *)F_PT(s,src);
    }
  }
} /* path_transport_hwv */


#ifdef EO
/* Stupid dslash routine that follows all the paths.
  Should optimize by precomputing sums of paths to all the displacements
  Use dslash_fn for actions that involve only +X and +X+X+X couplings.
*/
/* D_slash routine - sets dest. on each site equal to sum of
   sources parallel transported to site, with minus sign for transport
   from negative directions.  */
void dslash_eo( field_offset src, field_offset dest, int parity ) {
   register int i;
   register site *s;
   register int ipath,otherparity;
   register double x; /* coefficient of path */

    switch(parity){
	case EVEN:	otherparity=ODD; break;
	case ODD:	otherparity=EVEN; break;
	case EVENANDODD:	otherparity=EVENANDODD; break;
    }

    /* Parallel transport by all the paths in the action.  
       Multiply by coefficient in table
    */
    FORSOMEPARITY(i,s,parity){ clearvec( (su3_vector *)F_PT(s,dest) ); }

    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	    path_transport( src, F_OFFSET(tempvec[0]), parity,
		q_paths[ipath].dir, q_paths[ipath].length );
	    x=q_paths[ipath].coeff;
	    FORSOMEPARITY(i,s,parity){
		scalar_mult_add_su3_vector(  (su3_vector *)F_PT(s,dest),
		    &(s->tempvec[0]), x, (su3_vector *)F_PT(s,dest) );
	    }
    }   /* ipath */
} /* dslash_eo */
#endif /* ifdef EO */

#ifdef FN
/* Sum over paths connecting to nearest neighbor point (fat link) and to third
   nearest neighbor (longlinks) */
/* Doug Toussaint 2/4/98 */
/* modified to use t_longlinks, S. Gottlieb 7/13/01 */
/* long link calculating routine */
/* path_product() follows the path starting at step 0, and
   leaves the answer at the end of the path.  We want the answer
   at the site where the path begins.  So we look for paths with
   the opposite displacement from the displacement of the point
   that we want to transport to this site, and take the adjoint
   of the matrix at the end. clear? */
/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_longlinks() {
  register int i;
  register site *s;
  int ipath,dir;
  int disp[4];
  register su3_matrix *long1;

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_longlinks needs phases in\n");
    terminate(0);
  }
  for (dir=XUP; dir<=TUP; dir++){ /* loop over longlink directions */
    /* set longlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      long1 = &(t_longlink[4*i+dir]);
#else
      long1 = &(s->longlink[dir]);
#endif
      clear_su3mat( long1 );
    }

    /* loop over paths, checking for ones with total displacement 3*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=3,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("ipath = %d, found a path:  ",ipath);
for(j=0;j<q_paths[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  long1 = &(t_longlink[4*i+dir]);
#else
	  long1 = &(s->longlink[dir]);
#endif
          scalar_mult_add_su3_matrix( long1,
	    &(s->staple), -q_paths[ipath].coeff, long1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */

  } /* loop over directions */

  valid_longlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Long):  %e\n",dtime);
#endif
}  /* load_longlinks() */

/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_fatlinks() {
  register int i;
  register site *s;
  int dir;
  register su3_matrix *fat1;
#ifdef ASQ_OPTIMIZED_FATTENING
  int  nu,rho,sig ;
  double one_link ; /*needed to fix the problem with the Lepage term */
#else
  int ipath;
  int disp[4];
#endif

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_fatlinks needs phases in\n");
    terminate(0);
  }

#ifndef  ASQ_OPTIMIZED_FATTENING   /* general case code */
  for (dir=XUP; dir<=TUP; dir++){ /* loop over fatlink directions */
    /* set fatlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+dir]);
#else
      fat1 = &(s->fatlink[dir]);
#endif
      clear_su3mat( fat1 );
    }
    
    /* loop over paths, checking for ones with total displacement 1*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=1,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("dir = %d, found a path:  ",dir);
for(j=0;j<q_paths.[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  fat1 = &(t_fatlink[4*i+dir]);
#else
	  fat1 = &(s->fatlink[dir]);
#endif
          scalar_mult_add_su3_matrix( fat1,
	    &(s->staple), -q_paths[ipath].coeff, fat1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */
  } /* loop over directions */
#else	/* ASQ_OPTIMIZED_FATTENING, for Asq and Asqtad actions */
/*  Optimized fattening code for the Asq and Asqtad actions.           *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
 
 /* to fix up the Lepage term, included by a trick below */
 one_link = (act_path_coeff[0] - 6.0*act_path_coeff[5]) ; 
 
 for (dir=XUP; dir<=TUP; dir++){
   FORALLSITES(i,s) /* Intialize fat links with c_1*U_\mu(x) */
     {
#ifdef DSLASH_TMP_LINKS
       fat1 = &(t_fatlink[4*i+dir]);
#else
       fat1 = &(s->fatlink[dir]);
#endif
       scalar_mult_su3_matrix(&(s->link[dir]), one_link ,
			      fat1 );
     }
   for(nu=XUP; nu<=TUP; nu++) if(nu!=dir)
     {
       compute_gen_staple(F_OFFSET(staple),dir,nu,F_OFFSET(link[dir]),
			  act_path_coeff[2]);
       /* The Lepage term */
       /* Note this also involves modifying c_1 (above) */
       compute_gen_staple(NULL_FP,dir,nu,F_OFFSET(staple),act_path_coeff[5]);
       for(rho=XUP; rho<=TUP; rho++) if((rho!=dir)&&(rho!=nu))
	 {
	   compute_gen_staple(F_OFFSET(tempmat1),dir,rho,F_OFFSET(staple),
			      act_path_coeff[3]);
	   for(sig=XUP; sig<=TUP; sig++)
	     if((sig!=dir)&&(sig!=nu)&&(sig!=rho))
	       {
		 compute_gen_staple(NULL_FP,dir,sig,
				    F_OFFSET(tempmat1),
				    act_path_coeff[4]);
	       } /* sig */
	 } /* rho */
     } /* nu */
 }/* dir */  
#endif

  valid_fatlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Fat):  %e\n",dtime);
#endif
}  /* load_fatlinks() */
#endif /* ifdef FN */


/* compare two paths, return 1 if equal, else zero */
int is_path_equal( int *path1, int* path2, int length ){
   register int i;
   for(i=0;i<length;i++)if(path1[i]!=path2[i])return(0);
   return(1);
}


/* update the  momenta with the fermion force */
/* Assumes that the conjugate gradient has been run, with the answer in
   x_off, and dslash(x_off,x_off,ODD) has been run. (fills in x_off_odd) */
/* SEE LONG COMMENTS AT END */

#ifndef ASQ_OPTIMIZED_FORCE
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon, coeff;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  half_wilson_vector *hw_tmp0,*hw_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  hw_tmp0 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  hw_tmp1 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  /* Use half_wilson_vectors to store x_off transported from ends of
     path.  0 component from forward end, 1 component from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    path_transport( x_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[1] for transport from starting end */
    FORALLSITES(i,s){
      hw_tmp0[i].h[0]=s->tempvec[0];
      hw_tmp0[i].h[1]=*(su3_vector *)F_PT(s,x_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff = ferm_epsilon*q_paths[ipath].coeff;
      if( (ilink%2)==1 )coeff = -coeff;

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "half_wilson_vector" to handle pair of vectors -
	0 component is x_off from forward end, 1 component from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_su3_mat_hwvec( &(s->link[lastdir]),
	      &(hw_tmp0[i]), &(hw_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( hw_tmp1, 2*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     hw_tmp0[i] = *(half_wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( hw_tmp0, 2*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_su3_mat_hwvec( &(s->link[OPP_DIR(lastdir)]),
                (half_wilson_vector *)(gen_pt[0][i]),
		&(hw_tmp1[i]) );
          }
	  tmp_pt = hw_tmp0; hw_tmp0 = hw_tmp1; hw_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( hw_tmp0 ); free( hw_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */


void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off,
  int nflav2, field_offset x2_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/14/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon1, ferm_epsilon2, coeff1, coeff2;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  wilson_vector *w_tmp0,*w_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon1 = 2.0*(nflav1/4.0)*eps;
  ferm_epsilon2 = 2.0*(nflav2/4.0)*eps;
  w_tmp0 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  w_tmp1 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  /* Use wilson_vectors to store x_off transported from ends of
     path.  0 and 1 components from forward end, 2 and 3 components
     from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    /**
    path_transport( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    path_transport( x2_off, F_OFFSET(tempvec[1]),
      EVENANDODD, q_paths[ipath].dir, length );
    **/
/** WARNING!! Assumes xxx1 and xxx2 contiguous **/
if( x2_off-x1_off != sizeof(su3_vector) ){node0_printf("BOTCH\n"); exit(0);}
    path_transport_hwv( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[2] for transport from starting end */
    FORALLSITES(i,s){
      w_tmp0[i].d[0]=s->tempvec[0];
      w_tmp0[i].d[1]=s->tempvec[1];
      w_tmp0[i].d[2]=*(su3_vector *)F_PT(s,x1_off);
      w_tmp0[i].d[3]=*(su3_vector *)F_PT(s,x2_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff1 = ferm_epsilon1*q_paths[ipath].coeff;
      coeff2 = ferm_epsilon2*q_paths[ipath].coeff;
      if( (ilink%2)==1 ){ coeff1 = -coeff1; coeff2 = -coeff2;}

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "wilson_vector" to handle pair of vectors -
	0 component is x_off1 from forward end, 1 component is x_off2
	from forward end,  2 and 3  components are x_off1 and x_off2 
	from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_mat_wilson_vec( &(s->link[lastdir]),
	      &(w_tmp0[i]), &(w_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( w_tmp1, 4*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     w_tmp0[i] = *(wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( w_tmp0, 4*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_mat_wilson_vec( &(s->link[OPP_DIR(lastdir)]),
                (wilson_vector *)(gen_pt[0][i]),
		&(w_tmp1[i]) );
          }
	  tmp_pt = w_tmp0; w_tmp0 = w_tmp1; w_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( w_tmp0 ); free( w_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */



#else /* ASQ_OPTIMIZED_FORCE, for Asq and Asqtad actions */
/* Optimized force code for the Asq and Asqtad actions                 *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
#define Pmu          tempvec[0] 
#define Pnumu        tempvec[1]
#define Prhonumu     tempvec[2]
#define P7           tempvec[3]
#define P7rho        tempvec[4]              
#define P7rhonu      tempvec[5]
#define P5           tempvec[6]
#define P3           tempvec[7]
#define P5nu         tempvec[3]
#define P3mu         tempvec[3]
#define Popmu        tempvec[4]
#define Pmumumu      tempvec[4]
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double ferm_epsilon, coeff;
  double OneLink, Lepage, Naik, FiveSt, ThreeSt, SevenSt ;
  su3_vector *tempvec[8] ;
  su3_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  
  /* Path coefficients times fermion epsilon */
  OneLink = act_path_coeff[0]*ferm_epsilon ; 
  Naik    = act_path_coeff[1]*ferm_epsilon ;
  ThreeSt = act_path_coeff[2]*ferm_epsilon ;
  FiveSt  = act_path_coeff[3]*ferm_epsilon ;
  SevenSt = act_path_coeff[4]*ferm_epsilon ;
  Lepage  = act_path_coeff[5]*ferm_epsilon ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    tempvec[mu] = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

  /*copy x_off to a temporary vector */
  temp_x = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
  FORALLSITES(i,s) temp_x[i] = *(su3_vector *)F_PT(s,x_off) ;

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_force_to_mom(P3, Pmu, sig, -ThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_force_to_mom(P7, Prhonumu, sig, -SevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_fermion(P7, P7rho, rho);
		  side_link_force(rho,sig,SevenSt, Pnumu, P7, Prhonumu, P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff = SevenSt/FiveSt ; 
		  FORALLSITES(i,s)
		    scalar_mult_add_su3_vector(&(P5[i]),&(P7rho[i]),coeff,
					       &(P5[i]));
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_fermion(P5,P5nu, nu);
	      side_link_force(nu,sig,-FiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff = FiveSt/ThreeSt ; 
	      FORALLSITES(i,s)
		scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_fermion(P5,P5nu, mu);
	  side_link_force(mu, sig, -Lepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff = Lepage/ThreeSt ; 
	  FORALLSITES(i,s)
	    scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), -Naik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(tempvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      

#define Pmu          hwvec[0] 
#define Pnumu        hwvec[1]
#define Prhonumu     hwvec[2]
#define P7           hwvec[3]
#define P7rho        hwvec[4]              
#define P7rhonu      hwvec[5]
#define P5           hwvec[6]
#define P3           hwvec[7]
#define P5nu         hwvec[3]
#define P3mu         hwvec[3]
#define Popmu        hwvec[4]
#define Pmumumu      hwvec[4]
void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off, 
			  int nflav2, field_offset x2_off){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/15/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double coeff[2],ferm_epsilon ;
  double OneLink[2], Lepage[2], Naik[2], FiveSt[2], ThreeSt[2], SevenSt[2] ;
  double mNaik[2], mLepage[2], mFiveSt[2], mThreeSt[2], mSevenSt[2] ;
  half_wilson_vector *hwvec[8] ;
  half_wilson_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  
  /* Path coefficients times fermion epsilon */
  ferm_epsilon = 2.0*(nflav1/4.0)*eps;
  OneLink[0] = act_path_coeff[0]*ferm_epsilon ;
  Naik[0]    = act_path_coeff[1]*ferm_epsilon ; mNaik[0]    = -Naik[0] ;
  ThreeSt[0] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[0] = -ThreeSt[0] ;
  FiveSt[0]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[0]  = -FiveSt[0] ;
  SevenSt[0] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[0] = -SevenSt[0] ;
  Lepage[0]  = act_path_coeff[5]*ferm_epsilon ; mLepage[0]  = -Lepage[0] ;

  ferm_epsilon = 2.0*(nflav2/4.0)*eps;
  OneLink[1] = act_path_coeff[0]*ferm_epsilon ;
  Naik[1]    = act_path_coeff[1]*ferm_epsilon ; mNaik[1]    = -Naik[1] ;
  ThreeSt[1] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[1] = -ThreeSt[1] ;
  FiveSt[1]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[1]  = -FiveSt[1] ;
  SevenSt[1] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[1] = -SevenSt[1] ;
  Lepage[1]  = act_path_coeff[5]*ferm_epsilon ; mLepage[1]  = -Lepage[1] ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    hwvec[mu]= 
      (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));

  /*copy x_off to a temporary vector */
  temp_x= 
    (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
  FORALLSITES(i,s)
    {
      temp_x[i].h[0] = *(su3_vector *)F_PT(s,x1_off) ;
      temp_x[i].h[1] = *(su3_vector *)F_PT(s,x2_off) ;
    }

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_hw_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_3f_force_to_mom(P3, Pmu, sig, mThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_hw_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_3f_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_hw_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_hw_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_3f_force_to_mom(P7, Prhonumu, sig, mSevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_hw_fermion(P7, P7rho, rho);
		  side_link_3f_force(rho,sig,SevenSt,Pnumu,P7,Prhonumu,P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff[0] = SevenSt[0]/FiveSt[0] ; 
		  coeff[1] = SevenSt[1]/FiveSt[1] ; 
		  FORALLSITES(i,s)
		    {
		      scalar_mult_add_su3_vector(&(P5[i].h[0]),
						 &(P7rho[i].h[0]),coeff[0],
						 &(P5[i].h[0]));
		      scalar_mult_add_su3_vector(&(P5[i].h[1]),
						 &(P7rho[i].h[1]),coeff[1],
						 &(P5[i].h[1]));
		    }
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_hw_fermion(P5,P5nu, nu);
	      side_link_3f_force(nu,sig,mFiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff[0] = FiveSt[0]/ThreeSt[0] ; 
	      coeff[1] = FiveSt[1]/ThreeSt[1] ; 
	      FORALLSITES(i,s)
		{
		  scalar_mult_add_su3_vector(&(P3[i].h[0]),
					     &(P5nu[i].h[0]), coeff[0],
					     &(P3[i].h[0]));
		  scalar_mult_add_su3_vector(&(P3[i].h[1]),
					     &(P5nu[i].h[1]), coeff[1],
					     &(P3[i].h[1]));
		}
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_3f_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_hw_fermion(P5,P5nu, mu);
	  side_link_3f_force(mu, sig, mLepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff[0] = Lepage[0]/ThreeSt[0] ; 
	  coeff[1] = Lepage[1]/ThreeSt[1] ; 
	  FORALLSITES(i,s)
	    {
	      scalar_mult_add_su3_vector(&(P3[i].h[0]),
					 &(P5nu[i].h[0]),coeff[0],
					 &(P3[i].h[0]));
	      scalar_mult_add_su3_vector(&(P3[i].h[1]),
					 &(P5nu[i].h[1]),coeff[1],
					 &(P3[i].h[1]));
	    }

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_hw_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_3f_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_3f_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_3f_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), mNaik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_hw_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_3f_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_3f_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(hwvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      
#endif /* ASQ_OPTIMIZED_FORCE */


#ifdef  ASQ_OPTIMIZED_FATTENING   /* Asqtad action only, "_fn" executables */
#ifndef FN
BOMB THE COMPILE
#endif
void compute_gen_staple(field_offset staple, int mu, int nu, 
			field_offset link, double coef) {
  su3_matrix tmat1,tmat2;
  msg_tag *mtag0,*mtag1;
  su3_matrix *tempmat ;
  register site *s ;
  register int i ;
  register su3_matrix *fat1;

  /* Computes the staple :
                   mu
                +-------+
          nu	|	|
		|	|
		X	X
    Where the mu link can be any su3_matrix. The result is saved in staple.
    if staple==NULL_FP then the result is not saved.
    It also adds the computed staple to the fatlink[mu] with weight coef.
  */

  /* Upper staple */
  mtag0 = start_gather( link, sizeof(su3_matrix), nu, EVENANDODD, gen_pt[0] );
  mtag1 = start_gather( F_OFFSET(link[nu]), sizeof(su3_matrix), mu, 
			EVENANDODD, gen_pt[1] );
  wait_gather(mtag0);
  wait_gather(mtag1);
  
  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, (su3_matrix *)F_PT(s,staple) );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, &tmat2 );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix(fat1, &tmat2, coef,
				 fat1) ;
    }
  }
  cleanup_gather(mtag0);
  cleanup_gather(mtag1);

  /* lower staple */
  tempmat = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
  mtag0 = start_gather( F_OFFSET(link[nu]),
			sizeof(su3_matrix), mu, EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);
  FORALLSITES(i,s){
    mult_su3_an( &(s->link[nu]),(su3_matrix *)F_PT(s,link), &tmat1 );
    mult_su3_nn( &(tmat1),(su3_matrix *)gen_pt[0][i], &(tempmat[i]) );
  }
  cleanup_gather(mtag0);
  mtag0 = start_gather_from_temp( tempmat, sizeof(su3_matrix),
				  OPP_DIR(nu), EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);

  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      add_su3_matrix( (su3_matrix *)F_PT(s,staple),(su3_matrix *)gen_pt[0][i], 
		      (su3_matrix *)F_PT(s,staple) );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)F_PT(s,staple), coef, 
				 fat1 );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)gen_pt[0][i], coef, 
				 fat1 );
    }
  }

  free(tempmat);
  cleanup_gather(mtag0);
}
#endif  /* ASQ_OPTIMIZED_FATTENING   */


#ifdef  ASQ_OPTIMIZED_FORCE
/*   Covariant shift of the src fermion field in the direction dir  *
 *  by one unit. The result is stored in dest.                       */ 
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) {
  su3_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(su3_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_vec(&(s->link[dir]),(su3_vector *)(gen_pt[0][i]),
			 &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_vec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(su3_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(su3_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/*  Covariant shift of the src half wilson fermion field in the  *
 * direction dir   by one unit. The result is stored in dest.    */
void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) {
  half_wilson_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(half_wilson_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_hwvec(&(s->link[dir]),
			   (half_wilson_vector *)(gen_pt[0][i]), &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = 
	(half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_hwvec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(half_wilson_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(half_wilson_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/* Add in contribution to the force */
/* Put antihermitian traceless part into momentum */
void add_force_to_mom(su3_vector *back,su3_vector *forw,int dir,double coeff) {
  register site *s ;
  register int i ;  
  register double tmp_coeff ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff = -coeff ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD) 
      tmp_coeff = -coeff ;
    else
      tmp_coeff = coeff ;
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i]), &(forw[i]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff, &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/* Add in contribution to the force ( 3flavor case ) */
/* Put antihermitian traceless part into momentum */
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) {
  register site *s ;
  register int i ;  
  double tmp_coeff[2] ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff[0] = -coeff[0] ;
      coeff[1] = -coeff[1] ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD)
      {
	tmp_coeff[0] = -coeff[0] ;
	tmp_coeff[1] = -coeff[1] ;
      }
    else
      {
	tmp_coeff[0] = coeff[0] ;
	tmp_coeff[1] = coeff[1] ;
      }
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i].h[0]), &(forw[i].h[0]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[0], &tmat2 );
    su3_projector(&(back[i].h[1]), &(forw[i].h[1]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[1], &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/*  This routine is needed in order to add the force on the side link *
 * of the paths in the Asq and Asqtad actions. It gets as inputs the  *
 * direction mu of the side link and the direction nu of the Dslash   *
 * term we are dealing with. Then it takes also 4 fermion fields:     *
 * Path: the piece of the path with no hop in the nu or mu direction  *
 * Path_nu: the piece of the path with a hop in nu  but not in mu     *
 * Path_mu: is Path times the link mu                                 *
 * Path_numu: is Path_nu times the link mu                            */
void side_link_force(int mu, int nu, double coeff, 
		     su3_vector *Path   , su3_vector *Path_nu, 
		     su3_vector *Path_mu, su3_vector *Path_numu) {
  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_force_to_mom(Path, Path_numu, OPP_DIR(mu), -coeff );/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_nu, Path_mu, mu, -coeff) ; /* ? extra - */
      else
	add_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
 
/*  The 3 flavor version of side_link_force used *
 * to optimize fermion transports                */
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) {
  double m_coeff[2] ;

  m_coeff[0] = -coeff[0] ;
  m_coeff[1] = -coeff[1] ;

  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_3f_force_to_mom(Path,Path_numu,OPP_DIR(mu),m_coeff);/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_nu, Path_mu, mu, m_coeff) ; /* ? extra - */
      else
	add_3f_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
#endif  /* ASQ_OPTIMIZED_FORCE   */

/* LONG COMMENTS
   Here we have combined "xxx", (offset "x_off")  which is
(M_adjoint M)^{-1} phi, with Dslash times this vector, which goes in the
odd sites of xxx.  Recall that phi is defined only on even sites.  In
computing the fermion force, we are looking at

< X |  d/dt ( Dslash_eo Dslash_oe ) | X >
=
< X | d/dt Dslash_eo | T > + < T | d/dt Dslash_oe | X >
where T = Dslash X.

The subsequent manipulations to get the coefficent of H, the momentum
matrix, in the simulation time derivative above look the same for
the two terms, except for a minus sign at the end, if we simply stick
T, which lives on odd sites, into the odd sites of X

 Each path in the action contributes terms when any link of the path
is the link for which we are computing the force.  We get a minus sign
for odd numbered links in the path, since they connect sites of the
opposite parity from what it would be for an even numbered link.
Minus signs from "going around" plaquette - ie KS phases, are supposed
to be already encoded in the path coefficients.
Minus signs from paths that go backwards are supposed to be already
encoded in the path coefficients.

Here, for example, are comments reproduced from the force routine for
the one-link plus Naik plus single-staple-fat-link action:

 The three link force has three contributions, where the link that
was differentiated is the first, second, or third link in the 3-link
path, respectively.  Diagramatically, where "O" represents the momentum,
the solid line the link corresponding to the momentum, and the dashed
lines the other links:
 

	O______________ x ............ x ...............
+
	x..............O______________x.................
+
	x..............x..............O________________
Think of this as
	< xxx | O | UUUxxx >		(  xxx, UUUX_p3 )
+
	< xxx U | O | UUxxx >		( X_m1U , UUX_p2 )
+
	< xxx U U | O | Uxxx >		( X_m2UU , UX_p1 )
where "U" indicates parallel transport, "X_p3" is xxx displaced
by +3, etc.
Note the second contribution has a relative minus sign
because it effectively contributes to the <odd|even>, or M_adjoint,
part of the force when we work on an even site. i.e., for M on
an even site, this three link path begins on an odd site.

The staple force has six contributions from each plane containing the
link direction:
Call these diagrams A-F:


	x...........x		O____________x
		    .			     .
		    .			     .
		    .			     .
		    .			     .
		    .			     .
	O___________x		x............x
	   (A)			    (B)



	x	    x		O____________x
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	O___________x		x	     x
	   (C)			    (D)



	x...........x		O____________x
	.			.
	.			.
	.			.
	.			.
	.			.
	O___________x		x............x
	   (E)			    (F)

As with the Naik term, diagrams C and D have a relative minus
sign because they connect sites of the other parity.

Also note an overall minus sign in the staple terms relative to the
one link term because, with the KS phase factors included, the fat
link is  "U - w3 * UUU", or the straight link MINUS w3 times the staples.

Finally, diagrams B and E get one more minus sign because the link
we are differentiating is in the opposite direction from the staple
as a whole.  You can think of this as this "U" being a correction to
a "U_adjoint", but the derivative of U is iHU and the derivative
of U_adjoint is -iHU_adjoint.

*/
/* LONG COMMENT on sign conventions
In most of the program, the KS phases and antiperiodic boundary
conditions are absorbed into the link matrices.  This greatly simplfies
multiplying by the fermion matrix.  However, it requires care in
specifying the path coefficients.  Remember that each time you
encircle a plaquette, you pick up a net minus sign from the KS phases.
Thus, when you have more than one path to the same point, you generally
have a relative minus sign for each plaquette in a surface bounded by
this path and the basic path for that displacement.

Examples:
  Fat Link:
    Positive:	X-------X

    Negative     --------
	 	|	|
		|	|
		X	X

  Naik connection, smeared
    Positive:	X-------x-------x-------X

    Negative:	---------
		|	|
		|	|
		X	x-------x-------X

    Positive:	--------x--------
		|		|
		|		|
		X		x-------X

    Negative:	--------x-------x-------x
		|			|
		|			|
		X			X
*/



/* Comment on acceptable actions.
   We construct the backwards part of dslash by reversing all the
   paths in the forwards part.  So, for example, in the p4 action
   the forwards part includes +X+Y+Y

		X
		|
		|
		X
		|
		|
	X---->--X

  so we put -X-Y-Y in the backwards part.  But this isn't the adjoint
  of U_x(0)U_y(+x)U_y(+x+y).  Since much of the code assumes that the
  backwards hop is the adjoint of the forwards (for example, in
  preventing going to 8 flavors), the code only works for actions
  where this is true.  Roughly, this means that the fat link must
  be symmetric about reflection around its midpoint.  Equivalently,
  the paths in the backwards part of Dslash are translations of the
  paths in the forwards part.  In the case of the "P4" or knight's move
  action, this means that we have to have both paths
   +X+Y+Y and +Y+Y+X to the same point, with the same coefficients.
  Alternatively, we could just use the symmetric path +Y+X+Y.
*/

invivoquark_stuff_add_force_to_mom_1523

Real time: 2.660932e+09 -- Predicted time: 2.911636e+09

Callcount: 10240

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
4269	### 1	218.221154684729	2.870370e+05	2.751920e+05	4.1
336	### 2	2062.07324878866	2.815870e+05	2.674720e+05	5.0
3773	### 3	1484.91733503463	2.870200e+05	2.633280e+05	8.2
659	### 4	1290.31369013917	2.845620e+05	2.684560e+05	5.6
531	### 5	2114.66930448781	2.761890e+05	2.658760e+05	3.7
3059	### 6	1959.89018684581	2.973600e+05	2.667440e+05	10.2
234	### 7	1113.06701155685	2.768290e+05	2.644320e+05	4.4

                /****** quark_stuff.c  -- ******************/
/* MIMD version 6 */
/* quark action stuff for improved action
* D.T. 1/28/98, starting from gauge_stuff.c
* K.O. 3/99 Added optimized fattening for Asq actions
* D.T. 4/99 Combine force calculations for both mass quarks
* K.O. 4/99 Optimized force for Asq action
* S.G. 7/01, modified to use t_longlink and t_fatlink
* C.D. 10/02, consolidated quark_stuff.c and quark_stuff_tmp.c
*
* This code combines quark_stuff.c and quark_stuff_tmp.c
* with DSLASH_TMP_LINKS defined, puts links in field-major variables
* t_longlink and t_fatlink. Otherwise, puts them in the site structure.
*
* In this directory, assume all paths connect even to odd sites, etc.
* Tabulate "backwards" paths (e.g. "XDOWN" is backward path to "XUP")
* as separate parity transforms of the fundamental paths.  They will
* generally need a negative sign in Dslash.  See bottom for a long
* comment on sign conventions.
*/

/**#define FFTIME**/
/**#define LLTIME**/

#include "generic_ks_includes.h"	/* definitions files and prototypes */


#define NULL_FP -1 /* NULL field_offset to be used in the optimized version *
                    * of the load_fatlinks subroutine */

    /* Specify paths in orientation in which they appear in the
       forward part of the x component of dslash().  Rotations and
       reflections will be automatically included. Be careful
       about signs of coefficients.  See long comment at bottom. */
#include <quark_action.h>
    /* Include file specifies the basic paths */

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void printpath( int *path, int length );
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length );
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length );
#ifdef  ASQ_OPTIMIZED_FATTENING
void compute_gen_staple(field_offset staple, int mu, int nu,
                        field_offset link, double coef ) ;
#endif
#ifdef  ASQ_OPTIMIZED_FORCE
#ifndef FN
BOMB THE COMPILE
#endif
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) ;
void add_force_to_mom(su3_vector *back, su3_vector *forw, int dir, double coef);
void side_link_force(int mu, int nu, double coeff, su3_vector *Path,
		     su3_vector *Path_nu, su3_vector *Path_mu, 
		     su3_vector *Path_numu) ;

void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) ;
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) ;
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) ;
#endif

int path_num[MAX_BASIC_PATHS];	/* number of rotations/reflections for each 
					kind */
static double act_path_coeff[MAX_BASIC_PATHS]; /* actual path coefficient     *
                                               * it is equal to path_coeff   *
                                               * if not tadpole improvement  *
                                               * is specified                *
                                               * or path_coeff*u_0^(L-1) when*
                                               * tadpole improvement is      *
                                               * specified                   */

/* Array of structures, for each rotation and reflection of each kind of
	path.  */
struct {
    int dir[MAX_LENGTH];	/* directions in path */
    int length;		/* length of path */
    double coeff;	/* coefficient, including minus sign if backwards */
    double forwback;	/* +1 if in forward Dslash, -1 if in backward */
}   q_paths[MAX_NUM];
int num_q_paths;	/* number of paths in dslash */
int num_basic_paths;	/* number of paths before rotation/reflection */

int is_path_equal( int *path1, int* path2, int length );
int add_basic_path( int *vec, int length, double coeff );

/* Make table of paths in action */
void make_path_table() {

    int i,j;
#ifdef TADPOLE_IMPROVE
    int k;
#endif

    /* table of directions, 1 for each kind of path */
    /**int path_ind[MAX_BASIC_PATHS][MAX_LENGTH];**/
    /* table of coefficients in action, for each path */

    node0_printf("%s\n",quark_action_description);
    num_q_paths = 0;
    num_basic_paths = 0;


    /* add rots. and reflects to table, print out the path coefficients */
    node0_printf("path coefficients: npath  path_coeff  multiplicity\n");
    for(j=0;j<quark_action_npaths;j++) {
	double this_coeff;
	this_coeff = path_coeff[j];
#ifdef TADPOLE_IMPROVE
	for(k=1;k< path_length_in[j];k++)this_coeff /= u0;
#endif
	act_path_coeff[j] = this_coeff ;
	i = add_basic_path( path_ind[j], path_length_in[j],
	    this_coeff );
	node0_printf("                    %d      %e     %d\n",
	    j,this_coeff,i);
    }
}

/* add rotations and reflections of a path to the table.  Return
   multiplicity of paths added */
int add_basic_path( int *basic_vec, int length, double coeff ) {

    int perm[8],pp[8],ir[4];
    int j,path_num;
    int vec[MAX_LENGTH];
    int flag;

    path_num = 0;
    /* now fill the long table with all rotations and reflections
	of the fundamental path.  The path presented to us is for
        the positive x component of dslash, so if the x coordinate
        is reflected it will appear with a negative sign. */
      /* permutations */
      for(perm[0]=0;perm[0]<4;perm[0]++)
      for(perm[1]=0;perm[1]<4;perm[1]++)
      for(perm[2]=0;perm[2]<4;perm[2]++)
      for(perm[3]=0;perm[3]<4;perm[3]++){
	if(perm[0] != perm[1] && perm[0] != perm[2] 
	  && perm[0] != perm[3] && perm[1] != perm[2]
	  && perm[1] != perm[3] && perm[2] != perm[3] ) {
	  /* reflections*/
	  for(ir[0]=0;ir[0]<2;ir[0]++)
	  for(ir[1]=0;ir[1]<2;ir[1]++)
	  for(ir[2]=0;ir[2]<2;ir[2]++)
	  for(ir[3]=0;ir[3]<2;ir[3]++){
	    for(j=0;j<4;j++){
	      pp[j]=perm[j];

	      if(ir[j] == 1) pp[j]=OPP_DIR(pp[j]);
	      pp[OPP_DIR(j)]=OPP_DIR(pp[j]);
	    }
	    /* create new vector*/
	    for(j=0;j<length;j++) vec[j]=pp[basic_vec[j]];
	    for(j=length;j<MAX_LENGTH;j++) vec[j]=NODIR;

            flag=0;
	    /* check if it's a new set: */
	    for(j=0;j<num_q_paths;j++){
	      flag = is_path_equal( vec, q_paths[j].dir, MAX_LENGTH );
	      if(flag==1)break;
	    }
	    if(flag == 0 ){
	      if(num_q_paths>=MAX_NUM){
		node0_printf("OOPS: MAX_NUM too small\n");
		exit(0);
	      }
	      q_paths[num_q_paths].length=length;
	      for(j=0;j<MAX_LENGTH;j++) q_paths[num_q_paths].dir[j]=vec[j];
		/* remember to copy NODIR's, or comparison will get confused */
	      if(ir[0]==0){
		q_paths[num_q_paths].coeff =  coeff;
		q_paths[num_q_paths].forwback =  +1;
	      }
	      else{
		q_paths[num_q_paths].coeff = -coeff;
		q_paths[num_q_paths].forwback = -1;
	      }
	      num_q_paths++;
	      path_num++;
	      /**node0_printf("ADD PATH %d:  rx=%d ",num_q_paths-1,ir[0]);
		 printpath( vec, length );**/
	    }

	  } /* end reflection*/
        } /* end permutation if block */
      } /* end permutation */
    num_basic_paths++;
    return(path_num);
} /* add_basic_path */

/* parallel transport a vector to the current site along a path.
   For example, if the path is "XUP", bring in the vector from
   the +X direction to the current site. If KS phases are in lattice,
   this transport will automatically include them. 
   OK for src and dest to be the same.  OK for length=0.  */
/* NOT OPTIMIZED at the moment - do lots of extra copying.  Use temp.
   vectors rather than stuff in lattice.h */
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    su3_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    su3_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_dest = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_work = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(su3_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_vec( &(s->link[dir[j]]),
		    (su3_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_vec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(su3_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = *(su3_vector *)F_PT(s,src);
    }
  }
} /* path_transport */

/* Path transport a half_wilson_vector */
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    half_wilson_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    half_wilson_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (half_wilson_vector *)calloc(
      sites_on_node, sizeof(half_wilson_vector));
    tmp_dest = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));
    tmp_work = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(half_wilson_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_hwvec( &(s->link[dir[j]]),
		    (half_wilson_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_hwvec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(half_wilson_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) =
	  *(half_wilson_vector *)F_PT(s,src);
    }
  }
} /* path_transport_hwv */


#ifdef EO
/* Stupid dslash routine that follows all the paths.
  Should optimize by precomputing sums of paths to all the displacements
  Use dslash_fn for actions that involve only +X and +X+X+X couplings.
*/
/* D_slash routine - sets dest. on each site equal to sum of
   sources parallel transported to site, with minus sign for transport
   from negative directions.  */
void dslash_eo( field_offset src, field_offset dest, int parity ) {
   register int i;
   register site *s;
   register int ipath,otherparity;
   register double x; /* coefficient of path */

    switch(parity){
	case EVEN:	otherparity=ODD; break;
	case ODD:	otherparity=EVEN; break;
	case EVENANDODD:	otherparity=EVENANDODD; break;
    }

    /* Parallel transport by all the paths in the action.  
       Multiply by coefficient in table
    */
    FORSOMEPARITY(i,s,parity){ clearvec( (su3_vector *)F_PT(s,dest) ); }

    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	    path_transport( src, F_OFFSET(tempvec[0]), parity,
		q_paths[ipath].dir, q_paths[ipath].length );
	    x=q_paths[ipath].coeff;
	    FORSOMEPARITY(i,s,parity){
		scalar_mult_add_su3_vector(  (su3_vector *)F_PT(s,dest),
		    &(s->tempvec[0]), x, (su3_vector *)F_PT(s,dest) );
	    }
    }   /* ipath */
} /* dslash_eo */
#endif /* ifdef EO */

#ifdef FN
/* Sum over paths connecting to nearest neighbor point (fat link) and to third
   nearest neighbor (longlinks) */
/* Doug Toussaint 2/4/98 */
/* modified to use t_longlinks, S. Gottlieb 7/13/01 */
/* long link calculating routine */
/* path_product() follows the path starting at step 0, and
   leaves the answer at the end of the path.  We want the answer
   at the site where the path begins.  So we look for paths with
   the opposite displacement from the displacement of the point
   that we want to transport to this site, and take the adjoint
   of the matrix at the end. clear? */
/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_longlinks() {
  register int i;
  register site *s;
  int ipath,dir;
  int disp[4];
  register su3_matrix *long1;

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_longlinks needs phases in\n");
    terminate(0);
  }
  for (dir=XUP; dir<=TUP; dir++){ /* loop over longlink directions */
    /* set longlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      long1 = &(t_longlink[4*i+dir]);
#else
      long1 = &(s->longlink[dir]);
#endif
      clear_su3mat( long1 );
    }

    /* loop over paths, checking for ones with total displacement 3*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=3,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("ipath = %d, found a path:  ",ipath);
for(j=0;j<q_paths[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  long1 = &(t_longlink[4*i+dir]);
#else
	  long1 = &(s->longlink[dir]);
#endif
          scalar_mult_add_su3_matrix( long1,
	    &(s->staple), -q_paths[ipath].coeff, long1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */

  } /* loop over directions */

  valid_longlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Long):  %e\n",dtime);
#endif
}  /* load_longlinks() */

/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_fatlinks() {
  register int i;
  register site *s;
  int dir;
  register su3_matrix *fat1;
#ifdef ASQ_OPTIMIZED_FATTENING
  int  nu,rho,sig ;
  double one_link ; /*needed to fix the problem with the Lepage term */
#else
  int ipath;
  int disp[4];
#endif

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_fatlinks needs phases in\n");
    terminate(0);
  }

#ifndef  ASQ_OPTIMIZED_FATTENING   /* general case code */
  for (dir=XUP; dir<=TUP; dir++){ /* loop over fatlink directions */
    /* set fatlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+dir]);
#else
      fat1 = &(s->fatlink[dir]);
#endif
      clear_su3mat( fat1 );
    }
    
    /* loop over paths, checking for ones with total displacement 1*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=1,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("dir = %d, found a path:  ",dir);
for(j=0;j<q_paths.[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  fat1 = &(t_fatlink[4*i+dir]);
#else
	  fat1 = &(s->fatlink[dir]);
#endif
          scalar_mult_add_su3_matrix( fat1,
	    &(s->staple), -q_paths[ipath].coeff, fat1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */
  } /* loop over directions */
#else	/* ASQ_OPTIMIZED_FATTENING, for Asq and Asqtad actions */
/*  Optimized fattening code for the Asq and Asqtad actions.           *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
 
 /* to fix up the Lepage term, included by a trick below */
 one_link = (act_path_coeff[0] - 6.0*act_path_coeff[5]) ; 
 
 for (dir=XUP; dir<=TUP; dir++){
   FORALLSITES(i,s) /* Intialize fat links with c_1*U_\mu(x) */
     {
#ifdef DSLASH_TMP_LINKS
       fat1 = &(t_fatlink[4*i+dir]);
#else
       fat1 = &(s->fatlink[dir]);
#endif
       scalar_mult_su3_matrix(&(s->link[dir]), one_link ,
			      fat1 );
     }
   for(nu=XUP; nu<=TUP; nu++) if(nu!=dir)
     {
       compute_gen_staple(F_OFFSET(staple),dir,nu,F_OFFSET(link[dir]),
			  act_path_coeff[2]);
       /* The Lepage term */
       /* Note this also involves modifying c_1 (above) */
       compute_gen_staple(NULL_FP,dir,nu,F_OFFSET(staple),act_path_coeff[5]);
       for(rho=XUP; rho<=TUP; rho++) if((rho!=dir)&&(rho!=nu))
	 {
	   compute_gen_staple(F_OFFSET(tempmat1),dir,rho,F_OFFSET(staple),
			      act_path_coeff[3]);
	   for(sig=XUP; sig<=TUP; sig++)
	     if((sig!=dir)&&(sig!=nu)&&(sig!=rho))
	       {
		 compute_gen_staple(NULL_FP,dir,sig,
				    F_OFFSET(tempmat1),
				    act_path_coeff[4]);
	       } /* sig */
	 } /* rho */
     } /* nu */
 }/* dir */  
#endif

  valid_fatlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Fat):  %e\n",dtime);
#endif
}  /* load_fatlinks() */
#endif /* ifdef FN */


/* compare two paths, return 1 if equal, else zero */
int is_path_equal( int *path1, int* path2, int length ){
   register int i;
   for(i=0;i<length;i++)if(path1[i]!=path2[i])return(0);
   return(1);
}


/* update the  momenta with the fermion force */
/* Assumes that the conjugate gradient has been run, with the answer in
   x_off, and dslash(x_off,x_off,ODD) has been run. (fills in x_off_odd) */
/* SEE LONG COMMENTS AT END */

#ifndef ASQ_OPTIMIZED_FORCE
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon, coeff;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  half_wilson_vector *hw_tmp0,*hw_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  hw_tmp0 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  hw_tmp1 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  /* Use half_wilson_vectors to store x_off transported from ends of
     path.  0 component from forward end, 1 component from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    path_transport( x_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[1] for transport from starting end */
    FORALLSITES(i,s){
      hw_tmp0[i].h[0]=s->tempvec[0];
      hw_tmp0[i].h[1]=*(su3_vector *)F_PT(s,x_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff = ferm_epsilon*q_paths[ipath].coeff;
      if( (ilink%2)==1 )coeff = -coeff;

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "half_wilson_vector" to handle pair of vectors -
	0 component is x_off from forward end, 1 component from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_su3_mat_hwvec( &(s->link[lastdir]),
	      &(hw_tmp0[i]), &(hw_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( hw_tmp1, 2*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     hw_tmp0[i] = *(half_wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( hw_tmp0, 2*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_su3_mat_hwvec( &(s->link[OPP_DIR(lastdir)]),
                (half_wilson_vector *)(gen_pt[0][i]),
		&(hw_tmp1[i]) );
          }
	  tmp_pt = hw_tmp0; hw_tmp0 = hw_tmp1; hw_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( hw_tmp0 ); free( hw_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */


void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off,
  int nflav2, field_offset x2_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/14/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon1, ferm_epsilon2, coeff1, coeff2;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  wilson_vector *w_tmp0,*w_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon1 = 2.0*(nflav1/4.0)*eps;
  ferm_epsilon2 = 2.0*(nflav2/4.0)*eps;
  w_tmp0 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  w_tmp1 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  /* Use wilson_vectors to store x_off transported from ends of
     path.  0 and 1 components from forward end, 2 and 3 components
     from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    /**
    path_transport( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    path_transport( x2_off, F_OFFSET(tempvec[1]),
      EVENANDODD, q_paths[ipath].dir, length );
    **/
/** WARNING!! Assumes xxx1 and xxx2 contiguous **/
if( x2_off-x1_off != sizeof(su3_vector) ){node0_printf("BOTCH\n"); exit(0);}
    path_transport_hwv( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[2] for transport from starting end */
    FORALLSITES(i,s){
      w_tmp0[i].d[0]=s->tempvec[0];
      w_tmp0[i].d[1]=s->tempvec[1];
      w_tmp0[i].d[2]=*(su3_vector *)F_PT(s,x1_off);
      w_tmp0[i].d[3]=*(su3_vector *)F_PT(s,x2_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff1 = ferm_epsilon1*q_paths[ipath].coeff;
      coeff2 = ferm_epsilon2*q_paths[ipath].coeff;
      if( (ilink%2)==1 ){ coeff1 = -coeff1; coeff2 = -coeff2;}

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "wilson_vector" to handle pair of vectors -
	0 component is x_off1 from forward end, 1 component is x_off2
	from forward end,  2 and 3  components are x_off1 and x_off2 
	from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_mat_wilson_vec( &(s->link[lastdir]),
	      &(w_tmp0[i]), &(w_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( w_tmp1, 4*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     w_tmp0[i] = *(wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( w_tmp0, 4*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_mat_wilson_vec( &(s->link[OPP_DIR(lastdir)]),
                (wilson_vector *)(gen_pt[0][i]),
		&(w_tmp1[i]) );
          }
	  tmp_pt = w_tmp0; w_tmp0 = w_tmp1; w_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( w_tmp0 ); free( w_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */



#else /* ASQ_OPTIMIZED_FORCE, for Asq and Asqtad actions */
/* Optimized force code for the Asq and Asqtad actions                 *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
#define Pmu          tempvec[0] 
#define Pnumu        tempvec[1]
#define Prhonumu     tempvec[2]
#define P7           tempvec[3]
#define P7rho        tempvec[4]              
#define P7rhonu      tempvec[5]
#define P5           tempvec[6]
#define P3           tempvec[7]
#define P5nu         tempvec[3]
#define P3mu         tempvec[3]
#define Popmu        tempvec[4]
#define Pmumumu      tempvec[4]
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double ferm_epsilon, coeff;
  double OneLink, Lepage, Naik, FiveSt, ThreeSt, SevenSt ;
  su3_vector *tempvec[8] ;
  su3_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  
  /* Path coefficients times fermion epsilon */
  OneLink = act_path_coeff[0]*ferm_epsilon ; 
  Naik    = act_path_coeff[1]*ferm_epsilon ;
  ThreeSt = act_path_coeff[2]*ferm_epsilon ;
  FiveSt  = act_path_coeff[3]*ferm_epsilon ;
  SevenSt = act_path_coeff[4]*ferm_epsilon ;
  Lepage  = act_path_coeff[5]*ferm_epsilon ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    tempvec[mu] = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

  /*copy x_off to a temporary vector */
  temp_x = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
  FORALLSITES(i,s) temp_x[i] = *(su3_vector *)F_PT(s,x_off) ;

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_force_to_mom(P3, Pmu, sig, -ThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_force_to_mom(P7, Prhonumu, sig, -SevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_fermion(P7, P7rho, rho);
		  side_link_force(rho,sig,SevenSt, Pnumu, P7, Prhonumu, P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff = SevenSt/FiveSt ; 
		  FORALLSITES(i,s)
		    scalar_mult_add_su3_vector(&(P5[i]),&(P7rho[i]),coeff,
					       &(P5[i]));
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_fermion(P5,P5nu, nu);
	      side_link_force(nu,sig,-FiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff = FiveSt/ThreeSt ; 
	      FORALLSITES(i,s)
		scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_fermion(P5,P5nu, mu);
	  side_link_force(mu, sig, -Lepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff = Lepage/ThreeSt ; 
	  FORALLSITES(i,s)
	    scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), -Naik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(tempvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      

#define Pmu          hwvec[0] 
#define Pnumu        hwvec[1]
#define Prhonumu     hwvec[2]
#define P7           hwvec[3]
#define P7rho        hwvec[4]              
#define P7rhonu      hwvec[5]
#define P5           hwvec[6]
#define P3           hwvec[7]
#define P5nu         hwvec[3]
#define P3mu         hwvec[3]
#define Popmu        hwvec[4]
#define Pmumumu      hwvec[4]
void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off, 
			  int nflav2, field_offset x2_off){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/15/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double coeff[2],ferm_epsilon ;
  double OneLink[2], Lepage[2], Naik[2], FiveSt[2], ThreeSt[2], SevenSt[2] ;
  double mNaik[2], mLepage[2], mFiveSt[2], mThreeSt[2], mSevenSt[2] ;
  half_wilson_vector *hwvec[8] ;
  half_wilson_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  
  /* Path coefficients times fermion epsilon */
  ferm_epsilon = 2.0*(nflav1/4.0)*eps;
  OneLink[0] = act_path_coeff[0]*ferm_epsilon ;
  Naik[0]    = act_path_coeff[1]*ferm_epsilon ; mNaik[0]    = -Naik[0] ;
  ThreeSt[0] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[0] = -ThreeSt[0] ;
  FiveSt[0]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[0]  = -FiveSt[0] ;
  SevenSt[0] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[0] = -SevenSt[0] ;
  Lepage[0]  = act_path_coeff[5]*ferm_epsilon ; mLepage[0]  = -Lepage[0] ;

  ferm_epsilon = 2.0*(nflav2/4.0)*eps;
  OneLink[1] = act_path_coeff[0]*ferm_epsilon ;
  Naik[1]    = act_path_coeff[1]*ferm_epsilon ; mNaik[1]    = -Naik[1] ;
  ThreeSt[1] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[1] = -ThreeSt[1] ;
  FiveSt[1]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[1]  = -FiveSt[1] ;
  SevenSt[1] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[1] = -SevenSt[1] ;
  Lepage[1]  = act_path_coeff[5]*ferm_epsilon ; mLepage[1]  = -Lepage[1] ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    hwvec[mu]= 
      (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));

  /*copy x_off to a temporary vector */
  temp_x= 
    (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
  FORALLSITES(i,s)
    {
      temp_x[i].h[0] = *(su3_vector *)F_PT(s,x1_off) ;
      temp_x[i].h[1] = *(su3_vector *)F_PT(s,x2_off) ;
    }

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_hw_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_3f_force_to_mom(P3, Pmu, sig, mThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_hw_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_3f_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_hw_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_hw_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_3f_force_to_mom(P7, Prhonumu, sig, mSevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_hw_fermion(P7, P7rho, rho);
		  side_link_3f_force(rho,sig,SevenSt,Pnumu,P7,Prhonumu,P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff[0] = SevenSt[0]/FiveSt[0] ; 
		  coeff[1] = SevenSt[1]/FiveSt[1] ; 
		  FORALLSITES(i,s)
		    {
		      scalar_mult_add_su3_vector(&(P5[i].h[0]),
						 &(P7rho[i].h[0]),coeff[0],
						 &(P5[i].h[0]));
		      scalar_mult_add_su3_vector(&(P5[i].h[1]),
						 &(P7rho[i].h[1]),coeff[1],
						 &(P5[i].h[1]));
		    }
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_hw_fermion(P5,P5nu, nu);
	      side_link_3f_force(nu,sig,mFiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff[0] = FiveSt[0]/ThreeSt[0] ; 
	      coeff[1] = FiveSt[1]/ThreeSt[1] ; 
	      FORALLSITES(i,s)
		{
		  scalar_mult_add_su3_vector(&(P3[i].h[0]),
					     &(P5nu[i].h[0]), coeff[0],
					     &(P3[i].h[0]));
		  scalar_mult_add_su3_vector(&(P3[i].h[1]),
					     &(P5nu[i].h[1]), coeff[1],
					     &(P3[i].h[1]));
		}
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_3f_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_hw_fermion(P5,P5nu, mu);
	  side_link_3f_force(mu, sig, mLepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff[0] = Lepage[0]/ThreeSt[0] ; 
	  coeff[1] = Lepage[1]/ThreeSt[1] ; 
	  FORALLSITES(i,s)
	    {
	      scalar_mult_add_su3_vector(&(P3[i].h[0]),
					 &(P5nu[i].h[0]),coeff[0],
					 &(P3[i].h[0]));
	      scalar_mult_add_su3_vector(&(P3[i].h[1]),
					 &(P5nu[i].h[1]),coeff[1],
					 &(P3[i].h[1]));
	    }

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_hw_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_3f_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_3f_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_3f_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), mNaik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_hw_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_3f_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_3f_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(hwvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      
#endif /* ASQ_OPTIMIZED_FORCE */


#ifdef  ASQ_OPTIMIZED_FATTENING   /* Asqtad action only, "_fn" executables */
#ifndef FN
BOMB THE COMPILE
#endif
void compute_gen_staple(field_offset staple, int mu, int nu, 
			field_offset link, double coef) {
  su3_matrix tmat1,tmat2;
  msg_tag *mtag0,*mtag1;
  su3_matrix *tempmat ;
  register site *s ;
  register int i ;
  register su3_matrix *fat1;

  /* Computes the staple :
                   mu
                +-------+
          nu	|	|
		|	|
		X	X
    Where the mu link can be any su3_matrix. The result is saved in staple.
    if staple==NULL_FP then the result is not saved.
    It also adds the computed staple to the fatlink[mu] with weight coef.
  */

  /* Upper staple */
  mtag0 = start_gather( link, sizeof(su3_matrix), nu, EVENANDODD, gen_pt[0] );
  mtag1 = start_gather( F_OFFSET(link[nu]), sizeof(su3_matrix), mu, 
			EVENANDODD, gen_pt[1] );
  wait_gather(mtag0);
  wait_gather(mtag1);
  
  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, (su3_matrix *)F_PT(s,staple) );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, &tmat2 );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix(fat1, &tmat2, coef,
				 fat1) ;
    }
  }
  cleanup_gather(mtag0);
  cleanup_gather(mtag1);

  /* lower staple */
  tempmat = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
  mtag0 = start_gather( F_OFFSET(link[nu]),
			sizeof(su3_matrix), mu, EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);
  FORALLSITES(i,s){
    mult_su3_an( &(s->link[nu]),(su3_matrix *)F_PT(s,link), &tmat1 );
    mult_su3_nn( &(tmat1),(su3_matrix *)gen_pt[0][i], &(tempmat[i]) );
  }
  cleanup_gather(mtag0);
  mtag0 = start_gather_from_temp( tempmat, sizeof(su3_matrix),
				  OPP_DIR(nu), EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);

  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      add_su3_matrix( (su3_matrix *)F_PT(s,staple),(su3_matrix *)gen_pt[0][i], 
		      (su3_matrix *)F_PT(s,staple) );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)F_PT(s,staple), coef, 
				 fat1 );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)gen_pt[0][i], coef, 
				 fat1 );
    }
  }

  free(tempmat);
  cleanup_gather(mtag0);
}
#endif  /* ASQ_OPTIMIZED_FATTENING   */


#ifdef  ASQ_OPTIMIZED_FORCE
/*   Covariant shift of the src fermion field in the direction dir  *
 *  by one unit. The result is stored in dest.                       */ 
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) {
  su3_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(su3_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_vec(&(s->link[dir]),(su3_vector *)(gen_pt[0][i]),
			 &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_vec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(su3_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(su3_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/*  Covariant shift of the src half wilson fermion field in the  *
 * direction dir   by one unit. The result is stored in dest.    */
void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) {
  half_wilson_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(half_wilson_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_hwvec(&(s->link[dir]),
			   (half_wilson_vector *)(gen_pt[0][i]), &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = 
	(half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_hwvec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(half_wilson_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(half_wilson_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/* Add in contribution to the force */
/* Put antihermitian traceless part into momentum */
void add_force_to_mom(su3_vector *back,su3_vector *forw,int dir,double coeff) {
  register site *s ;
  register int i ;  
  register double tmp_coeff ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff = -coeff ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD) 
      tmp_coeff = -coeff ;
    else
      tmp_coeff = coeff ;
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i]), &(forw[i]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff, &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/* Add in contribution to the force ( 3flavor case ) */
/* Put antihermitian traceless part into momentum */
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) {
  register site *s ;
  register int i ;  
  double tmp_coeff[2] ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff[0] = -coeff[0] ;
      coeff[1] = -coeff[1] ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD)
      {
	tmp_coeff[0] = -coeff[0] ;
	tmp_coeff[1] = -coeff[1] ;
      }
    else
      {
	tmp_coeff[0] = coeff[0] ;
	tmp_coeff[1] = coeff[1] ;
      }
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i].h[0]), &(forw[i].h[0]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[0], &tmat2 );
    su3_projector(&(back[i].h[1]), &(forw[i].h[1]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[1], &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/*  This routine is needed in order to add the force on the side link *
 * of the paths in the Asq and Asqtad actions. It gets as inputs the  *
 * direction mu of the side link and the direction nu of the Dslash   *
 * term we are dealing with. Then it takes also 4 fermion fields:     *
 * Path: the piece of the path with no hop in the nu or mu direction  *
 * Path_nu: the piece of the path with a hop in nu  but not in mu     *
 * Path_mu: is Path times the link mu                                 *
 * Path_numu: is Path_nu times the link mu                            */
void side_link_force(int mu, int nu, double coeff, 
		     su3_vector *Path   , su3_vector *Path_nu, 
		     su3_vector *Path_mu, su3_vector *Path_numu) {
  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_force_to_mom(Path, Path_numu, OPP_DIR(mu), -coeff );/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_nu, Path_mu, mu, -coeff) ; /* ? extra - */
      else
	add_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
 
/*  The 3 flavor version of side_link_force used *
 * to optimize fermion transports                */
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) {
  double m_coeff[2] ;

  m_coeff[0] = -coeff[0] ;
  m_coeff[1] = -coeff[1] ;

  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_3f_force_to_mom(Path,Path_numu,OPP_DIR(mu),m_coeff);/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_nu, Path_mu, mu, m_coeff) ; /* ? extra - */
      else
	add_3f_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
#endif  /* ASQ_OPTIMIZED_FORCE   */

/* LONG COMMENTS
   Here we have combined "xxx", (offset "x_off")  which is
(M_adjoint M)^{-1} phi, with Dslash times this vector, which goes in the
odd sites of xxx.  Recall that phi is defined only on even sites.  In
computing the fermion force, we are looking at

< X |  d/dt ( Dslash_eo Dslash_oe ) | X >
=
< X | d/dt Dslash_eo | T > + < T | d/dt Dslash_oe | X >
where T = Dslash X.

The subsequent manipulations to get the coefficent of H, the momentum
matrix, in the simulation time derivative above look the same for
the two terms, except for a minus sign at the end, if we simply stick
T, which lives on odd sites, into the odd sites of X

 Each path in the action contributes terms when any link of the path
is the link for which we are computing the force.  We get a minus sign
for odd numbered links in the path, since they connect sites of the
opposite parity from what it would be for an even numbered link.
Minus signs from "going around" plaquette - ie KS phases, are supposed
to be already encoded in the path coefficients.
Minus signs from paths that go backwards are supposed to be already
encoded in the path coefficients.

Here, for example, are comments reproduced from the force routine for
the one-link plus Naik plus single-staple-fat-link action:

 The three link force has three contributions, where the link that
was differentiated is the first, second, or third link in the 3-link
path, respectively.  Diagramatically, where "O" represents the momentum,
the solid line the link corresponding to the momentum, and the dashed
lines the other links:
 

	O______________ x ............ x ...............
+
	x..............O______________x.................
+
	x..............x..............O________________
Think of this as
	< xxx | O | UUUxxx >		(  xxx, UUUX_p3 )
+
	< xxx U | O | UUxxx >		( X_m1U , UUX_p2 )
+
	< xxx U U | O | Uxxx >		( X_m2UU , UX_p1 )
where "U" indicates parallel transport, "X_p3" is xxx displaced
by +3, etc.
Note the second contribution has a relative minus sign
because it effectively contributes to the <odd|even>, or M_adjoint,
part of the force when we work on an even site. i.e., for M on
an even site, this three link path begins on an odd site.

The staple force has six contributions from each plane containing the
link direction:
Call these diagrams A-F:


	x...........x		O____________x
		    .			     .
		    .			     .
		    .			     .
		    .			     .
		    .			     .
	O___________x		x............x
	   (A)			    (B)



	x	    x		O____________x
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	O___________x		x	     x
	   (C)			    (D)



	x...........x		O____________x
	.			.
	.			.
	.			.
	.			.
	.			.
	O___________x		x............x
	   (E)			    (F)

As with the Naik term, diagrams C and D have a relative minus
sign because they connect sites of the other parity.

Also note an overall minus sign in the staple terms relative to the
one link term because, with the KS phase factors included, the fat
link is  "U - w3 * UUU", or the straight link MINUS w3 times the staples.

Finally, diagrams B and E get one more minus sign because the link
we are differentiating is in the opposite direction from the staple
as a whole.  You can think of this as this "U" being a correction to
a "U_adjoint", but the derivative of U is iHU and the derivative
of U_adjoint is -iHU_adjoint.

*/
/* LONG COMMENT on sign conventions
In most of the program, the KS phases and antiperiodic boundary
conditions are absorbed into the link matrices.  This greatly simplfies
multiplying by the fermion matrix.  However, it requires care in
specifying the path coefficients.  Remember that each time you
encircle a plaquette, you pick up a net minus sign from the KS phases.
Thus, when you have more than one path to the same point, you generally
have a relative minus sign for each plaquette in a surface bounded by
this path and the basic path for that displacement.

Examples:
  Fat Link:
    Positive:	X-------X

    Negative     --------
	 	|	|
		|	|
		X	X

  Naik connection, smeared
    Positive:	X-------x-------x-------X

    Negative:	---------
		|	|
		|	|
		X	x-------x-------X

    Positive:	--------x--------
		|		|
		|		|
		X		x-------X

    Negative:	--------x-------x-------x
		|			|
		|			|
		X			X
*/



/* Comment on acceptable actions.
   We construct the backwards part of dslash by reversing all the
   paths in the forwards part.  So, for example, in the p4 action
   the forwards part includes +X+Y+Y

		X
		|
		|
		X
		|
		|
	X---->--X

  so we put -X-Y-Y in the backwards part.  But this isn't the adjoint
  of U_x(0)U_y(+x)U_y(+x+y).  Since much of the code assumes that the
  backwards hop is the adjoint of the forwards (for example, in
  preventing going to 8 flavors), the code only works for actions
  where this is true.  Roughly, this means that the fat link must
  be symmetric about reflection around its midpoint.  Equivalently,
  the paths in the backwards part of Dslash are translations of the
  paths in the forwards part.  In the case of the "P4" or knight's move
  action, this means that we have to have both paths
   +X+Y+Y and +Y+Y+X to the same point, with the same coefficients.
  Alternatively, we could just use the symmetric path +Y+X+Y.
*/

invivoquark_stuff_compute_gen_staple_1396

Real time: 9.268518e+08 -- Predicted time: 0.000000e+00

Callcount: 1584

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
289	### 1	21.1714937861594	0.000000e+00	6.817040e+05	100.0
959	### 2	999.384637127631	0.000000e+00	5.802040e+05	100.0
1459	### 3	565.355245790587	0.000000e+00	5.727640e+05	100.0

                /****** quark_stuff.c  -- ******************/
/* MIMD version 6 */
/* quark action stuff for improved action
* D.T. 1/28/98, starting from gauge_stuff.c
* K.O. 3/99 Added optimized fattening for Asq actions
* D.T. 4/99 Combine force calculations for both mass quarks
* K.O. 4/99 Optimized force for Asq action
* S.G. 7/01, modified to use t_longlink and t_fatlink
* C.D. 10/02, consolidated quark_stuff.c and quark_stuff_tmp.c
*
* This code combines quark_stuff.c and quark_stuff_tmp.c
* with DSLASH_TMP_LINKS defined, puts links in field-major variables
* t_longlink and t_fatlink. Otherwise, puts them in the site structure.
*
* In this directory, assume all paths connect even to odd sites, etc.
* Tabulate "backwards" paths (e.g. "XDOWN" is backward path to "XUP")
* as separate parity transforms of the fundamental paths.  They will
* generally need a negative sign in Dslash.  See bottom for a long
* comment on sign conventions.
*/

/**#define FFTIME**/
/**#define LLTIME**/

#include "generic_ks_includes.h"	/* definitions files and prototypes */


#define NULL_FP -1 /* NULL field_offset to be used in the optimized version *
                    * of the load_fatlinks subroutine */

    /* Specify paths in orientation in which they appear in the
       forward part of the x component of dslash().  Rotations and
       reflections will be automatically included. Be careful
       about signs of coefficients.  See long comment at bottom. */
#include <quark_action.h>
    /* Include file specifies the basic paths */

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void printpath( int *path, int length );
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length );
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length );
#ifdef  ASQ_OPTIMIZED_FATTENING
void compute_gen_staple(field_offset staple, int mu, int nu,
                        field_offset link, double coef ) ;
#endif
#ifdef  ASQ_OPTIMIZED_FORCE
#ifndef FN
BOMB THE COMPILE
#endif
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) ;
void add_force_to_mom(su3_vector *back, su3_vector *forw, int dir, double coef);
void side_link_force(int mu, int nu, double coeff, su3_vector *Path,
		     su3_vector *Path_nu, su3_vector *Path_mu, 
		     su3_vector *Path_numu) ;

void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) ;
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) ;
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) ;
#endif

int path_num[MAX_BASIC_PATHS];	/* number of rotations/reflections for each 
					kind */
static double act_path_coeff[MAX_BASIC_PATHS]; /* actual path coefficient     *
                                               * it is equal to path_coeff   *
                                               * if not tadpole improvement  *
                                               * is specified                *
                                               * or path_coeff*u_0^(L-1) when*
                                               * tadpole improvement is      *
                                               * specified                   */

/* Array of structures, for each rotation and reflection of each kind of
	path.  */
struct {
    int dir[MAX_LENGTH];	/* directions in path */
    int length;		/* length of path */
    double coeff;	/* coefficient, including minus sign if backwards */
    double forwback;	/* +1 if in forward Dslash, -1 if in backward */
}   q_paths[MAX_NUM];
int num_q_paths;	/* number of paths in dslash */
int num_basic_paths;	/* number of paths before rotation/reflection */

int is_path_equal( int *path1, int* path2, int length );
int add_basic_path( int *vec, int length, double coeff );

/* Make table of paths in action */
void make_path_table() {

    int i,j;
#ifdef TADPOLE_IMPROVE
    int k;
#endif

    /* table of directions, 1 for each kind of path */
    /**int path_ind[MAX_BASIC_PATHS][MAX_LENGTH];**/
    /* table of coefficients in action, for each path */

    node0_printf("%s\n",quark_action_description);
    num_q_paths = 0;
    num_basic_paths = 0;


    /* add rots. and reflects to table, print out the path coefficients */
    node0_printf("path coefficients: npath  path_coeff  multiplicity\n");
    for(j=0;j<quark_action_npaths;j++) {
	double this_coeff;
	this_coeff = path_coeff[j];
#ifdef TADPOLE_IMPROVE
	for(k=1;k< path_length_in[j];k++)this_coeff /= u0;
#endif
	act_path_coeff[j] = this_coeff ;
	i = add_basic_path( path_ind[j], path_length_in[j],
	    this_coeff );
	node0_printf("                    %d      %e     %d\n",
	    j,this_coeff,i);
    }
}

/* add rotations and reflections of a path to the table.  Return
   multiplicity of paths added */
int add_basic_path( int *basic_vec, int length, double coeff ) {

    int perm[8],pp[8],ir[4];
    int j,path_num;
    int vec[MAX_LENGTH];
    int flag;

    path_num = 0;
    /* now fill the long table with all rotations and reflections
	of the fundamental path.  The path presented to us is for
        the positive x component of dslash, so if the x coordinate
        is reflected it will appear with a negative sign. */
      /* permutations */
      for(perm[0]=0;perm[0]<4;perm[0]++)
      for(perm[1]=0;perm[1]<4;perm[1]++)
      for(perm[2]=0;perm[2]<4;perm[2]++)
      for(perm[3]=0;perm[3]<4;perm[3]++){
	if(perm[0] != perm[1] && perm[0] != perm[2] 
	  && perm[0] != perm[3] && perm[1] != perm[2]
	  && perm[1] != perm[3] && perm[2] != perm[3] ) {
	  /* reflections*/
	  for(ir[0]=0;ir[0]<2;ir[0]++)
	  for(ir[1]=0;ir[1]<2;ir[1]++)
	  for(ir[2]=0;ir[2]<2;ir[2]++)
	  for(ir[3]=0;ir[3]<2;ir[3]++){
	    for(j=0;j<4;j++){
	      pp[j]=perm[j];

	      if(ir[j] == 1) pp[j]=OPP_DIR(pp[j]);
	      pp[OPP_DIR(j)]=OPP_DIR(pp[j]);
	    }
	    /* create new vector*/
	    for(j=0;j<length;j++) vec[j]=pp[basic_vec[j]];
	    for(j=length;j<MAX_LENGTH;j++) vec[j]=NODIR;

            flag=0;
	    /* check if it's a new set: */
	    for(j=0;j<num_q_paths;j++){
	      flag = is_path_equal( vec, q_paths[j].dir, MAX_LENGTH );
	      if(flag==1)break;
	    }
	    if(flag == 0 ){
	      if(num_q_paths>=MAX_NUM){
		node0_printf("OOPS: MAX_NUM too small\n");
		exit(0);
	      }
	      q_paths[num_q_paths].length=length;
	      for(j=0;j<MAX_LENGTH;j++) q_paths[num_q_paths].dir[j]=vec[j];
		/* remember to copy NODIR's, or comparison will get confused */
	      if(ir[0]==0){
		q_paths[num_q_paths].coeff =  coeff;
		q_paths[num_q_paths].forwback =  +1;
	      }
	      else{
		q_paths[num_q_paths].coeff = -coeff;
		q_paths[num_q_paths].forwback = -1;
	      }
	      num_q_paths++;
	      path_num++;
	      /**node0_printf("ADD PATH %d:  rx=%d ",num_q_paths-1,ir[0]);
		 printpath( vec, length );**/
	    }

	  } /* end reflection*/
        } /* end permutation if block */
      } /* end permutation */
    num_basic_paths++;
    return(path_num);
} /* add_basic_path */

/* parallel transport a vector to the current site along a path.
   For example, if the path is "XUP", bring in the vector from
   the +X direction to the current site. If KS phases are in lattice,
   this transport will automatically include them. 
   OK for src and dest to be the same.  OK for length=0.  */
/* NOT OPTIMIZED at the moment - do lots of extra copying.  Use temp.
   vectors rather than stuff in lattice.h */
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    su3_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    su3_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_dest = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_work = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(su3_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_vec( &(s->link[dir[j]]),
		    (su3_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_vec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(su3_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = *(su3_vector *)F_PT(s,src);
    }
  }
} /* path_transport */

/* Path transport a half_wilson_vector */
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    half_wilson_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    half_wilson_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (half_wilson_vector *)calloc(
      sites_on_node, sizeof(half_wilson_vector));
    tmp_dest = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));
    tmp_work = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(half_wilson_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_hwvec( &(s->link[dir[j]]),
		    (half_wilson_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_hwvec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(half_wilson_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) =
	  *(half_wilson_vector *)F_PT(s,src);
    }
  }
} /* path_transport_hwv */


#ifdef EO
/* Stupid dslash routine that follows all the paths.
  Should optimize by precomputing sums of paths to all the displacements
  Use dslash_fn for actions that involve only +X and +X+X+X couplings.
*/
/* D_slash routine - sets dest. on each site equal to sum of
   sources parallel transported to site, with minus sign for transport
   from negative directions.  */
void dslash_eo( field_offset src, field_offset dest, int parity ) {
   register int i;
   register site *s;
   register int ipath,otherparity;
   register double x; /* coefficient of path */

    switch(parity){
	case EVEN:	otherparity=ODD; break;
	case ODD:	otherparity=EVEN; break;
	case EVENANDODD:	otherparity=EVENANDODD; break;
    }

    /* Parallel transport by all the paths in the action.  
       Multiply by coefficient in table
    */
    FORSOMEPARITY(i,s,parity){ clearvec( (su3_vector *)F_PT(s,dest) ); }

    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	    path_transport( src, F_OFFSET(tempvec[0]), parity,
		q_paths[ipath].dir, q_paths[ipath].length );
	    x=q_paths[ipath].coeff;
	    FORSOMEPARITY(i,s,parity){
		scalar_mult_add_su3_vector(  (su3_vector *)F_PT(s,dest),
		    &(s->tempvec[0]), x, (su3_vector *)F_PT(s,dest) );
	    }
    }   /* ipath */
} /* dslash_eo */
#endif /* ifdef EO */

#ifdef FN
/* Sum over paths connecting to nearest neighbor point (fat link) and to third
   nearest neighbor (longlinks) */
/* Doug Toussaint 2/4/98 */
/* modified to use t_longlinks, S. Gottlieb 7/13/01 */
/* long link calculating routine */
/* path_product() follows the path starting at step 0, and
   leaves the answer at the end of the path.  We want the answer
   at the site where the path begins.  So we look for paths with
   the opposite displacement from the displacement of the point
   that we want to transport to this site, and take the adjoint
   of the matrix at the end. clear? */
/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_longlinks() {
  register int i;
  register site *s;
  int ipath,dir;
  int disp[4];
  register su3_matrix *long1;

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_longlinks needs phases in\n");
    terminate(0);
  }
  for (dir=XUP; dir<=TUP; dir++){ /* loop over longlink directions */
    /* set longlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      long1 = &(t_longlink[4*i+dir]);
#else
      long1 = &(s->longlink[dir]);
#endif
      clear_su3mat( long1 );
    }

    /* loop over paths, checking for ones with total displacement 3*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=3,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("ipath = %d, found a path:  ",ipath);
for(j=0;j<q_paths[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  long1 = &(t_longlink[4*i+dir]);
#else
	  long1 = &(s->longlink[dir]);
#endif
          scalar_mult_add_su3_matrix( long1,
	    &(s->staple), -q_paths[ipath].coeff, long1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */

  } /* loop over directions */

  valid_longlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Long):  %e\n",dtime);
#endif
}  /* load_longlinks() */

/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_fatlinks() {
  register int i;
  register site *s;
  int dir;
  register su3_matrix *fat1;
#ifdef ASQ_OPTIMIZED_FATTENING
  int  nu,rho,sig ;
  double one_link ; /*needed to fix the problem with the Lepage term */
#else
  int ipath;
  int disp[4];
#endif

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_fatlinks needs phases in\n");
    terminate(0);
  }

#ifndef  ASQ_OPTIMIZED_FATTENING   /* general case code */
  for (dir=XUP; dir<=TUP; dir++){ /* loop over fatlink directions */
    /* set fatlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+dir]);
#else
      fat1 = &(s->fatlink[dir]);
#endif
      clear_su3mat( fat1 );
    }
    
    /* loop over paths, checking for ones with total displacement 1*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=1,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("dir = %d, found a path:  ",dir);
for(j=0;j<q_paths.[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  fat1 = &(t_fatlink[4*i+dir]);
#else
	  fat1 = &(s->fatlink[dir]);
#endif
          scalar_mult_add_su3_matrix( fat1,
	    &(s->staple), -q_paths[ipath].coeff, fat1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */
  } /* loop over directions */
#else	/* ASQ_OPTIMIZED_FATTENING, for Asq and Asqtad actions */
/*  Optimized fattening code for the Asq and Asqtad actions.           *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
 
 /* to fix up the Lepage term, included by a trick below */
 one_link = (act_path_coeff[0] - 6.0*act_path_coeff[5]) ; 
 
 for (dir=XUP; dir<=TUP; dir++){
   FORALLSITES(i,s) /* Intialize fat links with c_1*U_\mu(x) */
     {
#ifdef DSLASH_TMP_LINKS
       fat1 = &(t_fatlink[4*i+dir]);
#else
       fat1 = &(s->fatlink[dir]);
#endif
       scalar_mult_su3_matrix(&(s->link[dir]), one_link ,
			      fat1 );
     }
   for(nu=XUP; nu<=TUP; nu++) if(nu!=dir)
     {
       compute_gen_staple(F_OFFSET(staple),dir,nu,F_OFFSET(link[dir]),
			  act_path_coeff[2]);
       /* The Lepage term */
       /* Note this also involves modifying c_1 (above) */
       compute_gen_staple(NULL_FP,dir,nu,F_OFFSET(staple),act_path_coeff[5]);
       for(rho=XUP; rho<=TUP; rho++) if((rho!=dir)&&(rho!=nu))
	 {
	   compute_gen_staple(F_OFFSET(tempmat1),dir,rho,F_OFFSET(staple),
			      act_path_coeff[3]);
	   for(sig=XUP; sig<=TUP; sig++)
	     if((sig!=dir)&&(sig!=nu)&&(sig!=rho))
	       {
		 compute_gen_staple(NULL_FP,dir,sig,
				    F_OFFSET(tempmat1),
				    act_path_coeff[4]);
	       } /* sig */
	 } /* rho */
     } /* nu */
 }/* dir */  
#endif

  valid_fatlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Fat):  %e\n",dtime);
#endif
}  /* load_fatlinks() */
#endif /* ifdef FN */


/* compare two paths, return 1 if equal, else zero */
int is_path_equal( int *path1, int* path2, int length ){
   register int i;
   for(i=0;i<length;i++)if(path1[i]!=path2[i])return(0);
   return(1);
}


/* update the  momenta with the fermion force */
/* Assumes that the conjugate gradient has been run, with the answer in
   x_off, and dslash(x_off,x_off,ODD) has been run. (fills in x_off_odd) */
/* SEE LONG COMMENTS AT END */

#ifndef ASQ_OPTIMIZED_FORCE
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon, coeff;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  half_wilson_vector *hw_tmp0,*hw_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  hw_tmp0 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  hw_tmp1 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  /* Use half_wilson_vectors to store x_off transported from ends of
     path.  0 component from forward end, 1 component from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    path_transport( x_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[1] for transport from starting end */
    FORALLSITES(i,s){
      hw_tmp0[i].h[0]=s->tempvec[0];
      hw_tmp0[i].h[1]=*(su3_vector *)F_PT(s,x_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff = ferm_epsilon*q_paths[ipath].coeff;
      if( (ilink%2)==1 )coeff = -coeff;

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "half_wilson_vector" to handle pair of vectors -
	0 component is x_off from forward end, 1 component from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_su3_mat_hwvec( &(s->link[lastdir]),
	      &(hw_tmp0[i]), &(hw_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( hw_tmp1, 2*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     hw_tmp0[i] = *(half_wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( hw_tmp0, 2*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_su3_mat_hwvec( &(s->link[OPP_DIR(lastdir)]),
                (half_wilson_vector *)(gen_pt[0][i]),
		&(hw_tmp1[i]) );
          }
	  tmp_pt = hw_tmp0; hw_tmp0 = hw_tmp1; hw_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( hw_tmp0 ); free( hw_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */


void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off,
  int nflav2, field_offset x2_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/14/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon1, ferm_epsilon2, coeff1, coeff2;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  wilson_vector *w_tmp0,*w_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon1 = 2.0*(nflav1/4.0)*eps;
  ferm_epsilon2 = 2.0*(nflav2/4.0)*eps;
  w_tmp0 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  w_tmp1 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  /* Use wilson_vectors to store x_off transported from ends of
     path.  0 and 1 components from forward end, 2 and 3 components
     from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    /**
    path_transport( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    path_transport( x2_off, F_OFFSET(tempvec[1]),
      EVENANDODD, q_paths[ipath].dir, length );
    **/
/** WARNING!! Assumes xxx1 and xxx2 contiguous **/
if( x2_off-x1_off != sizeof(su3_vector) ){node0_printf("BOTCH\n"); exit(0);}
    path_transport_hwv( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[2] for transport from starting end */
    FORALLSITES(i,s){
      w_tmp0[i].d[0]=s->tempvec[0];
      w_tmp0[i].d[1]=s->tempvec[1];
      w_tmp0[i].d[2]=*(su3_vector *)F_PT(s,x1_off);
      w_tmp0[i].d[3]=*(su3_vector *)F_PT(s,x2_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff1 = ferm_epsilon1*q_paths[ipath].coeff;
      coeff2 = ferm_epsilon2*q_paths[ipath].coeff;
      if( (ilink%2)==1 ){ coeff1 = -coeff1; coeff2 = -coeff2;}

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "wilson_vector" to handle pair of vectors -
	0 component is x_off1 from forward end, 1 component is x_off2
	from forward end,  2 and 3  components are x_off1 and x_off2 
	from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_mat_wilson_vec( &(s->link[lastdir]),
	      &(w_tmp0[i]), &(w_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( w_tmp1, 4*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     w_tmp0[i] = *(wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( w_tmp0, 4*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_mat_wilson_vec( &(s->link[OPP_DIR(lastdir)]),
                (wilson_vector *)(gen_pt[0][i]),
		&(w_tmp1[i]) );
          }
	  tmp_pt = w_tmp0; w_tmp0 = w_tmp1; w_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( w_tmp0 ); free( w_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */



#else /* ASQ_OPTIMIZED_FORCE, for Asq and Asqtad actions */
/* Optimized force code for the Asq and Asqtad actions                 *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
#define Pmu          tempvec[0] 
#define Pnumu        tempvec[1]
#define Prhonumu     tempvec[2]
#define P7           tempvec[3]
#define P7rho        tempvec[4]              
#define P7rhonu      tempvec[5]
#define P5           tempvec[6]
#define P3           tempvec[7]
#define P5nu         tempvec[3]
#define P3mu         tempvec[3]
#define Popmu        tempvec[4]
#define Pmumumu      tempvec[4]
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double ferm_epsilon, coeff;
  double OneLink, Lepage, Naik, FiveSt, ThreeSt, SevenSt ;
  su3_vector *tempvec[8] ;
  su3_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  
  /* Path coefficients times fermion epsilon */
  OneLink = act_path_coeff[0]*ferm_epsilon ; 
  Naik    = act_path_coeff[1]*ferm_epsilon ;
  ThreeSt = act_path_coeff[2]*ferm_epsilon ;
  FiveSt  = act_path_coeff[3]*ferm_epsilon ;
  SevenSt = act_path_coeff[4]*ferm_epsilon ;
  Lepage  = act_path_coeff[5]*ferm_epsilon ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    tempvec[mu] = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

  /*copy x_off to a temporary vector */
  temp_x = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
  FORALLSITES(i,s) temp_x[i] = *(su3_vector *)F_PT(s,x_off) ;

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_force_to_mom(P3, Pmu, sig, -ThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_force_to_mom(P7, Prhonumu, sig, -SevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_fermion(P7, P7rho, rho);
		  side_link_force(rho,sig,SevenSt, Pnumu, P7, Prhonumu, P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff = SevenSt/FiveSt ; 
		  FORALLSITES(i,s)
		    scalar_mult_add_su3_vector(&(P5[i]),&(P7rho[i]),coeff,
					       &(P5[i]));
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_fermion(P5,P5nu, nu);
	      side_link_force(nu,sig,-FiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff = FiveSt/ThreeSt ; 
	      FORALLSITES(i,s)
		scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_fermion(P5,P5nu, mu);
	  side_link_force(mu, sig, -Lepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff = Lepage/ThreeSt ; 
	  FORALLSITES(i,s)
	    scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), -Naik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(tempvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      

#define Pmu          hwvec[0] 
#define Pnumu        hwvec[1]
#define Prhonumu     hwvec[2]
#define P7           hwvec[3]
#define P7rho        hwvec[4]              
#define P7rhonu      hwvec[5]
#define P5           hwvec[6]
#define P3           hwvec[7]
#define P5nu         hwvec[3]
#define P3mu         hwvec[3]
#define Popmu        hwvec[4]
#define Pmumumu      hwvec[4]
void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off, 
			  int nflav2, field_offset x2_off){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/15/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double coeff[2],ferm_epsilon ;
  double OneLink[2], Lepage[2], Naik[2], FiveSt[2], ThreeSt[2], SevenSt[2] ;
  double mNaik[2], mLepage[2], mFiveSt[2], mThreeSt[2], mSevenSt[2] ;
  half_wilson_vector *hwvec[8] ;
  half_wilson_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  
  /* Path coefficients times fermion epsilon */
  ferm_epsilon = 2.0*(nflav1/4.0)*eps;
  OneLink[0] = act_path_coeff[0]*ferm_epsilon ;
  Naik[0]    = act_path_coeff[1]*ferm_epsilon ; mNaik[0]    = -Naik[0] ;
  ThreeSt[0] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[0] = -ThreeSt[0] ;
  FiveSt[0]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[0]  = -FiveSt[0] ;
  SevenSt[0] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[0] = -SevenSt[0] ;
  Lepage[0]  = act_path_coeff[5]*ferm_epsilon ; mLepage[0]  = -Lepage[0] ;

  ferm_epsilon = 2.0*(nflav2/4.0)*eps;
  OneLink[1] = act_path_coeff[0]*ferm_epsilon ;
  Naik[1]    = act_path_coeff[1]*ferm_epsilon ; mNaik[1]    = -Naik[1] ;
  ThreeSt[1] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[1] = -ThreeSt[1] ;
  FiveSt[1]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[1]  = -FiveSt[1] ;
  SevenSt[1] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[1] = -SevenSt[1] ;
  Lepage[1]  = act_path_coeff[5]*ferm_epsilon ; mLepage[1]  = -Lepage[1] ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    hwvec[mu]= 
      (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));

  /*copy x_off to a temporary vector */
  temp_x= 
    (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
  FORALLSITES(i,s)
    {
      temp_x[i].h[0] = *(su3_vector *)F_PT(s,x1_off) ;
      temp_x[i].h[1] = *(su3_vector *)F_PT(s,x2_off) ;
    }

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_hw_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_3f_force_to_mom(P3, Pmu, sig, mThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_hw_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_3f_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_hw_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_hw_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_3f_force_to_mom(P7, Prhonumu, sig, mSevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_hw_fermion(P7, P7rho, rho);
		  side_link_3f_force(rho,sig,SevenSt,Pnumu,P7,Prhonumu,P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff[0] = SevenSt[0]/FiveSt[0] ; 
		  coeff[1] = SevenSt[1]/FiveSt[1] ; 
		  FORALLSITES(i,s)
		    {
		      scalar_mult_add_su3_vector(&(P5[i].h[0]),
						 &(P7rho[i].h[0]),coeff[0],
						 &(P5[i].h[0]));
		      scalar_mult_add_su3_vector(&(P5[i].h[1]),
						 &(P7rho[i].h[1]),coeff[1],
						 &(P5[i].h[1]));
		    }
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_hw_fermion(P5,P5nu, nu);
	      side_link_3f_force(nu,sig,mFiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff[0] = FiveSt[0]/ThreeSt[0] ; 
	      coeff[1] = FiveSt[1]/ThreeSt[1] ; 
	      FORALLSITES(i,s)
		{
		  scalar_mult_add_su3_vector(&(P3[i].h[0]),
					     &(P5nu[i].h[0]), coeff[0],
					     &(P3[i].h[0]));
		  scalar_mult_add_su3_vector(&(P3[i].h[1]),
					     &(P5nu[i].h[1]), coeff[1],
					     &(P3[i].h[1]));
		}
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_3f_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_hw_fermion(P5,P5nu, mu);
	  side_link_3f_force(mu, sig, mLepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff[0] = Lepage[0]/ThreeSt[0] ; 
	  coeff[1] = Lepage[1]/ThreeSt[1] ; 
	  FORALLSITES(i,s)
	    {
	      scalar_mult_add_su3_vector(&(P3[i].h[0]),
					 &(P5nu[i].h[0]),coeff[0],
					 &(P3[i].h[0]));
	      scalar_mult_add_su3_vector(&(P3[i].h[1]),
					 &(P5nu[i].h[1]),coeff[1],
					 &(P3[i].h[1]));
	    }

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_hw_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_3f_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_3f_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_3f_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), mNaik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_hw_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_3f_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_3f_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(hwvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      
#endif /* ASQ_OPTIMIZED_FORCE */


#ifdef  ASQ_OPTIMIZED_FATTENING   /* Asqtad action only, "_fn" executables */
#ifndef FN
BOMB THE COMPILE
#endif
void compute_gen_staple(field_offset staple, int mu, int nu, 
			field_offset link, double coef) {
  su3_matrix tmat1,tmat2;
  msg_tag *mtag0,*mtag1;
  su3_matrix *tempmat ;
  register site *s ;
  register int i ;
  register su3_matrix *fat1;

  /* Computes the staple :
                   mu
                +-------+
          nu	|	|
		|	|
		X	X
    Where the mu link can be any su3_matrix. The result is saved in staple.
    if staple==NULL_FP then the result is not saved.
    It also adds the computed staple to the fatlink[mu] with weight coef.
  */

  /* Upper staple */
  mtag0 = start_gather( link, sizeof(su3_matrix), nu, EVENANDODD, gen_pt[0] );
  mtag1 = start_gather( F_OFFSET(link[nu]), sizeof(su3_matrix), mu, 
			EVENANDODD, gen_pt[1] );
  wait_gather(mtag0);
  wait_gather(mtag1);
  
  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, (su3_matrix *)F_PT(s,staple) );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, &tmat2 );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix(fat1, &tmat2, coef,
				 fat1) ;
    }
  }
  cleanup_gather(mtag0);
  cleanup_gather(mtag1);

  /* lower staple */
  tempmat = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
  mtag0 = start_gather( F_OFFSET(link[nu]),
			sizeof(su3_matrix), mu, EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);
  FORALLSITES(i,s){
    mult_su3_an( &(s->link[nu]),(su3_matrix *)F_PT(s,link), &tmat1 );
    mult_su3_nn( &(tmat1),(su3_matrix *)gen_pt[0][i], &(tempmat[i]) );
  }
  cleanup_gather(mtag0);
  mtag0 = start_gather_from_temp( tempmat, sizeof(su3_matrix),
				  OPP_DIR(nu), EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);

  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      add_su3_matrix( (su3_matrix *)F_PT(s,staple),(su3_matrix *)gen_pt[0][i], 
		      (su3_matrix *)F_PT(s,staple) );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)F_PT(s,staple), coef, 
				 fat1 );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)gen_pt[0][i], coef, 
				 fat1 );
    }
  }

  free(tempmat);
  cleanup_gather(mtag0);
}
#endif  /* ASQ_OPTIMIZED_FATTENING   */


#ifdef  ASQ_OPTIMIZED_FORCE
/*   Covariant shift of the src fermion field in the direction dir  *
 *  by one unit. The result is stored in dest.                       */ 
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) {
  su3_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(su3_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_vec(&(s->link[dir]),(su3_vector *)(gen_pt[0][i]),
			 &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_vec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(su3_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(su3_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/*  Covariant shift of the src half wilson fermion field in the  *
 * direction dir   by one unit. The result is stored in dest.    */
void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) {
  half_wilson_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(half_wilson_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_hwvec(&(s->link[dir]),
			   (half_wilson_vector *)(gen_pt[0][i]), &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = 
	(half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_hwvec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(half_wilson_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(half_wilson_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/* Add in contribution to the force */
/* Put antihermitian traceless part into momentum */
void add_force_to_mom(su3_vector *back,su3_vector *forw,int dir,double coeff) {
  register site *s ;
  register int i ;  
  register double tmp_coeff ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff = -coeff ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD) 
      tmp_coeff = -coeff ;
    else
      tmp_coeff = coeff ;
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i]), &(forw[i]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff, &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/* Add in contribution to the force ( 3flavor case ) */
/* Put antihermitian traceless part into momentum */
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) {
  register site *s ;
  register int i ;  
  double tmp_coeff[2] ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff[0] = -coeff[0] ;
      coeff[1] = -coeff[1] ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD)
      {
	tmp_coeff[0] = -coeff[0] ;
	tmp_coeff[1] = -coeff[1] ;
      }
    else
      {
	tmp_coeff[0] = coeff[0] ;
	tmp_coeff[1] = coeff[1] ;
      }
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i].h[0]), &(forw[i].h[0]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[0], &tmat2 );
    su3_projector(&(back[i].h[1]), &(forw[i].h[1]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[1], &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/*  This routine is needed in order to add the force on the side link *
 * of the paths in the Asq and Asqtad actions. It gets as inputs the  *
 * direction mu of the side link and the direction nu of the Dslash   *
 * term we are dealing with. Then it takes also 4 fermion fields:     *
 * Path: the piece of the path with no hop in the nu or mu direction  *
 * Path_nu: the piece of the path with a hop in nu  but not in mu     *
 * Path_mu: is Path times the link mu                                 *
 * Path_numu: is Path_nu times the link mu                            */
void side_link_force(int mu, int nu, double coeff, 
		     su3_vector *Path   , su3_vector *Path_nu, 
		     su3_vector *Path_mu, su3_vector *Path_numu) {
  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_force_to_mom(Path, Path_numu, OPP_DIR(mu), -coeff );/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_nu, Path_mu, mu, -coeff) ; /* ? extra - */
      else
	add_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
 
/*  The 3 flavor version of side_link_force used *
 * to optimize fermion transports                */
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) {
  double m_coeff[2] ;

  m_coeff[0] = -coeff[0] ;
  m_coeff[1] = -coeff[1] ;

  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_3f_force_to_mom(Path,Path_numu,OPP_DIR(mu),m_coeff);/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_nu, Path_mu, mu, m_coeff) ; /* ? extra - */
      else
	add_3f_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
#endif  /* ASQ_OPTIMIZED_FORCE   */

/* LONG COMMENTS
   Here we have combined "xxx", (offset "x_off")  which is
(M_adjoint M)^{-1} phi, with Dslash times this vector, which goes in the
odd sites of xxx.  Recall that phi is defined only on even sites.  In
computing the fermion force, we are looking at

< X |  d/dt ( Dslash_eo Dslash_oe ) | X >
=
< X | d/dt Dslash_eo | T > + < T | d/dt Dslash_oe | X >
where T = Dslash X.

The subsequent manipulations to get the coefficent of H, the momentum
matrix, in the simulation time derivative above look the same for
the two terms, except for a minus sign at the end, if we simply stick
T, which lives on odd sites, into the odd sites of X

 Each path in the action contributes terms when any link of the path
is the link for which we are computing the force.  We get a minus sign
for odd numbered links in the path, since they connect sites of the
opposite parity from what it would be for an even numbered link.
Minus signs from "going around" plaquette - ie KS phases, are supposed
to be already encoded in the path coefficients.
Minus signs from paths that go backwards are supposed to be already
encoded in the path coefficients.

Here, for example, are comments reproduced from the force routine for
the one-link plus Naik plus single-staple-fat-link action:

 The three link force has three contributions, where the link that
was differentiated is the first, second, or third link in the 3-link
path, respectively.  Diagramatically, where "O" represents the momentum,
the solid line the link corresponding to the momentum, and the dashed
lines the other links:
 

	O______________ x ............ x ...............
+
	x..............O______________x.................
+
	x..............x..............O________________
Think of this as
	< xxx | O | UUUxxx >		(  xxx, UUUX_p3 )
+
	< xxx U | O | UUxxx >		( X_m1U , UUX_p2 )
+
	< xxx U U | O | Uxxx >		( X_m2UU , UX_p1 )
where "U" indicates parallel transport, "X_p3" is xxx displaced
by +3, etc.
Note the second contribution has a relative minus sign
because it effectively contributes to the <odd|even>, or M_adjoint,
part of the force when we work on an even site. i.e., for M on
an even site, this three link path begins on an odd site.

The staple force has six contributions from each plane containing the
link direction:
Call these diagrams A-F:


	x...........x		O____________x
		    .			     .
		    .			     .
		    .			     .
		    .			     .
		    .			     .
	O___________x		x............x
	   (A)			    (B)



	x	    x		O____________x
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	O___________x		x	     x
	   (C)			    (D)



	x...........x		O____________x
	.			.
	.			.
	.			.
	.			.
	.			.
	O___________x		x............x
	   (E)			    (F)

As with the Naik term, diagrams C and D have a relative minus
sign because they connect sites of the other parity.

Also note an overall minus sign in the staple terms relative to the
one link term because, with the KS phase factors included, the fat
link is  "U - w3 * UUU", or the straight link MINUS w3 times the staples.

Finally, diagrams B and E get one more minus sign because the link
we are differentiating is in the opposite direction from the staple
as a whole.  You can think of this as this "U" being a correction to
a "U_adjoint", but the derivative of U is iHU and the derivative
of U_adjoint is -iHU_adjoint.

*/
/* LONG COMMENT on sign conventions
In most of the program, the KS phases and antiperiodic boundary
conditions are absorbed into the link matrices.  This greatly simplfies
multiplying by the fermion matrix.  However, it requires care in
specifying the path coefficients.  Remember that each time you
encircle a plaquette, you pick up a net minus sign from the KS phases.
Thus, when you have more than one path to the same point, you generally
have a relative minus sign for each plaquette in a surface bounded by
this path and the basic path for that displacement.

Examples:
  Fat Link:
    Positive:	X-------X

    Negative     --------
	 	|	|
		|	|
		X	X

  Naik connection, smeared
    Positive:	X-------x-------x-------X

    Negative:	---------
		|	|
		|	|
		X	x-------x-------X

    Positive:	--------x--------
		|		|
		|		|
		X		x-------X

    Negative:	--------x-------x-------x
		|			|
		|			|
		X			X
*/



/* Comment on acceptable actions.
   We construct the backwards part of dslash by reversing all the
   paths in the forwards part.  So, for example, in the p4 action
   the forwards part includes +X+Y+Y

		X
		|
		|
		X
		|
		|
	X---->--X

  so we put -X-Y-Y in the backwards part.  But this isn't the adjoint
  of U_x(0)U_y(+x)U_y(+x+y).  Since much of the code assumes that the
  backwards hop is the adjoint of the forwards (for example, in
  preventing going to 8 flavors), the code only works for actions
  where this is true.  Roughly, this means that the fat link must
  be symmetric about reflection around its midpoint.  Equivalently,
  the paths in the backwards part of Dslash are translations of the
  paths in the forwards part.  In the case of the "P4" or knight's move
  action, this means that we have to have both paths
   +X+Y+Y and +Y+Y+X to the same point, with the same coefficients.
  Alternatively, we could just use the symmetric path +Y+X+Y.
*/

invivoquark_stuff_u_shift_fermion_1466

Real time: 2.684497e+08 -- Predicted time: 2.560370e+08

Callcount: 9880

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
759	### 1	1549.10522548318	2.646600e+04	2.794000e+04	5.2
31	### 2	20.8675845086188	2.678700e+04	3.573600e+04	25.0
1500	### 3	4409.8626596544	2.518700e+04	2.662000e+04	5.3
350	### 4	3900.13700324726	2.651400e+04	2.586800e+04	2.4

                /****** quark_stuff.c  -- ******************/
/* MIMD version 6 */
/* quark action stuff for improved action
* D.T. 1/28/98, starting from gauge_stuff.c
* K.O. 3/99 Added optimized fattening for Asq actions
* D.T. 4/99 Combine force calculations for both mass quarks
* K.O. 4/99 Optimized force for Asq action
* S.G. 7/01, modified to use t_longlink and t_fatlink
* C.D. 10/02, consolidated quark_stuff.c and quark_stuff_tmp.c
*
* This code combines quark_stuff.c and quark_stuff_tmp.c
* with DSLASH_TMP_LINKS defined, puts links in field-major variables
* t_longlink and t_fatlink. Otherwise, puts them in the site structure.
*
* In this directory, assume all paths connect even to odd sites, etc.
* Tabulate "backwards" paths (e.g. "XDOWN" is backward path to "XUP")
* as separate parity transforms of the fundamental paths.  They will
* generally need a negative sign in Dslash.  See bottom for a long
* comment on sign conventions.
*/

/**#define FFTIME**/
/**#define LLTIME**/

#include "generic_ks_includes.h"	/* definitions files and prototypes */


#define NULL_FP -1 /* NULL field_offset to be used in the optimized version *
                    * of the load_fatlinks subroutine */

    /* Specify paths in orientation in which they appear in the
       forward part of the x component of dslash().  Rotations and
       reflections will be automatically included. Be careful
       about signs of coefficients.  See long comment at bottom. */
#include <quark_action.h>
    /* Include file specifies the basic paths */

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void printpath( int *path, int length );
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length );
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length );
#ifdef  ASQ_OPTIMIZED_FATTENING
void compute_gen_staple(field_offset staple, int mu, int nu,
                        field_offset link, double coef ) ;
#endif
#ifdef  ASQ_OPTIMIZED_FORCE
#ifndef FN
BOMB THE COMPILE
#endif
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) ;
void add_force_to_mom(su3_vector *back, su3_vector *forw, int dir, double coef);
void side_link_force(int mu, int nu, double coeff, su3_vector *Path,
		     su3_vector *Path_nu, su3_vector *Path_mu, 
		     su3_vector *Path_numu) ;

void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) ;
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) ;
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) ;
#endif

int path_num[MAX_BASIC_PATHS];	/* number of rotations/reflections for each 
					kind */
static double act_path_coeff[MAX_BASIC_PATHS]; /* actual path coefficient     *
                                               * it is equal to path_coeff   *
                                               * if not tadpole improvement  *
                                               * is specified                *
                                               * or path_coeff*u_0^(L-1) when*
                                               * tadpole improvement is      *
                                               * specified                   */

/* Array of structures, for each rotation and reflection of each kind of
	path.  */
struct {
    int dir[MAX_LENGTH];	/* directions in path */
    int length;		/* length of path */
    double coeff;	/* coefficient, including minus sign if backwards */
    double forwback;	/* +1 if in forward Dslash, -1 if in backward */
}   q_paths[MAX_NUM];
int num_q_paths;	/* number of paths in dslash */
int num_basic_paths;	/* number of paths before rotation/reflection */

int is_path_equal( int *path1, int* path2, int length );
int add_basic_path( int *vec, int length, double coeff );

/* Make table of paths in action */
void make_path_table() {

    int i,j;
#ifdef TADPOLE_IMPROVE
    int k;
#endif

    /* table of directions, 1 for each kind of path */
    /**int path_ind[MAX_BASIC_PATHS][MAX_LENGTH];**/
    /* table of coefficients in action, for each path */

    node0_printf("%s\n",quark_action_description);
    num_q_paths = 0;
    num_basic_paths = 0;


    /* add rots. and reflects to table, print out the path coefficients */
    node0_printf("path coefficients: npath  path_coeff  multiplicity\n");
    for(j=0;j<quark_action_npaths;j++) {
	double this_coeff;
	this_coeff = path_coeff[j];
#ifdef TADPOLE_IMPROVE
	for(k=1;k< path_length_in[j];k++)this_coeff /= u0;
#endif
	act_path_coeff[j] = this_coeff ;
	i = add_basic_path( path_ind[j], path_length_in[j],
	    this_coeff );
	node0_printf("                    %d      %e     %d\n",
	    j,this_coeff,i);
    }
}

/* add rotations and reflections of a path to the table.  Return
   multiplicity of paths added */
int add_basic_path( int *basic_vec, int length, double coeff ) {

    int perm[8],pp[8],ir[4];
    int j,path_num;
    int vec[MAX_LENGTH];
    int flag;

    path_num = 0;
    /* now fill the long table with all rotations and reflections
	of the fundamental path.  The path presented to us is for
        the positive x component of dslash, so if the x coordinate
        is reflected it will appear with a negative sign. */
      /* permutations */
      for(perm[0]=0;perm[0]<4;perm[0]++)
      for(perm[1]=0;perm[1]<4;perm[1]++)
      for(perm[2]=0;perm[2]<4;perm[2]++)
      for(perm[3]=0;perm[3]<4;perm[3]++){
	if(perm[0] != perm[1] && perm[0] != perm[2] 
	  && perm[0] != perm[3] && perm[1] != perm[2]
	  && perm[1] != perm[3] && perm[2] != perm[3] ) {
	  /* reflections*/
	  for(ir[0]=0;ir[0]<2;ir[0]++)
	  for(ir[1]=0;ir[1]<2;ir[1]++)
	  for(ir[2]=0;ir[2]<2;ir[2]++)
	  for(ir[3]=0;ir[3]<2;ir[3]++){
	    for(j=0;j<4;j++){
	      pp[j]=perm[j];

	      if(ir[j] == 1) pp[j]=OPP_DIR(pp[j]);
	      pp[OPP_DIR(j)]=OPP_DIR(pp[j]);
	    }
	    /* create new vector*/
	    for(j=0;j<length;j++) vec[j]=pp[basic_vec[j]];
	    for(j=length;j<MAX_LENGTH;j++) vec[j]=NODIR;

            flag=0;
	    /* check if it's a new set: */
	    for(j=0;j<num_q_paths;j++){
	      flag = is_path_equal( vec, q_paths[j].dir, MAX_LENGTH );
	      if(flag==1)break;
	    }
	    if(flag == 0 ){
	      if(num_q_paths>=MAX_NUM){
		node0_printf("OOPS: MAX_NUM too small\n");
		exit(0);
	      }
	      q_paths[num_q_paths].length=length;
	      for(j=0;j<MAX_LENGTH;j++) q_paths[num_q_paths].dir[j]=vec[j];
		/* remember to copy NODIR's, or comparison will get confused */
	      if(ir[0]==0){
		q_paths[num_q_paths].coeff =  coeff;
		q_paths[num_q_paths].forwback =  +1;
	      }
	      else{
		q_paths[num_q_paths].coeff = -coeff;
		q_paths[num_q_paths].forwback = -1;
	      }
	      num_q_paths++;
	      path_num++;
	      /**node0_printf("ADD PATH %d:  rx=%d ",num_q_paths-1,ir[0]);
		 printpath( vec, length );**/
	    }

	  } /* end reflection*/
        } /* end permutation if block */
      } /* end permutation */
    num_basic_paths++;
    return(path_num);
} /* add_basic_path */

/* parallel transport a vector to the current site along a path.
   For example, if the path is "XUP", bring in the vector from
   the +X direction to the current site. If KS phases are in lattice,
   this transport will automatically include them. 
   OK for src and dest to be the same.  OK for length=0.  */
/* NOT OPTIMIZED at the moment - do lots of extra copying.  Use temp.
   vectors rather than stuff in lattice.h */
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    su3_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    su3_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_dest = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_work = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(su3_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_vec( &(s->link[dir[j]]),
		    (su3_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_vec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(su3_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = *(su3_vector *)F_PT(s,src);
    }
  }
} /* path_transport */

/* Path transport a half_wilson_vector */
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    half_wilson_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    half_wilson_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (half_wilson_vector *)calloc(
      sites_on_node, sizeof(half_wilson_vector));
    tmp_dest = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));
    tmp_work = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(half_wilson_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_hwvec( &(s->link[dir[j]]),
		    (half_wilson_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_hwvec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(half_wilson_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) =
	  *(half_wilson_vector *)F_PT(s,src);
    }
  }
} /* path_transport_hwv */


#ifdef EO
/* Stupid dslash routine that follows all the paths.
  Should optimize by precomputing sums of paths to all the displacements
  Use dslash_fn for actions that involve only +X and +X+X+X couplings.
*/
/* D_slash routine - sets dest. on each site equal to sum of
   sources parallel transported to site, with minus sign for transport
   from negative directions.  */
void dslash_eo( field_offset src, field_offset dest, int parity ) {
   register int i;
   register site *s;
   register int ipath,otherparity;
   register double x; /* coefficient of path */

    switch(parity){
	case EVEN:	otherparity=ODD; break;
	case ODD:	otherparity=EVEN; break;
	case EVENANDODD:	otherparity=EVENANDODD; break;
    }

    /* Parallel transport by all the paths in the action.  
       Multiply by coefficient in table
    */
    FORSOMEPARITY(i,s,parity){ clearvec( (su3_vector *)F_PT(s,dest) ); }

    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	    path_transport( src, F_OFFSET(tempvec[0]), parity,
		q_paths[ipath].dir, q_paths[ipath].length );
	    x=q_paths[ipath].coeff;
	    FORSOMEPARITY(i,s,parity){
		scalar_mult_add_su3_vector(  (su3_vector *)F_PT(s,dest),
		    &(s->tempvec[0]), x, (su3_vector *)F_PT(s,dest) );
	    }
    }   /* ipath */
} /* dslash_eo */
#endif /* ifdef EO */

#ifdef FN
/* Sum over paths connecting to nearest neighbor point (fat link) and to third
   nearest neighbor (longlinks) */
/* Doug Toussaint 2/4/98 */
/* modified to use t_longlinks, S. Gottlieb 7/13/01 */
/* long link calculating routine */
/* path_product() follows the path starting at step 0, and
   leaves the answer at the end of the path.  We want the answer
   at the site where the path begins.  So we look for paths with
   the opposite displacement from the displacement of the point
   that we want to transport to this site, and take the adjoint
   of the matrix at the end. clear? */
/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_longlinks() {
  register int i;
  register site *s;
  int ipath,dir;
  int disp[4];
  register su3_matrix *long1;

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_longlinks needs phases in\n");
    terminate(0);
  }
  for (dir=XUP; dir<=TUP; dir++){ /* loop over longlink directions */
    /* set longlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      long1 = &(t_longlink[4*i+dir]);
#else
      long1 = &(s->longlink[dir]);
#endif
      clear_su3mat( long1 );
    }

    /* loop over paths, checking for ones with total displacement 3*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=3,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("ipath = %d, found a path:  ",ipath);
for(j=0;j<q_paths[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  long1 = &(t_longlink[4*i+dir]);
#else
	  long1 = &(s->longlink[dir]);
#endif
          scalar_mult_add_su3_matrix( long1,
	    &(s->staple), -q_paths[ipath].coeff, long1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */

  } /* loop over directions */

  valid_longlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Long):  %e\n",dtime);
#endif
}  /* load_longlinks() */

/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_fatlinks() {
  register int i;
  register site *s;
  int dir;
  register su3_matrix *fat1;
#ifdef ASQ_OPTIMIZED_FATTENING
  int  nu,rho,sig ;
  double one_link ; /*needed to fix the problem with the Lepage term */
#else
  int ipath;
  int disp[4];
#endif

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_fatlinks needs phases in\n");
    terminate(0);
  }

#ifndef  ASQ_OPTIMIZED_FATTENING   /* general case code */
  for (dir=XUP; dir<=TUP; dir++){ /* loop over fatlink directions */
    /* set fatlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+dir]);
#else
      fat1 = &(s->fatlink[dir]);
#endif
      clear_su3mat( fat1 );
    }
    
    /* loop over paths, checking for ones with total displacement 1*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=1,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("dir = %d, found a path:  ",dir);
for(j=0;j<q_paths.[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  fat1 = &(t_fatlink[4*i+dir]);
#else
	  fat1 = &(s->fatlink[dir]);
#endif
          scalar_mult_add_su3_matrix( fat1,
	    &(s->staple), -q_paths[ipath].coeff, fat1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */
  } /* loop over directions */
#else	/* ASQ_OPTIMIZED_FATTENING, for Asq and Asqtad actions */
/*  Optimized fattening code for the Asq and Asqtad actions.           *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
 
 /* to fix up the Lepage term, included by a trick below */
 one_link = (act_path_coeff[0] - 6.0*act_path_coeff[5]) ; 
 
 for (dir=XUP; dir<=TUP; dir++){
   FORALLSITES(i,s) /* Intialize fat links with c_1*U_\mu(x) */
     {
#ifdef DSLASH_TMP_LINKS
       fat1 = &(t_fatlink[4*i+dir]);
#else
       fat1 = &(s->fatlink[dir]);
#endif
       scalar_mult_su3_matrix(&(s->link[dir]), one_link ,
			      fat1 );
     }
   for(nu=XUP; nu<=TUP; nu++) if(nu!=dir)
     {
       compute_gen_staple(F_OFFSET(staple),dir,nu,F_OFFSET(link[dir]),
			  act_path_coeff[2]);
       /* The Lepage term */
       /* Note this also involves modifying c_1 (above) */
       compute_gen_staple(NULL_FP,dir,nu,F_OFFSET(staple),act_path_coeff[5]);
       for(rho=XUP; rho<=TUP; rho++) if((rho!=dir)&&(rho!=nu))
	 {
	   compute_gen_staple(F_OFFSET(tempmat1),dir,rho,F_OFFSET(staple),
			      act_path_coeff[3]);
	   for(sig=XUP; sig<=TUP; sig++)
	     if((sig!=dir)&&(sig!=nu)&&(sig!=rho))
	       {
		 compute_gen_staple(NULL_FP,dir,sig,
				    F_OFFSET(tempmat1),
				    act_path_coeff[4]);
	       } /* sig */
	 } /* rho */
     } /* nu */
 }/* dir */  
#endif

  valid_fatlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Fat):  %e\n",dtime);
#endif
}  /* load_fatlinks() */
#endif /* ifdef FN */


/* compare two paths, return 1 if equal, else zero */
int is_path_equal( int *path1, int* path2, int length ){
   register int i;
   for(i=0;i<length;i++)if(path1[i]!=path2[i])return(0);
   return(1);
}


/* update the  momenta with the fermion force */
/* Assumes that the conjugate gradient has been run, with the answer in
   x_off, and dslash(x_off,x_off,ODD) has been run. (fills in x_off_odd) */
/* SEE LONG COMMENTS AT END */

#ifndef ASQ_OPTIMIZED_FORCE
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon, coeff;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  half_wilson_vector *hw_tmp0,*hw_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  hw_tmp0 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  hw_tmp1 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  /* Use half_wilson_vectors to store x_off transported from ends of
     path.  0 component from forward end, 1 component from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    path_transport( x_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[1] for transport from starting end */
    FORALLSITES(i,s){
      hw_tmp0[i].h[0]=s->tempvec[0];
      hw_tmp0[i].h[1]=*(su3_vector *)F_PT(s,x_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff = ferm_epsilon*q_paths[ipath].coeff;
      if( (ilink%2)==1 )coeff = -coeff;

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "half_wilson_vector" to handle pair of vectors -
	0 component is x_off from forward end, 1 component from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_su3_mat_hwvec( &(s->link[lastdir]),
	      &(hw_tmp0[i]), &(hw_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( hw_tmp1, 2*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     hw_tmp0[i] = *(half_wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( hw_tmp0, 2*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_su3_mat_hwvec( &(s->link[OPP_DIR(lastdir)]),
                (half_wilson_vector *)(gen_pt[0][i]),
		&(hw_tmp1[i]) );
          }
	  tmp_pt = hw_tmp0; hw_tmp0 = hw_tmp1; hw_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( hw_tmp0 ); free( hw_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */


void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off,
  int nflav2, field_offset x2_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/14/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon1, ferm_epsilon2, coeff1, coeff2;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  wilson_vector *w_tmp0,*w_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon1 = 2.0*(nflav1/4.0)*eps;
  ferm_epsilon2 = 2.0*(nflav2/4.0)*eps;
  w_tmp0 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  w_tmp1 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  /* Use wilson_vectors to store x_off transported from ends of
     path.  0 and 1 components from forward end, 2 and 3 components
     from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    /**
    path_transport( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    path_transport( x2_off, F_OFFSET(tempvec[1]),
      EVENANDODD, q_paths[ipath].dir, length );
    **/
/** WARNING!! Assumes xxx1 and xxx2 contiguous **/
if( x2_off-x1_off != sizeof(su3_vector) ){node0_printf("BOTCH\n"); exit(0);}
    path_transport_hwv( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[2] for transport from starting end */
    FORALLSITES(i,s){
      w_tmp0[i].d[0]=s->tempvec[0];
      w_tmp0[i].d[1]=s->tempvec[1];
      w_tmp0[i].d[2]=*(su3_vector *)F_PT(s,x1_off);
      w_tmp0[i].d[3]=*(su3_vector *)F_PT(s,x2_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff1 = ferm_epsilon1*q_paths[ipath].coeff;
      coeff2 = ferm_epsilon2*q_paths[ipath].coeff;
      if( (ilink%2)==1 ){ coeff1 = -coeff1; coeff2 = -coeff2;}

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "wilson_vector" to handle pair of vectors -
	0 component is x_off1 from forward end, 1 component is x_off2
	from forward end,  2 and 3  components are x_off1 and x_off2 
	from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_mat_wilson_vec( &(s->link[lastdir]),
	      &(w_tmp0[i]), &(w_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( w_tmp1, 4*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     w_tmp0[i] = *(wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( w_tmp0, 4*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_mat_wilson_vec( &(s->link[OPP_DIR(lastdir)]),
                (wilson_vector *)(gen_pt[0][i]),
		&(w_tmp1[i]) );
          }
	  tmp_pt = w_tmp0; w_tmp0 = w_tmp1; w_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( w_tmp0 ); free( w_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */



#else /* ASQ_OPTIMIZED_FORCE, for Asq and Asqtad actions */
/* Optimized force code for the Asq and Asqtad actions                 *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
#define Pmu          tempvec[0] 
#define Pnumu        tempvec[1]
#define Prhonumu     tempvec[2]
#define P7           tempvec[3]
#define P7rho        tempvec[4]              
#define P7rhonu      tempvec[5]
#define P5           tempvec[6]
#define P3           tempvec[7]
#define P5nu         tempvec[3]
#define P3mu         tempvec[3]
#define Popmu        tempvec[4]
#define Pmumumu      tempvec[4]
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double ferm_epsilon, coeff;
  double OneLink, Lepage, Naik, FiveSt, ThreeSt, SevenSt ;
  su3_vector *tempvec[8] ;
  su3_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  
  /* Path coefficients times fermion epsilon */
  OneLink = act_path_coeff[0]*ferm_epsilon ; 
  Naik    = act_path_coeff[1]*ferm_epsilon ;
  ThreeSt = act_path_coeff[2]*ferm_epsilon ;
  FiveSt  = act_path_coeff[3]*ferm_epsilon ;
  SevenSt = act_path_coeff[4]*ferm_epsilon ;
  Lepage  = act_path_coeff[5]*ferm_epsilon ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    tempvec[mu] = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

  /*copy x_off to a temporary vector */
  temp_x = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
  FORALLSITES(i,s) temp_x[i] = *(su3_vector *)F_PT(s,x_off) ;

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_force_to_mom(P3, Pmu, sig, -ThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_force_to_mom(P7, Prhonumu, sig, -SevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_fermion(P7, P7rho, rho);
		  side_link_force(rho,sig,SevenSt, Pnumu, P7, Prhonumu, P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff = SevenSt/FiveSt ; 
		  FORALLSITES(i,s)
		    scalar_mult_add_su3_vector(&(P5[i]),&(P7rho[i]),coeff,
					       &(P5[i]));
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_fermion(P5,P5nu, nu);
	      side_link_force(nu,sig,-FiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff = FiveSt/ThreeSt ; 
	      FORALLSITES(i,s)
		scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_fermion(P5,P5nu, mu);
	  side_link_force(mu, sig, -Lepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff = Lepage/ThreeSt ; 
	  FORALLSITES(i,s)
	    scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), -Naik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(tempvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      

#define Pmu          hwvec[0] 
#define Pnumu        hwvec[1]
#define Prhonumu     hwvec[2]
#define P7           hwvec[3]
#define P7rho        hwvec[4]              
#define P7rhonu      hwvec[5]
#define P5           hwvec[6]
#define P3           hwvec[7]
#define P5nu         hwvec[3]
#define P3mu         hwvec[3]
#define Popmu        hwvec[4]
#define Pmumumu      hwvec[4]
void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off, 
			  int nflav2, field_offset x2_off){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/15/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double coeff[2],ferm_epsilon ;
  double OneLink[2], Lepage[2], Naik[2], FiveSt[2], ThreeSt[2], SevenSt[2] ;
  double mNaik[2], mLepage[2], mFiveSt[2], mThreeSt[2], mSevenSt[2] ;
  half_wilson_vector *hwvec[8] ;
  half_wilson_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  
  /* Path coefficients times fermion epsilon */
  ferm_epsilon = 2.0*(nflav1/4.0)*eps;
  OneLink[0] = act_path_coeff[0]*ferm_epsilon ;
  Naik[0]    = act_path_coeff[1]*ferm_epsilon ; mNaik[0]    = -Naik[0] ;
  ThreeSt[0] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[0] = -ThreeSt[0] ;
  FiveSt[0]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[0]  = -FiveSt[0] ;
  SevenSt[0] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[0] = -SevenSt[0] ;
  Lepage[0]  = act_path_coeff[5]*ferm_epsilon ; mLepage[0]  = -Lepage[0] ;

  ferm_epsilon = 2.0*(nflav2/4.0)*eps;
  OneLink[1] = act_path_coeff[0]*ferm_epsilon ;
  Naik[1]    = act_path_coeff[1]*ferm_epsilon ; mNaik[1]    = -Naik[1] ;
  ThreeSt[1] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[1] = -ThreeSt[1] ;
  FiveSt[1]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[1]  = -FiveSt[1] ;
  SevenSt[1] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[1] = -SevenSt[1] ;
  Lepage[1]  = act_path_coeff[5]*ferm_epsilon ; mLepage[1]  = -Lepage[1] ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    hwvec[mu]= 
      (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));

  /*copy x_off to a temporary vector */
  temp_x= 
    (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
  FORALLSITES(i,s)
    {
      temp_x[i].h[0] = *(su3_vector *)F_PT(s,x1_off) ;
      temp_x[i].h[1] = *(su3_vector *)F_PT(s,x2_off) ;
    }

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_hw_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_3f_force_to_mom(P3, Pmu, sig, mThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_hw_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_3f_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_hw_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_hw_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_3f_force_to_mom(P7, Prhonumu, sig, mSevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_hw_fermion(P7, P7rho, rho);
		  side_link_3f_force(rho,sig,SevenSt,Pnumu,P7,Prhonumu,P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff[0] = SevenSt[0]/FiveSt[0] ; 
		  coeff[1] = SevenSt[1]/FiveSt[1] ; 
		  FORALLSITES(i,s)
		    {
		      scalar_mult_add_su3_vector(&(P5[i].h[0]),
						 &(P7rho[i].h[0]),coeff[0],
						 &(P5[i].h[0]));
		      scalar_mult_add_su3_vector(&(P5[i].h[1]),
						 &(P7rho[i].h[1]),coeff[1],
						 &(P5[i].h[1]));
		    }
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_hw_fermion(P5,P5nu, nu);
	      side_link_3f_force(nu,sig,mFiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff[0] = FiveSt[0]/ThreeSt[0] ; 
	      coeff[1] = FiveSt[1]/ThreeSt[1] ; 
	      FORALLSITES(i,s)
		{
		  scalar_mult_add_su3_vector(&(P3[i].h[0]),
					     &(P5nu[i].h[0]), coeff[0],
					     &(P3[i].h[0]));
		  scalar_mult_add_su3_vector(&(P3[i].h[1]),
					     &(P5nu[i].h[1]), coeff[1],
					     &(P3[i].h[1]));
		}
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_3f_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_hw_fermion(P5,P5nu, mu);
	  side_link_3f_force(mu, sig, mLepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff[0] = Lepage[0]/ThreeSt[0] ; 
	  coeff[1] = Lepage[1]/ThreeSt[1] ; 
	  FORALLSITES(i,s)
	    {
	      scalar_mult_add_su3_vector(&(P3[i].h[0]),
					 &(P5nu[i].h[0]),coeff[0],
					 &(P3[i].h[0]));
	      scalar_mult_add_su3_vector(&(P3[i].h[1]),
					 &(P5nu[i].h[1]),coeff[1],
					 &(P3[i].h[1]));
	    }

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_hw_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_3f_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_3f_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_3f_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), mNaik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_hw_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_3f_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_3f_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(hwvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      
#endif /* ASQ_OPTIMIZED_FORCE */


#ifdef  ASQ_OPTIMIZED_FATTENING   /* Asqtad action only, "_fn" executables */
#ifndef FN
BOMB THE COMPILE
#endif
void compute_gen_staple(field_offset staple, int mu, int nu, 
			field_offset link, double coef) {
  su3_matrix tmat1,tmat2;
  msg_tag *mtag0,*mtag1;
  su3_matrix *tempmat ;
  register site *s ;
  register int i ;
  register su3_matrix *fat1;

  /* Computes the staple :
                   mu
                +-------+
          nu	|	|
		|	|
		X	X
    Where the mu link can be any su3_matrix. The result is saved in staple.
    if staple==NULL_FP then the result is not saved.
    It also adds the computed staple to the fatlink[mu] with weight coef.
  */

  /* Upper staple */
  mtag0 = start_gather( link, sizeof(su3_matrix), nu, EVENANDODD, gen_pt[0] );
  mtag1 = start_gather( F_OFFSET(link[nu]), sizeof(su3_matrix), mu, 
			EVENANDODD, gen_pt[1] );
  wait_gather(mtag0);
  wait_gather(mtag1);
  
  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, (su3_matrix *)F_PT(s,staple) );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, &tmat2 );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix(fat1, &tmat2, coef,
				 fat1) ;
    }
  }
  cleanup_gather(mtag0);
  cleanup_gather(mtag1);

  /* lower staple */
  tempmat = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
  mtag0 = start_gather( F_OFFSET(link[nu]),
			sizeof(su3_matrix), mu, EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);
  FORALLSITES(i,s){
    mult_su3_an( &(s->link[nu]),(su3_matrix *)F_PT(s,link), &tmat1 );
    mult_su3_nn( &(tmat1),(su3_matrix *)gen_pt[0][i], &(tempmat[i]) );
  }
  cleanup_gather(mtag0);
  mtag0 = start_gather_from_temp( tempmat, sizeof(su3_matrix),
				  OPP_DIR(nu), EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);

  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      add_su3_matrix( (su3_matrix *)F_PT(s,staple),(su3_matrix *)gen_pt[0][i], 
		      (su3_matrix *)F_PT(s,staple) );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)F_PT(s,staple), coef, 
				 fat1 );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)gen_pt[0][i], coef, 
				 fat1 );
    }
  }

  free(tempmat);
  cleanup_gather(mtag0);
}
#endif  /* ASQ_OPTIMIZED_FATTENING   */


#ifdef  ASQ_OPTIMIZED_FORCE
/*   Covariant shift of the src fermion field in the direction dir  *
 *  by one unit. The result is stored in dest.                       */ 
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) {
  su3_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(su3_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_vec(&(s->link[dir]),(su3_vector *)(gen_pt[0][i]),
			 &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_vec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(su3_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(su3_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/*  Covariant shift of the src half wilson fermion field in the  *
 * direction dir   by one unit. The result is stored in dest.    */
void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) {
  half_wilson_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(half_wilson_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_hwvec(&(s->link[dir]),
			   (half_wilson_vector *)(gen_pt[0][i]), &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = 
	(half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_hwvec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(half_wilson_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(half_wilson_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/* Add in contribution to the force */
/* Put antihermitian traceless part into momentum */
void add_force_to_mom(su3_vector *back,su3_vector *forw,int dir,double coeff) {
  register site *s ;
  register int i ;  
  register double tmp_coeff ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff = -coeff ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD) 
      tmp_coeff = -coeff ;
    else
      tmp_coeff = coeff ;
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i]), &(forw[i]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff, &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/* Add in contribution to the force ( 3flavor case ) */
/* Put antihermitian traceless part into momentum */
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) {
  register site *s ;
  register int i ;  
  double tmp_coeff[2] ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff[0] = -coeff[0] ;
      coeff[1] = -coeff[1] ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD)
      {
	tmp_coeff[0] = -coeff[0] ;
	tmp_coeff[1] = -coeff[1] ;
      }
    else
      {
	tmp_coeff[0] = coeff[0] ;
	tmp_coeff[1] = coeff[1] ;
      }
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i].h[0]), &(forw[i].h[0]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[0], &tmat2 );
    su3_projector(&(back[i].h[1]), &(forw[i].h[1]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[1], &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/*  This routine is needed in order to add the force on the side link *
 * of the paths in the Asq and Asqtad actions. It gets as inputs the  *
 * direction mu of the side link and the direction nu of the Dslash   *
 * term we are dealing with. Then it takes also 4 fermion fields:     *
 * Path: the piece of the path with no hop in the nu or mu direction  *
 * Path_nu: the piece of the path with a hop in nu  but not in mu     *
 * Path_mu: is Path times the link mu                                 *
 * Path_numu: is Path_nu times the link mu                            */
void side_link_force(int mu, int nu, double coeff, 
		     su3_vector *Path   , su3_vector *Path_nu, 
		     su3_vector *Path_mu, su3_vector *Path_numu) {
  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_force_to_mom(Path, Path_numu, OPP_DIR(mu), -coeff );/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_nu, Path_mu, mu, -coeff) ; /* ? extra - */
      else
	add_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
 
/*  The 3 flavor version of side_link_force used *
 * to optimize fermion transports                */
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) {
  double m_coeff[2] ;

  m_coeff[0] = -coeff[0] ;
  m_coeff[1] = -coeff[1] ;

  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_3f_force_to_mom(Path,Path_numu,OPP_DIR(mu),m_coeff);/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_nu, Path_mu, mu, m_coeff) ; /* ? extra - */
      else
	add_3f_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
#endif  /* ASQ_OPTIMIZED_FORCE   */

/* LONG COMMENTS
   Here we have combined "xxx", (offset "x_off")  which is
(M_adjoint M)^{-1} phi, with Dslash times this vector, which goes in the
odd sites of xxx.  Recall that phi is defined only on even sites.  In
computing the fermion force, we are looking at

< X |  d/dt ( Dslash_eo Dslash_oe ) | X >
=
< X | d/dt Dslash_eo | T > + < T | d/dt Dslash_oe | X >
where T = Dslash X.

The subsequent manipulations to get the coefficent of H, the momentum
matrix, in the simulation time derivative above look the same for
the two terms, except for a minus sign at the end, if we simply stick
T, which lives on odd sites, into the odd sites of X

 Each path in the action contributes terms when any link of the path
is the link for which we are computing the force.  We get a minus sign
for odd numbered links in the path, since they connect sites of the
opposite parity from what it would be for an even numbered link.
Minus signs from "going around" plaquette - ie KS phases, are supposed
to be already encoded in the path coefficients.
Minus signs from paths that go backwards are supposed to be already
encoded in the path coefficients.

Here, for example, are comments reproduced from the force routine for
the one-link plus Naik plus single-staple-fat-link action:

 The three link force has three contributions, where the link that
was differentiated is the first, second, or third link in the 3-link
path, respectively.  Diagramatically, where "O" represents the momentum,
the solid line the link corresponding to the momentum, and the dashed
lines the other links:
 

	O______________ x ............ x ...............
+
	x..............O______________x.................
+
	x..............x..............O________________
Think of this as
	< xxx | O | UUUxxx >		(  xxx, UUUX_p3 )
+
	< xxx U | O | UUxxx >		( X_m1U , UUX_p2 )
+
	< xxx U U | O | Uxxx >		( X_m2UU , UX_p1 )
where "U" indicates parallel transport, "X_p3" is xxx displaced
by +3, etc.
Note the second contribution has a relative minus sign
because it effectively contributes to the <odd|even>, or M_adjoint,
part of the force when we work on an even site. i.e., for M on
an even site, this three link path begins on an odd site.

The staple force has six contributions from each plane containing the
link direction:
Call these diagrams A-F:


	x...........x		O____________x
		    .			     .
		    .			     .
		    .			     .
		    .			     .
		    .			     .
	O___________x		x............x
	   (A)			    (B)



	x	    x		O____________x
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	O___________x		x	     x
	   (C)			    (D)



	x...........x		O____________x
	.			.
	.			.
	.			.
	.			.
	.			.
	O___________x		x............x
	   (E)			    (F)

As with the Naik term, diagrams C and D have a relative minus
sign because they connect sites of the other parity.

Also note an overall minus sign in the staple terms relative to the
one link term because, with the KS phase factors included, the fat
link is  "U - w3 * UUU", or the straight link MINUS w3 times the staples.

Finally, diagrams B and E get one more minus sign because the link
we are differentiating is in the opposite direction from the staple
as a whole.  You can think of this as this "U" being a correction to
a "U_adjoint", but the derivative of U is iHU and the derivative
of U_adjoint is -iHU_adjoint.

*/
/* LONG COMMENT on sign conventions
In most of the program, the KS phases and antiperiodic boundary
conditions are absorbed into the link matrices.  This greatly simplfies
multiplying by the fermion matrix.  However, it requires care in
specifying the path coefficients.  Remember that each time you
encircle a plaquette, you pick up a net minus sign from the KS phases.
Thus, when you have more than one path to the same point, you generally
have a relative minus sign for each plaquette in a surface bounded by
this path and the basic path for that displacement.

Examples:
  Fat Link:
    Positive:	X-------X

    Negative     --------
	 	|	|
		|	|
		X	X

  Naik connection, smeared
    Positive:	X-------x-------x-------X

    Negative:	---------
		|	|
		|	|
		X	x-------x-------X

    Positive:	--------x--------
		|		|
		|		|
		X		x-------X

    Negative:	--------x-------x-------x
		|			|
		|			|
		X			X
*/



/* Comment on acceptable actions.
   We construct the backwards part of dslash by reversing all the
   paths in the forwards part.  So, for example, in the p4 action
   the forwards part includes +X+Y+Y

		X
		|
		|
		X
		|
		|
	X---->--X

  so we put -X-Y-Y in the backwards part.  But this isn't the adjoint
  of U_x(0)U_y(+x)U_y(+x+y).  Since much of the code assumes that the
  backwards hop is the adjoint of the forwards (for example, in
  preventing going to 8 flavors), the code only works for actions
  where this is true.  Roughly, this means that the fat link must
  be symmetric about reflection around its midpoint.  Equivalently,
  the paths in the backwards part of Dslash are translations of the
  paths in the forwards part.  In the case of the "P4" or knight's move
  action, this means that we have to have both paths
   +X+Y+Y and +Y+Y+X to the same point, with the same coefficients.
  Alternatively, we could just use the symmetric path +Y+X+Y.
*/

invivoquark_stuff_u_shift_fermion_1460

Real time: 1.350528e+09 -- Predicted time: 1.685396e+09

Callcount: 9880

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
1047	### 1	5618.53210736178	1.687980e+05	1.344240e+05	20.3
754	### 2	4270.97307865845	1.725600e+05	1.329800e+05	22.9

                /******************  su3_adjoint.c  (in su3.a) **************************
*									*
* void su3_adjoint( su3_matrix *a, su3_matrix *b )			*
* B  <- A_adjoint,  adjoint of an SU3 matrix 				*
*/
#include "config.h"
#include "complex.h"
#include "su3.h"

/* adjoint of an SU3 matrix */
void su3_adjoint( su3_matrix *a, su3_matrix *b ){
register int i,j;
    for(i=0;i<3;i++)for(j=0;j<3;j++){
	CONJG( a->e[j][i], b->e[i][j] );
    }
}

invivosu3_adjoint_su3_adjoint_13

Real time: 1.957137e+08 -- Predicted time: 1.618735e+09

Callcount: 5872000

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
1154412	### 1	464866.97584333	4.240000e+02	3.600000e+01	91.5
4206853	### 2	2082477.62107202	2.400000e+02	3.200000e+01	86.6
2126167	### 3	1676260.44500142	2.580000e+02	4.000000e+01	84.4
4775737	### 4	1201693.75890198	3.130000e+02	2.800000e+01	91.0
4688773	### 5	100208.041128655	1.080000e+02	4.800000e+01	55.5
713093	### 6	292429.540583389	2.950000e+02	4.400000e+01	85.0
1756479	### 7	50761.3881957432	3.180000e+02	5.200000e+01	83.6

                /****************  s_m_a_vec.c  (in su3.a) ******************************
*									*
* void scalar_mult_add_su3_vector( su3_vector *a, su3_vector *b,	*
*	double s, su3_vector *c)						*
* C <- A + s*B,   A,B and C vectors 					*
*/
#include "config.h"
#include "complex.h"
#include "su3.h"

/* c <- a + s*b, vectors */

void scalar_mult_add_su3_vector(su3_vector *a, su3_vector *b, double s,
	su3_vector *c){

#ifndef NATIVEDOUBLE
  register int i;
  for(i=0;i<3;i++){
    c->c[i].real = a->c[i].real + s*b->c[i].real;
    c->c[i].imag = a->c[i].imag + s*b->c[i].imag;
  }
  
#else /* RS6000 version */
  
  register double ss;
  
  ss = s;

  c->c[0].real = a->c[0].real + ss*b->c[0].real;
  c->c[0].imag = a->c[0].imag + ss*b->c[0].imag;
  c->c[1].real = a->c[1].real + ss*b->c[1].real;
  c->c[1].imag = a->c[1].imag + ss*b->c[1].imag;
  c->c[2].real = a->c[2].real + ss*b->c[2].real;
  c->c[2].imag = a->c[2].imag + ss*b->c[2].imag;
  
#endif
}

invivos_m_a_vec_scalar_mult_add_su3_vector_18

Real time: 3.093487e+08 -- Predicted time: 2.135326e+09

Callcount: 12848000

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
2525865	### 1	12863409.1666667	1.660000e+02	2.400000e+01	85.5

                /****** quark_stuff.c  -- ******************/
/* MIMD version 6 */
/* quark action stuff for improved action
* D.T. 1/28/98, starting from gauge_stuff.c
* K.O. 3/99 Added optimized fattening for Asq actions
* D.T. 4/99 Combine force calculations for both mass quarks
* K.O. 4/99 Optimized force for Asq action
* S.G. 7/01, modified to use t_longlink and t_fatlink
* C.D. 10/02, consolidated quark_stuff.c and quark_stuff_tmp.c
*
* This code combines quark_stuff.c and quark_stuff_tmp.c
* with DSLASH_TMP_LINKS defined, puts links in field-major variables
* t_longlink and t_fatlink. Otherwise, puts them in the site structure.
*
* In this directory, assume all paths connect even to odd sites, etc.
* Tabulate "backwards" paths (e.g. "XDOWN" is backward path to "XUP")
* as separate parity transforms of the fundamental paths.  They will
* generally need a negative sign in Dslash.  See bottom for a long
* comment on sign conventions.
*/

/**#define FFTIME**/
/**#define LLTIME**/

#include "generic_ks_includes.h"	/* definitions files and prototypes */


#define NULL_FP -1 /* NULL field_offset to be used in the optimized version *
                    * of the load_fatlinks subroutine */

    /* Specify paths in orientation in which they appear in the
       forward part of the x component of dslash().  Rotations and
       reflections will be automatically included. Be careful
       about signs of coefficients.  See long comment at bottom. */
#include <quark_action.h>
    /* Include file specifies the basic paths */

#define GOES_FORWARDS(dir) (dir<=TUP)
#define GOES_BACKWARDS(dir) (dir>TUP)

void printpath( int *path, int length );
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length );
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length );
#ifdef  ASQ_OPTIMIZED_FATTENING
void compute_gen_staple(field_offset staple, int mu, int nu,
                        field_offset link, double coef ) ;
#endif
#ifdef  ASQ_OPTIMIZED_FORCE
#ifndef FN
BOMB THE COMPILE
#endif
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) ;
void add_force_to_mom(su3_vector *back, su3_vector *forw, int dir, double coef);
void side_link_force(int mu, int nu, double coeff, su3_vector *Path,
		     su3_vector *Path_nu, su3_vector *Path_mu, 
		     su3_vector *Path_numu) ;

void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) ;
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) ;
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) ;
#endif

int path_num[MAX_BASIC_PATHS];	/* number of rotations/reflections for each 
					kind */
static double act_path_coeff[MAX_BASIC_PATHS]; /* actual path coefficient     *
                                               * it is equal to path_coeff   *
                                               * if not tadpole improvement  *
                                               * is specified                *
                                               * or path_coeff*u_0^(L-1) when*
                                               * tadpole improvement is      *
                                               * specified                   */

/* Array of structures, for each rotation and reflection of each kind of
	path.  */
struct {
    int dir[MAX_LENGTH];	/* directions in path */
    int length;		/* length of path */
    double coeff;	/* coefficient, including minus sign if backwards */
    double forwback;	/* +1 if in forward Dslash, -1 if in backward */
}   q_paths[MAX_NUM];
int num_q_paths;	/* number of paths in dslash */
int num_basic_paths;	/* number of paths before rotation/reflection */

int is_path_equal( int *path1, int* path2, int length );
int add_basic_path( int *vec, int length, double coeff );

/* Make table of paths in action */
void make_path_table() {

    int i,j;
#ifdef TADPOLE_IMPROVE
    int k;
#endif

    /* table of directions, 1 for each kind of path */
    /**int path_ind[MAX_BASIC_PATHS][MAX_LENGTH];**/
    /* table of coefficients in action, for each path */

    node0_printf("%s\n",quark_action_description);
    num_q_paths = 0;
    num_basic_paths = 0;


    /* add rots. and reflects to table, print out the path coefficients */
    node0_printf("path coefficients: npath  path_coeff  multiplicity\n");
    for(j=0;j<quark_action_npaths;j++) {
	double this_coeff;
	this_coeff = path_coeff[j];
#ifdef TADPOLE_IMPROVE
	for(k=1;k< path_length_in[j];k++)this_coeff /= u0;
#endif
	act_path_coeff[j] = this_coeff ;
	i = add_basic_path( path_ind[j], path_length_in[j],
	    this_coeff );
	node0_printf("                    %d      %e     %d\n",
	    j,this_coeff,i);
    }
}

/* add rotations and reflections of a path to the table.  Return
   multiplicity of paths added */
int add_basic_path( int *basic_vec, int length, double coeff ) {

    int perm[8],pp[8],ir[4];
    int j,path_num;
    int vec[MAX_LENGTH];
    int flag;

    path_num = 0;
    /* now fill the long table with all rotations and reflections
	of the fundamental path.  The path presented to us is for
        the positive x component of dslash, so if the x coordinate
        is reflected it will appear with a negative sign. */
      /* permutations */
      for(perm[0]=0;perm[0]<4;perm[0]++)
      for(perm[1]=0;perm[1]<4;perm[1]++)
      for(perm[2]=0;perm[2]<4;perm[2]++)
      for(perm[3]=0;perm[3]<4;perm[3]++){
	if(perm[0] != perm[1] && perm[0] != perm[2] 
	  && perm[0] != perm[3] && perm[1] != perm[2]
	  && perm[1] != perm[3] && perm[2] != perm[3] ) {
	  /* reflections*/
	  for(ir[0]=0;ir[0]<2;ir[0]++)
	  for(ir[1]=0;ir[1]<2;ir[1]++)
	  for(ir[2]=0;ir[2]<2;ir[2]++)
	  for(ir[3]=0;ir[3]<2;ir[3]++){
	    for(j=0;j<4;j++){
	      pp[j]=perm[j];

	      if(ir[j] == 1) pp[j]=OPP_DIR(pp[j]);
	      pp[OPP_DIR(j)]=OPP_DIR(pp[j]);
	    }
	    /* create new vector*/
	    for(j=0;j<length;j++) vec[j]=pp[basic_vec[j]];
	    for(j=length;j<MAX_LENGTH;j++) vec[j]=NODIR;

            flag=0;
	    /* check if it's a new set: */
	    for(j=0;j<num_q_paths;j++){
	      flag = is_path_equal( vec, q_paths[j].dir, MAX_LENGTH );
	      if(flag==1)break;
	    }
	    if(flag == 0 ){
	      if(num_q_paths>=MAX_NUM){
		node0_printf("OOPS: MAX_NUM too small\n");
		exit(0);
	      }
	      q_paths[num_q_paths].length=length;
	      for(j=0;j<MAX_LENGTH;j++) q_paths[num_q_paths].dir[j]=vec[j];
		/* remember to copy NODIR's, or comparison will get confused */
	      if(ir[0]==0){
		q_paths[num_q_paths].coeff =  coeff;
		q_paths[num_q_paths].forwback =  +1;
	      }
	      else{
		q_paths[num_q_paths].coeff = -coeff;
		q_paths[num_q_paths].forwback = -1;
	      }
	      num_q_paths++;
	      path_num++;
	      /**node0_printf("ADD PATH %d:  rx=%d ",num_q_paths-1,ir[0]);
		 printpath( vec, length );**/
	    }

	  } /* end reflection*/
        } /* end permutation if block */
      } /* end permutation */
    num_basic_paths++;
    return(path_num);
} /* add_basic_path */

/* parallel transport a vector to the current site along a path.
   For example, if the path is "XUP", bring in the vector from
   the +X direction to the current site. If KS phases are in lattice,
   this transport will automatically include them. 
   OK for src and dest to be the same.  OK for length=0.  */
/* NOT OPTIMIZED at the moment - do lots of extra copying.  Use temp.
   vectors rather than stuff in lattice.h */
void path_transport( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    su3_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    su3_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_dest = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
    tmp_work = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(su3_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_vec( &(s->link[dir[j]]),
		    (su3_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_vec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work, sizeof(su3_vector),
	        dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(su3_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(su3_vector *)F_PT(s,dest) = *(su3_vector *)F_PT(s,src);
    }
  }
} /* path_transport */

/* Path transport a half_wilson_vector */
void path_transport_hwv( field_offset src, field_offset dest, int parity,
    int *dir, int length ){
    register int i;
    register site *s;
    msg_tag *mtag0;
    int j;
    half_wilson_vector *tmp_src,*tmp_dest,*tmp_work; /*source, dest and workspace*/
    half_wilson_vector *tmp_pt; /* scratch */
    int tmp_parity, tmp_otherparity; /* parity for this step */

  if( length > 0 ){
    tmp_src = (half_wilson_vector *)calloc(
      sites_on_node, sizeof(half_wilson_vector));
    tmp_dest = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));
    tmp_work = (half_wilson_vector *)calloc(
       sites_on_node, sizeof(half_wilson_vector));

    for( j=length-1; j>=0; j-- ){
	/* figure out parities for this step */
	if( j%2==0 ){
	    tmp_parity = parity;
	    switch(tmp_parity){
		case EVEN: tmp_otherparity=ODD; break;
		case ODD: tmp_otherparity=EVEN; break;
		case EVENANDODD: tmp_otherparity=EVENANDODD; break;
	    }
	}
	else { /* odd # step */
	    tmp_otherparity = parity;
	    switch(tmp_otherparity){
		case EVEN: tmp_parity=ODD; break;
		case ODD: tmp_parity=EVEN; break;
		case EVENANDODD: tmp_parity=EVENANDODD; break;
	    }
	}

	if( j==length-1 ){
	    FORSOMEPARITY(i,s,tmp_otherparity){
	        tmp_src[i] = *(half_wilson_vector *)F_PT(s,src);
	    }
	}

	if( GOES_FORWARDS(dir[j]) ) {
	    mtag0 = start_gather_from_temp( tmp_src,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		mult_su3_mat_hwvec( &(s->link[dir[j]]),
		    (half_wilson_vector *)(gen_pt[0][i]),
		    &(tmp_dest[i]) );
	    }
	    cleanup_gather(mtag0);
	}

	else{ /* GOES_BACKWARDS(dir[j]) */
	    FORSOMEPARITY(i,s,tmp_otherparity){
		mult_adj_su3_mat_hwvec( &(s->link[OPP_DIR(dir[j])]),
		    &(tmp_src[i]), &(tmp_work[i]) );
	    }
	    mtag0 = start_gather_from_temp( tmp_work,
		sizeof(half_wilson_vector), dir[j], tmp_parity, gen_pt[0] );
	    wait_gather(mtag0);
	    FORSOMEPARITY(i,s,tmp_parity){
		 tmp_dest[i] = *(half_wilson_vector *)gen_pt[0][i];
	    }
	    cleanup_gather(mtag0);
	}
	
	/* src for next step is dest for this one. */
	tmp_pt=tmp_src; tmp_src=tmp_dest; tmp_dest=tmp_pt;
    }  /* j=link in path */
    /* done, copy result into real dest. (tmp_src now points to result) */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) = tmp_src[i];
    }
    free(tmp_src); free(tmp_dest); free(tmp_work);
  } /* end if(length>0) */
  else if( src != dest ){ /* for length=0 */
    FORSOMEPARITY(i,s,parity){
        *(half_wilson_vector *)F_PT(s,dest) =
	  *(half_wilson_vector *)F_PT(s,src);
    }
  }
} /* path_transport_hwv */


#ifdef EO
/* Stupid dslash routine that follows all the paths.
  Should optimize by precomputing sums of paths to all the displacements
  Use dslash_fn for actions that involve only +X and +X+X+X couplings.
*/
/* D_slash routine - sets dest. on each site equal to sum of
   sources parallel transported to site, with minus sign for transport
   from negative directions.  */
void dslash_eo( field_offset src, field_offset dest, int parity ) {
   register int i;
   register site *s;
   register int ipath,otherparity;
   register double x; /* coefficient of path */

    switch(parity){
	case EVEN:	otherparity=ODD; break;
	case ODD:	otherparity=EVEN; break;
	case EVENANDODD:	otherparity=EVENANDODD; break;
    }

    /* Parallel transport by all the paths in the action.  
       Multiply by coefficient in table
    */
    FORSOMEPARITY(i,s,parity){ clearvec( (su3_vector *)F_PT(s,dest) ); }

    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	    path_transport( src, F_OFFSET(tempvec[0]), parity,
		q_paths[ipath].dir, q_paths[ipath].length );
	    x=q_paths[ipath].coeff;
	    FORSOMEPARITY(i,s,parity){
		scalar_mult_add_su3_vector(  (su3_vector *)F_PT(s,dest),
		    &(s->tempvec[0]), x, (su3_vector *)F_PT(s,dest) );
	    }
    }   /* ipath */
} /* dslash_eo */
#endif /* ifdef EO */

#ifdef FN
/* Sum over paths connecting to nearest neighbor point (fat link) and to third
   nearest neighbor (longlinks) */
/* Doug Toussaint 2/4/98 */
/* modified to use t_longlinks, S. Gottlieb 7/13/01 */
/* long link calculating routine */
/* path_product() follows the path starting at step 0, and
   leaves the answer at the end of the path.  We want the answer
   at the site where the path begins.  So we look for paths with
   the opposite displacement from the displacement of the point
   that we want to transport to this site, and take the adjoint
   of the matrix at the end. clear? */
/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_longlinks() {
  register int i;
  register site *s;
  int ipath,dir;
  int disp[4];
  register su3_matrix *long1;

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_longlinks needs phases in\n");
    terminate(0);
  }
  for (dir=XUP; dir<=TUP; dir++){ /* loop over longlink directions */
    /* set longlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      long1 = &(t_longlink[4*i+dir]);
#else
      long1 = &(s->longlink[dir]);
#endif
      clear_su3mat( long1 );
    }

    /* loop over paths, checking for ones with total displacement 3*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=3,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("ipath = %d, found a path:  ",ipath);
for(j=0;j<q_paths[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  long1 = &(t_longlink[4*i+dir]);
#else
	  long1 = &(s->longlink[dir]);
#endif
          scalar_mult_add_su3_matrix( long1,
	    &(s->staple), -q_paths[ipath].coeff, long1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */

  } /* loop over directions */

  valid_longlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Long):  %e\n",dtime);
#endif
}  /* load_longlinks() */

/* KS phases and APBC must be in the links. See long comment at bottom*/
void load_fatlinks() {
  register int i;
  register site *s;
  int dir;
  register su3_matrix *fat1;
#ifdef ASQ_OPTIMIZED_FATTENING
  int  nu,rho,sig ;
  double one_link ; /*needed to fix the problem with the Lepage term */
#else
  int ipath;
  int disp[4];
#endif

#ifdef LLTIME
double dtime;
dtime=-dclock();
#endif
  if( phases_in != 1){
    node0_printf("BOTCH: load_fatlinks needs phases in\n");
    terminate(0);
  }

#ifndef  ASQ_OPTIMIZED_FATTENING   /* general case code */
  for (dir=XUP; dir<=TUP; dir++){ /* loop over fatlink directions */
    /* set fatlink to zero */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+dir]);
#else
      fat1 = &(s->fatlink[dir]);
#endif
      clear_su3mat( fat1 );
    }
    
    /* loop over paths, checking for ones with total displacement 1*dir */
    for( ipath=0; ipath<num_q_paths; ipath++ ){  /* loop over paths */
	/* compute total displacement of path */
	for(i=XUP;i<=TUP;i++)disp[i]=0;
	for( i=0; i<q_paths[ipath].length; i++){
	  if( GOES_FORWARDS(q_paths[ipath].dir[i]) )
	    disp[        q_paths[ipath].dir[i]  ]++;
	  else
	    disp[OPP_DIR(q_paths[ipath].dir[i]) ]--;
	}
	for( disp[dir]+=1,i=XUP; i<=TUP; i++)if(disp[i]!=0)break;
	if( i<=TUP )continue;  /* skip if path doesn't go to right place */
/**printf("dir = %d, found a path:  ",dir);
for(j=0;j<q_paths.[ipath].length;j++)printf("\t%d", q_paths[ipath].dir[j]);
printf("\n");**/

	path_product( q_paths[ipath].dir, q_paths[ipath].length );
	FORALLSITES(i,s){
	  su3_adjoint( &(s->tempmat1), &(s->staple) );
#ifdef DSLASH_TMP_LINKS
	  fat1 = &(t_fatlink[4*i+dir]);
#else
	  fat1 = &(s->fatlink[dir]);
#endif
          scalar_mult_add_su3_matrix( fat1,
	    &(s->staple), -q_paths[ipath].coeff, fat1 );
		/* minus sign in coeff. because we used backward path*/
	}
    } /* ipath */
  } /* loop over directions */
#else	/* ASQ_OPTIMIZED_FATTENING, for Asq and Asqtad actions */
/*  Optimized fattening code for the Asq and Asqtad actions.           *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
 
 /* to fix up the Lepage term, included by a trick below */
 one_link = (act_path_coeff[0] - 6.0*act_path_coeff[5]) ; 
 
 for (dir=XUP; dir<=TUP; dir++){
   FORALLSITES(i,s) /* Intialize fat links with c_1*U_\mu(x) */
     {
#ifdef DSLASH_TMP_LINKS
       fat1 = &(t_fatlink[4*i+dir]);
#else
       fat1 = &(s->fatlink[dir]);
#endif
       scalar_mult_su3_matrix(&(s->link[dir]), one_link ,
			      fat1 );
     }
   for(nu=XUP; nu<=TUP; nu++) if(nu!=dir)
     {
       compute_gen_staple(F_OFFSET(staple),dir,nu,F_OFFSET(link[dir]),
			  act_path_coeff[2]);
       /* The Lepage term */
       /* Note this also involves modifying c_1 (above) */
       compute_gen_staple(NULL_FP,dir,nu,F_OFFSET(staple),act_path_coeff[5]);
       for(rho=XUP; rho<=TUP; rho++) if((rho!=dir)&&(rho!=nu))
	 {
	   compute_gen_staple(F_OFFSET(tempmat1),dir,rho,F_OFFSET(staple),
			      act_path_coeff[3]);
	   for(sig=XUP; sig<=TUP; sig++)
	     if((sig!=dir)&&(sig!=nu)&&(sig!=rho))
	       {
		 compute_gen_staple(NULL_FP,dir,sig,
				    F_OFFSET(tempmat1),
				    act_path_coeff[4]);
	       } /* sig */
	 } /* rho */
     } /* nu */
 }/* dir */  
#endif

  valid_fatlinks = 1;
#ifdef LLTIME
dtime += dclock();
node0_printf("LLTIME(Fat):  %e\n",dtime);
#endif
}  /* load_fatlinks() */
#endif /* ifdef FN */


/* compare two paths, return 1 if equal, else zero */
int is_path_equal( int *path1, int* path2, int length ){
   register int i;
   for(i=0;i<length;i++)if(path1[i]!=path2[i])return(0);
   return(1);
}


/* update the  momenta with the fermion force */
/* Assumes that the conjugate gradient has been run, with the answer in
   x_off, and dslash(x_off,x_off,ODD) has been run. (fills in x_off_odd) */
/* SEE LONG COMMENTS AT END */

#ifndef ASQ_OPTIMIZED_FORCE
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon, coeff;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  half_wilson_vector *hw_tmp0,*hw_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  hw_tmp0 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  hw_tmp1 = (half_wilson_vector *)
    calloc(sites_on_node, sizeof(half_wilson_vector));
  /* Use half_wilson_vectors to store x_off transported from ends of
     path.  0 component from forward end, 1 component from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    path_transport( x_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[1] for transport from starting end */
    FORALLSITES(i,s){
      hw_tmp0[i].h[0]=s->tempvec[0];
      hw_tmp0[i].h[1]=*(su3_vector *)F_PT(s,x_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff = ferm_epsilon*q_paths[ipath].coeff;
      if( (ilink%2)==1 )coeff = -coeff;

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "half_wilson_vector" to handle pair of vectors -
	0 component is x_off from forward end, 1 component from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_su3_mat_hwvec( &(s->link[lastdir]),
	      &(hw_tmp0[i]), &(hw_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( hw_tmp1, 2*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     hw_tmp0[i] = *(half_wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( hw_tmp0, 2*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_su3_mat_hwvec( &(s->link[OPP_DIR(lastdir)]),
                (half_wilson_vector *)(gen_pt[0][i]),
		&(hw_tmp1[i]) );
          }
	  tmp_pt = hw_tmp0; hw_tmp0 = hw_tmp1; hw_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(hw_tmp0[i].h[0]), &(hw_tmp0[i].h[1]), &tmat );
	  if( s->parity==EVEN )scalar_mult_add_su3_matrix(
	    &tmat2, &tmat, -coeff, &tmat2 );
	  else		 scalar_mult_add_su3_matrix(
	    &tmat2, &tmat,  coeff, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( hw_tmp0 ); free( hw_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */


void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off,
  int nflav2, field_offset x2_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/14/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i,dir,lastdir,ipath,ilink;
  register site *s;
  int length;
  su3_matrix tmat,tmat2;
  double ferm_epsilon1, ferm_epsilon2, coeff1, coeff2;
#ifdef FFTIME
  double dtime;
#endif
  msg_tag *mtag0;
  wilson_vector *w_tmp0,*w_tmp1,*tmp_pt;

#ifdef FFTIME
dtime=-dclock();
#endif
  ferm_epsilon1 = 2.0*(nflav1/4.0)*eps;
  ferm_epsilon2 = 2.0*(nflav2/4.0)*eps;
  w_tmp0 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  w_tmp1 = (wilson_vector *)
    calloc(sites_on_node, sizeof(wilson_vector));
  /* Use wilson_vectors to store x_off transported from ends of
     path.  0 and 1 components from forward end, 2 and 3 components
     from back end */

  /* loop over paths, and loop over links in path */
  for( ipath=0; ipath<num_q_paths; ipath++ ){
    if(q_paths[ipath].forwback== -1)continue;	/* skip backwards dslash */
    length = q_paths[ipath].length;

    /* path transport x_off and Dslash*x_off from far end.  Sometimes
	we need them at the start point of the path, and sometimes
	one link into the path - an optimization for later */
    /**
    path_transport( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    path_transport( x2_off, F_OFFSET(tempvec[1]),
      EVENANDODD, q_paths[ipath].dir, length );
    **/
/** WARNING!! Assumes xxx1 and xxx2 contiguous **/
if( x2_off-x1_off != sizeof(su3_vector) ){node0_printf("BOTCH\n"); exit(0);}
    path_transport_hwv( x1_off, F_OFFSET(tempvec[0]),
      EVENANDODD, q_paths[ipath].dir, length );
    /* use tempvec[2] for transport from starting end */
    FORALLSITES(i,s){
      w_tmp0[i].d[0]=s->tempvec[0];
      w_tmp0[i].d[1]=s->tempvec[1];
      w_tmp0[i].d[2]=*(su3_vector *)F_PT(s,x1_off);
      w_tmp0[i].d[3]=*(su3_vector *)F_PT(s,x2_off);
    }

    /* A path has (length+1) points, counting the ends.  At first
	 point, no "down" direction links have their momenta "at this
	 point". At last, no "up" ... */
    for( ilink=0; ilink<=length; ilink++ ){
      if(ilink<length)dir = q_paths[ipath].dir[ilink];
      else dir=NODIR;
      coeff1 = ferm_epsilon1*q_paths[ipath].coeff;
      coeff2 = ferm_epsilon2*q_paths[ipath].coeff;
      if( (ilink%2)==1 ){ coeff1 = -coeff1; coeff2 = -coeff2;}

      /* path transport x_off and Dslash*x_off from previous point */
      /* Use "wilson_vector" to handle pair of vectors -
	0 component is x_off1 from forward end, 1 component is x_off2
	from forward end,  2 and 3  components are x_off1 and x_off2 
	from back end */
      /* sometimes we don't need them */
      if( (ilink>0&&ilink<length) || 
	(ilink==length && GOES_BACKWARDS(lastdir)) ){

	if( GOES_FORWARDS(lastdir) ){
	  FORALLSITES(i,s){
            mult_adj_mat_wilson_vec( &(s->link[lastdir]),
	      &(w_tmp0[i]), &(w_tmp1[i]) );
	  }
	  mtag0 = start_gather_from_temp( w_tmp1, 4*sizeof(su3_vector),
             OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
	     w_tmp0[i] = *(wilson_vector *)gen_pt[0][i];
	  }
          cleanup_gather(mtag0);
	}
	else{   /* GOES_BACKWARDS(lastdir) */
          mtag0 = start_gather_from_temp( w_tmp0, 4*sizeof(su3_vector),
               OPP_DIR(lastdir), EVENANDODD, gen_pt[0] );
          wait_gather(mtag0);
          FORALLSITES(i,s){
            mult_mat_wilson_vec( &(s->link[OPP_DIR(lastdir)]),
                (wilson_vector *)(gen_pt[0][i]),
		&(w_tmp1[i]) );
          }
	  tmp_pt = w_tmp0; w_tmp0 = w_tmp1; w_tmp1 = tmp_pt;
          cleanup_gather(mtag0);
	}
      }

      /* add in contribution to the force */
      /* Put antihermitian traceless part into momentum */
      FORALLSITES(i,s){
        if( ilink<length && GOES_FORWARDS(dir) ){
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[dir]) );
	  if( s->parity==ODD ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
	if( ilink>0 && GOES_BACKWARDS(lastdir) ){
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
          uncompress_anti_hermitian( &(s->mom[OPP_DIR(lastdir)]), &tmat2 );
	  su3_projector( &(w_tmp0[i].d[0]), &(w_tmp0[i].d[2]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff1, &tmat2 );
	  su3_projector( &(w_tmp0[i].d[1]), &(w_tmp0[i].d[3]), &tmat );
	  scalar_mult_add_su3_matrix( &tmat2, &tmat,  coeff2, &tmat2 );
          make_anti_hermitian( &tmat2, &(s->mom[OPP_DIR(lastdir)]) );
	  if( s->parity==EVEN ){coeff1 *= -1.0; coeff2 *= -1.0; }
        }
      }
      lastdir = dir;
    } /* end loop over links in path */
  } /* end loop over paths */

  free( w_tmp0 ); free( w_tmp1 );

#ifdef FFTIME
dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */



#else /* ASQ_OPTIMIZED_FORCE, for Asq and Asqtad actions */
/* Optimized force code for the Asq and Asqtad actions                 *
 * I assume that path 0 is the one link path 2 the 3-staple            *
 * path 3 the 5-staple path 4 the 7-staple and path 5 the Lapage term. *
 * Path 1 is the Naik term.                                            */
#define Pmu          tempvec[0] 
#define Pnumu        tempvec[1]
#define Prhonumu     tempvec[2]
#define P7           tempvec[3]
#define P7rho        tempvec[4]              
#define P7rhonu      tempvec[5]
#define P5           tempvec[6]
#define P3           tempvec[7]
#define P5nu         tempvec[3]
#define P3mu         tempvec[3]
#define Popmu        tempvec[4]
#define Pmumumu      tempvec[4]
void eo_fermion_force( double eps, int nflavors, field_offset x_off ){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double ferm_epsilon, coeff;
  double OneLink, Lepage, Naik, FiveSt, ThreeSt, SevenSt ;
  su3_vector *tempvec[8] ;
  su3_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  ferm_epsilon = 2.0*(nflavors/4.0)*eps;
  
  /* Path coefficients times fermion epsilon */
  OneLink = act_path_coeff[0]*ferm_epsilon ; 
  Naik    = act_path_coeff[1]*ferm_epsilon ;
  ThreeSt = act_path_coeff[2]*ferm_epsilon ;
  FiveSt  = act_path_coeff[3]*ferm_epsilon ;
  SevenSt = act_path_coeff[4]*ferm_epsilon ;
  Lepage  = act_path_coeff[5]*ferm_epsilon ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    tempvec[mu] = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));

  /*copy x_off to a temporary vector */
  temp_x = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
  FORALLSITES(i,s) temp_x[i] = *(su3_vector *)F_PT(s,x_off) ;

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_force_to_mom(P3, Pmu, sig, -ThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_force_to_mom(P7, Prhonumu, sig, -SevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_fermion(P7, P7rho, rho);
		  side_link_force(rho,sig,SevenSt, Pnumu, P7, Prhonumu, P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff = SevenSt/FiveSt ; 
		  FORALLSITES(i,s)
		    scalar_mult_add_su3_vector(&(P5[i]),&(P7rho[i]),coeff,
					       &(P5[i]));
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_fermion(P5,P5nu, nu);
	      side_link_force(nu,sig,-FiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff = FiveSt/ThreeSt ; 
	      FORALLSITES(i,s)
		scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_fermion(P5,P5nu, mu);
	  side_link_force(mu, sig, -Lepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff = Lepage/ThreeSt ; 
	  FORALLSITES(i,s)
	    scalar_mult_add_su3_vector(&(P3[i]),&(P5nu[i]),coeff,&(P3[i]));

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), -Naik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(tempvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      

#define Pmu          hwvec[0] 
#define Pnumu        hwvec[1]
#define Prhonumu     hwvec[2]
#define P7           hwvec[3]
#define P7rho        hwvec[4]              
#define P7rhonu      hwvec[5]
#define P5           hwvec[6]
#define P3           hwvec[7]
#define P5nu         hwvec[3]
#define P3mu         hwvec[3]
#define Popmu        hwvec[4]
#define Pmumumu      hwvec[4]
void eo_fermion_force_3f( double eps, int nflav1, field_offset x1_off, 
			  int nflav2, field_offset x2_off){
  /* note CG_solution and Dslash * solution are combined in "x_off" */
  /* New version 1/21/99.  Use forward part of Dslash to get force */
  /* 4/15/99 combine force from two different mass quarks, (eg 2+1flavors) */
  /* see long comment at end */
  /* For each link we need x_off transported from both ends of path. */
  register int i ;
  register site *s;
  int mu,nu,rho,sig ;
  int DirectLinks[8] ;
  double coeff[2],ferm_epsilon ;
  double OneLink[2], Lepage[2], Naik[2], FiveSt[2], ThreeSt[2], SevenSt[2] ;
  double mNaik[2], mLepage[2], mFiveSt[2], mThreeSt[2], mSevenSt[2] ;
  half_wilson_vector *hwvec[8] ;
  half_wilson_vector *temp_x ;

#ifdef FFTIME
  double dtime;

  dtime=-dclock();
#endif
  
  /* Path coefficients times fermion epsilon */
  ferm_epsilon = 2.0*(nflav1/4.0)*eps;
  OneLink[0] = act_path_coeff[0]*ferm_epsilon ;
  Naik[0]    = act_path_coeff[1]*ferm_epsilon ; mNaik[0]    = -Naik[0] ;
  ThreeSt[0] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[0] = -ThreeSt[0] ;
  FiveSt[0]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[0]  = -FiveSt[0] ;
  SevenSt[0] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[0] = -SevenSt[0] ;
  Lepage[0]  = act_path_coeff[5]*ferm_epsilon ; mLepage[0]  = -Lepage[0] ;

  ferm_epsilon = 2.0*(nflav2/4.0)*eps;
  OneLink[1] = act_path_coeff[0]*ferm_epsilon ;
  Naik[1]    = act_path_coeff[1]*ferm_epsilon ; mNaik[1]    = -Naik[1] ;
  ThreeSt[1] = act_path_coeff[2]*ferm_epsilon ; mThreeSt[1] = -ThreeSt[1] ;
  FiveSt[1]  = act_path_coeff[3]*ferm_epsilon ; mFiveSt[1]  = -FiveSt[1] ;
  SevenSt[1] = act_path_coeff[4]*ferm_epsilon ; mSevenSt[1] = -SevenSt[1] ;
  Lepage[1]  = act_path_coeff[5]*ferm_epsilon ; mLepage[1]  = -Lepage[1] ;
  /* *************************************** */

  /* Initialize the DirectLink flags */
  for(mu=0;mu<8;mu++)
    DirectLinks[mu] = 0 ;

  /* Allocate temporary vectors */
  for(mu=0;mu<8;mu++)
    hwvec[mu]= 
      (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));

  /*copy x_off to a temporary vector */
  temp_x= 
    (half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
  FORALLSITES(i,s)
    {
      temp_x[i].h[0] = *(su3_vector *)F_PT(s,x1_off) ;
      temp_x[i].h[1] = *(su3_vector *)F_PT(s,x2_off) ;
    }

  for(sig=0;sig<8;sig++)
    {
      for(mu=0;mu<8;mu++)if((mu!=sig)&&(mu!=OPP_DIR(sig)))
	{
	  u_shift_hw_fermion(temp_x, Pmu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pmu, P3, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu]:         x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 1 link in the path: - (numbering starts form 0) */
	      add_3f_force_to_mom(P3, Pmu, sig, mThreeSt) ;
	    }
	  for(nu=0;nu<8;nu++)if((nu!=mu )&&(nu!=OPP_DIR(mu ))&&
				(nu!=sig)&&(nu!=OPP_DIR(sig)))
	    {
	      u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(nu));
	      u_shift_hw_fermion(Pnumu, P5, sig);
	      if(GOES_FORWARDS(sig))
		{
		  /* Add the force F_sig[x+mu+nu]:      x--+             *
		   *                                   |   |             *
		   *                                   o   o             *
		   * the 2 link in the path: + (numbering starts form 0) */
		  add_3f_force_to_mom(P5, Pnumu, sig, FiveSt);
		}
	      for(rho=0;rho<8;rho++)if((rho!=mu )&&(rho!=OPP_DIR(mu ))&&
				       (rho!=nu )&&(rho!=OPP_DIR(nu ))&&
				       (rho!=sig)&&(rho!=OPP_DIR(sig)))
		{
		  u_shift_hw_fermion(Pnumu, Prhonumu, OPP_DIR(rho));
		  /* Length 7 paths */
		  u_shift_hw_fermion(Prhonumu, P7,sig);
		  if(GOES_FORWARDS(sig))
		    {
		      /* Add the force F_sig[x+mu+nu+rho]:  x--+             *
		       *                                   |   |             *
		       *                                   o   o             *
		       * the 3 link in the path: - (numbering starts form 0) */
		      add_3f_force_to_mom(P7, Prhonumu, sig, mSevenSt ) ;
		    }
		  /*Add the force F_rho the 2(4) link in the path: +     */
		  u_shift_hw_fermion(P7, P7rho, rho);
		  side_link_3f_force(rho,sig,SevenSt,Pnumu,P7,Prhonumu,P7rho);
		  /* Add the P7rho vector to P5 */
		  coeff[0] = SevenSt[0]/FiveSt[0] ; 
		  coeff[1] = SevenSt[1]/FiveSt[1] ; 
		  FORALLSITES(i,s)
		    {
		      scalar_mult_add_su3_vector(&(P5[i].h[0]),
						 &(P7rho[i].h[0]),coeff[0],
						 &(P5[i].h[0]));
		      scalar_mult_add_su3_vector(&(P5[i].h[1]),
						 &(P7rho[i].h[1]),coeff[1],
						 &(P5[i].h[1]));
		    }
		}/* rho */
	      /* Length 5 paths */
	      /*Add the force F_nu the 1(3) link in the path: -     */
	      u_shift_hw_fermion(P5,P5nu, nu);
	      side_link_3f_force(nu,sig,mFiveSt,Pmu,P5, 
			      Pnumu,P5nu) ;
	      /* Add the P5nu vector to P3 */
	      coeff[0] = FiveSt[0]/ThreeSt[0] ; 
	      coeff[1] = FiveSt[1]/ThreeSt[1] ; 
	      FORALLSITES(i,s)
		{
		  scalar_mult_add_su3_vector(&(P3[i].h[0]),
					     &(P5nu[i].h[0]), coeff[0],
					     &(P3[i].h[0]));
		  scalar_mult_add_su3_vector(&(P3[i].h[1]),
					     &(P5nu[i].h[1]), coeff[1],
					     &(P3[i].h[1]));
		}
	    }/* nu */

	  /* Now the Lepage term... It is the same with 5-link paths with
             nu=mu and FiveSt=Lepage. So Pnumu is really Pmumu */
	  u_shift_hw_fermion(Pmu, Pnumu, OPP_DIR(mu));
	  u_shift_hw_fermion(Pnumu, P5, sig);
	  if(GOES_FORWARDS(sig))
	    {
	      /* Add the force F_sig[x+mu+nu]:      x--+             *
	       *                                   |   |             *
	       *                                   o   o             *
	       * the 2 link in the path: + (numbering starts form 0) */
	      add_3f_force_to_mom(P5, Pnumu, sig, Lepage) ;
	    }
	  /*Add the force F_nu the 1(3) link in the path: -     */
	  u_shift_hw_fermion(P5,P5nu, mu);
	  side_link_3f_force(mu, sig, mLepage, Pmu, P5, Pnumu, P5nu) ;
	  /* Add the P5nu vector to P3 */
	  coeff[0] = Lepage[0]/ThreeSt[0] ; 
	  coeff[1] = Lepage[1]/ThreeSt[1] ; 
	  FORALLSITES(i,s)
	    {
	      scalar_mult_add_su3_vector(&(P3[i].h[0]),
					 &(P5nu[i].h[0]),coeff[0],
					 &(P3[i].h[0]));
	      scalar_mult_add_su3_vector(&(P3[i].h[1]),
					 &(P5nu[i].h[1]),coeff[1],
					 &(P3[i].h[1]));
	    }

	  /* Length 3 paths (Not the Naik term) */
	  /*Add the force F_mu the 0(2) link in the path: +     */
	  if(GOES_FORWARDS(mu)) 
	    u_shift_hw_fermion(P3,P3mu, mu );
	  /* The above shift is not needed if mu is backwards */
	  side_link_3f_force(mu, sig, ThreeSt, temp_x, P3, Pmu, P3mu);

	  /* Finally the OneLink and the Naik term */
	  /* Check if this direction is not already done */
	  if( (!DirectLinks[mu]) ){
	    DirectLinks[mu]=1 ;
	    if(GOES_BACKWARDS(mu))/* Do only the forward terms in the Dslash */
	      {
		/* Because I have shifted with OPP_DIR(mu) Pmu is a forward *
		 * shift.                                                   */
		/* The one link */
		add_3f_force_to_mom(Pmu, temp_x, OPP_DIR(mu), OneLink) ;
		/* For the same reason Pnumu is the forward double link */

		/* Popmu is a backward shift */
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* The Naik */
		/* link no 1: - */
		add_3f_force_to_mom(Pnumu, Popmu, OPP_DIR(mu), mNaik) ;
		/*Pmumumu can overwrite Popmu which is no longer needed */
		u_shift_hw_fermion(Pnumu, Pmumumu, OPP_DIR(mu));
		/* link no 0: + */
		add_3f_force_to_mom(Pmumumu, temp_x, OPP_DIR(mu), Naik);
	      }
	    else /* The rest of the Naik terms */
	      {
		u_shift_hw_fermion(temp_x, Popmu, mu);
		/* link no 2: + */
		/* Pnumu is double backward shift */
		add_3f_force_to_mom(Popmu, Pnumu, mu, Naik) ;
	      }
	  }
	}/* mu */
      /* Here we have to do together the Naik term and the one link term */
    }/*sig */

  /* Free temporary vectors */
  free(temp_x) ;
  for(mu=0;mu<8;mu++)
    free(hwvec[mu]) ;

#ifdef FFTIME
  dtime += dclock();
node0_printf("FFTIME:  %e\n",dtime);
/**printf("TLENGTH: %d\n",tlength);**/
#endif
} /* eo_fermion_force_3f(version 6) */
#undef Pmu          
#undef Pnumu        
#undef Prhonumu     
#undef P7           
#undef P7rho        
#undef P7rhonu      
#undef P5           
#undef P3           
#undef P5nu         
#undef P3mu         
#undef Popmu        
#undef Pmumumu      
#endif /* ASQ_OPTIMIZED_FORCE */


#ifdef  ASQ_OPTIMIZED_FATTENING   /* Asqtad action only, "_fn" executables */
#ifndef FN
BOMB THE COMPILE
#endif
void compute_gen_staple(field_offset staple, int mu, int nu, 
			field_offset link, double coef) {
  su3_matrix tmat1,tmat2;
  msg_tag *mtag0,*mtag1;
  su3_matrix *tempmat ;
  register site *s ;
  register int i ;
  register su3_matrix *fat1;

  /* Computes the staple :
                   mu
                +-------+
          nu	|	|
		|	|
		X	X
    Where the mu link can be any su3_matrix. The result is saved in staple.
    if staple==NULL_FP then the result is not saved.
    It also adds the computed staple to the fatlink[mu] with weight coef.
  */

  /* Upper staple */
  mtag0 = start_gather( link, sizeof(su3_matrix), nu, EVENANDODD, gen_pt[0] );
  mtag1 = start_gather( F_OFFSET(link[nu]), sizeof(su3_matrix), mu, 
			EVENANDODD, gen_pt[1] );
  wait_gather(mtag0);
  wait_gather(mtag1);
  
  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, (su3_matrix *)F_PT(s,staple) );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
      mult_su3_na( (su3_matrix *)gen_pt[0][i],
		   (su3_matrix *)gen_pt[1][i], &tmat1 );
      mult_su3_nn( &(s->link[nu]), &tmat1, &tmat2 );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix(fat1, &tmat2, coef,
				 fat1) ;
    }
  }
  cleanup_gather(mtag0);
  cleanup_gather(mtag1);

  /* lower staple */
  tempmat = (su3_matrix *)calloc(sites_on_node, sizeof(su3_matrix));
  mtag0 = start_gather( F_OFFSET(link[nu]),
			sizeof(su3_matrix), mu, EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);
  FORALLSITES(i,s){
    mult_su3_an( &(s->link[nu]),(su3_matrix *)F_PT(s,link), &tmat1 );
    mult_su3_nn( &(tmat1),(su3_matrix *)gen_pt[0][i], &(tempmat[i]) );
  }
  cleanup_gather(mtag0);
  mtag0 = start_gather_from_temp( tempmat, sizeof(su3_matrix),
				  OPP_DIR(nu), EVENANDODD, gen_pt[0] );
  wait_gather(mtag0);

  if(staple!=NULL_FP){/* Save the staple */
    FORALLSITES(i,s){
      add_su3_matrix( (su3_matrix *)F_PT(s,staple),(su3_matrix *)gen_pt[0][i], 
		      (su3_matrix *)F_PT(s,staple) );
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)F_PT(s,staple), coef, 
				 fat1 );
    }
  }
  else{ /* No need to save the staple. Add it to the fatlinks */
    FORALLSITES(i,s){
#ifdef DSLASH_TMP_LINKS
      fat1 = &(t_fatlink[4*i+mu]);
#else
      fat1 = &(s->fatlink[mu]);
#endif
      scalar_mult_add_su3_matrix( fat1,
				 (su3_matrix *)gen_pt[0][i], coef, 
				 fat1 );
    }
  }

  free(tempmat);
  cleanup_gather(mtag0);
}
#endif  /* ASQ_OPTIMIZED_FATTENING   */


#ifdef  ASQ_OPTIMIZED_FORCE
/*   Covariant shift of the src fermion field in the direction dir  *
 *  by one unit. The result is stored in dest.                       */ 
void u_shift_fermion(su3_vector *src, su3_vector *dest, int dir ) {
  su3_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(su3_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_vec(&(s->link[dir]),(su3_vector *)(gen_pt[0][i]),
			 &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = (su3_vector *)calloc(sites_on_node, sizeof(su3_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_vec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(su3_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(su3_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/*  Covariant shift of the src half wilson fermion field in the  *
 * direction dir   by one unit. The result is stored in dest.    */
void u_shift_hw_fermion(half_wilson_vector *src, 
			half_wilson_vector *dest, int dir ) {
  half_wilson_vector *tmpvec ; 
  msg_tag *mtag ;
  register site *s ;
  register int i ;
  
  if(GOES_FORWARDS(dir)) /* forward shift */
    {
      mtag = start_gather_from_temp(src, sizeof(half_wilson_vector), 
				    dir, EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      FORALLSITES(i,s)
	mult_su3_mat_hwvec(&(s->link[dir]),
			   (half_wilson_vector *)(gen_pt[0][i]), &(dest[i]));
      cleanup_gather(mtag);
    }
  else /* backward shift */
    {
      tmpvec = 
	(half_wilson_vector *)calloc(sites_on_node, sizeof(half_wilson_vector));
      FORALLSITES(i,s)
	mult_adj_su3_mat_hwvec(&(s->link[OPP_DIR(dir)]),&(src[i]), &tmpvec[i]);
      mtag = start_gather_from_temp(tmpvec, sizeof(half_wilson_vector), dir, 
				    EVENANDODD, gen_pt[0]);
      wait_gather(mtag);
      /* copy the gen_pt to the dest */
      FORALLSITES(i,s)
	dest[i] = *(half_wilson_vector *)gen_pt[0][i];
      cleanup_gather(mtag);
      free(tmpvec) ;
    }
}

/* Add in contribution to the force */
/* Put antihermitian traceless part into momentum */
void add_force_to_mom(su3_vector *back,su3_vector *forw,int dir,double coeff) {
  register site *s ;
  register int i ;  
  register double tmp_coeff ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff = -coeff ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD) 
      tmp_coeff = -coeff ;
    else
      tmp_coeff = coeff ;
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i]), &(forw[i]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff, &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/* Add in contribution to the force ( 3flavor case ) */
/* Put antihermitian traceless part into momentum */
void add_3f_force_to_mom(half_wilson_vector *back,
			 half_wilson_vector *forw, int dir, double coeff[2]) {
  register site *s ;
  register int i ;  
  double tmp_coeff[2] ;

  su3_matrix tmat,tmat2;

  if(GOES_BACKWARDS(dir))
    {
      dir = OPP_DIR(dir) ; 
      coeff[0] = -coeff[0] ;
      coeff[1] = -coeff[1] ;
    }
  FORALLSITES(i,s){
    if(s->parity==ODD)
      {
	tmp_coeff[0] = -coeff[0] ;
	tmp_coeff[1] = -coeff[1] ;
      }
    else
      {
	tmp_coeff[0] = coeff[0] ;
	tmp_coeff[1] = coeff[1] ;
      }
    uncompress_anti_hermitian( &(s->mom[dir]), &tmat2 );
    su3_projector(&(back[i].h[0]), &(forw[i].h[0]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[0], &tmat2 );
    su3_projector(&(back[i].h[1]), &(forw[i].h[1]), &tmat);
    scalar_mult_add_su3_matrix(&tmat2, &tmat,  tmp_coeff[1], &tmat2 );
    make_anti_hermitian( &tmat2, &(s->mom[dir]) ); 
  }
}

/*  This routine is needed in order to add the force on the side link *
 * of the paths in the Asq and Asqtad actions. It gets as inputs the  *
 * direction mu of the side link and the direction nu of the Dslash   *
 * term we are dealing with. Then it takes also 4 fermion fields:     *
 * Path: the piece of the path with no hop in the nu or mu direction  *
 * Path_nu: the piece of the path with a hop in nu  but not in mu     *
 * Path_mu: is Path times the link mu                                 *
 * Path_numu: is Path_nu times the link mu                            */
void side_link_force(int mu, int nu, double coeff, 
		     su3_vector *Path   , su3_vector *Path_nu, 
		     su3_vector *Path_mu, su3_vector *Path_numu) {
  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_force_to_mom(Path, Path_numu, OPP_DIR(mu), -coeff );/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_force_to_mom(Path_nu, Path_mu, mu, -coeff) ; /* ? extra - */
      else
	add_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
 
/*  The 3 flavor version of side_link_force used *
 * to optimize fermion transports                */
void side_link_3f_force(int mu, int nu, double coeff[2], 
			half_wilson_vector *Path   , 
			half_wilson_vector *Path_nu, 
			half_wilson_vector *Path_mu, 
			half_wilson_vector *Path_numu) {
  double m_coeff[2] ;

  m_coeff[0] = -coeff[0] ;
  m_coeff[1] = -coeff[1] ;

  if(GOES_FORWARDS(mu))
    {
      /*                    nu           * 
       * Add the force :  +----+         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  o    o         */
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_numu, Path, mu, coeff ) ;
      else
	add_3f_force_to_mom(Path,Path_numu,OPP_DIR(mu),m_coeff);/* ? extra - */
    }
  else /*GOES_BACKWARDS(mu)*/
    {
      /* Add the force :  o    o         *
       *               mu |    |         *
       *                  x    (x)       *
       *                  +----+         *
       *                    nu           */ 
      if(GOES_FORWARDS(nu))
	add_3f_force_to_mom(Path_nu, Path_mu, mu, m_coeff) ; /* ? extra - */
      else
	add_3f_force_to_mom(Path_mu, Path_nu, OPP_DIR(mu), coeff) ;
    }
}
#endif  /* ASQ_OPTIMIZED_FORCE   */

/* LONG COMMENTS
   Here we have combined "xxx", (offset "x_off")  which is
(M_adjoint M)^{-1} phi, with Dslash times this vector, which goes in the
odd sites of xxx.  Recall that phi is defined only on even sites.  In
computing the fermion force, we are looking at

< X |  d/dt ( Dslash_eo Dslash_oe ) | X >
=
< X | d/dt Dslash_eo | T > + < T | d/dt Dslash_oe | X >
where T = Dslash X.

The subsequent manipulations to get the coefficent of H, the momentum
matrix, in the simulation time derivative above look the same for
the two terms, except for a minus sign at the end, if we simply stick
T, which lives on odd sites, into the odd sites of X

 Each path in the action contributes terms when any link of the path
is the link for which we are computing the force.  We get a minus sign
for odd numbered links in the path, since they connect sites of the
opposite parity from what it would be for an even numbered link.
Minus signs from "going around" plaquette - ie KS phases, are supposed
to be already encoded in the path coefficients.
Minus signs from paths that go backwards are supposed to be already
encoded in the path coefficients.

Here, for example, are comments reproduced from the force routine for
the one-link plus Naik plus single-staple-fat-link action:

 The three link force has three contributions, where the link that
was differentiated is the first, second, or third link in the 3-link
path, respectively.  Diagramatically, where "O" represents the momentum,
the solid line the link corresponding to the momentum, and the dashed
lines the other links:
 

	O______________ x ............ x ...............
+
	x..............O______________x.................
+
	x..............x..............O________________
Think of this as
	< xxx | O | UUUxxx >		(  xxx, UUUX_p3 )
+
	< xxx U | O | UUxxx >		( X_m1U , UUX_p2 )
+
	< xxx U U | O | Uxxx >		( X_m2UU , UX_p1 )
where "U" indicates parallel transport, "X_p3" is xxx displaced
by +3, etc.
Note the second contribution has a relative minus sign
because it effectively contributes to the <odd|even>, or M_adjoint,
part of the force when we work on an even site. i.e., for M on
an even site, this three link path begins on an odd site.

The staple force has six contributions from each plane containing the
link direction:
Call these diagrams A-F:


	x...........x		O____________x
		    .			     .
		    .			     .
		    .			     .
		    .			     .
		    .			     .
	O___________x		x............x
	   (A)			    (B)



	x	    x		O____________x
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	.	    .		.	     .
	O___________x		x	     x
	   (C)			    (D)



	x...........x		O____________x
	.			.
	.			.
	.			.
	.			.
	.			.
	O___________x		x............x
	   (E)			    (F)

As with the Naik term, diagrams C and D have a relative minus
sign because they connect sites of the other parity.

Also note an overall minus sign in the staple terms relative to the
one link term because, with the KS phase factors included, the fat
link is  "U - w3 * UUU", or the straight link MINUS w3 times the staples.

Finally, diagrams B and E get one more minus sign because the link
we are differentiating is in the opposite direction from the staple
as a whole.  You can think of this as this "U" being a correction to
a "U_adjoint", but the derivative of U is iHU and the derivative
of U_adjoint is -iHU_adjoint.

*/
/* LONG COMMENT on sign conventions
In most of the program, the KS phases and antiperiodic boundary
conditions are absorbed into the link matrices.  This greatly simplfies
multiplying by the fermion matrix.  However, it requires care in
specifying the path coefficients.  Remember that each time you
encircle a plaquette, you pick up a net minus sign from the KS phases.
Thus, when you have more than one path to the same point, you generally
have a relative minus sign for each plaquette in a surface bounded by
this path and the basic path for that displacement.

Examples:
  Fat Link:
    Positive:	X-------X

    Negative     --------
	 	|	|
		|	|
		X	X

  Naik connection, smeared
    Positive:	X-------x-------x-------X

    Negative:	---------
		|	|
		|	|
		X	x-------x-------X

    Positive:	--------x--------
		|		|
		|		|
		X		x-------X

    Negative:	--------x-------x-------x
		|			|
		|			|
		X			X
*/



/* Comment on acceptable actions.
   We construct the backwards part of dslash by reversing all the
   paths in the forwards part.  So, for example, in the p4 action
   the forwards part includes +X+Y+Y

		X
		|
		|
		X
		|
		|
	X---->--X

  so we put -X-Y-Y in the backwards part.  But this isn't the adjoint
  of U_x(0)U_y(+x)U_y(+x+y).  Since much of the code assumes that the
  backwards hop is the adjoint of the forwards (for example, in
  preventing going to 8 flavors), the code only works for actions
  where this is true.  Roughly, this means that the fat link must
  be symmetric about reflection around its midpoint.  Equivalently,
  the paths in the backwards part of Dslash are translations of the
  paths in the forwards part.  In the case of the "P4" or knight's move
  action, this means that we have to have both paths
   +X+Y+Y and +Y+Y+X to the same point, with the same coefficients.
  Alternatively, we could just use the symmetric path +Y+X+Y.
*/

invivoquark_stuff_compute_gen_staple_1375

Real time: 5.598377e+08 -- Predicted time: 6.111380e+08

Callcount: 792

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
571	### 1	65.2588684688535	7.982980e+05	7.487200e+05	6.2
100	### 2	186.072256214468	7.520980e+05	7.197720e+05	4.2
244	### 3	137.013110428675	7.997330e+05	7.307160e+05	8.6
271	### 4	244.023111872738	7.650640e+05	7.095920e+05	7.2
229	### 5	159.824428135886	7.685320e+05	6.992920e+05	9.0

                /*****************  su3_proj.c  (in su3.a) ******************************
*									*
* void su3_projector( su3_vector *a, su3_vector *b, su3_matrix *c )	*
* C  <- outer product of A and B					*
*  C_ij = A_i * B_adjoint_j						*
*/
#include "config.h"
#include "complex.h"
#include "su3.h"

#ifndef FAST
void su3_projector( su3_vector *a, su3_vector *b, su3_matrix *c ){
register int i,j;
    for(i=0;i<3;i++)for(j=0;j<3;j++){
	CMUL_J( a->c[i], b->c[j], c->e[i][j] );
    }
}

#else
#ifdef NATIVEDOUBLE   /* RS6000 version */

void su3_projector( su3_vector *a, su3_vector *b, su3_matrix *c ){

  register int i,j;
  register double ar,ai,br,bi;

    for(i=0;i<3;i++){
	ar=a->c[i].real;  ai=a->c[i].imag;
	for(j=0;j<3;j++){
	    br=b->c[j].real;  bi=b->c[j].imag;
	    c->e[i][j].real = ar*br + ai*bi;
	    c->e[i][j].imag = ai*br - ar*bi;
	}
    }
}
#else

void su3_projector( su3_vector *a, su3_vector *b, su3_matrix *c ){
register int i,j;
register double tmp,tmp2;
    for(i=0;i<3;i++)for(j=0;j<3;j++){
	tmp2 = a->c[i].real * b->c[j].real;
	tmp = a->c[i].imag * b->c[j].imag;
	c->e[i][j].real = tmp + tmp2;
	tmp2 = a->c[i].real * b->c[j].imag;
	tmp = a->c[i].imag * b->c[j].real;
	c->e[i][j].imag = tmp - tmp2;
    }
}
#endif  /* End of "#ifdef NATIVEDOUBLE" */
#endif /* end ifdef FAST */

invivosu3_proj_su3_projector_41

Real time: 1.206690e+09 -- Predicted time: 1.243209e+10

Callcount: 20480000

Invocation	Cluster	Part	Invitro (cycles)	Invivo (cycles)	Error (%)
4026285	### 1	4795611.08273581	5.720000e+02	6.000000e+01	89.5
14672405	### 2	11570948.7919343	6.880000e+02	5.600000e+01	91.8
19881290	### 3	128292.843495411	4.920000e+02	6.800000e+01	86.1
16192533	### 4	2062112.7655764	3.560000e+02	5.200000e+01	85.3
6231313	### 5	239109.839300044	4.210000e+02	8.800000e+01	79.0
13774462	### 6	507979.544319422	4.360000e+02	7.200000e+01	83.4
15709268	### 7	219286.489680654	5.130000e+02	8.000000e+01	84.4
1431678	### 8	736966.557842648	5.420000e+02	6.400000e+01	88.1
11621090	### 9	254946.161834331	3.800000e+02	8.400000e+01	77.8

433-milc

Exec Time : 1.330863e+10 cycles

Matching : 97.2%

__invivo__quark_stuff_u_shift_fermion_1452

Real time: 1.434033e+09 -- Predicted time: 1.555542e+09

Callcount: 10160

__invivo__com_vanilla_start_gather_from_temp_581

Real time: 1.784641e+08 -- Predicted time: 2.017281e+08

Callcount: 26472

__invivo__m_mat_an_mult_su3_an_39

Real time: 5.050794e+08 -- Predicted time: 2.255115e+09

Callcount: 3216000

__invivo__control_main_32

Real time: 1.329345e+10 -- Predicted time: 2.289945e+07

Callcount: 1

__invivo__quark_stuff_load_fatlinks_566

Real time: 2.371161e+09 -- Predicted time: 2.412550e+09

Callcount: 22

__invivo__update_update_40

Real time: 1.279301e+10 -- Predicted time: 1.363889e+10

Callcount: 2

__invivo__quark_stuff_compute_gen_staple_1406

Real time: 1.792900e+08 -- Predicted time: 0.000000e+00

Callcount: 792

__invivo__su3mat_copy_su3mat_copy_13

Real time: 4.508012e+08 -- Predicted time: 1.863417e+09

Callcount: 7984000

__invivo__m_mat_nn_mult_su3_nn_73

Real time: 2.073895e+09 -- Predicted time: 8.405498e+09

Callcount: 13114000

__invivo__gauge_stuff_imp_gauge_force_230

Real time: 3.316111e+09 -- Predicted time: 3.830944e+09

Callcount: 10

__invivo__path_product_path_product_156

Real time: 1.662913e+08 -- Predicted time: 1.056162e+08

Callcount: 1080

__invivo__s_m_a_mat_scalar_mult_add_su3_matrix_17

Real time: 1.332137e+09 -- Predicted time: 6.356343e+09

Callcount: 30688000

__invivo__gauge_stuff_g_measure_345

Real time: 1.484978e+08 -- Predicted time: 1.511139e+08

Callcount: 2

__invivo__m_amatvec_mult_adj_su3_mat_vec_114

Real time: 1.575511e+09 -- Predicted time: 8.569147e+09

Callcount: 19760000

__invivo__quark_stuff_eo_fermion_force_946

Real time: 6.211676e+09 -- Predicted time: 6.656005e+09

Callcount: 10

__invivo__quark_stuff_compute_gen_staple_1368

Real time: 4.745753e+08 -- Predicted time: 4.991175e+08

Callcount: 792

__invivo__update_u_update_u_40

Real time: 2.115407e+08 -- Predicted time: 2.083390e+08

Callcount: 30

__invivo__path_product_path_product_49

Real time: 2.967069e+09 -- Predicted time: 2.734133e+09

Callcount: 2076

__invivo__m_matvec_mult_su3_mat_vec_67

Real time: 1.642143e+09 -- Predicted time: 1.018734e+10

Callcount: 20320000

__invivo__m_mat_na_mult_su3_na_31

Real time: 2.244397e+09 -- Predicted time: 8.754871e+09

Callcount: 13344000

__invivo__quark_stuff_add_force_to_mom_1523

Real time: 2.660932e+09 -- Predicted time: 2.911636e+09

Callcount: 10240

__invivo__quark_stuff_compute_gen_staple_1396

Real time: 9.268518e+08 -- Predicted time: 0.000000e+00

Callcount: 1584

__invivo__quark_stuff_u_shift_fermion_1466

Real time: 2.684497e+08 -- Predicted time: 2.560370e+08

Callcount: 9880

__invivo__quark_stuff_u_shift_fermion_1460

Real time: 1.350528e+09 -- Predicted time: 1.685396e+09

Callcount: 9880

__invivo__su3_adjoint_su3_adjoint_13

Real time: 1.957137e+08 -- Predicted time: 1.618735e+09

Callcount: 5872000

__invivo__s_m_a_vec_scalar_mult_add_su3_vector_18

Real time: 3.093487e+08 -- Predicted time: 2.135326e+09

invivoquark_stuff_u_shift_fermion_1452

invivocom_vanilla_start_gather_from_temp_581

invivom_mat_an_mult_su3_an_39

invivocontrol_main_32

invivoquark_stuff_load_fatlinks_566

invivoupdate_update_40

invivoquark_stuff_compute_gen_staple_1406

invivosu3mat_copy_su3mat_copy_13

invivom_mat_nn_mult_su3_nn_73

invivogauge_stuff_imp_gauge_force_230

invivopath_product_path_product_156

invivos_m_a_mat_scalar_mult_add_su3_matrix_17

invivogauge_stuff_g_measure_345

invivom_amatvec_mult_adj_su3_mat_vec_114

invivoquark_stuff_eo_fermion_force_946

invivoquark_stuff_compute_gen_staple_1368

invivoupdate_u_update_u_40

invivopath_product_path_product_49

invivom_matvec_mult_su3_mat_vec_67

invivom_mat_na_mult_su3_na_31

invivoquark_stuff_add_force_to_mom_1523

invivoquark_stuff_compute_gen_staple_1396

invivoquark_stuff_u_shift_fermion_1466

invivoquark_stuff_u_shift_fermion_1460

invivosu3_adjoint_su3_adjoint_13

invivos_m_a_vec_scalar_mult_add_su3_vector_18

invivoquark_stuff_compute_gen_staple_1375

invivosu3_proj_su3_projector_41