/* 
 * -- High Performance Computing Linpack Benchmark (HPL)                
 *    HPL - 2.0 - September 10, 2008                          
 *    Antoine P. Petitet                                                
 *    University of Tennessee, Knoxville                                
 *    Innovative Computing Laboratory                                 
 *    (C) Copyright 2000-2008 All Rights Reserved                       
 *                                                                      
 * -- Copyright notice and Licensing terms:                             
 *                                                                      
 * Redistribution  and  use in  source and binary forms, with or without
 * modification, are  permitted provided  that the following  conditions
 * are met:                                                             
 *                                                                      
 * 1. Redistributions  of  source  code  must retain the above copyright
 * notice, this list of conditions and the following disclaimer.        
 *                                                                      
 * 2. Redistributions in binary form must reproduce  the above copyright
 * notice, this list of conditions,  and the following disclaimer in the
 * documentation and/or other materials provided with the distribution. 
 *                                                                      
 * 3. All  advertising  materials  mentioning  features  or  use of this
 * software must display the following acknowledgement:                 
 * This  product  includes  software  developed  at  the  University  of
 * Tennessee, Knoxville, Innovative Computing Laboratory.             
 *                                                                      
 * 4. The name of the  University,  the name of the  Laboratory,  or the
 * names  of  its  contributors  may  not  be used to endorse or promote
 * products  derived   from   this  software  without  specific  written
 * permission.                                                          
 *                                                                      
 * -- Disclaimer:                                                       
 *                                                                      
 * THIS  SOFTWARE  IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,  INCLUDING,  BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY
 * OR  CONTRIBUTORS  BE  LIABLE FOR ANY  DIRECT,  INDIRECT,  INCIDENTAL,
 * SPECIAL,  EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES  (INCLUDING,  BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA OR PROFITS; OR BUSINESS INTERRUPTION)  HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT,  STRICT LIABILITY,  OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 * ---------------------------------------------------------------------
 */ 
/*
 * Include files
 */
#include "hpl.h"
#include "RCCE.h"
int MPI_Send(void *, int, int, int, int, RCCE_COMM);
int MPI_Recv(void *, int, int, int, int, RCCE_COMM, int *);


#ifndef HPL_COPY_L       /* and also want to avoid the copy of L ... */
#define HPL_COPY_L   /* well, sorry, can not do that: force the copy */
#endif

#ifdef STDC_HEADERS
int HPL_binit_1ring
(
   HPL_T_panel *              PANEL
)
#else
int HPL_binit_1ring( PANEL )
   HPL_T_panel *              PANEL;
#endif
{

/* ..
 * .. Executable Statements ..
 */
   if( PANEL == NULL )           { return( HPL_SUCCESS ); }
   if( PANEL->grid->npcol <= 1 ) { return( HPL_SUCCESS ); }

/*
 * Force the copy of the panel into a contiguous buffer
 */
   HPL_copyL( PANEL );

   return( HPL_SUCCESS );
}

#define   _M_BUFF     (void *)(PANEL->L2)
#define   _M_COUNT    PANEL->len
#define   _M_TYPE     MPI_DOUBLE

#ifdef STDC_HEADERS
int HPL_bcast_1ring
(
   HPL_T_panel                * PANEL,
   int                        * IFLAG
)
#else
int HPL_bcast_1ring( PANEL, IFLAG )
   HPL_T_panel                * PANEL;
   int                        * IFLAG;
#endif
{
/*
 * .. Local Variables ..
 */
   MPI_Comm                   comm;
   int                        ierr, go, next, msgid, prev, rank, root,
                              size;
/* ..
 * .. Executable Statements ..
 */
   if( PANEL == NULL ) { *IFLAG = HPL_SUCCESS; return( HPL_SUCCESS ); }
   if( ( size = PANEL->grid->npcol ) <= 1 )
   {                     *IFLAG = HPL_SUCCESS; return( HPL_SUCCESS ); }
/*
 * Cast phase:  If I am the root process, start spreading the panel.  If
 * I am not the root process, probe for message. If the message is here,
 * then receive it, and  if I am not the last process of the ring, then
 * forward it to the next.  Otherwise, inform the caller that the panel
 * has still not been received.
 */
   rank = PANEL->grid->mycol; comm  = PANEL->grid->row_comm;
   root = PANEL->pcol;        msgid = PANEL->msgid;

   if( rank == root )
   {
      ierr = MPI_Send( _M_BUFF, _M_COUNT, _M_TYPE, MModAdd1( rank,
                       size ), msgid, comm );
   }
   else
   {
      prev = MModSub1( rank, size );

//RvdW: remove probe code
//      ierr = MPI_Iprobe( prev, msgid, comm, &go, &PANEL->status[0] );      
      ierr = MPI_SUCCESS; go = 1;

      if( ierr == MPI_SUCCESS )
      {
         if( go != 0 )
         {
            ierr = MPI_Recv( _M_BUFF, _M_COUNT, _M_TYPE, prev, msgid,
                             comm, &PANEL->status[0] );
            next = MModAdd1( rank, size );
            if( ( ierr == MPI_SUCCESS ) && ( next != root ) )
            {
               ierr = MPI_Send( _M_BUFF, _M_COUNT, _M_TYPE, next,
                                msgid, comm );
            }
         }
         else { *IFLAG = HPL_KEEP_TESTING; return( *IFLAG ); }
      }
   }
/*
 * If the message was received and being forwarded,  return HPL_SUCCESS.
 * If an error occured in an MPI call, return HPL_FAILURE.
 */  
   *IFLAG = ( ierr == MPI_SUCCESS ? HPL_SUCCESS : HPL_FAILURE );

   return( *IFLAG );
}

#ifdef STDC_HEADERS
int HPL_bwait_1ring
(
   HPL_T_panel *              PANEL
)
#else
int HPL_bwait_1ring( PANEL )
   HPL_T_panel *              PANEL;
#endif
{

/* ..
 * .. Executable Statements ..
 */
   if( PANEL == NULL )           { return( HPL_SUCCESS ); }
   if( PANEL->grid->npcol <= 1 ) { return( HPL_SUCCESS ); }
/*
 * Release the arrays of request / status / data-types and buffers 
 */

   return( HPL_SUCCESS );

}
