//
// MPI input for MITgcm
// domain-specific  hack version
//
// This file gets included into readtile_mpiio.c, twice,
// once with
//   #define dataElementType  float
//   #define mpi_dataElementType  MPI_FLOAT
// and once with
//   #define dataElementType  double
//   #define mpi_dataElementType  MPI_DOUBLE
//


///////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////

#define MPI_ARRAY_ORDER  MPI_ORDER_FORTRAN


// Use token pasting to produce type-specific names for these routines
#define CONCAT(AA,BB) AA ## BB
#define CAT(AA,BB)  CONCAT(AA,BB)
#define tileOffsetInField  CAT(tileOffsetInField_,dataElementType)
#define readField  CAT(readField_,dataElementType)
#define readfield_ CAT(CAT(readfield_,dataElementType),_)


// Tile numbers are rastered West-to-East, South-to-North.
// Return value is the byte offset from the begining of the field
// to the first byte of the first z-level of the given tile.
// For the simple rectangular domain, this is a straight forward
// calculation.  For something like the full global llc model,
// this is more complicated.
long int
tileOffsetInField (int tileID)
{
    long int rowOffset, columnOffset;

    // N.B.: tileID is 1-based, not 0-based
    tileID -= 1;

    // Offset in tiles
    rowOffset = tileID / numTilesInX;
    columnOffset = tileID % numTilesInX;

    // Offset in elements
    rowOffset *= numTilesInX * (tileSizeX * tileSizeY);
    columnOffset *= tileSizeX;

    // Offset in bytes
    rowOffset *= sizeof(dataElementType);
    columnOffset *= sizeof(dataElementType);

    return rowOffset + columnOffset;
}


void
readField (
  MPI_Comm  appComm,
  char *filename,
  MPI_Offset fieldOffsetInFile,
  dataElementType *tileBuf,
  int tileID,
  int zLevels)
{
    static int oldNumZLevels = -1;
    static MPI_Datatype fullTileInMemoryShape;
    static MPI_Datatype fullTileInFileShape;
    static MPI_Info  ioHints;

    int  appCommRank;
    MPI_Comm_rank(appComm, &appCommRank);

if (0 == appCommRank) fprintf (stdout,
"Begin MPI-IO read for '%s',  byte offset %ld\n", filename, fieldOffsetInFile);

    MPI_Offset tileOffsetInFile = fieldOffsetInFile + tileOffsetInField (tileID);
    MPI_File fh;
    MPI_Status status;
    int kk, count;


    // If the number of zLevels for this read is the *same* as the previous
    // one, just reuse the MPI_Datatypes.  If *not*, make new ones.
    // N.B.: we are assuming that the zLevels check is sufficient.
    // In particular, this means that "xGhosts" and "yGhosts" must be
    // constant (no mixing of ghosted and un-ghosted reads).
    // If that changes, the check for "sameness" will need to be expanded.

    if (zLevels != oldNumZLevels) {

        char blockSize[64];
        long int stride;
        MPI_Datatype tileShapeInMemory2D;
        MPI_Datatype tileShapeInFile2D;

        // Free the old Datatypes (if any)
        if (oldNumZLevels > 0) {
          MPI_Type_free (&fullTileInMemoryShape);
          MPI_Type_free (&fullTileInFileShape);
        }


        ///////////////////////////////////////////////////////////////////
        // Create a type with the "shape" of a tile in memory,
        // one z-level deep.
        if (MPI_ARRAY_ORDER == MPI_ORDER_FORTRAN) {
          int fullDims[2] = {tileSizeX + 2*xGhosts, tileSizeY + 2*yGhosts};
          int tileDims[2] = {tileSizeX, tileSizeY};
          int startElements[2] = {xGhosts, yGhosts};
          MPI_Type_create_subarray (
                  2, fullDims, tileDims, startElements, MPI_ARRAY_ORDER,
                  mpi_dataElementType, &tileShapeInMemory2D);
if (0 == appCommRank) fprintf (stdout,
  "In memory shape: {%d,%d} ({%d,%d}, start at %d,%d)\n",
  tileDims[0], tileDims[1], fullDims[0], fullDims[1], startElements[0], startElements[1]);
        }
        else {
          int fullDims[2] = {tileSizeY + 2*yGhosts, tileSizeX + 2*xGhosts};
          int tileDims[2] = {tileSizeY, tileSizeX};
          int startElements[2] = {yGhosts, xGhosts};
          MPI_Type_create_subarray (
                  2, fullDims, tileDims, startElements, MPI_ARRAY_ORDER,
                  mpi_dataElementType, &tileShapeInMemory2D);
if (0 == appCommRank) fprintf (stdout,
  "In memory shape: {%d,%d} ({%d,%d}, start at %d,%d)\n",
  tileDims[0], tileDims[1], fullDims[0], fullDims[1], startElements[0], startElements[1]);
        }
        MPI_Type_commit(&tileShapeInMemory2D);

        // Create a type with the "shape" of a tile in memory,
        // with the given number of z-levels.
        // "stride" is expressed in units of bytes when calling
        // MPI_Type_hvector
        stride = sizeof(dataElementType) *
                 (tileSizeX + 2*xGhosts) * (tileSizeY + 2*yGhosts);
        MPI_Type_hvector (zLevels, 1, stride,
                          tileShapeInMemory2D, &fullTileInMemoryShape);
        MPI_Type_commit (&fullTileInMemoryShape);
        MPI_Type_free(&tileShapeInMemory2D);

if (0 == appCommRank) fprintf (stdout,
  "In memory shape: %d zLevels, stride %ld\n", zLevels, stride);


        ///////////////////////////////////////////////////////////////////
        // Create a type with the "shape" of a tile in the file,
        // one z-level deep
        // "stride" is expressed in units of dataElementType when calling
        // MPI_Type_vector
        stride = tileSizeX * numTilesInX;
        MPI_Type_vector (tileSizeY, tileSizeX, (int)stride,
                         mpi_dataElementType, &tileShapeInFile2D);
        MPI_Type_commit (&tileShapeInFile2D);

        // Create a type with the "shape" of a tile in the file,
        // with the given number of z-levels.
        // "stride" is expressed in units of bytes when calling
        // MPI_Type_hvector
        stride = stride * tileSizeY * numTilesInY * sizeof(dataElementType);
        MPI_Type_hvector (zLevels, 1, stride,
                          tileShapeInFile2D, &fullTileInFileShape);
        MPI_Type_commit (&fullTileInFileShape);
        MPI_Type_free(&tileShapeInFile2D);

if (0 == appCommRank) fprintf (stdout,
  "In file shape: {%d,%d}, %d zLevels, stride %ld\n",
  tileSizeX*numTilesInX, tileSizeY*numTilesInY, zLevels, stride);


        ///////////////////////////////////////////////////////////////////
        // Set up some possible hints
        MPI_Info_create (&ioHints);
        MPI_Info_set (ioHints, "collective_buffering", "true");
        sprintf (blockSize, "%ld", (long)tileSizeX * tileSizeY * numTilesInX);
        MPI_Info_set (ioHints, "cb_block_size", blockSize);

        // Since we got to this point, we assume things worked,
        // so set the flag to reuse the types if possible
        oldNumZLevels = zLevels;

    }

    // Do the read
    MPI_File_open (appComm, filename, MPI_MODE_RDONLY, ioHints, &fh);
    MPI_File_set_errhandler (fh, MPI_ERRORS_ARE_FATAL);
    MPI_File_set_view (fh, tileOffsetInFile, mpi_dataElementType,
                       fullTileInFileShape, "native", ioHints);
    kk = MPI_File_read_all (fh, tileBuf, 1, fullTileInMemoryShape, &status);
    MPI_File_close(&fh);

if (0 == appCommRank) fprintf (stdout,
"End MPI-IO read for '%s',  byte offset %ld\n", filename, fieldOffsetInFile);

}



// Fortran interface
// This uses the "usual" method for passing Fortran strings:
// the string length is passed, by value, as an extra "hidden" argument
// after the end of the normal argument list.  So for example, this
// routine would be invoked on the Fortran side like this:
//     call readField(comm, filename, offset, tilebuf, tileid, zlevels)
// This method of passing FOrtran strings is NOT defined by the Fortran
// standard, but it is the method of choice for many compilers, including
// gcc (GNU/Linux), and icc (Intel).
//
// In this routine, a "record" is synonomous with a contiguous 1D array
// of length "tileSizeX" of the elemental type.  This is the terminology
// that the mds_read routines use.
//
// PLEASE NOTE that the "fieldOffsetInFileInRecords" is of type "MPI_Offset"
// which is synonymous with the Fortran type "integer(kind=MPI_OFFSET_KIND)".
// This will typically be integer*8.  But in particular it is almost
// certainly NOT of type "default integer", which means in particular
// that you CANNOT simply pass a constant (e.g. "0") as the argument,
// since that type will be of the wrong size. (Unless the caller has a
// visible prototype for the routine.)
void
readfield_ (
  MPI_Fint  *fortranAppComm,
  char *fortranFilename,
  MPI_Offset  *fieldOffsetInFileInRecords,
  void *tileBuf,
  int *tileID,
  int *zLevels,
  int filenameLength)
{
    int i;
    char namebuf[filenameLength+1];
    char *filename = namebuf;

    MPI_Offset fieldOffsetInFile = (*fieldOffsetInFileInRecords) *
                                    tileSizeX * sizeof(dataElementType);

    // Translate the MPI communicator from a Fortran-style handle
    // into a C-style handle.
    MPI_Comm appComm = MPI_Comm_f2c(*fortranAppComm);

    // Translate the Fortran-style string into a C-style string
    strncpy(filename, fortranFilename, filenameLength);
    for (i = filenameLength;  (i > 0) && (' ' == filename[i-1]);  --i) ;
    filename[i] = '\0';
    assert(strlen(filename) > 0);

    // Make the translated call
    readField (appComm, filename, fieldOffsetInFile, tileBuf, *tileID, *zLevels);
}


