CBOI
C
C !TITLE: MITGCM KERNEL CODE SYNOPSIS
C !AUTHORS: mitgcm developers ( support@mitgcm.org )
C !AFFILIATION: Massachussetts Institute of Technology
C !DATE:
C !INTRODUCTION: Kernel dynamical routines
C This document summarises MITgcm code under the model/ subdirectory.
C The code under model/ ( src/ and inc/ ) contains most of
C the driver routines for the baseline forms of the kernel equations in the
C MITgcm algorithm. Numerical code for much of the baseline forms of
C these equations is also under the model/ directory. Other numerical code
C used for the kernel equations is contained in packages in the pkg/
C directory tree.
C Code for auxiliary equations and alternate discretizations of the kernel
C equations and algorithm can also be found in the pkg/ directory tree.
C
C \subsection{Getting Help and Reporting Errors and Problems}
C If you have questions please subscribe and e-mail support@mitgcm.org.
C We also welcome reports of errors and inconsistencies in the code or
C in the accompanying documentation. Please feel free to send these
C to support@mitgcm.org. For further information and to review
C problems reported to support@mitgcm.org please visit http://mitgcm.org.
C
C \subsection{MITgcm Kernel Code Calling Sequence}
C \bv
C
C Invocation from WRAPPER level...
C
C  |
C  |-THE_MODEL_MAIN :: Primary driver for the MITgcm algorithm
C    |              :: Called from WRAPPER level numerical
C    |              :: code invocation routine. On entry
C    |              :: to THE_MODEL_MAIN separate thread and
C    |              :: separate processes will have been established.
C    |              :: Each thread and process will have a unique ID
C    |              :: but as yet it will not be associated with a
C    |              :: specific region in decomposed discrete space.
C    |
C    |-INITIALISE_FIXED :: Set fixed model arrays such as topography,
C    | |                :: grid, solver matrices etc..
C    | |
C    | |-INI_PARMS :: Routine to set kernel model parameters.
C    | |           :: Kernel parameters are read from file "data"
C    | |           :: in directory in which code executes.
C    | |
C    | |-PACAKGES_BOOT      :: Start up the optional package environment.
C    | |                    :: Runtime selection of active packages.
C    | |-PACKAGES_READPARMS :: read all packages input parameter file
C    | | |- ${PKG}_READPARMS
C    | |
C    | |-INI_MODEL_IO  :: Initialise Input/Ouput setting
C    | |
C    | |-INI_GRID      :: Control grid array (vert. and hori.) initialisation.
C    | |               :: Grid arrays are held and described in GRID.h.
C    | |
C    | |-INI_DEPTHS    :: Read (from "bathyFile") or set bathymetry/orography.
C    | |-INI_MASKS_ETC :: Derive horizontal and vertical cell fractions and
C    | |               :: land masking for solid-fluid boundaries.
C    | |
C    | |-PACKAGES_INIT_FIXED  :: do all packages fixed-initialisation setting
C    | | |- ${PKG}_INIT_FIXED
C    | |
C    | |-CONFIG_SUMMARY       :: Provide synopsis of kernel setup. Includes
C    | |                      :: annotated table of kernel parameter settings.
C    | |
C    | |-PACKAGES_CHECK       :: call each package configuration checking S/R
C    | | |- ${PKG}_CHECK
C    | |
C    | |-CONFIG_CHECK         :: Check config and parameter consistency.
C    |
C    |-CTRL_UNPACK     :: Control vector support package. see pkg/ctrl
C    |
C    |-ADTHE_MAIN_LOOP :: Derivative evaluating form of main time stepping loop
C    !                 :: Automatically generated by TAMC/TAF.
C    |
C    |-THE_MAIN_LOOP   :: Main timestepping loop routine.
C    | |
C    | |-INITIALISE_VARIA :: Set the initial conditions for time evolving
C    | | |-INI_DYNVARS  :: set common block variable to zero
C    | | |-INI_NH_VARS  :: set common block variable to zero
C    | | |
C    | | |-INI_FIELDS   :: Control initialising model fields to non-zero
C    | | | |-INI_VEL,_THETA,_SALT,_PSURF, ...
C    | | | |-READ_PICKUP
C    | | |
C    | | |-INI_FORCING  :: initialise forcing fields
C    | | |
C    | | |-PACKAGES_INIT_VARIABLES :: Does initialisation of time evolving
C    | | | | ${PKG}_INIT_VARIA     :: package data.
C    | | |
C    | | |-MONITOR          :: Monitor state (see pkg/monitor)
C    | | |-STATE_SUMMARY    :: Summarise model prognostic variables.
C    | | |-DO_THE_MODEL_IO  :: Standard diagnostic I/O.
C    | |
C====|>| ****************************
C====|>| BEGIN MAIN TIMESTEPPING LOOP
C====|>| ****************************
C    | |-COST_AVERAGESFIELDS :: time-averaged Cost function terms (see pkg/cost)
C    | |
C/\  | |-FORWARD_STEP        :: Step forward a time-step ( AT LAST !!! )
C/\  | | |
C/\  | | |-LOAD_FIELDS_DRIVER :: control loading of input fields from files
C/\  | | |
C/\  | | |-CPL_EXPORT_MY_DATA :: Send coupling fields to coupler
C/\  | | |-CPL_IMPORT_EXTERNAL_DATA :: Receive coupling fields from coupler
C/\  | | |
C/\  | | |-DO_ATMOSPHERIC_PHYS :: Atmospheric physics computation
C/\  | | |
C/\  | | |-DO_OCEANIC_PHYS     :: Oceanic (& seaice) physics computation
C/\  | | | |-OBCS_CALC         :: Open boundary. package (see pkg/obcs).
C/\  | | |
C/\  | | |-GCHEM_CALC_TENDENCY :: geochemistry driver routine (see pkg/gchem)
C/\  | | |
C/\  | | |-THERMODYNAMICS      :: (synchronous time-stepping)
C/\  | | |                        theta, salt + tracer equations driver.
C/\  | | | |-EXTERNAL_FORCING_SURF:: Accumulates appropriately dimensioned
C/\  | | | |                      :: forcing terms.
C/\  | | | |-GAD_ADVECTION        :: Generalised advection driver (multi-dim
C/\  | | | |                         advection case) (see pkg/gad).
C/\  | | | |-CALC_GT              :: Calculate the temperature tendency terms
C/\  | | | |-TIMESTEP_TRACER      :: Step tracer field forward in time
C/\  | | | |-CALC_GS              :: Calculate the salinity tendency terms
C/\  | | | |-TIMESTEP_TRACER      :: Step tracer field forward in time
C/\  | | | |-PTRACERS_INTEGRATE   :: Integrate other tracer(s) (see pkg/ptracers).
C/\  | | | |-IMPLDIFF             :: Solve vertical implicit diffusion equation.
C/\  | | | |-OBCS_APPLY_TS        :: Open boundary package (see pkg/obcs ).
C/\  | | |
C/\  | | |-DYNAMICS       :: Momentum equations driver.
C/\  | | | |
C/\  | | | |-CALC_GRAD_PHI_SURF :: Calculate the gradient of the surface
C/\  | | | |                       Potential anomaly.
C/\  | | | |-CALC_VISCOSITY   :: Calculate net vertical viscosity
C/\  | | | |-CALC_PHI_HYD     :: Integrate the hydrostatic relation.
C/\  | | | |-MOM_FLUXFORM     :: Flux form mom eqn. package  (pkg/mom_fluxform)
C/\  | | | |-MOM_VECINV       :: Vector invariant form mom eqn (pkg/mom_vecinv)
C/\  | | | |-TIMESTEP         :: Step momentum fields forward in time
C/\  | | | |-OBCS_APPLY_UV    :: Open boundary package (see pkg/obcs).
C/\  | | | |-IMPLDIFF         :: Solve vertical implicit diffusion equation.
C/\  | | | |-CALC_GW          :: vert. momentum tendency terms (Non-Hydrostatic)
C/\  | | |
C/\  | | |-UPDATE_SURF_DR :: Update the surface-level thickness fraction.
C/\  | | |-UPDATE_R_STAR  :: Update the level thickness fraction.
C/\  | | |-UPDATE_CG2D    :: Update 2d conjugate grad. for Free-Surf.
C/\  | | |
C/\  | | |-SOLVE_FOR_PRESSURE  :: Find surface pressure.
C/\  | | | |-CG2D              :: Two-dim pre-con. conjugate-gradient.
C/\  | | | |-CG3D              :: Three-dim pre-con. conjugate-gradient solver.
C/\  | | |
C/\  | | |-MOMENTUM_CORRECTION_STEP :: Finalise momentum stepping
C/\  | | | |-CALC_GRAD_PHI_SURF :: Return DDx and DDy of surface pressure
C/\  | | | |-CORRECTION_STEP    :: Pressure correction to momentum
C/\  | | | |-OBCS_APPLY_UV      :: Open boundary package (see pkg/obcs).
C/\  | | |
C/\  | | |-INTEGR_CONTINUITY    :: Integrate continuity equation
C/\  | | |
C/\  | | |-THERMODYNAMICS       :: (staggered time-stepping)
C/\  | | |                         theta, salt + tracer equations driver.
C/\  | | |
C/\  | | |-TRACERS_CORRECTION_STEP :: Finalise tracer stepping
C/\  | | |
C/\  | | |-GCHEM_FORCING_SEP :: tracer forcing for gchem pkg (if tracer
C/\  | | |                      dependent tendencies calculated separately)
C/\  | | |
C/\  | | |-DO_FIELDS_BLOCKING_EXCHANGES :: Sync up overlap regions.
C/\  | | |
C/\  | | |-MONITOR          :: Monitor package (pkg/monitor).
C/\  | | |-DO_THE_MODEL_IO  :: Standard diagnostic I/O.
C/\  | | |
C/\  | | |-DO_WRITE_PICKUP  :: Write restart files.
C    | |
C<===|=| **************************
C<===|=| END MAIN TIMESTEPPING LOOP
C<===|=| **************************
C    | |
C    | |-COST_AVERAGESFIELDS :: time-averaged Cost function terms (see pkg/cost)
C    | |-COST_FINAL          :: Cost function package. (see pkg/cost)
C    |
C    |-CTRL_PACK       :: Control vector support package. see pkg/ctrl
C    |
C    |-GRDCHK_MAIN     :: Gradient check package. see pkg/grdchk
C    |
C    |-TIMER_PRINTALL  :: Computational timing summary
C    |
C    |-COMM_STATS      :: Summarise inter-proc and inter-thread communication
C                      :: events.
C \ev
C
CEOI

#include "PACKAGES_CONFIG.h"
#include "CPP_OPTIONS.h"
#include "AD_CONFIG.h"
#ifdef ALLOW_OPENAD
# include "OPENAD_OPTIONS.h"
#endif
#ifdef ALLOW_AUTODIFF
# include "AUTODIFF_OPTIONS.h"
#endif
#ifdef ALLOW_CTRL
# include "CTRL_OPTIONS.h"
#endif
#ifdef ALLOW_STREAMICE
# include "STREAMICE_OPTIONS.h"
#endif

CBOP
C     !ROUTINE: THE_MODEL_MAIN

C     !INTERFACE:
      SUBROUTINE THE_MODEL_MAIN(myThid)

C     !DESCRIPTION: \bv
C     *==========================================================*
C     | SUBROUTINE THE_MODEL_MAIN
C     | o Master controlling routine for model using the MITgcm
C     |   UV parallel wrapper.
C     *==========================================================*
C     | THE_MODEL_MAIN is invoked by the MITgcm UV parallel
C     | wrapper with a single integer argument "myThid". This
C     | variable identifies the thread number of an instance of
C     | THE_MODEL_MAIN. Each instance of THE_MODEL_MAIN works
C     | on a particular region of the models domain and
C     | synchronises with other instances as necessary. The
C     | routine has to "understand" the MITgcm parallel
C     | environment and the numerical algorithm. Editing this
C     | routine is best done with some knowledge of both aspects.
C     | Notes
C     | =====
C     | C*P* comments indicating place holders for which code is
C     |      presently being developed.
C     *==========================================================*
C     \ev

C     !CALLING SEQUENCE:
C     THE_MODEL_MAIN()
C       |
C       |
C       |--INITIALISE_FIXED
C       |   o Set model configuration (fixed arrays)
C       |     Topography, hydrography, timestep, grid, etc..
C       |
C       |--CTRL_UNPACK      o Derivative mode. Unpack control vector.
C       |
C       |--ADTHE_MAIN_LOOP  o Main timestepping loop for combined
C       |                     prognostic and reverse mode integration.
C       |
C       |--THE_MAIN_LOOP    o Main timestepping loop for pure prognostic
C       |                     integration.
C       |
C       |--CTRL_PACK        o Derivative mode. Unpack control vector.
C       |
C       |--GRDCHK_MAIN      o Gradient check control routine.
C       |
C       |--TIMER_PRINTALL   o Print out timing statistics.
C       |
C       |--COMM_STATS       o Print out communication statistics.

C     !USES:
      IMPLICIT NONE

C     == Global variables ===
C -->> OpenAD
      use OAD_active
      use OAD_rev
      use OAD_tape
      use OAD_cp
#include "cost.h"
C <<-- OpenAD
#include "SIZE.h"
#include "EEPARAMS.h"
#include "PARAMS.h"
#include "DYNVARS.h"
#include "FFIELDS.h"

#ifdef ALLOW_AUTODIFF_TAMC
# include "tamc.h"
#endif
#ifdef ALLOW_CTRL
# include "ctrl.h"
# include "optim.h"
# include "CTRL_GENARR.h"
#endif

C     !INPUT/OUTPUT PARAMETERS:
C     == Routine arguments ==
C     myThid :: Thread number for this instance of the routine.
      INTEGER myThid

C     !LOCAL VARIABLES:
C     == Local variables ==
C     Note: Under the multi-threaded model myIter and myTime are local
C           variables passed around as routine arguments.
C           Although this is fiddly it saves the need to impose
C           additional synchronisation points when they are updated.
C     myTime :: Time counter for this thread
C     myIter :: Iteration counter for this thread
      INTEGER myIter
      _RL     myTime
      LOGICAL exst
      LOGICAL lastdiva
C -->> OpenAD
      _RL foo(1-olx:snx+olx,1-oly:sny+oly,nr,nsx,nsy)
      _RL foo2D(1-olx:snx+olx,1-oly:sny+oly,nsx,nsy)
      CHARACTER*(10) suff
      CHARACTER*(MAX_LEN_FNAM) fname
C     Temprarily change precision to agree with ctrlprec
      INTEGER tmpprec
      INTEGER ik, il
#ifdef OAD_DEBUG
      INTEGER i1, i2, i3, i4, i5
#endif
C <<-- OpenAD

C     !EXTERNAL VARIABLES:
c     == external ==
      integer  ilnblnk
      external ilnblnk

CEOP

C--   set default:
      exst     = .TRUE.
      lastdiva = .TRUE.
C -->> OpenAD
C-    Set the execution mode
      our_rev_mode%arg_store=.FALSE.
      our_rev_mode%arg_restore=.FALSE.
      our_rev_mode%res_store=.FALSE.
      our_rev_mode%res_restore=.FALSE.
      our_rev_mode%plain=.TRUE.
      our_rev_mode%tape=.FALSE.
      our_rev_mode%adjoint=.FALSE.
      our_rev_mode%switchedToCheckpoint=.FALSE.
C-    Initialize the tape
      call oad_tape_init()
C-    Initialize the checkpoint areas
      call cp_init()
C <<-- OpenAD

#ifdef ALLOW_PETSC
      call streamice_initialize_petsc
#endif

#ifdef ALLOW_DEBUG
      IF (debugMode) CALL DEBUG_ENTER('THE_MODEL_MAIN',myThid)
#endif

#if defined(USE_PAPI) || defined(USE_PCL_FLOPS_SFP) || defined(USE_PCL_FLOPS) || defined(USE_PCL)
      CALL TIMER_CONTROL('','INIT','THE_MODEL_MAIN',myThid)
#endif
C--   This timer encompasses the whole code
      CALL TIMER_START('ALL                    [THE_MODEL_MAIN]',myThid)

#ifdef ALLOW_DEBUG
      IF (debugMode) CALL DEBUG_CALL('INITIALISE_FIXED',myThid)
#endif
C--   Set model configuration (fixed arrays)
      CALL TIMER_START('INITIALISE_FIXED       [THE_MODEL_MAIN]',myThid)
C -->> OpenAD
c     CALL INITIALISE_FIXED( myThid )
      CALL OpenAD_INITIALISE_FIXED( myThid )
C <<-- OpenAD
      CALL TIMER_STOP ('INITIALISE_FIXED       [THE_MODEL_MAIN]',myThid)

      myTime = startTime
      myIter = nIter0

#if ( defined (ALLOW_ADMTLM) )

      STOP 'should never get here; ADMTLM_DSVD calls ADMTLM_DRIVER'

#elif ( defined (ALLOW_AUTODIFF))

# ifdef  ALLOW_CTRL
# ifndef EXCLUDE_CTRL_PACK
      IF (useCTRL) THEN
         inquire( file='costfinal', exist=exst )
         IF ( .NOT. exst ) THEN
            IF ( (optimcycle.NE.0 .OR. .NOT.doinitxx)
     &           .AND. doMainUnpack ) THEN
               CALL TIMER_START('CTRL_UNPACK   [THE_MODEL_MAIN]',myThid)
               CALL CTRL_UNPACK( .TRUE. , myThid )
               CALL TIMER_STOP ('CTRL_UNPACK   [THE_MODEL_MAIN]',myThid)
            ENDIF
         ENDIF
      ENDIF
# endif /* EXCLUDE_CTRL_PACK */
# endif /* ALLOW_CTRL */

# ifdef ALLOW_COST
      CALL COST_DEPENDENT_INIT ( myThid )
# endif

# if ( defined (ALLOW_TANGENTLINEAR_RUN) )

#  ifdef ALLOW_DEBUG
      IF (debugMode) CALL DEBUG_CALL('G_THE_MAIN_LOOP',myThid)
#  endif
      CALL TIMER_START('G_THE_MAIN_LOOP           [TANGENT RUN]',myThid)
      CALL G_THE_MAIN_LOOP ( myTime, myIter, myThid )
      CALL TIMER_STOP ('G_THE_MAIN_LOOP           [TANGENT RUN]',myThid)

# elif ( defined (ALLOW_ADJOINT_RUN) || \
         defined (ALLOW_ECCO_OPTIMIZATION) )

#  ifdef ALLOW_DIVIDED_ADJOINT
C-- The following assumes the TAF option '-pure'
      inquire( file='costfinal', exist=exst )
      IF ( .NOT. exst) THEN
#   ifdef ALLOW_DEBUG
         IF (debugMode) CALL DEBUG_CALL('MDTHE_MAIN_LOOP',myThid)
#   endif
         CALL TIMER_START('MDTHE_MAIN_LOOP            [MD RUN]', myThid)
         CALL MDTHE_MAIN_LOOP ( myTime, myIter, myThid )
         CALL TIMER_STOP ('MDTHE_MAIN_LOOP            [MD RUN]', myThid)
         CALL COST_FINAL_STORE ( myThid, lastdiva )
      ELSE
#   ifdef ALLOW_DEBUG
         IF (debugMode) CALL DEBUG_CALL('ADTHE_MAIN_LOOP',myThid)
#   endif
         CALL TIMER_START('ADTHE_MAIN_LOOP       [ADJOINT RUN]', myThid)
         CALL ADTHE_MAIN_LOOP (  myThid )
         CALL TIMER_STOP ('ADTHE_MAIN_LOOP       [ADJOINT RUN]', myThid)
         CALL COST_FINAL_RESTORE ( myThid, lastdiva )
      ENDIF

#  else /* ALLOW_DIVIDED_ADJOINT undef */
#   ifndef ALLOW_OPENAD
#    ifdef ALLOW_DEBUG
      IF (debugMode) CALL DEBUG_CALL('ADTHE_MAIN_LOOP',myThid)
#    endif
      CALL TIMER_START('ADTHE_MAIN_LOOP          [ADJOINT RUN]', myThid)
      CALL ADTHE_MAIN_LOOP ( myThid )
      CALL TIMER_STOP ('ADTHE_MAIN_LOOP          [ADJOINT RUN]', myThid)
#   else /* ALLOW_OPENAD defined */
C -->> OpenAD
#   ifdef ALLOW_DEBUG
      IF (debugMode) CALL DEBUG_CALL('THE_MAIN_LOOP',myThid)
#   endif
      CALL TIMER_START('THE_MAIN_LOOP (F)      [THE_MODEL_MAIN]',myThid)
      our_rev_mode%plain=.FALSE.
      our_rev_mode%tape=.TRUE.
      call timeratio()
      CALL OpenAD_THE_MAIN_LOOP( myTime, myIter, myThid )
      CALL TIMER_STOP ('THE_MAIN_LOOP (F)      [THE_MODEL_MAIN]',myThid)
      CALL TIMER_START('THE_MAIN_LOOP (A)      [THE_MODEL_MAIN]',myThid)
      our_rev_mode%arg_store=.FALSE.
      our_rev_mode%arg_restore=.FALSE.
      our_rev_mode%plain=.FALSE.
      our_rev_mode%tape=.FALSE.
      our_rev_mode%adjoint=.TRUE.
      IF (myProcID .EQ. 0) THEN
        fc%d=1.0
      ENDIF
      call timeratio()
      CALL OpenAD_THE_MAIN_LOOP( myTime, myIter, myThid )
      call timeratio()
      our_rev_mode%arg_store=.FALSE.
      our_rev_mode%arg_restore=.FALSE.
      our_rev_mode%plain=.TRUE.
      our_rev_mode%tape=.FALSE.
      our_rev_mode%adjoint=.FALSE.
#   ifdef OAD_DEBUG
#    if (defined (ALLOW_THETA0_CONTROL) && defined (ALLOW_SALT0_CONTROL))
      do i1=1-olx,snx+olx
        do i2=1-oly,sny+oly
          do i3=1,nr
            do i4=1,nsx
              do i5=1,nsy
                 write (standardmessageunit,
     +'(A,5(I3,A),E25.17E3,A,E25.17E3)')
     +'OAD: (',
     +i1,',',i2,',',i3,',',i4,',',i5,') salt/theta ',
     +xx_salt(i1,i2,i3,i4,i5)%d,'/',xx_theta(i1,i2,i3,i4,i5)%d
              end do
            end do
          end do
        end do
      end do
#    endif
#   endif /* OAD_DEBUG */
C     Temporarily change setting of writeBinaryPrec
      tmpprec = writeBinaryPrec
      writeBinaryPrec = ctrlprec
      WRITE(suff,'(I10.10)') optimcycle
#   ifndef ALLOW_OPENAD_ACTIVE_READ_XYZ
#   ifdef ALLOW_THETA0_CONTROL
      foo=xx_theta%d
      il=ilnblnk( xx_theta_file )
      write(fname(1:MAX_LEN_FNAM),'(3a)')
     &     'ad',xx_theta_file(1:il),'.'
      call write_fld_xyz_rl(fname,suff,foo,myIter,1)
#   endif
#   ifdef ALLOW_SALT0_CONTROL
      foo=xx_salt%d
      il=ilnblnk( xx_salt_file )
      write(fname(1:MAX_LEN_FNAM),'(3a)')
     &     'ad',xx_salt_file(1:il),'.'
      call write_fld_xyz_rl(fname,suff,foo,myIter,1)
#   endif
#   ifdef ALLOW_DIFFKR_CONTROL
      foo=diffkr%d
      il=ilnblnk( xx_diffkr_file )
      write(fname(1:MAX_LEN_FNAM),'(3a)')
     &     'ad',xx_diffkr_file(1:il),'.'
      call write_fld_xyz_rl(fname,suff,foo,myIter,1)
#   endif
#   endif /* ALLOW_OPENAD_ACTIVE_READ_XYZ */

#   ifdef ALLOW_TAUU0_CONTROL
      foo2D=fu%d
      il=ilnblnk( xx_tauu_file )
      write(fname(1:MAX_LEN_FNAM),'(3a)')
     &     'ad',xx_tauu_file(1:il),'.'
      call write_fld_xy_rl(fname,suff,foo2D,myIter,1)
#   endif
#   ifdef ALLOW_TAUV0_CONTROL
      foo2D=fv%d
      il=ilnblnk( xx_tauv_file )
      write(fname(1:MAX_LEN_FNAM),'(3a)')
     &     'ad',xx_tauv_file(1:il),'.'
      call write_fld_xy_rl(fname,suff,foo2D,myIter,1)
#   endif
#   ifdef ALLOW_HFLUX0_CONTROL
      foo2D=qnet%d
      il=ilnblnk( xx_hflux_file )
      write(fname(1:MAX_LEN_FNAM),'(3a)')
     &     'ad',xx_hflux_file(1:il),'.'
      call write_fld_xy_rl(fname,suff,foo2D,myIter,1)
#   endif
#   ifdef ALLOW_SFLUX0_CONTROL
      foo2D=empmr%d
      il=ilnblnk( xx_sflux_file )
      write(fname(1:MAX_LEN_FNAM),'(3a)')
     &     'ad',xx_sflux_file(1:il),'.'
      call write_fld_xy_rl(fname,suff,foo2D,myIter,1)
#   endif
#   ifdef ALLOW_HFLUXM_CONTROL
      foo2D=xx_hfluxm%d
      il=ilnblnk( xx_hfluxm_file )
      write(fname(1:MAX_LEN_FNAM),'(3a)')
     &     'ad',xx_hfluxm_file(1:il),'.'
      call write_fld_xy_rl(fname,suff,foo2D,myIter,1)
#   endif
#   ifdef ALLOW_ETAN0_CONTROL
      foo2D=etan%d
      il=ilnblnk( xx_etan_file )
      write(fname(1:MAX_LEN_FNAM),'(3a)')
     &     'ad',xx_etan_file(1:il),'.'
      call write_fld_xy_rl(fname,suff,foo2D,myIter,1)
#   endif
cc#   ifdef ALLOW_GENARR2D_CONTROL
cc      do ik = 1, maxCtrlArr2D
cc        foo2d=xx_genarr2d(:,:,:,:,ik)%d
cc        write(fname,'(A,I2.2,A)') 'adxx_genarr2d_',ik,'.'
cc        call write_fld_xy_rl(fname,suff,foo2D,myIter,1)
cc      enddo
cc#   endif
cc#   ifdef ALLOW_GENTIM2D_CONTROL
cc      do ik = 1, maxCtrlTim2D
cc        foo2d=xx_gentim2d(:,:,:,:,ik)%d
cc        write(fname,'(A,I2.2,A)') 'adxx_gentim2d_',ik,'.'
cc        call write_fld_xy_rl(fname,suff,foo2D,myIter,1)
cc      enddo
cc#   endif
cc#   ifdef ALLOW_GENARR3D_CONTROL
cc      do ik = 1, maxCtrlArr3D
cc        foo=xx_genarr3d(:,:,:,:,:,ik)%d
cc        write(fname,'(A,I2.2,A)') 'adxx_genarr3d_',ik,'.'
cc        call write_fld_xyz_rl(fname,suff,foo,myIter,1)
cc      enddo
cc#   endif
C     Change back to original writeBinaryPrec
      writeBinaryPrec = tmpprec
      our_rev_mode%plain=.TRUE.
      our_rev_mode%tape=.FALSE.
      our_rev_mode%adjoint=.FALSE.
      CALL TIMER_STOP ('THE_MAIN_LOOP (A)      [THE_MODEL_MAIN]',myThid)

C <<-- OpenAD
#   endif /* ALLOW_OPENAD */
#  endif /* ALLOW_DIVIDED_ADJOINT */

# else /* forward run only within AD setting */

#  ifdef ALLOW_DEBUG
      IF (debugMode) CALL DEBUG_CALL('THE_MAIN_LOOP',myThid)
#  endif
C--   Call time stepping loop of full model
      CALL TIMER_START('THE_MAIN_LOOP          [THE_MODEL_MAIN]',myThid)
      CALL THE_MAIN_LOOP( myTime, myIter, myThid )
      CALL TIMER_STOP ('THE_MAIN_LOOP          [THE_MODEL_MAIN]',myThid)

# endif /* forward run only within AD setting */

# ifdef  ALLOW_CTRL
# ifndef EXCLUDE_CTRL_PACK
#  ifdef ALLOW_OPENAD
cph-- ad hoc fix for OpenAD time stepping counter lagging one step
cph-- after final adjoint step
      myIter=nIter0
#  endif
      IF (useCTRL) THEN
      IF ( lastdiva .AND. doMainPack ) THEN
         CALL TIMER_START('CTRL_PACK           [THE_MODEL_MAIN]',myThid)
         CALL CTRL_PACK( .FALSE. , myThid )
         CALL TIMER_STOP ('CTRL_PACK           [THE_MODEL_MAIN]',myThid)
         IF ( ( optimcycle.EQ.0 .OR. (.NOT. doMainUnpack) )
     &        .AND. myIter.EQ.nIter0 ) THEN
            CALL TIMER_START('CTRL_PACK     [THE_MODEL_MAIN]',myThid)
            CALL CTRL_PACK( .TRUE. , myThid )
            CALL TIMER_STOP ('CTRL_PACK     [THE_MODEL_MAIN]',myThid)
         ENDIF
      ENDIF
      ENDIF
# endif /* EXCLUDE_CTRL_PACK */
# endif /* ALLOW_CTRL */

# ifdef ALLOW_GRDCHK
      IF ( useGrdchk .AND. lastdiva ) THEN
         CALL TIMER_START('GRDCHK_MAIN         [THE_MODEL_MAIN]',myThid)
         CALL GRDCHK_MAIN( myThid )
         CALL TIMER_STOP ('GRDCHK_MAIN         [THE_MODEL_MAIN]',myThid)
      ENDIF
# endif

#else /* ALL AD-related undef */

# ifdef ALLOW_DEBUG
      IF (debugMode) CALL DEBUG_CALL('THE_MAIN_LOOP',myThid)
# endif
C--   Call time stepping loop of full model
      CALL TIMER_START('THE_MAIN_LOOP          [THE_MODEL_MAIN]',myThid)
      CALL THE_MAIN_LOOP( myTime, myIter, myThid )
      CALL TIMER_STOP ('THE_MAIN_LOOP          [THE_MODEL_MAIN]',myThid)

#endif /* ALLOW_TANGENTLINEAR_RUN ALLOW_ADJOINT_RUN ALLOW_ADMTLM */

#ifdef ALLOW_PETSC
      call streamice_finalize_petsc
#endif

#ifdef ALLOW_MNC
      IF (useMNC) THEN
C       Close all open NetCDF files
        _BEGIN_MASTER( myThid )
        CALL MNC_FILE_CLOSE_ALL( myThid )
        _END_MASTER( myThid )
      ENDIF
#endif

C--   This timer encompasses the whole code
      CALL TIMER_STOP ('ALL                    [THE_MODEL_MAIN]',myThid)

C--   Write timer statistics
      IF ( myThid .EQ. 1 ) THEN
       CALL TIMER_PRINTALL( myThid )
       CALL COMM_STATS
      ENDIF

C--   Check threads synchronization :
      CALL BAR_CHECK( 9, myThid )

#ifdef ALLOW_DEBUG
      IF (debugMode) CALL DEBUG_LEAVE('THE_MODEL_MAIN',myThid)
#endif

      RETURN
      END
