DataWaveProject · mondus · Jan 4, 2023 · Jan 5, 2023 · Jan 5, 2023 · Mar 2, 2023
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -0,0 +1,130 @@
+cmake_minimum_required(VERSION 3.2)
+#policy CMP0076 - target_sources source files are relative to file where target_sources is run
+cmake_policy (SET CMP0076 NEW)
+
+option(BUILD_PT_COUPLING "Build the PyTorch coupling code")
+option(BUILD_TF_COUPLING "Build the TensorFlow coupling code")
+option(BUILD_FORPY_COUPLING "Build the Forpy coupling code")
+
+# MiMA claims to only compile with ifort / icc currently.
+set ( CMAKE_Fortran_COMPILER "ifort" )
+set ( CMAKE_C_COMPILER "icc" )
+project(MiMA Fortran C)
+
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Debug CACHE STRING "" FORCE)
+endif()
+
+# Generate the main mima.x executable with dirs, libs, and opts
+add_executable ( mima.x )
+target_include_directories( mima.x PUBLIC src/shared/include src/shared/mpp/include )
+add_library( mima_c ) # The C parts of MiMA, so we can apply different options for them.
+target_compile_definitions( mima_c PRIVATE __IFC )
+target_compile_definitions( mima.x PRIVATE use_libMPI use_netCDF gFortran ) # gFortran appears to be unused
+
+# Also generate the postprocessing executable
+add_executable ( mppnccombine postprocessing/mppnccombine.c )
+
+#Add cmake directory to the environment module variable
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
+# Set to install in bin directory as per current MiMA behaviour
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/bin" CACHE PATH "..." FORCE)
+endif()
+
+# Find MPI and OpenMP libraries and link
+find_package (MPI REQUIRED)
+target_link_libraries( mima.x PRIVATE MPI::MPI_Fortran )
+
+find_package (OpenMP REQUIRED COMPONENTS Fortran)
+target_link_libraries( mima.x PRIVATE OpenMP::OpenMP_Fortran )
+
+# Find any libraries as neccesary for ML and link
+# FIXME this is a bit clunky.  I think we will have to have
+# #ifdefs in the code, so may as well use them for this.
+# Also this setup precludes having multiple methods compiled in.
+if(BUILD_PT_COUPLING)
+   find_package(FTorch)
+   target_link_libraries( mima.x PRIVATE FTorch::ftorch )
+   message(STATUS "Building with the Fortran PyTorch coupling")
+
+    # point the pytorch coupling routines at mima
+    target_sources( mima.x PRIVATE src/atmos_param/cg_drag/pytorch.f90)
+endif()
+if(BUILD_TF_COUPLING)
+    find_package(FortranTensorFlow)
+    target_link_libraries( mima.x PRIVATE FortranTensorFlow::fortran-tf)
+    message( STATUS "Building with Fortran TensorFlow coupling")
+
+    # point the pytorch coupling routines at mima
+    target_sources( mima.x PRIVATE src/atmos_param/cg_drag/tensorflow.f90)
+endif()
+if(BUILD_FORPY_COUPLING)
+   # Make sure python present
+   find_package (Python REQUIRED COMPONENTS Development)
+   target_link_libraries( mima.x PRIVATE Python::Python )
+
+   target_sources( mima.x PRIVATE src/shared/forpy/forpy_mod.f90)
+   target_sources( mima.x PRIVATE src/atmos_param/cg_drag/forpy.f90)
+endif()
+if(NOT (BUILD_PT_COUPLING OR BUILD_TF_COUPLING OR BUILD_FORPY_COUPLING))
+    target_sources( mima.x PRIVATE src/atmos_param/cg_drag/null.f90)
+endif()
+
+# Find the NetCDF installations and set the relevant variables for compilation
+# Then link to executables
+# Requires more legwork as NetCDF not provided by default
+find_package(PkgConfig)
+pkg_search_module(NETCDF_FORTRAN netcdf-fortran)
+if (NETCDF_FORTRAN_FOUND)
+    set(NETCDF_LIBRARIES "${NETCDF_FORTRAN_LDFLAGS}")
+    set(NETCDF_INCLUDES "${NETCDF_FORTRAN_INCLUDE_DIRS}")
+else()
+    set(NETCDF_F90 "YES")
+    find_package(NetCDF REQUIRED)
+endif()
+pkg_search_module(NETCDF_C netcdf)
+if (NETCDF_C_FOUND)
+    list(APPEND NETCDF_LIBRARIES "${NETCDF_C_LDFLAGS}")
+    list(APPEND NETCDF_INCLUDES "${NETCDF_C_INCLUDE_DIRS}")
+endif()
+
+target_link_libraries( mima.x PRIVATE mima_c ${NETCDF_LIBRARIES} )
+target_include_directories( mima.x PRIVATE ${NETCDF_INCLUDES} )
+target_link_libraries( mppnccombine PRIVATE ${NETCDF_LIBRARIES} )
+target_include_directories( mppnccombine PRIVATE ${NETCDF_INCLUDES} )
+
+# Add various subdirectories with long lists of source files
+add_subdirectory( src/coupler )
+add_subdirectory( src/atmos_coupled )
+add_subdirectory( src/atmos_param )
+
+set_source_files_properties (
+# The following files do nothing but assign very large arrays.
+# For some reason when compiling with ifort and optimisation
+# the compilation will take a very long time (10s of minutes).
+# Since the code doesn't actually *do* anything there's no
+# need to waste time having the compiler apply probably
+# meaningless optimisation.  So we disable optimisation for these
+# files.
+src/atmos_param/rrtm_radiation/rrtmg_sw/gcm_model/src/rrtmg_sw_k_g.f90
+src/atmos_param/rrtm_radiation/rrtmg_lw/gcm_model/src/rrtmg_lw_k_g.f90
+PROPERTIES
+COMPILE_FLAGS -O0
+)
+add_subdirectory( src/atmos_shared )
+add_subdirectory( src/atmos_spectral )
+add_subdirectory( src/ice_param )
+# include/fms_platform.h
+add_subdirectory( src/shared )
+
+# Set coompile options for executable
+target_compile_options( mima.x PRIVATE
+-fpp
+-safe-cray-ptr
+-ftz
+-assume byterecl
+-i4
+-r8
+)
diff --git a/README.md b/README.md
@@ -26,3 +26,36 @@ AM2 is distributed under a GNU GPLv2 license. That means you have permission to
 RRTM/RRTMG: Copyright © 2002-2010, Atmospheric and Environmental Research, Inc. (AER, Inc.). This software
 may be used, copied, or redistributed as long as it is not sold and this copyright notice is reproduced
 on each copy made. This model is provided as is without any express or implied warranties.
+
+
+## Building for Machine Learning
+
+### PyTorch
+* Requires the fortran-pytorch library available [here](https://github.com/Cambridge-ICCS/fortran-pytorch-lib)
+* Build with: 
+
+    cmake -DTorch_DIR=<Path-to-venv>/lib/python3.11/site-packages/torch/share/cmake/Torc
+
+* Build MiMA using:
+
+### TensorFlow
+* Requires the fortran-pytorch library available [here](https://github.com/Cambridge-ICCS/fortran-tf-lib)
+* Requires TensorFlow C API:
+
+    FILENAME=libtensorflow-cpu-linux-x86_64-2.11.0.tar.gz
+    wget -q --no-check-certificate https://storage.googleapis.com/tensorflow/libtensorflow/${FILENAME}
+    tar -C <PATH_TO_INSTALL_LOCATION> -xzf ${FILENAME}
+
+* Build with:
+
+    cmake .. -DTENSORFLOW_LOCATION=<PATH_TO_TF_C_API_INSTALL_LOCATION> -DCMAKE_Fortran_COMPILER=ifort -DCMAKE_C_COMPILER=icc -DCMAKE_BUILD_TYPE=Release
+
+### Forpy
+* Requires a python venv with TensorFlow/PyTorch (and any other requirements) installed.
+
+### MiMA
+* Build with 
+
+    cmake -DFLAG_FOR_ML ..
+
+where `-DFLAG_FOR_ML` is one of `-DBUILD_PT_COUPLING`, `-DBUILD_TF_COUPLING`, `-DBUILD_FORPY_COUPLING`
diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake
@@ -0,0 +1,78 @@
+# - Find NetCDF
+# Find the native NetCDF includes and library
+#
+#  NETCDF_INCLUDES    - where to find netcdf.h, etc
+#  NETCDF_LIBRARIES   - Link these libraries when using NetCDF
+#  NETCDF_FOUND       - True if NetCDF found including required interfaces (see below)
+#
+# Your package can require certain interfaces to be FOUND by setting these
+#
+#  NETCDF_CXX         - require the C++ interface and link the C++ library
+#  NETCDF_F77         - require the F77 interface and link the fortran library
+#  NETCDF_F90         - require the F90 interface and link the fortran library
+#
+# The following are not for general use and are included in
+# NETCDF_LIBRARIES if the corresponding option above is set.
+#
+#  NETCDF_LIBRARIES_C    - Just the C interface
+#  NETCDF_LIBRARIES_CXX  - C++ interface, if available
+#  NETCDF_LIBRARIES_F77  - Fortran 77 interface, if available
+#  NETCDF_LIBRARIES_F90  - Fortran 90 interface, if available
+#
+# Normal usage would be:
+#  set (NETCDF_F90 "YES")
+#  find_package (NetCDF REQUIRED)
+#  target_link_libraries (uses_f90_interface ${NETCDF_LIBRARIES})
+#  target_link_libraries (only_uses_c_interface ${NETCDF_LIBRARIES_C})
+
+if (NETCDF_INCLUDES AND NETCDF_LIBRARIES)
+  # Already in cache, be silent
+  set (NETCDF_FIND_QUIETLY TRUE)
+endif (NETCDF_INCLUDES AND NETCDF_LIBRARIES)
+
+#set(CMAKE_FIND_DEBUG_MODE TRUE)
+find_path (NETCDF_INCLUDES netcdf.h netcdf.mod netcdf.inc
+  HINTS NETCDF_DIR ENV NETCDF_DIR ENV CPATH ENV FPATH)
+#set(CMAKE_FIND_DEBUG_MODE FALSE)
+message (STATUS "so far NETCDF_INCLUDES: ${NETCDF_INCLUDES}")
+
+
+find_library (NETCDF_LIBRARIES_C       NAMES netcdf netcdff
+    HINTS ENV LD_LIBRARY_PATH LIBRARY_PATH)
+mark_as_advanced(NETCDF_LIBRARIES_C)
+message (STATUS "so far NETCDF_LIBRARIES_C: ${NETCDF_LIBRARIES_C}")
+
+
+set (NetCDF_has_interfaces "YES") # will be set to NO if we're missing any interfaces
+set (NetCDF_libs "${NETCDF_LIBRARIES_C}")
+
+get_filename_component (NetCDF_lib_dirs "${NETCDF_LIBRARIES_C}" PATH)
+
+macro (NetCDF_check_interface lang header libs)
+  if (NETCDF_${lang})
+    find_path (NETCDF_INCLUDES_${lang} NAMES ${header}
+      HINTS "${NETCDF_INCLUDES}" NO_DEFAULT_PATH)
+    find_library (NETCDF_LIBRARIES_${lang} NAMES ${libs}
+      HINTS "${NetCDF_lib_dirs}" NO_DEFAULT_PATH)
+    mark_as_advanced (NETCDF_INCLUDES_${lang} NETCDF_LIBRARIES_${lang})
+    if (NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang})
+      list (INSERT NetCDF_libs 0 ${NETCDF_LIBRARIES_${lang}}) # prepend so that -lnetcdf is last
+    else (NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang})
+      set (NetCDF_has_interfaces "NO")
+      message (STATUS "Failed to find NetCDF interface for ${lang}")
+    endif (NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang})
+  endif (NETCDF_${lang})
+endmacro (NetCDF_check_interface)
+
+NetCDF_check_interface (CXX netcdfcpp.h netcdf_c++)
+NetCDF_check_interface (F77 netcdf.inc  netcdff)
+NetCDF_check_interface (F90 netcdf.mod  netcdff)
+
+set (NETCDF_LIBRARIES "${NetCDF_libs}" CACHE STRING "All NetCDF libraries required for interface level")
+
+# handle the QUIETLY and REQUIRED arguments and set NETCDF_FOUND to TRUE if
+# all listed variables are TRUE
+include (FindPackageHandleStandardArgs)
+find_package_handle_standard_args (NetCDF DEFAULT_MSG NETCDF_LIBRARIES NETCDF_INCLUDES NetCDF_has_interfaces)
+
+mark_as_advanced (NETCDF_LIBRARIES NETCDF_INCLUDES)
diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md
@@ -16,13 +16,57 @@ Get the latest version from [GitHub](https://github.com/mjucker/MiMA/releases/la
   * The code in its present form will only compile with Intel `ifort` and `icc` compilers. This description will assume `ifort` and `icc` are available.
   * MiMA reads and writes to `netCDF`, so `netCDF` needs to be installed on the system.
   * Being parallel, `MPI` needs to be there too.
-  * The flags need to be adjusted in the `bin/mkmf.template.$PLATFORM` file of choice. Typically, `$PLATFORM` will be slightly different on each machine, as libraries may not be found at the same location.
 
-* Compilation flags: The relevant flags are defined in `bin/mkmf.template.$PLATFORM`, and might or might not use environment variables. For instance, `netCDF` libraries or debug flags could be read from environment variables for more dynamic compilation. The first thing to do is to create an appropriate `mkmf.template.something`, which contains the relevant flags. Look at some of the template files that are already there to get an idea how to set the flags.
+* Build Systems
+  * There are two build systems that may be used to compile MiMA, CMake, and `mkmf`.
+  * Instructions for building with both of these are provided below.
 
-* Compile script: A compilescript is provided in `exp/compilescript.csh`. Make sure to set the first variable, `platform`, to whatever name you gave the mkmf template in the previous step. In our example, set it to `something`. The output executable will be in `exp/exec.$PLATFORM/mima.x`
+### CMake
+Building using CMake should follow the same process, regardless of the platform MiMA is being built on.
 
-* Adding files: If you work on your own version of MiMA, make sure every extension is in a new file, so as to not disturb the main branch and any other fork that might exist. When adding a source file, add the path to the file in `exp/path_names`, and it will be compiled the next time you run `./compilescript.csh`.
+* Dependencies
+  * In addition to the above list, building with CMake requires `cmake` to be installed on the system.
+
+* To build MiMA using CMake after cloning the repository and navigating into it (e.g. via `cd MiMA/`) run the following commands:
+  ```
+  mkdir build
+  cd build
+  cmake ..
+  make
+  ```
+  This takes you into the MiMA directory, creates a build directory, runs the CMake script `CMakeLists.txt` to generate a makefile for the system and then builds using the makefile.
+
+* The output executable will be at `build/mima.x`
+
+### mkmf
+Building using mkmf requires the user to amend the `compilescript.csh` scipt as appropriate for the platform they are building on, and possibly defining a mkmf template for their platform.
+
+* MiMA can be built using mkmf via the following steps:
+
+  * Select the appropriate mkmf template file for your platform from `bin/`.
+    These are of the form `bin/mkmf.template.$PLATFORM`, where `$PLATFORM` is typically slightly different on each machine as libraries may not be found at the same locations.
+
+  * Adjust the relevant compilation flags in the `bin/mkmf.template.$PLATFORM` file of choice as appropriate. 
+    * These may or may not use environment variables. For instance, `netCDF` libraries or debug flags could be read from environment variables for more dynamic compilation. The first thing to do is to create an appropriate `mkmf.template.something`, which contains the relevant flags. Look at some of the template files that are already there to get an idea how to set the flags.
+
+  * Compile script: A compilescript is provided in `exp/compilescript.csh`. Make sure to set the first variable, `platform`, to match the `$PLATFORM` of the mkmf template in the previous step. In our example, set it to `something`.
+
+  * MiMA can now be built with the following commands:
+    ```
+    cd exp
+    ./compilescript.csh
+    ```
+
+* The output executable will be in `exp/exec.$PLATFORM/mima.x`
+
+
+### Adding files to the build process
+
+* If you work on your own version of MiMA, make sure every extension is in a new file, so as to not disturb the main branch and any other fork that might exist.
+* When adding a source file you should:
+  * add the file to the `CMakeLists.txt` file in its local directory,
+  * add the path to the file in `exp/path_names`,
+  To ensure that it will be compiled the next time you build using CMake or run `./compilescript.csh`.
 
 
 ## Test run

diff --git a/input/input.nml b/input/input.nml
@@ -128,7 +128,10 @@
      weightminus2 = 0.02,
      source_level_pressure = 315.e+02,
      damp_level_pressure = 0.85e+02,
-     cg_drag_freq = 21600 /
+     cg_drag_freq = 21600,
+     runML = .true.,
+     model_dir = 'unknown',
+     model_name = 'unknown.ext'/
 
  &moist_processes_nml
      do_bm           =.true.,