How to use MSS-Chem

Installation

Required dependencies

• Python 2.7, 3.4, 3.5, or 3.6
• numpy (1.7 or later)
• netcdf4-python

Instructions

Using conda

MSS-Chem itself is a pure Python package, but its dependencies are not. The easiest way to get everything installed is to use conda. To install mss-chem using the conda command line tool:

$ conda install -c andreas-h mss-chem

Using pip

If you don’t use conda, be sure you have the required dependencies (see above) installed first. Then, install mss-chem with pip:

$ pip install mss-chem

Running MSS-Chem

usage: msschem-dl [-h] (-m MODEL | -a) [-d DATE] [-c CONFIG] [-q | -v]

MSS-Chem downloader

optional arguments:
  -h, --help            show this help message and exit
  -m MODEL, --model MODEL
                        Model to download
  -a, --all             Download data from all configured models
  -d DATE, --date DATE  Date to download data for (YYYY-MM-DD)
  -c CONFIG, --config CONFIG
                        MSS-Chem configuration file
  -q, --quiet           No output except for errors
  -v, --verbosity       Increase output verbosity (can be supplied multiple times)

Configuration of MSS-Chem

The location of a configuration file can be passed to the msschem-dl script with the --config option. Alternatively, MSS-Chem tries to import a module msschem_settings. In that case, the file msschem_settings.py has to be somewhere in your $PYTHONPATH.

The configuration file could look like this:

import os.path
from subprocess import check_call
from nco import Nco

from msschem import DataNotAvailable

from msschem.models import CAMSRegDriver
from msschem.download import CAMSRegDownload

from msschem.models.cams_global import CAMSGlobDriver
from msschem.download import CAMSGlobDownload

from msschem.models import EMEPDriver
from msschem.download import FTPDownload

from msschem.models import SilamDriver
from msschem.download import SilamDownload

from msschem.download import SCPDownload, FilesystemDownload
from msschem.download import DictFormatter


datasources = {
    'CAMSReg_ENSEMBLE': CAMSRegDriver(
        dict(
            dldriver=CAMSRegDownload(
                password='__M0bChV6QsoOFqHz31VRqnpr4GhWPtcpaRy3oeZjBNSg__',
                modelname='ENSEMBLE',
                n_tries=3),
            force=False,
            basepath=os.path.expanduser('~/tmp/mss/data/'),
            name='CAMSReg-ENSEMBLE',
            temppath=None,
            species=['CO', 'NH3', 'NMVOC', 'NO2', 'NO', 'O3', 'PANS', 'PM10',
                     'PM25', 'SO2'],
        )
    ),
    'CAMSGlob': CAMSGlobDriver(
        dict(
            dldriver=CAMSGlobDownload(
                username="andreas.hilboll",
                password="bMmGkpWn",
                host="dissemination.ecmwf.int",
                n_tries=1),
            force=False,
            basepath=os.path.expanduser('~/tmp/mss/data/'),
            name='CAMSGlob',
            temppath=None,
            species=['CO', 'O3', 'HCHO', 'HNO3', 'NO', 'NO2', 'OH', 'PANS',
                     'SO2'],
        )
    ),
    'SILAM': SilamDriver(
        dict(
            dldriver=SilamDownload(n_tries=1),
            force=False,
            basepath=os.path.expanduser('~/tmp/mss/data/'),
            name='SILAM',
            temppath=None,
            species=['CO', 'NO2', 'NO', 'NMVOC', 'O3', 'PANS', 'PM10', 'SO2'],
        )
    ),
    'EMEP': EMEPDriver(
        dict(
            dldriver=SCPDownload(
                host='glogin',
                path='.',
                fnpattern='CWF_12-{fcinit:%Y%m%d}_hourInst.nc',
                username='metno',
                password='EdEh2gS.'),
            force=False,
            basepath=os.path.expanduser('~/tmp/mss/data/'),
            name='EMEP',
            temppath='/home2/hilboll/code/mss-chem/tmp',
            species=['NO2', 'PM25'],
        )
    ),
}


EMEP_PERT_CITIES = ['AmsRot', 'London', 'Paris', 'PoVall', 'Ruhr']


def preprocess_emep_perturbation(**kwargs):
    
    varlist = ['D3_ug_CO', 'PS', 'hyam', 'hybm', 'P0']

    species = kwargs['species']

    options_string = ['-7', '-O']  # for NETCDF4_CLASSIC output
    nco = Nco()

    city = kwargs.get('city')
    fn_city = os.path.join(kwargs.get('path'),
                           DictFormatter().format(kwargs.get('fnpattern'),
                                                  city_id='{}_ALL_P15_'.format(city)))
    if not os.path.isfile(fn_city):
        raise DataNotAvailable
    if species == 'co':
        fn_orig = os.path.join(kwargs.get('path'), DictFormatter().format(kwargs.get('fnpattern'), city_id=''))
        if not os.path.isfile(fn_orig):
            raise DataNotAvailable
        fn_tmp_orig = os.path.join(kwargs.get('path_out'), '{}_orig.nc'.format(city))
        fn_tmp_city = os.path.join(kwargs.get('path_out'), '{}_pert.nc'.format(city))
        options_string += ['-v' + ','.join(varlist)]
        nco.ncks(input=fn_orig, output=fn_tmp_orig, options=options_string)
        nco.ncks(input=fn_city, output=fn_tmp_city, options=options_string)
        fn_out_tracer = os.path.join(kwargs.get('path_out'), 'emep_pert_{}_co.nc'.format(city))
        check_call(['ncdiff', '-7', '-O', '-vD3_ug_CO', fn_tmp_orig,
                    fn_tmp_city, fn_out_tracer])
        os.remove(fn_tmp_city)
        os.remove(fn_tmp_orig)
    elif species == 'pressure':
        options_string += ['-v' + ','.join(varlist[1:])]
        fn_out_pressure = os.path.join(kwargs.get('path_out'), 'emep_pert_{}_pressure.nc'.format(city))
        nco.ncks(input=fn_city, output=fn_out_pressure, options=options_string)
    else:
        raise ValueError

    # extract variables of interest
    # create pressure


for city in EMEP_PERT_CITIES:
    driver = EMEPDriver(
            dict(dldriver=FilesystemDownload(
                    path=os.path.expanduser('~/tmp/mss/data/tmp'),
                    fnpattern='emep_pert_{}_{{species}}.nc'.format(city),
                    do_copy=False,
                    pre_filter_hook=(preprocess_emep_perturbation,
                                     dict(city=city,
                                          fnpattern='CWF_12-{fcinit:%Y%m%d}_{city_id}hourInst.nc'))),
                                        #'/home/home/metno',
                                        #os.path.expanduser('~/tmp/mss/data/tmp'),
                                        #os.path.expanduser('~/tmp/mss/data/tmp')])),
            force=False,
            basepath=os.path.expanduser('~/tmp/mss/data/'),
            name='EMEP_' + city,
            temppath=os.path.expanduser('~/tmp/mss/data/tmp'),
            species=['CO'],
            )
    )
    driver.need_to_convert_to_nc4c = False
    driver.name = 'EMEP_' + city
    datasources['EMEP_' + city] = driver


class CAMSTracerDriver(CAMSGlobDriver):

    def fix_dataset(self, fn_out, species, fcinit):
        from netCDF4 import Dataset, num2date, date2num
        # read vertical coordinates
        with Dataset(fn_out, 'a', format='NETCDF4_CLASSIC') as nc:
            # fix time variable
            t_obj = num2date(nc.variables['time'][:],
                             nc.variables['time'].units)
            t_unit = 'hours since {:%Y-%m-%dT%H:%M:%S}'.format(fcinit)
            nc.variables['time'][:] = date2num(t_obj, t_unit)
            nc.variables['time'].setncattr('units', t_unit)
            nc.variables['time'].setncattr('standard_name', 'time')

            nc.variables['level'].setncattr('standard_name',
                                            'atmosphere_pressure_coordinate')
            # convert to Pa; otherwise, vsec plotting doesn't work
            nc.variables['level'][:] *= 100.
            nc.variables['level'].setncattr('units', 'Pa')

            self.set_standard_name(nc, species)

            # TODO add history


CAMS_TRACER = [('London', 'p26.212', '{fcinit:%Y%m%d}_tracer4emerge_pl.nc', (10, 101, 177), 'CO'),
               ('Ruhr', 'p27.212', '{fcinit:%Y%m%d}_tracer4emerge_pl.nc', (10, 101, 177), 'CO'),
               ('Berlin', 'p28.212', '{fcinit:%Y%m%d}_tracer4emerge_pl.nc', (10, 101, 177), 'CO'),
               ('PoValley', 'p29.212', '{fcinit:%Y%m%d}_tracer4emerge_pl.nc', (10, 101, 177), 'CO'),
               ('Madrid', 'p30.212', '{fcinit:%Y%m%d}_tracer4emerge_pl.nc', (10, 101, 177), 'CO'),
               ('Paris', 'p31.212', '{fcinit:%Y%m%d}_tracer4emerge_pl.nc', (10, 101, 177), 'CO'),
               ('Amsterdam', 'p33.212', '{fcinit:%Y%m%d}_tracer4emerge_pl.nc', (10, 101, 177), 'CO'),
               ('Rome', 'p34.212', '{fcinit:%Y%m%d}_tracer4emerge_pl.nc', (10, 101, 177), 'CO'),
               ('BB-NA', 'p32.212', '{fcinit:%Y%m%d}_tracer4emerge_nh_pl.nc', (10, 226, 900), 'CO'),
               ('BB-Sib', 'p25.212', '{fcinit:%Y%m%d}_tracer4emerge_nh_pl.nc', (10, 226, 900), 'CO'),
               ('O3strat', 'o3s', '{fcinit:%Y%m%d}_tracer4emerge_o3s_pl.nc', (10, 101, 177), 'O3'),
               ('AllCities', 'alltracer', '{fcinit:%Y%m%d}_alltracer4emerge_pl.nc', (10, 101, 177), 'CO'),
]


for name, varname, fn, dims, species in CAMS_TRACER:
    #print(name, varname, fn, dims, species)
    driver = CAMSTracerDriver(
            dict(
                dldriver=FilesystemDownload(
                    path=os.path.expanduser('/misc/gomzo2/home2/annebl/Tracer-Runs/'),
                    fnpattern=fn,
                    do_copy=False,
                    n_tries=1),
                force=False,
                basepath=os.path.expanduser('~/tmp/mss/data/'),
                name='CAMSTr_' + name,
                temppath=None,
                species=[species],
            ),
        )

    # set variable name
    driver.species = {species: dict(varname=varname, urlname=varname)}

    # set layer type
    driver.layer_type = 'pl'

    # remove AIR_PRESSURE - we don't need it for pressure_level data
    if 'AIR_PRESSURE' in driver.cfg['species']:
        driver.cfg['species'].pop(driver.cfg['species'].index('AIR_PRESSURE'))

    # set lat/lon dimensions so that sanity check doesn't fail
    driver.dims = [('t', None), ('z', dims[0]), ('y', dims[1]), ('x', dims[2])]

    assert 'CAMSTr_' + name not in datasources.keys()

    driver.name = 'CAMSTr_' + name
    datasources['CAMSTr_' + name] = driver

    driver = None
    del driver

Configuration of MSS

The first MSS version to fully support the model data downloaded and prepared by MSS-Chem is MSS 1.6.2.

MSS is configured via the file mss_wms_settings.py, as described in the MSS documentation. To include CTM data downloaded with MSS-Chem in the MSS WMS setup, the following steps need to be taken:

• The path to the model data has to be specified in the datapath dict, e.g.,

  datapath = {
      "camsglobal": "/path/to/mss/data/camsg",
      "camsregional": "/path/to/mss/data/camsr",
      "silam": "/path/to/mss/data/silam",
  }
  

• The data classes have to be defined in the data dict, e.g.,

  data = {
      "camsglobal": mslib.mswms.dataaccess.DefaultDataAccess(datapath["camsglobal"], "ml"),
      "camsregional": mslib.mswms.dataaccess.DefaultDataAccess(datapath["camsregional"], "al"),
      "silam": mslib.mswms.dataaccess.DefaultDataAccess(datapath["silam"], "al"),
  }
  

• The MSS plotting styles have to be instantiated for both horizontal (maps) and vertical (cross-section) plots. This is done using the make_msschem_class functions from the modules mslib.mswms.mpl_hsec_styles.py and mslib.mswms.mpl_vsec_styles.py.

  For example, in order to instantiate (i.e., make available) plotting styles for model data on altitude, pressure, and model levels, one could do the following:

  # import plot style modules
  from mslib.mswms import mpl_hsec_styles
  from mslib.mswms import mpl_vsec_styles
  # import chemical species
  from mslib.mswms.msschem import MSSChemTargets
  
  # create horizontal (map) plot styles
  for vert in ["al", "pl", "ml"]:
      for stdname, props in list(MSSChemTargets.items()):
          name, qty, units, scale = props
          key = "HS_MSSChemStyle_" + vert.upper() + "_" + name + "_" + qty
          globals()[key] = mpl_hsec_styles.make_msschem_class(stdname, name, vert, units, scale)
  
  # create vertical (cross-section) plot styles
  for vert in ["al", "pl", "ml"]:
      for stdname, props in list(MSSChemTargets.items()):
          name, qty, units, scale = props
          key = "VS_MSSChemStyle_" + vert.upper() + "_" + name + "_" + qty
          globals()[key] = mpl_vsec_styles.make_msschem_class(stdname, name, vert, units, scale)
  

  These MSS layer styles are classes which follow the following naming scheme: XS_MSSChemStyle_LAYERTYPE_SPECIES_QUANTITY.

  Here,

  • XS is either HS for horizontal or VS for vertical plots
  • LAYERTYPE is one of AL, PL, ML for model data on altitude, pressure, and model levels, respectively,
  • SPECIES is the species to be plotted (in upper case)
  • QUANTITY identifies the quantity to be plotted, which can be one of mfrac for mass fractions (in kg/kg) and mconc for mass concentrations (in kg/m³)

  For example, the plotting style for horizontal plots of PM10 mass concentrations defined on model levels would be HS_MSSChemStyle_ML_PM10_mconc.

• The horizontal plot styles which should be offered by the MSS WMS server have to be registered in the register_horizontal_layers list, e.g.,

  register_horizontal_layers = [
      (HS_MSSChemStyle_ML_NO2_mfrac, ["camsglobal"]),
      (HS_MSSChemStyle_AL_NO2_mconc, ["camsregional"]),
      (HS_MSSChemStyle_AL_PM10_mconc, ["silam"]),
  ]
  

• The vertical plot styles which should be offered by the MSS WMS server have to be registered in the register_horizontal_layers list, e.g.,

  register_vertical_layers = [
      (VS_MSSChemStyle_ML_NO2_mfrac, ["camsglobal"]),
      (VS_MSSChemStyle_AL_NO2_mconc, ["camsregional"]),
      (VS_MSSChemStyle_ALWithPressure_PM10_mconc, ["silam"]),
  ]