Commit 7b45ff93 authored by Liesbeth Florentie's avatar Liesbeth Florentie
Browse files

first round of bug fixes after testing on cartesius. Now crashes when TM5 compilation starts...

parent 625d10cd
*.pyc
*/*.pyc*.pyc
*/*.pyc*.pyc
*/*.pyc*.pyc
*/*.pyc*.pyc
*/*.pyc*.pyc
*/*.pyc*.pyc
*/*.pyc*.pyc
*/*.pyc
......@@ -19,7 +19,6 @@ import shutil
import logging
import netCDF4
import numpy as np
from string import join
from datetime import datetime, timedelta
sys.path.append('../../')
from da.tools.general import date2num, num2date
......
......@@ -18,7 +18,6 @@ sys.path.append('../../')
rootdir = os.getcwd().split('da/')[0]
analysisdir = os.path.join(rootdir, 'da/analysis')
from string import join, split
from numpy import array, identity, zeros, arange, dot
import da.tools.io4 as io
......
......@@ -26,7 +26,6 @@ import logging
import datetime as dtm
import numpy as np
from string import strip
from numpy import array, logical_and, sqrt
sys.path.append(os.getcwd())
sys.path.append('../../')
......
......@@ -23,9 +23,7 @@ File created on 28 Jul 2010.
import os
import sys
import logging
#from da.baseclasses.statevector import filename
import datetime as dtm
from string import strip
from numpy import array, logical_and
sys.path.append(os.getcwd())
......
......@@ -24,7 +24,6 @@ import os
import sys
import logging
import datetime as dtm
#from string import strip
from numpy import array, logical_and
sys.path.append(os.getcwd())
......
......@@ -25,7 +25,6 @@ import sys
import logging
import datetime as dtm
from string import strip
from numpy import array, logical_and
sys.path.append(os.getcwd())
sys.path.append('../../')
......
......@@ -25,7 +25,6 @@ import sys
import logging
import datetime as dtm
#from string import strip
from numpy import array, logical_and, sqrt
sys.path.append(os.getcwd())
sys.path.append('../../')
......
......@@ -26,7 +26,6 @@ import sys
import logging
import datetime as dtm
from string import strip
from numpy import array, logical_and, sqrt
sys.path.append(os.getcwd())
sys.path.append('../../')
......@@ -115,8 +114,8 @@ class ObsPackObservations(Observations):
logging.debug("ObsPack dataset info read, proceeding with %d netcdf files" % len(ncfilelist))
for ncfile in ncfilelist:
if ncfile not in valid_sites: continue
if ncfile not in valid_sites: continue
infile = os.path.join(self.obspack_dir, 'data', 'nc', ncfile + '.nc')
ncf = io.ct_read(infile, 'read')
......@@ -296,27 +295,27 @@ class ObsPackObservations(Observations):
savedict['missing_value'] = '!'
f.add_data(savedict)
data = self.getvalues('obs')
savedict = io.std_savedict.copy()
savedict['name'] = "observed"
savedict['long_name'] = "observedvalues"
savedict['units'] = "mol mol-1"
savedict['dims'] = dimid
savedict['values'] = data.tolist()
savedict['comment'] = 'Observations used in optimization'
f.add_data(savedict)
data = self.getvalues('mdm')
savedict = io.std_savedict.copy()
savedict['name'] = "modeldatamismatch"
savedict['long_name'] = "modeldatamismatch"
savedict['units'] = "[mol mol-1]"
savedict['dims'] = dimid
savedict['values'] = data.tolist()
savedict['comment'] = 'Standard deviation of mole fractions resulting from model-data mismatch'
f.add_data(savedict)
data = self.getvalues('obs')
savedict = io.std_savedict.copy()
savedict['name'] = "observed"
savedict['long_name'] = "observedvalues"
savedict['units'] = "mol mol-1"
savedict['dims'] = dimid
savedict['values'] = data.tolist()
savedict['comment'] = 'Observations used in optimization'
f.add_data(savedict)
data = self.getvalues('mdm')
savedict = io.std_savedict.copy()
savedict['name'] = "modeldatamismatch"
savedict['long_name'] = "modeldatamismatch"
savedict['units'] = "[mol mol-1]"
savedict['dims'] = dimid
savedict['values'] = data.tolist()
savedict['comment'] = 'Standard deviation of mole fractions resulting from model-data mismatch'
f.add_data(savedict)
f.close()
logging.debug("Successfully wrote data to obs file")
......
......@@ -89,9 +89,9 @@ class CO2GriddedStateVector(StateVector):
if 'pco2' in file or 'cov_ocean' in file:
parnr = list(range(9805,9835))
if 'ocn.optimize' in dacycle.dasystem and not dacycle.dasystem['ocn.optimize']:
cov = np.identity(30)*1E-20
logging.debug('Prior ocean covariance matrix = Identity matrix')
if 'ocn.optimize' in dacycle.dasystem and not dacycle.dasystem['ocn.optimize']:
cov = np.identity(30)*1E-20
logging.debug('Prior ocean covariance matrix = Identity matrix')
else:
cov_ocn = f.get_variable('CORMAT')
cov = cov_ocn
......
......@@ -132,9 +132,9 @@ class CartesiusPlatform(Platform):
logging.info("A new task will be started (%s)" % cmd)
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
logging.info(output)
print 'output', output
print('output', output)
jobid = output.split()[-1]
print 'jobid', jobid
print('jobid', jobid)
else:
cmd = ["sbatch", jobfile]
logging.info("A new job will be submitted (%s)" % cmd)
......
!!! Info for the CarbonTracker data assimilation system
datadir : /projects/0/ctdas/input/ctdas_2012
! XCO2 samples
obs.column.input.dir : ${datadir}/xco2/oco2_l2_lite_fp_9r_10secs/
obs.column.ncfile : oco2_LtCO2_<YYYYMMDD>_10sec.nc
obs.column.selection.variables: land_water_indicator, operation_mode
obs.column.selection.criteria : == 0, <2
obs.column.extra.variables : pressure_weight
obs.column.mdm.weight : 1
obs.column.rc : ${datadir}/xco2/oco2_weights.rc
obs.column.mdm.calculation : parametrization
! For ObsPack
obspack.input.id : obspack_co2_111_MERGE_GLOBALVIEWplus_v4.2_NRT_v4.4_2019-05-16
obspack.input.dir : ${datadir}/obspacks/${obspack.input.id}
obs.sites.rc : /projects/0/ctdas/liesbeth/ctdas_xco2/sites_weights_validation_noaircraft.rc
obspack.mdm.weight : 1
obs.scaling.factor : 1e6
ocn.covariance : ${datadir}/oceans/oif/cov_ocean.2000.01.nc
deltaco2.prefix : oif_p3_era40.dpco2
bio.cov.dir : ${datadir}/covariances/gridded_NH/
bio.cov.prefix : cov_ecoregion
regtype : gridded_oif30
nparameters : 9835
random.seed : 4385
regionsfile : ${datadir}/covariances/gridded_NH/griddedNHparameters.nc
!random.seed.init: ${datadir}/randomseedinit.pickle
! Include a naming scheme for the variables
#include NamingScheme.wp_Mar2011.rc
! Info on the sites file used
ocn.optimize : True
......@@ -38,7 +38,6 @@ import logging
import shutil
import datetime
import subprocess
from string import join
import glob
sys.path.append(os.getcwd())
sys.path.append("../../")
......@@ -191,7 +190,7 @@ class TM5ObservationOperator(ObservationOperator):
else: cmd = ['python', 'setup_tm5', '--%s' % self.dacycle.daplatform.give_queue_type(), rcfilename]
logging.info('Starting the external TM5 setup script')
logging.info('using command ... %s' % join(cmd))
logging.info('using command ... %s' % ''.join(cmd))
retcode = subprocess.call(cmd)
os.chdir(self.dacycle['dir.da_submit'])
......
......@@ -75,7 +75,6 @@ import copy
import getopt
import pickle
import numpy as np
#from string import join
import da.tools.rc as rc
from da.tools.general import create_dirs, to_datetime, advance_time
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment