DreqPy (Data Request Python API)
CMIP Phase 6 Data Request Python Interface
IPython NoteBook created by Martin Schupfner, DKRZ
References:
Download this website.
Synopsis¶
0 - Basic Information
1 - Basic Imports
2 - dq.coll Examples
3 - dq.inx Examples
4 - Presentation of dq.coll sections
5 - Advanced Example
6a - Data Request Volume Estimate
6b - Data Request Volume Estimate (Legacy - outdated)
7 - Additional Functions
8 - Official Web Tools
In [1]:
from IPython.display import Image, display
display(Image(filename='files/Folie1.PNG'))
In [2]:
display(Image(filename='files/Folie2.PNG'))
In [3]:
display(Image(filename='files/Folie3.PNG'))
In [4]:
display(Image(filename='files/Folie4.PNG'))
In [5]:
#!/usr/bin/python
# -*- coding: utf-8 -*-
# IPython NoteBook created by Martin Schupfner, DKRZ
# Reference: Martin Juckes 2016 -
# dreqPy (Data Request Python API) & dreqPy User's Guide
from dreqPy import dreq
print("Using DreqPy (Data Request Python API) in version %s"\
% str(dreq.version))
# Initialisation
dq = dreq.loadDreq()
In [6]:
# dq.coll
# Content Object dq.coll is a dictionary containing the data request sections,
# with 3 elements (named tuples) for each section:
# - header : named tuple with info such as title, lable, etc.
# - attDefn: dictionary containing record attribute definitions
# - items : list of records
# Print all entries of dq.coll
print("dq.coll Entries:\n", ", ".join(dq.coll.keys()))
In [7]:
# header content (Example MIP Variable):
print(".header content Example 'var':\n------------------------------")
for name in dq.coll['var'].header._fields: print("%-15s : %s" \
%(name, getattr(dq.coll['var'].header, name)))
In [8]:
# attDefn content (Example MIP Variable):
print(".attDefn content Example 'var':\n-------------------------------")
for key in dq.coll['var'].attDefn.keys():
print("%-15s : %s" %(key, dq.coll['var'].attDefn[key]))
In [9]:
# items content (Example MIP Variable):
print(".items content Example 'var':\n-----------------------------")
for key in dq.coll['var'].attDefn.keys():
print("%-15s : %s" %(key, getattr(dq.coll['var'].items[0], key)))
In [10]:
# Index dq.inx is a simple lookup table
# dq.inx.uid[UID] returns the record corresponding to 'UID'
# Example from above:
item = dq.inx.uid['0baf6a333b91c4db341b515c28cd2c05']
print(item)
print("Label:", item.label)
print("Title:", item.title)
print("Units:", item.units)
In [11]:
# dq.inx.iref_by_uid[UID]
# gives a list of IDs of objects linked to 'UID',
# as a tuple (SECT, ID)
# Example from above:
id_list = dq.inx.iref_by_uid['0baf6a333b91c4db341b515c28cd2c05']
for ID in id_list:
print("# Section: %-10s\n UID: %15s\n %s\n"\
%(ID[0], ID[1], dq.inx.uid[ID[1]]))
In [12]:
# dq.inx.iref_by_sect[UID].a[SECT]
# gives a list of IDs of 'SECT' linked to 'UID'
# Example from above:
id_list = dq.inx.iref_by_sect['0baf6a333b91c4db341b515c28cd2c05'].a['CMORvar']
for ID in id_list:
item = dq.inx.uid[ID]
print("%15s | %10s | %s" \
%(item.label, item.mipTable, item.vid))
In [13]:
# The same can be achieved using dq.coll, though:
# Example from above:
CMORvar_list = [ item for item in dq.coll['CMORvar'].items \
if item.vid=='0baf6a333b91c4db341b515c28cd2c05']
for item in CMORvar_list:
print("%15s | %10s | %s" \
%(item.label, item.mipTable, item.vid))
In [14]:
from IPython.display import Image, display
# Display Structure of DreqPy
display(Image(filename='files/DreqPyStructure.png'))
In [15]:
def showSect(sect, desc=False, uid=""):
"""
Print the Content of dq.coll's sections and one example item
Arg: sect - str - section key
desc - boolean - also print description
(Default: False)
uid - str - provide the uid of the example item if desired,
else, the example item will be chosen automatically
"""
# print header title
print(dq.coll[sect].header.title)
print("-"*len(dq.coll[sect].header.title))
# print Section name and description
for subsect in dq.coll[sect].attDefn.keys():
if desc==True:
print("# %15s | %s%s" \
%(subsect, dq.coll[sect].attDefn[subsect].title, \
"\n"+dq.coll[sect].attDefn[subsect].description if \
(str(dq.coll[sect].attDefn[subsect].description)!="" or \
subsect in str(dq.coll[sect].attDefn[subsect].description)) \
else ""))
else:
print("# %15s | %s" \
%(subsect, dq.coll[sect].attDefn[subsect].title))
# print Example of first item in list
print("-------\nExample\n-------")
for subsect in dq.coll[sect].attDefn.keys():
if uid=="":
print("# %15s | %s"\
%(subsect, getattr(dq.coll[sect].items[0], subsect) if \
subsect not in str(getattr(dq.coll[sect].items[0], subsect)) \
else ""))
else:
print("# %15s | %s"\
%(subsect, getattr(dq.inx.uid[uid], subsect) if \
subsect not in str(getattr(dq.coll[sect].items[0], subsect)) \
else ""))
In [16]:
# MIPs
print("Chicken or egg? MIP or Objective?\n"\
"The CMIP endorsed MIP (Model Intercomparison Project) is linked to scientific objectives.")
display(Image(filename='files/DreqPyStructure_1.png'))
showSect('mip')
#showSect('mip', desc=True) #Activate for additional info if there is any
In [17]:
# Objective
showSect('objective')
#showSect('objective', desc=True) #Activate for additional info if there is any
In [18]:
# Experiments
print("MIPs may define experiments or solely request data from experiments.")
print("Experiments are organised in experiment groups.")
display(Image(filename='files/DreqPyStructure_2.png'))
showSect('experiment')
#showSect('experiment', desc=True) #Activate for additional info if there is any
In [19]:
# Experiment Groups
showSect('exptgroup')
#showSect('exptgroup', desc=True) #Activate for additional info if there is any
In [20]:
# Request Item
print("The request items build up the data request.")
print("They are linked to (or: requested by) either MIPs, experiments or experiment groups.")
print("Request items hold information about the time slice of the requested variables they include\n"\
"as well as the possibility to set their priority and tier.")
display(Image(filename='files/DreqPyStructure_3.png'))
showSect('requestItem', True)
#showSect('requestItem', desc=True) #Activate for additional info if there is any
In [21]:
# Time Slice
showSect('timeSlice')
#showSect('timeSlice', desc=True) #Activate for additional info if there is any
In [22]:
# Request Variable Group
print("The request variable groups are defined by MIPs.")
display(Image(filename='files/DreqPyStructure_4.png'))
showSect('requestVarGroup')
#showSect('requestVarGroup', desc=True) #Activate for additional info if there is any
In [23]:
# Request Link
print("Each request item is linked (via request link) to a request variable group.")
print("Several request items can link to the same request variable group\n"\
"for a different set of experiments, time slices, priorities, etc.")
print("Of course a request variable group can be requested by MIPs that did not define it.")
display(Image(filename='files/DreqPyStructure_5.png'))
showSect('requestLink',True)
#showSect('requestLink', desc=True) #Activate for additional info if there is any
In [24]:
# Objective Link
print("Request items are linked to scientific objectives (via objective link and request link).")
display(Image(filename='files/DreqPyStructure_6.png'))
showSect('objectiveLink')
#showSect('objectiveLink', desc=True) #Activate for additional info if there is any
In [25]:
# Request Variable
print("A request variable group holds a list of request variables.")
print("Request variables are basically links to CMOR variables with an\n"\
"additional information about the CMOR variables' priority.")
display(Image(filename='files/DreqPyStructure_7.png'))
showSect('requestVar')
#showSect('requestVar', desc=True) #Activate for additional info if there is any
In [26]:
# CMOR Variable
showSect('CMORvar')
#showSect('CMORvar', desc=True) #Activate for additional info if there is any
#showSect('CMORvar', uid=[i.uid for i in dq.coll['CMORvar'].items if i.mipTable=="Esubhr" and i.label=="pr"][0])
#showSect('CMORvar', uid=[i.uid for i in dq.coll['CMORvar'].items if i.mipTable=="CFsubhr" and i.label=="pr"][0])
In [27]:
# MIP Variable
print("MIP variables are defined by a MIP and linked to CMOR variables.")
print("They hold information like the variables' unit, etc.")
display(Image(filename='files/DreqPyStructure_8.png'))
showSect('var')
#showSect('var', desc=True) #Activate for additional info if there is any
In [28]:
# Variable Choice Links
print("CMOR variables can be subject to a variable choice.")
print("Those choices are either defined by a hierarchy (ranked choice),\n"\
"or the choice is made via the model configuration (configuration choice).")
display(Image(filename='files/DreqPyStructure_9.png'))
showSect('varChoiceLinkC')
#showSect('varChoiceLinkC', desc=True) #Activate for additional info if there is any
print("\n"*2)
showSect('varChoiceLinkR')
#showSect('varChoiceLinkR', desc=True) #Activate for additional info if there is any
In [29]:
# Model Configuration
showSect('modelConfig')
#showSect('modelConfig', desc=True) #Activate for additional info if there is any
#Variable Choice
print("\n"*2)
showSect('varChoice')
#showSect('varChoice', desc=True) #Activate for additional info if there is any
In [30]:
# Structure
print("CMOR variables are linked to the structure, that holds information about dimensions ...")
display(Image(filename='files/DreqPyStructure_10.png'))
showSect('structure')
#showSect('structure', desc=True) #Activate for additional info if there is any
In [31]:
# Temporal and Spatial Shape
print("... such as the spatial and temporal shape.")
display(Image(filename='files/DreqPyStructure_11.png'))
showSect('temporalShape')
#showSect('temporalShape', desc=True) #Activate for additional info if there is any
print("\n"*2)
showSect('spatialShape')
#showSect('spatialShape', desc=True) #Activate for additional info if there is any
In [32]:
# Grids/Dimensions
print("Spatial and temporal shape are linked to a grid entry.")
print("Structure also links to the cell methods.")
display(Image(filename='files/DreqPyStructure_12.png'))
showSect('grids')
#showSect('grids', desc=True) #Activate for additional info if there is any
In [33]:
# Cell Methods
showSect('cellMethods')
#showSect('cellMethods, desc=True) #Activate for additional info if there is any if there is any
In [34]:
# Standard Name
print("Grids and MIP variables have a defined standard name.")
print("This concludes the list with the most important objects \n"\
"of the DreqPy CMIP6 Data Request structure.")
display(Image(filename='files/DreqPyStructure.png'))
showSect('standardname')
#showSect('standardname', desc=True) #Activate for additional info if there is any
5 - Advanced Example
Get a list of all variables requested by the CMIP6 Endorsed MIP 'C4MIP':
To attain the requested variables for an experiment, experiment group, or MIP, the first step is to find all requestItem-items (or requestLink-items) related to this experiment/experiment group/MIP.
Then proceed to find all requestVarGroup-items by following the Request Links. The requestVarGroup-items hold the requestVar-items, which finally are linked to the desired CMORvar-items.
In [35]:
# We want a list of all CMOR variables requested by C4MIP
# 1 Collect all 'requestLink'-items, for requests made by 'C4MIP':
C4MIP_rL = [ rLink for rLink in dq.coll['requestLink'].items\
if rLink.mip=='C4MIP']
print("%i requestLink-items found for 'C4MIP'.\n" % len(C4MIP_rL))
In [36]:
# 2.1 Find the 'requestVarGroup'-items, associated with these requestLinks,
# (not all requestVarGroups have to be defined by C4MIP!):
C4MIP_rVG = set([ dq.inx.uid[rLink.refid] for rLink in C4MIP_rL ])
print("%i requestVarGroup-items found for 'C4MIP'.\n" % len(C4MIP_rVG))
# 2.2 Get the UIDs of the requestVarGroup-items:
C4MIP_rVG_UIDs = [ rVG.uid for rVG in C4MIP_rVG ]
# 3 Find the 'requestVar'-items, associated with these requestVarGroups:
C4MIP_rV = set([ rV for rV in dq.coll['requestVar'].items\
if rV.vgid in C4MIP_rVG_UIDs])
print("%i requestVar-items found for 'C4MIP'.\n" % len(C4MIP_rV))
In [37]:
# 4.1 Get the CMORvar-UIDs from the requestVar-items:
C4MIP_rV_VIDs = [ rV.vid for rV in C4MIP_rV ]
# 4.2 Find the 'CMORvar'-items, associated with these requestVars:
C4MIP_CMORvars = set([ cmvar for cmvar in dq.coll['CMORvar'].items\
if cmvar.uid in C4MIP_rV_VIDs ])
print("%i CMORvar-items found for 'C4MIP'\n" % len(C4MIP_CMORvars))
print("Here are the first 10 list entries:")
C4MIP_CMORvars=list(C4MIP_CMORvars) # set -> list conversion
for i in range(0, 10):
if i<len(C4MIP_CMORvars):
cmvar = C4MIP_CMORvars[i]
print(" # %-15s | mipTable: %-10s | Frequency: %-2s\n LongName: %s\n"\
%(cmvar.label, cmvar.mipTable, cmvar.frequency, cmvar.title))
In [38]:
# Now we only want the list of variables requested by C4MIP for a certain experiment.
# We use the "requestItems" for this purpose:
# Find all experiments, C4MIP is requesting data from:
# We select all requestItems of C4MIP (via the link rlid to the requestLinks)
# and list all experiments that C4MIP is requesting data from
C4MIP_rI_esids = [ item.esid for item in dq.coll['requestItem'].items \
if dq.inx.uid[item.rlid].mip=='C4MIP']
print("%i requestItem-experiment-identifiers found for 'C4MIP'.\n" % len(set(C4MIP_rI_esids)))
# The collected "esids" are the unique identifiers of either
# experiments, experiment groups or MIPs.
# Experiment groups are self explanatory.
# If the "esid" links to a MIP, it means that the respective requestItem is
# valid for all the experiments defined by that MIP
# -> Now let us translate the esids to only experiment ids:
C4MIP_rI_expids = list()
for itemid in set(C4MIP_rI_esids):
if type(dq.inx.uid[itemid])==type(dq.coll['experiment'].items[0]):
C4MIP_rI_expids.append(dq.inx.uid[itemid].uid)
elif type(dq.inx.uid[itemid])==type(dq.coll['exptgroup'].items[0]):
for exp in dq.coll['experiment'].items:
if exp.egid==itemid: C4MIP_rI_expids.append(exp.uid)
elif type(dq.inx.uid[itemid])==type(dq.coll['mip'].items[0]):
for exp in dq.coll['experiment'].items:
if exp.mip==itemid: C4MIP_rI_expids.append(exp.uid)
else:
print("Experiment identifier %s links to neither experiment, experiment "\
"group nor MIP." % itemid)
print("C4MIP requests data from %i experiments.\n" % len(set(C4MIP_rI_expids)))
In [39]:
# Is there one experiment not defined by C4MIP?
for expid in set(C4MIP_rI_expids):
if dq.inx.uid[expid].mip!="C4MIP":
req_expid=expid
req_explabel=dq.inx.uid[req_expid].label
print(req_explabel)
break
In [40]:
# Which variables are requested for this experiment by C4MIP?
# 1 Collect all possible experiment identifiers for this experiment:
# -> experiment uid, experiment group uid, mip uid
exp_esids=[req_expid, dq.inx.uid[req_expid].egid, dq.inx.uid[req_expid].mip]
# 2 Find the requestLinks with requests by C4MIP, linking to requestItems
# that are requested from one of our experiment identifiers:
exp_rl=[dq.inx.uid[rItem.rlid] for rItem in dq.coll['requestItem'].items \
if dq.inx.uid[rItem.rlid].mip=='C4MIP' and rItem.esid in exp_esids]
print("%i requestItems found for 'C4MIP' and '%s'.\n" % (len(exp_rl), req_explabel))
# 3.1 Find the 'requestVarGroup'-items, associated with these requestLinks,
# (not all requestVarGroups have to be defined by C4MIP!):
exp_rVG = set([ dq.inx.uid[rLink.refid].uid for rLink in exp_rl ])
print("%i requestVarGroup-items found for 'C4MIP' and '%s'.\n" % (len(exp_rVG),req_explabel))
# 4 Find the 'requestVar'-items, associated with these requestVarGroups:
exp_rV = set([ rV for rV in dq.coll['requestVar'].items \
if rV.vgid in exp_rVG])
print("%i requestVar-items found for 'C4MIP' and '%s'.\n" % (len(exp_rV),req_explabel))
# 5 Find the 'CMORvar'-items, associated with these requestVars:
exp_CMORvars = set([ cmvar for cmvar in dq.coll['CMORvar'].items \
if cmvar.uid in [rV_i.vid for rV_i in exp_rV] ])
print("%i CMORvar-items found for 'C4MIP' and '%s'.\n" % (len(set(exp_CMORvars)),req_explabel))
In [41]:
# Import
from dreqPy import scope
# Initialisation
sc = scope.dreqQuery()
print("Using DreqPy (Data Request Python API) in version %s"\
% str(sc.dq.version))
# Type Storage Allocation
#--------------------------
# char 1 Byte
# bool 1 Byte
# short 2 Byte
# int 4 Byte
# long 4 Byte
# float 4 Byte
# double 8 Byte
# Assume 50% compression:
compression = 50
bytesPerFloat = compression/100.*4.
# Set maximum tier [1,4]
tierMax = 4
# Set maximum priority [1,3]
priorityMax = 3
#Force native grid or set native grid as default for more realistic estimates.
# else many requests will be estimated for a 1-degree grid.
sc.gridPolicyDefaultNative=True
#sc.gridPolicyForce='native'
# Print Volume Estimate Configuration settings
print("\nDefault model configuration settings:")
for key in sc.mcfg.keys():
print("%-5s %s" %(str(key), str(sc.mcfg[key])))
# Configure Volume Estimate:
# Example MPI-ESM1.2-HR
#----------------------
# MPIOM Ocean (Res. TP04L40, ~0.4°)
nho = 6811850/40 # Nr. of hor. mesh points in ocean
nlo = 40 # Nr. of vertical levels in ocean
# ECHAM 6.3 Atmosphere (Res T127L95, ~1°)
nha = 7004160/95 # Nr. of hor. mesh points in atm.
nla = 95 # Nr. of vertical levels in atm.
nlas = 43 #guess # Nr. of vertical levels in StrSph.
nh1 = 3*128 # Nr. of latitude points
# JSBACH Land Model
nls = 5 # Nr. of vertical levels in soil model
#Description
mcfg_desc={}
mcfg_desc['nho'] = "Number of horizontal mesh points in ocean"
mcfg_desc['nlo'] = "Number of vertical levels in ocean"
mcfg_desc['nha'] = "Number of horizontal mesh points in atmosphere"
mcfg_desc['nla'] = "Number of vertical levels in atmosphere"
mcfg_desc['nlas']= "Number of vertical levels in stratosphere"
mcfg_desc['nh1'] = "Number of latitude points"
mcfg_desc['nls'] = "Number of vertical levels in soil"
# Apply settings
MCFG={"nho":nho, "nlo":nlo, "nha":nha, "nla":nla, "nlas":nlas, "nls":nls, "nh1":nh1}
sc.setMcfg([int(MCFG[key]) for key in sc.mcfg])
# Print Volume Estimate Configuration settings
print("\nUpdated model configuration settings (Check!):")
for key in sc.mcfg.keys():
print("%-5s %-15s - %s" %(str(key), str(sc.mcfg[key]), mcfg_desc[key]))
In [42]:
# Select the MIPs
mips=["CMIP", "ScenarioMIP"]
# Select for example all experiments defined by these MIPs for the selected tiers (tierMax or sc.setTierMax below)
experiments=[exp.uid for exp in dq.coll['experiment'].items if exp.mip in mips and min(exp.tier)<tierMax]
# Selected MIPs and experiments:
print("Selected MIPs:\n-----------------------------")
for mip in mips:
print("Entry %s:" % mip)
for key in dq.coll['mip'].attDefn.keys():
print(" %-15s : %s" %(key, getattr(dq.inx.uid[mip], key)))
print("\n\nSelected Experiments:\n-----------------------------")
for exp in experiments:
print("Entry %s:" % exp)
for key in dq.coll['experiment'].attDefn.keys():
if key not in ['label', 'title']: continue
print(" %-15s : %s" %(key, getattr(dq.inx.uid[exp], key)))
In [43]:
# Filter (=whitelist) variables, specifying a list of CMORvar UIDs
WL = []
# Example: whitelist all variables but "tas"
#WL = [cmvar.uid for cmvar in dq.coll['CMORvar'].items if not cmvar.label=="tas"]
# Alternatively blacklist variables, specifying a list of CMORvar UIDs
BL = []
# Example: blacklist variable "tas"
#BL = [cmvar.uid for cmvar in dq.coll['CMORvar'].items if cmvar.label=="tas"]
In [44]:
# Volsum Class - create object vs
from dreqPy import volsum
def VolumeEstimateWBL(MIP, Expt, WL, BL, priorityMax=3, tierMax=4, bytesPerFloat=2.):
"""
Function to set up a configuration for
"dreqPy.scope.dreqQuery"
and call its function volByMip
Args:
MIP: set(str) - Set of MIP UIDs
Expt: set(str) - Set of Experiment UIDs
WL: list(str) - List of CMORvar UIDs
BL: list(str) - List of CMORvar UIDs
priorityMax : int - Maximum priority of the variables
taken into account [1,3]
Default: 3
tierMax: int - Maximum supported tier [1,4]
bytesPerFloat: float - Compression factor
Default: 2. # 50% compression
"""
# Set the experiment filter
sc.exptFilter=set(Expt)
# Alternatively, there is also a blacklist-filter
#sc.exptFilterBlack=set([exp.uid for exp in dq.coll['experiment'].items if exp.uid not in Expt])
#Set maximum supported tier
sc.setTierMax(tierMax)
# Initialize volsum object
vs = volsum.vsum(sc, scope.odsz, scope.npy, odir="/home/k/k204212/tmp/")
# Detailed Volume Estimate (filtered by Experiments and MIPs)
ss = 0
result = list()
resultmip = list()
for mip in mips:
sm=0
#run volsum class functions
vs.run(mip, pmax=priorityMax, doxlsx=False)
vs.anal(makeTabs=False, olab=mip)
result.append(list())
for exp in experiments:
x = vs.res['ve'][exp]
if x>1:
result[-1].append([mip, dq.inx.uid[exp].label, x*1.e-12*bytesPerFloat, exp])
ss+=x
sm+=x
resultmip.append([mip, sm*1.e-12*bytesPerFloat])
# For all MIPs
vs.run(set(mips), pmax=priorityMax, doxlsx=False)
vs.anal(makeTabs=False, olab='TOTAL')
z=0
for mi in vs.res['vm']:
z+=vs.res['vm'][mi]
for i in range(0, len(result)):
result[i]=sorted(result[i], key=lambda x: x[2], reverse=True)
result.sort(key=lambda x:sum([y[2] for y in x]), reverse=True)
resultmip.sort(key=lambda x: x[1], reverse=True)
# Get an estimate for whitelisted or without blacklisted variables
wbz=0.
if WL!=[]:
wbz=sum([vs.res['vu'][uid] for uid in vs.res['vu'].keys() if uid in WL])
elif BL!=[]:
wbz=sum([vs.res['vu'][uid] for uid in vs.res['vu'].keys() if not uid in BL])
# Print result for MIPs
print("Request of every single MIP:")
for mip_and_res in resultmip:
print('%-15s: %5.1f Tb' % (mip_and_res[0], mip_and_res[1]))
# Print sum of all MIPs
print("\nSum for all MIPs: %5.1f Tb" % float(ss*1.e-12*bytesPerFloat))
# Print the estimate for the combined request:
# as MIPs may request the same data from the same experiments,
# there might be an overlap (Sum of all requests >= Combined request)
print("Combined Request: %5.1f Tb" % float(z*1.e-12*bytesPerFloat))
print("Overlap : %5.1f Tb" % float((ss-z)*1.e-12*bytesPerFloat))
# Same for all experiments
print("\nVolume estimate of the request for every single Experiment:")
VolByE = list()
resultexp = list()
for i in range(0, len(Expt)):
# Calculate volByMip and sum up for every experiment
# if statement to avoid bug in version 01.beta.41 and older
Esize=vs.res['ve'][Expt[i]]
resultexp.append([dq.inx.uid[Expt[i]].label, Esize*1.e-12*bytesPerFloat])
VolByE.append(Esize)
resultexp.sort(key=lambda x: x[1], reverse=True)
for exp_and_res in resultexp:
print('%-20s: %5.1f Tb' % (exp_and_res[0], exp_and_res[1]))
ss=0.
for val in VolByE: ss+=val
print("-> Combined Request : %5.1f Tb" % (ss*1.e-12*bytesPerFloat))
# Print estimate including application of Black- or Whitelist
if BL!=[]:
print("\nVolume estimate without blacklisted variables:", wbz*1.e-12*bytesPerFloat)
elif WL!=[]:
print("\nVolume estimate for whitelisted variables:", wbz*1.e-12*bytesPerFloat)
# Call the function with above specifications
VolumeEstimateWBL(mips, experiments, WL, BL, priorityMax, tierMax, bytesPerFloat)
In [45]:
# Import
from dreqPy import scope
# Initialisation
sc = scope.dreqQuery()
print("Using DreqPy (Data Request Python API) in version %s"\
% str(sc.dq.version))
# Type Storage Allocation
#--------------------------
# char 1 Byte
# bool 1 Byte
# short 2 Byte
# int 4 Byte
# long 4 Byte
# float 4 Byte
# double 8 Byte
# Assume 50% compression:
bytesPerFloat = 2.
# Set maximum tier [1,4]
tierMax = 4
# Set maximum priority [1,3]
priorityMax = 3
# Print Volume Estimate Configuration settings
print("\nDefault model configuration settings:")
for key in sc.mcfg.keys():
print("%-5s %s" %(str(key), str(sc.mcfg[key])))
# Configure Volume Estimate:
# Example MPI-ESM1.2-HR
#----------------------
# MPIOM Ocean (Res. TP04L40, ~0.4°)
nho = 6811850/40 # Nr. of hor. mesh points in ocean
nlo = 40 # Nr. of vertical levels in ocean
# ECHAM 6.3 Atmosphere (Res T127L95, ~1°)
nha = 7004160/95 # Nr. of hor. mesh points in atm.
nla = 95 # Nr. of vertical levels in atm.
nlas = 43 #guess # Nr. of vertical levels in StrSph.
nh1 = 3*128 # Nr. of latitude points
# JSBACH Land Model
nls = 5 # Nr. of vertical levels in soil model
# Apply settings
MCFG=[nho, nlo, nha, nla, nlas, nls, nh1]
sc.setMcfg([int(i) for i in MCFG])
# Print Volume Estimate Configuration settings
print("\nUpdated model configuration settings (Check!):")
for key in sc.mcfg.keys():
print("%-5s %s" %(str(key), str(sc.mcfg[key])))
In [46]:
def VolumeEstimateLegacy(MIPorExpt, isMIP=True, priorityMax=3, tierMax=4, bytesPerFloat=2.):
"""
Function to set up a configuration for
"dreqPy.scope.dreqQuery"
and call its function volByMip
Args:
MIPorExpt: set(str) - Set of MIP or Experiment UIDs
isMIP: bool - MIPorExpt is MIP or Experiment
Default: True (MIPorExpt is MIP!)
priorityMax : int - Maximum priority of the variables
taken into account [1,3]
Default: 3
tierMax : int - Maximum tier of the experiments
taken into account [1,2,3,4]
Default: 4
bytesPerFloat: float - Compression factor
Default: 2. # 50% compression
"""
#Apply tier setting
sc.setTierMax(tierMax)
# Use dreqPy.scope.dreqQuery-function volByMip
if isMIP==True:
ss = 0.
for mip in MIPorExpt:
# Calculate volByMip and sum up
x = sc.volByMip(mip, pmax=priorityMax)\
*1.e-12*bytesPerFloat
print('%-15s: %5.1fTb' % (mip,x))
ss += x
z = sc.volByMip( set(MIPorExpt), pmax=priorityMax)\
*1.e-12*bytesPerFloat
# Use dreqPy.scope.dreqQuery-function volByMip with
# additional experiment arg
else:
VolByE = list()
MIPorExpt = list(MIPorExpt)
for i in range(0, len(MIPorExpt)):
#VolByE.append(list)
Esize = 0.
# Calculate volByMip and sum up for every experiment
# if statement to avoid bug in version 01.beta.41 and older
Esize+=sc.volByMip(set([mip.uid for mip in dq.coll['mip'].items \
if not mip.uid in ['CMIP6', 'DECK', \
'PDRMIP', 'SolarMIP', 'SPECS', 'CCMI', 'CMIP5']]), \
pmax=priorityMax, exptid=MIPorExpt[i])\
*1.e-12*bytesPerFloat
print('%-15s: %5.1fTb' % (dq.inx.uid[MIPorExpt[i]].label, Esize))
VolByE.append(Esize)
ss=0.
for val in VolByE: ss+=val
z = sc.volByMip(set([mip.uid for mip in dq.coll['mip'].items \
if not mip.uid in ['CMIP6', 'DECK', \
'PDRMIP', 'SolarMIP', 'SPECS', 'CCMI', 'CMIP5']]), \
pmax=priorityMax, exptid=set(MIPorExpt))\
*1.e-12*bytesPerFloat
# Print Info and calculate Overlap (Overlap is 0 for Experiments only!)
print( 'Combined: %5.1fTb' % z )
print( 'Overlap: %5.1fTb' % (ss-z) )
In [47]:
# Volume Estimate by MIP
print ( '######### All variables ###########' )
VolumeEstimateLegacy(set(["AerChemMIP", "C4MIP", "LUMIP"]), isMIP=True, \
priorityMax=3, tierMax=4, bytesPerFloat=2.)
print ( '\n######### Top priority variables ###########' )
VolumeEstimateLegacy(set(["AerChemMIP", "C4MIP", "LUMIP"]), isMIP=True, \
priorityMax=1, tierMax=4, bytesPerFloat=2.)
In [48]:
# Legacy function sc.volByMip is being used in above examples (outdated)
print(sc.volByMip(set(["C4MIP", "CFMIP"]), pmax=3, exptid=set([dq.coll['experiment'].items[i].uid for i in range(1,4)])))
In [49]:
# Volume Estimate by Experiments
# Get Experiment ids:
ExpIDs=list()
print("%-15s %-8s %-36s %s\n%s" %("Experiment", "Ens.Size", "UID", "MIP", "-"*75))
for exp in dq.coll['experiment'].items:
if exp.label in ['historical', 'amip', 'abrupt-4xCO2', '1pctCO2', 'piControl', \
'ssp245', 'ssp585', 'ssp370', 'ssp126']:
ExpIDs.append(exp.uid)
print("%-15s %8s %36s %s" % (exp.label, ",".join([str(i) for i in exp.ensz]), exp.uid, exp.mip))
# Get Volume Estimates
print ( '\n######### All variables ###########' )
VolumeEstimateLegacy(set(ExpIDs), isMIP=False, \
priorityMax=3, tierMax=4, bytesPerFloat=2.)
print ( '\n######### Top priority variables ###########' )
VolumeEstimateLegacy(set(ExpIDs), isMIP=False, \
priorityMax=1, tierMax=1, bytesPerFloat=2.)
In [50]:
# Auxiliary Functions to find request variables for request Links or request Items
def requestVarTOrequestLink(rv, toScreen=False):
"""
Args:
+ rv - 'dreqPy.dreq.dreqItem_requestVar'-object
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding:
- 'dreqPy.dreq.dreqItem_requestLink'-objects
"""
#requestVar -> requestVarGroup.uid
rvg= set([ y.uid for y in dq.coll['requestVarGroup'].items if y.uid==rv.vgid ])
#requestVarGroup.uid -> requestLink
rl = set([ x for x in dq.coll['requestLink'].items if x.refid in rvg])
#print to std.out
if toScreen==True:
print("\nrequestLink(s) of requestVar '%s' (uid: %s):" \
% (rv.label, rv.uid))
for i in rl: print(" %s (uid: %s) requested by mip '%s'" \
% (i.title, i.uid, i.mip))
print()
#return list of requestLink-objects
return list(rl)
def requestVarTOrequestItem(rv, toScreen=False):
"""
Args:
+ rv - 'dreqPy.dreq.dreqItem_requestVar'-object
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding:
- 'dreqPy.dreq.dreqItem_requestItem'-objects
"""
#requestVar->requestLink
rl=set([ x.uid for x in requestVarTOrequestLink(rv) ])
#requestLink->requestItem
ri=set([ x for x in dq.coll['requestItem'].items if x.rlid in rl ])
#print to std.out
if toScreen==True:
print("\nrequestItem(s) of requestVar '%s' (uid: %s):" \
% (rv.label, rv.uid))
for i in ri: print("%s (uid: %s) requested by mip '%s'" \
% (i.title, i.uid, i.mip))
print()
#return list of requestItem-objects
return list(ri)
# Examples
rl = requestVarTOrequestLink(dq.coll['requestVar'].items[9], True)
ri = requestVarTOrequestItem(dq.coll['requestVar'].items[9], True)
In [51]:
# Find all MIPs requesting a CMOR Variable
def CMORvarTOmip(cmvar, toScreen=False):
"""
Args:
+ cmvar - 'dreqPy.dreq.dreqItem_CMORvar'-object
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding 'dreqPy.dreq.dreqItem_mip'-objects
"""
#CMORvar -> requestVar -> mip.label
mips=set([ x.mip for x in dq.coll['requestVar'].items if x.vid==cmvar.uid ])
#mip.label -> mip
mips=set([ x for x in dq.coll['mip'].items if x.label in mips ])
#print to stdout
if toScreen==True:
print()
print(str(cmvar.label), ":", [mip.label for mip in mips])
print()
#return matches
return list(mips)
# Example for arbitrary CMOR Variable UID
mips = CMORvarTOmip(dq.inx.uid['d22da9f2-4a9f-11e6-b84e-ac72891c3257'], True)
In [52]:
#Find all experiments requesting a CMOR Variable
def CMORvarTOexperiment(cmvar, toScreen=False):
"""
Args:
+ cmvar - 'dreqPy.dreq.dreqItem_CMORvar'-object
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding 'dreqPy.dreq.dreqItem_experiment'-objects
"""
#CMORvar -> requestVar
rv=set([ x for x in dq.coll['requestVar'].items if x.vid==cmvar.uid ])
#requestVar -> requestItem
ri=list()
for i in rv:
for j in requestVarTOrequestItem(i): ri.append(j)
ri = set(ri)
#requestItem -> mip,experiment,exptgroup -> experiment
exp=list()
for i in ri:
if type(dq.inx.uid[i.esid])==type(dq.coll['experiment'].items[0]):
exp.append(dq.inx.uid[i.esid])
elif type(dq.inx.uid[i.esid])==type(dq.coll['exptgroup'].items[0]):
for e in dq.coll['experiment'].items:
if e.egid==i.esid: exp.append(e)
elif type(dq.inx.uid[i.esid])==type(dq.coll['mip'].items[0]):
#print "mip", dq.inx.uid[i.esid].uid, dq.inx.uid[i.esid].label
currmip=dq.inx.uid[i.esid].uid
for item in dq.coll['experiment'].items:
if item.mip==currmip:
exp.append(item)
else:
raise TypeError("Type must be dreqPy.dreq.dreqItem_experiment, \
dreqPy.dreq.dreqItem_exptgroup or dreqPy.dreq.dreqItem_mip!")
sys.exit(1)
#print to stdout
exp=set(exp)
if toScreen==True:
print("\n%i Experiments linked to CMORvar '%s':" % (len(exp), cmvar.label))
for i in exp: print(" - %-15s : %s" % (i.label, i.description))
print()
#return matches
return list(exp)
# Example for arbitrary CMOR Variable UID
exps = CMORvarTOexperiment(dq.inx.uid['8bb09104-4a5b-11e6-9cd2-ac72891c3257'], True)
In [53]:
# Find objectives linked to a CMOR variable
def CMORvarTOobjectives(cmvar, toScreen=False):
"""
Args:
+ cmvar - 'dreqPy.dreq.dreqItem_CMORvar'-object
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding:
- 'dreqPy.dreq.dreqItem_objective'-objects
"""
#CMORvar -> requestVar -> requestVarGroup.uid
rvg = set([ x.vgid for x in dq.coll['requestVar'].items if x.vid==cmvar.uid ])
#requestVarGroup.uid -> requestLink.uid
rl = set([ x.uid for x in dq.coll['requestLink'].items if x.refid in rvg ])
#requestLink.uid -> objectiveLink -> objective
ob = set([ dq.inx.uid[x.oid] for x in dq.coll['objectiveLink'].items if x.rid in rl ])
#print to std.out
if toScreen==True:
print()
for i in ob: print(str(i.label), i.description)
print()
#return objective(s)
return list(ob)
# Example for arbitrary CMOR Variable UID
objectives = CMORvarTOobjectives(dq.inx.uid['d22da9f2-4a9f-11e6-b84e-ac72891c3257'], True)
In [54]:
# List structure/dimensions connected to a CMOR Variable
def CMORvarTOstructure(cmvar, toScreen=False):
"""
Args:
+ cmvar - 'dreqPy.dreq.dreqItem_CMORvar'-object
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding:
- 'dreqPy.dreq.dreqItem_CMORvar'-object
- 'dreqPy.dreq.dreqItem_cellMethods'-object
- 'dreqPy.dreq.dreqItem_temporalShape'-object
- 'dreqPy.dreq.dreqItem_spathialShape'-object
- List of corresponding 'dreqPy.dreq.dreqItem_grids'-objects
"""
#Follow links to related objects:
struc=dq.inx.uid[cmvar.stid]
cm=dq.inx.uid[struc.cmid]
ts=dq.inx.uid[struc.tmid]
ss=dq.inx.uid[struc.spid]
grid=list()
grid.append(dq.inx.uid[ts.dimid])
for i in ss.dimids: grid.append(dq.inx.uid[i])
#Print info to std.out
if toScreen==True:
print()
print("Dimension Info for CMORvar "+str(cmvar.label))
print("cellMethods :" , cm.cell_methods)
print("temporalShape:", ts.label, "|", \
ts.dimensions, "|", ts.description)
print("spatialShape :", ss.label, "|", \
ss.dimensions)
print("grids :\n------")
for i in grid: print("", i.label, "|", i.units, "|", \
i.standardName, "|", i.description)
print()
#Return List of objects
return [cmvar, cm, ts, ss, grid]
# Example for arbitrary CMOR Variable UID
structure_info = CMORvarTOstructure(dq.inx.uid['19bebf2a-81b1-11e6-92de-ac72891c3257'], True)
In [55]:
# Find priorities linked to a CMOR Variable
def CMORvarTOpriority(cmvar, toScreen=False):
"""
Args:
+ cmvar - 'dreqPy.dreq.dreqItem_CMORvar'-object
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ integer List of corresponding priorities
"""
#priority stored in requestVar and as preset in requestItem
#CMORvar->requestVar
rv=[ x for x in dq.coll['requestVar'].items if x.vid==cmvar.uid ]
#requestVar->requestItem
ri=list()
for i in rv:
for j in requestVarTOrequestItem(i):
ri.append(j)
#add all priorities from requestVar and requestItem
plist=[ x.preset for x in ri ]
for i in [ x.priority for x in rv ]: plist.append(i)
#remove double entries and "-1", the default preset value
plist=set(plist)
plist.remove(-1)
#print to std.out
if toScreen==True:
pliststr=[str(i) for i in plist]
prioritystring=", ".join(pliststr)
print("\nThe CMORvar %s is linked to \n \
- %i requestVar-items\n - %i requestItem-items.\n\
Its priorities are: \033[1m%s\033[0m\n" \
% (cmvar.label, len(rv), len(ri), prioritystring))
#return list of integers/priorities
return list(sorted(plist))
# Example for arbitrary CMOR Variable UID
prts = CMORvarTOpriority(dq.inx.uid['19bebf2a-81b1-11e6-92de-ac72891c3257'], True)
In [56]:
# Find time slices linked to CMOR Variables for any request items
def CMORvarTOtimeSlice(cmvar, toScreen=False):
"""
Args:
+ cmvar - 'dreqPy.dreq.dreqItem_CMORvar'-object
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding 'dreqPy.dreq.dreqItem_timeSlice'-objects
beware: str("notSet") will be one item in the list,
if any requestItems requests the CMORvar for
the entire simulation length!
"""
#CMORvar -> requestVar
rv=set([ x for x in dq.coll['requestVar'].items if x.vid==cmvar.uid ])
#requestVar -> requestItem
ri=list()
for i in rv:
for j in requestVarTOrequestItem(i): ri.append(j)
ri = list(set(ri))
#requestItem -> timeSlice
ts=list()
for i in ri:
try:
ts.append(dq.inx.uid[i.tslice])
except KeyError:
ts.append("notSet")
tsset =list(set(ts))
#print to stdout
if toScreen==True:
#Group timeSlices and requestItems:
riset=list()
n=0
for i in tsset:
riset.append(list())
for j in ri:
try:
if i.uid==j.tslice:
if j.label not in riset[n]: riset[n].append(j.label)
except AttributeError:
if j.label not in riset[n]: riset[n].append(j.label)
n+=1
#print
ristr=riset
for i in range(0, len(ristr)): ristr[i]=", ".join(ristr[i])
print("\n%i different timeSlices linked to CMORvar '%s' (%s):" % (len(tsset), cmvar.label, cmvar.uid))
j=0
for i in tsset:
if type(i)==str:
print(" - %s (= entire simulation length)\n \
for requestItems %s" %(i, ristr[j]))
else:
print(" - %-15s : %s\n %s %s %s %s\n for requestItems %s" \
%(i.label, i.description \
if not "description" in str(i.description) \
else "", \
"Start: "+str(i.start) \
if not "start" in str(i.start) \
else "", \
"End: "+str(i.end) \
if not "end" in str(i.end) \
else "", \
"Step [a]: "+str(i.step) \
if not "step" in str(i.step) \
else "", \
"SliceLen: "+str(i.sliceLen) \
if not "slice" in str(i.sliceLen) \
else "", ristr[j]))
j+=1
print()
#return matches
return tsset
# Example for arbitrary CMOR Variable UID
ts = CMORvarTOtimeSlice(dq.inx.uid['bab5d898-e5dd-11e5-8482-ac72891c3257'], True)
In [57]:
# Find Variable Choices the CMOR Variable is linked to
def CMORvarTOvarChoice(cmvar, toScreen=False):
"""
Args:
+ cmvar - 'dreqPy.dreq.dreqItem_CMORvar'-object
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding 'dreqPy.dreq.dreqItem_varChoice'-objects
"""
#CMORvar -> var.uid
#var=set([ x.uid for x in dq.coll['var'].items if x.uid==cmvar.vid ])
var=set([x.uid for x in dq.coll['CMORvar'].items])
#var -> varChoiceLinkC, varChoiceLinkR -> varChoice
vclc=list(set([ x for x in dq.coll['varChoiceLinkC'].items if x.vid in var ]))
vclr=list(set([ x for x in dq.coll['varChoiceLinkR'].items if x.vid in var ]))
vcc=[ dq.inx.uid[x.cid] for x in vclc ]
vcr=[ dq.inx.uid[x.cid] for x in vclr ]
#print to std.out
if toScreen==True:
print("%i configuration varChoices found for CMORvar '%s' (%s)" \
%(len(vcc), cmvar.label, cmvar.uid))
for i in range(0, len(vcc)):
print(" - %s:\n %s\n %s\n for cfg value %s in modelConfig %s (%s)" \
%(str(vcc[i].label), \
str(vcc[i].varList), \
str(vcc[i].optionList), \
str(vclc[i].cfg), \
str(dq.inx.uid[vclc[i].cfgid].label), \
str(vclc[i].cfgid)))
print()
print("%i rank varChoices found for CMORvar '%s' (%s)" \
%(len(vcr), cmvar.label, cmvar.uid))
for i in range(0, len(vcr)):
print(" - %s:\n %s\n %s\n for rank %s" \
%(str(vcr[i].label), \
str(vcr[i].varList), \
str(vcr[i].optionList), \
str(vclr[i].rank)))
print()
#return varChoice-objects
return vcc+vcr
# Example for arbitrary CMOR Variable UID
vc = CMORvarTOvarChoice(dq.inx.uid['bab5d898-e5dd-11e5-8482-ac72891c3257'], True)
In [58]:
# Find all CMORvars requested by a MIP
def mipTOCMORvar(mip, toScreen=False, info=False):
"""
Args:
+ mip - 'dreqPy.dreq.dreqItem_mip'-object
+ toScreen - Boolean (print result to std.out or not)
+ info - Boolean (print number of matches to std.out or not)
Returns:
+ List of corresponding:
- 'dreqPy.dreq.dreqItem_CMORvar'-objects
"""
#mip.label -> requestLink -> requestVarGroup.uid
requestVarGroup_uid=set([ x.refid for x in dq.coll['requestLink'].items if x.mip==mip.uid ])
#requestVarGroup.uid -> requestVar -> CMORvar.uid
CMORvar_uid=set([ x.vid for x in dq.coll['requestVar'].items if x.vgid in requestVarGroup_uid ])
#print result to std.out
if toScreen==True:
print("\nCMORvars in mip %s:" % mip.label)
for uid in CMORvar_uid: print(dq.inx.uid[uid].label, end=', ')
print("\n")
if info==True:
print("[Info] mipTOCMORvar : %i CMORvars have been found for mip '%s'." \
% (len(CMORvar_uid), mip.label))
#return List of CMORvar objects
return list(set([dq.inx.uid[uid] for uid in CMORvar_uid]))
# Example for arbitrary MIP
cmvars = mipTOCMORvar(dq.inx.uid["VolMIP"], True, True)
In [59]:
# Find all CMORvars requested by all MIPs for a certain experiment
def experimentTOCMORvar(exp, toScreen=False, info=False):
"""
Args:
+ exp - 'dreqPy.dreq.dreqItem_experiment'-object
+ toScreen - Boolean (print result to std.out or not)
+ info - Boolean (print number of matches to std.out or not)
Returns:
+ List of corresponding:
- 'dreqPy.dreq.dreqItem_CMORvar'-objects
"""
#experiment -> exptgroup.uid
egid=list(set([ x.uid for x in dq.coll['exptgroup'].items \
if exp.egid==x.uid ]))
#experiment -> requestItem.uid
#allitems=set([ x.uid for x in dq.coll['requestItem'].items if x.esid==exp.uid ])
#experiment,exptgroup.uid -> requestLink.uid
alllinks=set([ x.rlid for x in dq.coll['requestItem'].items \
if exp.uid==x.esid or x.esid in egid ])
#requestLink.uid -> requestVarGroup.uid
allvargroups=set([ x.refid for x in dq.coll['requestLink'].items \
if x.uid in alllinks ])
#Find requestVar from requestVarGroup:requestVarGroup.uid -> CMORvar.uid
allcmorvars=set([ x.vid for x in dq.coll['requestVar'].items \
if x.vgid in allvargroups ])
#CMORvar -> var.uid
#allvars=set([ x.vid for x in dq.coll['CMORvar'].items if x.uid in allcmorvars])
#print results to std.out
if len(egid)==1:
egid=egid[0]
else:
raise ValueError("Multiple experiment groups found for experiment! \
An experiment should only belong to one group of experiments!")
if toScreen==True:
print("\nThe following CMORvars are requested for experiment %s (%s):" \
% (exp.label, exp.uid))
for i in allcmorvars: print(dq.inx.uid[i].label, end=', ')
print
if info==True:
print("\n[Info] experimentTOCMORvar : "\
"Your specified experiment '%s' is in exptgroup '%s'." \
% (exp.label, dq.inx.uid[egid].label))
#print len(allitems), "RequestItems found for your specifications." \
# % (len(allitems), exp.label)
print("[Info] experimentTOCMORvar : "\
"%i RequestLinks found for experiment '%s'." \
% (len(alllinks), exp.label))
print("[Info] experimentTOCMORvar : "\
"%i RequestVarGroups found for experiment '%s'." \
% (len(allvargroups), exp.label))
#print("[Info] experimentTOCMORvar : \
# %i requested Variables found for experiment '%s'." \
# % (len(allvars), exp.label))
print("[Info] experimentTOCMORvar : "\
"%i CMORvariables found for experiment '%s'." \
% (len(allcmorvars), exp.label))
#return List of CMORvars
return list(allcmorvars)
# Example for arbitrary Experiment
cmvars = experimentTOCMORvar(dq.inx.uid["f16f6188-dd9e-11e6-b89b-ac72891c3257"], False, True)
In [60]:
# Find all CMOR Variables linked to a certain cellMethod
def cellMethodsTOCMORvar(cm, toScreen=False, info=False):
"""
Args:
+ cm - 'dreqPy.dreq.dreqItem_cellMethods'-object
+ toScreen - Boolean (print result to std.out or not)
+ info - Boolean (print number of matches to std.out or not)
Returns:
+ List of corresponding:
- 'dreqPy.dreq.dreqItem_CMORvar'-objects
"""
#cellMethod -> structure.uid
struc=set([ x.uid for x in dq.coll['structure'].items if x.cmid==cm.uid ])
#structure.uid -> CMORvar
cmvar=set([ x for x in dq.coll['CMORvar'].items if x.stid in struc ])
#print results to std.out
if toScreen==True:
print("\nThe following CMORvars have the cellMethod %s (%s):" \
% (cm.label, cm.cell_methods))
for i in cmvar: print(i.label, end=', ')
print()
if info==True:
print("\n[Info] cellMethodsTOCMORvar : "\
"%i CMORvars have been found for cellMethod '%s' (%s)." \
% (len(cmvar), cm.label, cm.cell_methods))
#return list of CMORvars
return list(cmvar)
# Example for an arbitrary cell method
cmvars = cellMethodsTOCMORvar(dq.coll['cellMethods'].items[4], True, True)
In [61]:
# Find Objectives linked to a MIP
def mipTOobjective(mip, toScreen=False):
"""
Args:
+ mip - 'dreqPy.dreq.dreqItem_mip'-object
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding:
- 'dreqPy.dreq.dreqItem_objective'-objects
"""
#mip -> objective
ob=set([ x for x in dq.coll['objective'].items if x.mip==mip.label ])
#print to std.out
if toScreen==True:
print("\nThe MIP '%s' has the following objective(s):" % mip.label)
for i in ob: print(" - %-15s: %s" % (i.label, i.description))
print()
#return List of objective-objects
return list(ob)
# Example for arbitrary MIP
objectives = mipTOobjective(dq.inx.uid["VolMIP"], True)
In [62]:
# Find objectives linked to an experiment
def experimentTOobjective(exp, toScreen=False, info=False):
"""
Args:
+ exp - 'dreqPy.dreq.dreqItem_experiment'-object
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding:
- 'dreqPy.dreq.dreqItem_objective'-objects
"""
#experiment -> exptgroup.uid
egid=list(set([ x.uid for x in dq.coll['exptgroup'].items \
if exp.egid==x.uid ]))
#experiment -> requestItem.uid
#allitems=set([ x.uid for x in dq.coll['requestItem'].items \
# if x.esid==exp.uid ])
#experiment,exptgroup.uid -> requestLink.uid
alllinks=set([ x.rlid for x in dq.coll['requestItem'].items \
if x.esid==exp.uid or x.esid in egid])
#requestLink.uid -> requestVarGroup.uid
ob=set([ dq.inx.uid[x.oid] for x in dq.coll['objectiveLink'].items \
if x.rid in alllinks ])
#print results to std.out
if len(egid)==1:
egid=egid[0]
else:
raise ValueError("Multiple experiment groups found for experiment!" \
" An experiment should only belong to one group of experiments!")
if toScreen==True:
print("\nThe following %i objective(s) are found for experiment %s (%s):" \
% (len(ob), exp.label, exp.uid))
for i in ob: print(" - %-15s: %s" % (i.label, i.description))
print()
if info==True:
print("[Info] experimentTOobjective : Your specified experiment "\
"'%s' is in exptgroup '%s'." \
% (exp.label, dq.inx.uid[egid].label))
print("[Info] experimentTOobjective : "\
"%i RequestLinks found for experiment '%s'." \
% (len(alllinks), exp.label))
print("[Info] experimentTOobjective : "\
"%i objectives found for experiment '%s'." \
% (len(ob), exp.label))
#return List of objectives
return list(ob)
# Example for arbitrary experiment
objectives = experimentTOobjective(dq.inx.uid["f16f6188-dd9e-11e6-b89b-ac72891c3257"], True, True)
In [63]:
# Find all experiments
def mipTOexperimentBYtier(mip, tier=[1,2,3,4], toScreen=False):
"""
Args:
+ mip - 'dreqPy.dreq.dreqItem_mip'-object
+ tier - List containing desired tiers of the experiment, eg. [2] or [1,2]
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding 'dreqPy.dreq.dreqItem_experiment'-objects
"""
#mip -> requestItem -> [mip/experiment/exptgroup.uid, [tier,reset]]
exps=[ [x.esid, [] if "Option to override" in str(x.treset) \
else x.treset] for x in dq.coll['requestItem'].items if x.mip==mip.label]
#[mip/experiment/exptgroup.uid, [tier,reset]] -> [experiment, [tier, reset]]
expstier=list()
exp=list()
for i in exps:
if type(dq.inx.uid[i[0]])==type(dq.coll['experiment'].items[0]):
exp.append([dq.inx.uid[i[0]], i[1]])
elif type(dq.inx.uid[i[0]])==type(dq.coll['exptgroup'].items[0]):
for e in dq.coll['experiment'].items:
if e.egid==i[0]: exp.append([e, i[1]])
elif type(dq.inx.uid[i[0]])==type(dq.coll['mip'].items[0]):
currmip=dq.inx.uid[i[0]].uid
for item in dq.coll['experiment'].items:
if item.mip==currmip:
exp.append([item, i[1]])
elif type(dq.inx.uid[i[0]])==type(dq.coll['remarks'].items[0]):
continue
else:
raise TypeError("Type must be dreqPy.dreq.dreqItem_experiment, "\
"dreqPy.dreq.dreqItem_exptgroup or "\
"dreqPy.dreq.dreqItem_mip!\n Type is %s" % (str(type(dq.inx.uid[i[0]]))))
#Filter experiments by requested tier
for i in exp:
#if treset performs a experiment.tier override
if type(i[1])==type(list()) and i[1]!=[]:
for t in tier:
if t in i[1]:
expstier.append(i[0])
#if the experiment.tier specifies the tier
else:
for t in tier:
if t in i[0].tier:
expstier.append(i[0])
#print to std.out
expstier=set(expstier)
if toScreen==True:
tierstring=[str(t) for t in tier]
tierstring=", ".join(tierstring)
print("\n%i Experiments linked to MIP '%s' for tier %s:" \
% (len(expstier), mip.uid, tierstring))
for i in expstier: print(" - %-15s : %s" % (i.label, i.description))
print()
#Return experiment-objects of requested tier
return list(expstier)
# Example for an arbitrary MIP
exps = mipTOexperimentBYtier(dq.inx.uid["VolMIP"], tier=[1], toScreen=True)
In [64]:
# Find all CMOR Variables linked to a certain standardname
def standardnameTOCMORvar(sn, toScreen=False):
"""
Args:
+ sn - 'dreqPy.dreq.dreqItem_var'.sn or 'dreqPy.dreq.dreqItem_standardname'.uid,
which is the name of a standardname
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding 'dreqPy.dreq.dreqItem_CMORvar'-objects
"""
#standardname->var.uid
vl=set([ x.uid for x in dq.coll['var'].items if x.sn==sn ])
#var->CMORvar
cmvar=set([ x for x in dq.coll['CMORvar'].items if x.vid in vl ])
#print to std.out
if toScreen==True:
print("\nThe following %i CMORvars have been found for standardname '%s':" \
% (len(cmvar), sn))
for i in cmvar: print(" - %-10s (%-15s)" \
% (str(i.label), str(i.mipTable)))
print()
#return corresponding 'dreqPy.dreq.dreqItem_CMORvar'-objects
return list(cmvar)
# Example for an arbitrary standard name
cmvars = standardnameTOCMORvar('atmosphere_absorption_optical_thickness_due_to_ambient_aerosol_particles', True)
In [65]:
# Find all CMOR Variables whose standard name includes a certain part
def partOFstandardnameTOCMORvar(sn, toScreen=False):
"""
Args:
+ sn - 'dreqPy.dreq.dreqItem_var'.sn or 'dreqPy.dreq.dreqItem_standardname'.uid,
which is the name of a standardname
+ toScreen - Boolean (print result to std.out or not)
Returns:
+ List of corresponding 'dreqPy.dreq.dreqItem_CMORvar'-objects
"""
#standardname->var.uid
vl=set([ x.uid for x in dq.coll['var'].items if sn.lower() in x.sn.lower() ])
#var->CMORvar
cmvar=set([ (x, dq.inx.uid[x.vid].sn) \
for x in dq.coll['CMORvar'].items if x.vid in vl ])
#print to std.out
if toScreen==True:
print("\nThe following %i CMORvars have a standardname including '%s':" \
% (len(cmvar), sn.lower()))
for i in cmvar: print( " - %-10s (%-15s): %s" \
% (str(i[0].label), str(i[0].mipTable), str(i[1])))
print()
#return corresponding 'dreqPy.dreq.dreqItem_CMORvar'-objects
cmvar=[x[0] for x in cmvar]
return cmvar
# Example for an arbitrary part of a standardname
cmvars = partOFstandardnameTOCMORvar("Wet_Deposition", True)
In [ ]: