...
Pythontemplate (FC2K) is composed of one, large file that is hard to maintain- Lots of information between components is passed via dictionaries instead of strictly defined interfaces
- Logic of the actor is driven by if-else based approach - it's hard to add new features that are not supported at the momentmoment
- Lots of hidden assumptions are made in the code (e.g. temporary files creation, arguments that are responsible for script execution, etc.)
- Error handling is not quite ideal
...
- We can improve the whole solution such way it is based purely on
IMASbasedPython HLI - Once index based (file handler like) exchange of data workflows will require less time and memory to work - there will be no need of exchanging data back and forth between the calls
References to PyAL inside Python based template generated by FC2K
| Code Block |
|---|
...
from pyal import ALEnv, IDS, IDSRef ! features from pyal module
...
import mpi_actors ! this is one is responsible for saving input.txt, running mpi code and loading output.txt back
# only input arguments, outputs are returned (as a list if more than 1)
# kwargs for additional arguments
# TODO what about code parameters?
! maybe we can improve this part a little bit. Maybe we can pass arguments into function
! instead of passing dictionary
def ${actor}(
<#if arguments??>
<#list arguments as arg>
<#if arg.in>${arg.name},</#if>
</#list>
</#if>
<#if hasCodeParams>codeparams=None, </#if>
exec_type='${default_exec_type}',
**kwargs):
...
...
ids_tmp = ALEnv().ids_tmp ! temporary database
...
...
...
...
arg['cval'] = IDSRef() ! IDS class (wrapper provided by PyAL)
# the important fieds
# we use the temporary shot/run
arg['cval'].idx = ids_tmp.expIdx ! we are accessing temporary db
# TODO can we get the IDS name from the ids object?
# so that fc2k does not have to fill this in the template
arg['cval'].ids = '${arg.idsname}'
# the accurrence is just for temporary storage
# conflicts will be resolved later
arg['cval'].occurrence = 0
# these fields are redundant
arg['cval'].shot = ids_tmp.shot ! we are accessing temporary db
arg['cval'].run = ids_tmp.run ! we are accessing temporary db
arg['cval'].machine = ids_tmp.machine ! we are accessing temporary db
arg['cval'].user = ids_tmp.user ! we are accessing temporary db
arg['cval'].version = ids_tmp.version ! we are accessing temporary db
...
...
...
# check conflicting occurences and store data
occ_dict = {}
for arg in arguments_dict.values():
if isinstance(arg['cval'], IDSRef): ! IDS wrapper provided by PyAL
occ_dict[arg['cval'].ids] = 1 + occ_dict.get(arg['cval'].ids, -1)
arg['cval'].occurrence = occ_dict[arg['cval'].ids]
# store input data
if arg['in']:
ids_tmp.put(arg['value'], arg['cval'].occurrence, single_slice=True) ! we are accessing temporary db
...
...
...
<#if isTimeSliceActor>
# adding timein, timeout and iter
is_time_slice_actor = True
timein = None
for arg_name in arguments_order:
arg = arguments_dict[arg_name]
if isinstance(arg['cval'], IDSRef) and arg['in']: ! IDS wrapper provided by PyAL
if timein is None:
if len(arg['value'].time):
timein = float(arg['value'].time)
elif timein != arg['value'].time:
logger_${actor}.warning('time %g of %s is different from the first ids, using %g' %
(arg['value'].time, arg_name, timein))
...
...
...
! MPI based execution
if is_time_slice_actor:
mpi_out = mpi_actors.mpi_run_actor(_actor_name, _actor_executable, arguments_dict,
arguments_order, arguments_order_out,
codeparams_str,
xsd_file=_xsd_file,
default_xml_file=_default_xml_file,
mpi_command='mpirun',
mpi_params=[('-np', str(mpi_processes))],
mpi_env={},
rm_tmp_dir=rm_tmp_dir, time_in=timein, iter_in=iter_)
else:
mpi_out = mpi_actors.mpi_run_actor(_actor_name, _actor_executable, arguments_dict,
arguments_order, arguments_order_out,
codeparams_str,
xsd_file=_xsd_file,
default_xml_file=_default_xml_file,
mpi_command='mpirun',
mpi_params=[('-np', str(mpi_processes))],
mpi_env={},
rm_tmp_dir=rm_tmp_dir)
...
...
...
# get output data
results = []
for arg_name in arguments_order_out:
arg = arguments_dict[arg_name]
if isinstance(arg['cval'], IDSRef): ! IDS wrapper provided by PyAL
# read IDS data
if kwargs.get('get_output_idss', True):
if is_time_slice_actor: ! we are accessing temporary db
results.append(ids_tmp.get(arg['cval'].ids, arg['cval'].occurrence, time=timeout))
else:
results.append(ids_tmp.get(arg['cval'].ids, arg['cval'].occurrence))
...
...
... |