filesys_mapper workaround (was Re: [Swift-devel] [Bug 223] New: wildcard fails in the pattern parameter for simple_mapper)
Allan Espinosa
aespinosa at cs.uchicago.edu
Wed Mar 24 20:02:09 CDT 2010
The filesys_wrapper workaround starts to break when the type is a
struct. filesys_mapper is not supposed to work with structs so that's
reasonable. Hence, we should fix this bug :)
typical workaround:
type messagefile;
(messagefile o) write(messagefile i) {
app {
echo @filename(i) stdout=@filename(o);
}
}
messagefile infile[] <filesys_mapper;
prefix="071-singlefilemapper-input", suffix=".in",
location="gsiftp://ff-grid.unl.edu//panfs/panasas/CMS/data/engage/swift">;
messagefile outfile <"071-singlefilemapper-input.out">;
trace(@infile[0]);
outfile = write(infile[0]);
used in structs:
type SgtDim;
type Variation;
type Seismogram;
type PeakValue;
type Station {
string name;
float lat;
float lon;
int erf;
int variation_scenario;
}
type Sgt {
SgtDim x;
SgtDim y;
}
type Rupture {
int source;
int index;
int size;
}
/* some constants used by the apps*/
global int num_time_steps = 3000;
global string spectra_period1 = "all";
global float filter_highhz = 5.0;
global float simulation_timeskip = 0.1;
app (Sgt _ext) extract(Sgt _sgt, Station _stat, Variation _var) {
jbsim3d @strcat("stat=", _stat.name) "extract_sgt=1"
@strcat("slon=", _stat.lon) @strcat("slat=", _stat.lat)
@strcat("rupmodfile=", @filename(_var))
@strcat("sgt_xfile=", @filename(_sgt.x))
@strcat("sgt_yfile=", @filename(_sgt.y))
@strcat("extract_sgt_xfile=", @filename(_ext.x))
@strcat("extract_sgt_yfile=", @filename(_ext.y));
}
app (Seismogram _seis) seismogram(Sgt _sgt, Variation _var, Station _stat) {
jbsim3d @strcat("stat=", _stat.name) "extract_sgt=0"
@strcat("slon=", _stat.lon) @strcat("slat=", _stat.lat)
"outputBinary=1" "mergeOutput=1" @strcat("ntout=", num_time_steps)
@strcat("rupmodfile=", @filename(_var))
@strcat("sgt_xfile=", @filename(_sgt.x))
@strcat("sgt_yfile=", @filename(_sgt.y))
@strcat("seis_file=", @filename(_seis));
}
app (PeakValue _peak) peak_calc(Seismogram _seis, Variation _var) {
// touch @_peak;
surfeis_rspectra "simulation_out_pointsX=2" "simulation_out_pointsY=1"
"surfseis_rspectra_seismogram_units=cmpersec"
"surfseis_rspectra_output_units=cmpersec2"
"surfseis_rspectra_output_type=aa"
"surfseis_rspectra_apply_byteswap=no"
@strcat("simulation_out_timesamples=", num_time_steps)
@strcat("simulation_out_timeskip=", simulation_timeskip)
@strcat("surfseis_rspectra_period=", spectra_period1)
@strcat(" surfseis_rspectra_apply_filter_highHZ=", filter_highhz)
@strcat("in=", @filename(_seis))
@strcat("out=", @filename(_peak));
}
// Auxillary functions for the mappers
type StationFile;
app (StationFile _stat) getsite_file(int _run_id) {
getsite _run_id stdout=@filename(_stat);
}
(Station _stat) get_site(int _run_id) {
StationFile file<"site_tmp">;
file = getsite_file(_run_id);
_stat = readData(file);
}
type RuptureFile;
app (RuptureFile _rup) getrupture_file(int _run_id) {
getrupture _run_id stdout=@filename(_rup);
}
(Rupture _rup[]) get_ruptures(int _run_id) {
RuptureFile file<"rup_tmp">;
file = getrupture_file(_run_id);
_rup = readData(file);
}
/* TODO: data management zip jobs */
int run_id = 644;
Station site = get_site(run_id);
Sgt sgt_var[] <filesys_mapper; noauto="true",
location="gsiftp://ff-grid.unl.edu//panfs/panasas/CMS/data/engage/scec/data/SgtFiles/TEST/",
prefix=@strcat(site.name, "_f"), suffix=@strcat("_", run_id,".sgt")>;
Rupture rups[] = get_ruptures(run_id);
foreach rup in rups {
/* TODO: insert location parameters */
string datadir =
"gsiftp://ff-grid.unl.edu//panfs/panasas/CMS/data/engage/swift";
string loc_sub = @strcat(rup.index, "/", rup.source);
Sgt sub[] <filesys_mapper; location=loc_sub, suffix=".sgt",
prefix=@strcat(site.name, "_", rup.source, "_", rup.index, "_subf")>;
Variation vars[] <ext; exec="variation_mapper.rb", e=site.erf,
v=site.variation_scenario, s=rup.source, r=rup.index>;
sub[0] = extract(sgt_var[0], site, vars[rup.size-1]);
foreach var,i in vars {
Seismogram seis[] <filesys_mapper; location=loc_sub,
prefix=@strcat("Seismogram_", site.name,
"_", rup.source, "_", rup.index, "_", i), suffix=".grm">;
PeakValue peak[] <filesys_mapper; location=loc_sub,
prefix=@strcat("PeakVals_", site.name,
"_", rup.source, "_", rup.index, "_", i), suffix=".bsa">;
seis[0] = seismogram(sub[0], var, site);
peak[0] = peak_calc(seis[0], var);
}
}
error message:
swift-r3264 cog-r2728
RunID: testing
Progress:
Progress: Active:1 Checking status:1
Ex098
java.lang.RuntimeException: Inconsistency between type declaration and
handle for field 'y'
at org.griphyn.vdl.mapping.AbstractDataNode.getFringePaths(AbstractDataNode.java:372)
at org.griphyn.vdl.mapping.AbstractDataNode.getFringePaths(AbstractDataNode.java:353)
at org.griphyn.vdl.karajan.lib.FringePaths.function(FringePaths.java:29)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:67)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:33)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:334)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:134)
at org.globus.cog.karajan.workflow.events.EventBus.sendHooked(EventBus.java:108)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:176)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:298)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.AbstractFunction.post(AbstractFunction.java:28)
at org.globus.cog.karajan.workflow.nodes.Sequential.startNext(Sequential.java:51)
at org.globus.cog.karajan.workflow.nodes.Sequential.executeChildren(Sequential.java:27)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.execute(FlowContainer.java:63)
at org.globus.cog.karajan.workflow.nodes.FlowNode.restart(FlowNode.java:233)
at org.globus.cog.karajan.workflow.nodes.FlowNode.start(FlowNode.java:280)
at org.globus.cog.karajan.workflow.nodes.FlowNode.controlEvent(FlowNode.java:393)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:331)
at org.globus.cog.karajan.workflow.FlowElementWrapper.event(FlowElementWrapper.java:229)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:134)
at org.globus.cog.karajan.workflow.events.EventBus.sendHooked(EventBus.java:108)
at org.globus.cog.karajan.workflow.events.EventTargetPair.run(EventTargetPair.java:43)
at edu.emory.mathcs.backport.java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:431)
at edu.emory.mathcs.backport.java.util.concurrent.FutureTask.run(FutureTask.java:166)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:643)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:668)
at java.lang.Thread.run(Thread.java:595)
Execution failed:
java.lang.RuntimeException: Data set initialization failed for
org.griphyn.vdl.mapping.DataNode identifier
tag:benc at ci.uchicago.edu,2008:swift:dataset:20100324-1959-n2vdgijf:720000000115
type Sgt with no value at dataset=sgt_var path=[0] (closed). Missing
required field: y
I also attached the logfile.
-Allan
2010/3/16 <bugzilla-daemon at mcs.anl.gov>:
> https://bugzilla.mcs.anl.gov/swift/show_bug.cgi?id=223
>
> Summary: wildcard fails in the pattern parameter for
> simple_mapper
> Product: Swift
> Version: unspecified
> Platform: PC
> OS/Version: Windows
> Status: NEW
> Severity: normal
> Priority: P2
> Component: SwiftScript language
> AssignedTo: benc at hawaga.org.uk
> ReportedBy: skenny at uchicago.edu
>
>
> the simple_mapper is meant to take a 'pattern' parameter which can contain
> wildcards, however...
>
> the following code:
>
> file physdata<simple_mapper;location="rawphys/",pattern="*017*",suffix=".txt">;
> trace(@filename(physdata));
>
> produces this error:
>
> Execution failed:
> java.lang.IllegalStateException: mapper.existing() returned a path
> .r33subj017 that it cannot subsequently map
>
>
> though, as i understand it, it should map a single file from this dir:
>
> $ ls rawphys/
> r33subj003.txt r33subj017.txt
>
>
> (workaround is to use filesys_mapper, so perhaps if this cannot be fixed it
> could be altered in the documentation)
>
> --
> Configure bugmail: https://bugzilla.mcs.anl.gov/swift/userprefs.cgi?tab=email
> ------- You are receiving this mail because: -------
> You are watching the assignee of the bug.
-------------- next part --------------
A non-text attachment was scrubbed...
Name: postproc-testing.log
Type: text/x-log
Size: 107155 bytes
Desc: not available
URL: <http://lists.mcs.anl.gov/pipermail/swift-devel/attachments/20100324/526fc90b/attachment.bin>
More information about the Swift-devel
mailing list