filesys_mapper workaround (was Re: [Swift-devel] [Bug 223] New: wildcard fails in the pattern parameter for simple_mapper)
Mihael Hategan
hategan at mcs.anl.gov
Wed Mar 24 21:02:10 CDT 2010
Would you like more virtual cookies?
On Wed, 2010-03-24 at 20:02 -0500, Allan Espinosa wrote:
> The filesys_wrapper workaround starts to break when the type is a
> struct. filesys_mapper is not supposed to work with structs so that's
> reasonable. Hence, we should fix this bug :)
>
>
> typical workaround:
> type messagefile;
>
> (messagefile o) write(messagefile i) {
> app {
> echo @filename(i) stdout=@filename(o);
> }
> }
>
> messagefile infile[] <filesys_mapper;
> prefix="071-singlefilemapper-input", suffix=".in",
> location="gsiftp://ff-grid.unl.edu//panfs/panasas/CMS/data/engage/swift">;
>
>
> messagefile outfile <"071-singlefilemapper-input.out">;
>
> trace(@infile[0]);
> outfile = write(infile[0]);
>
>
> used in structs:
> type SgtDim;
> type Variation;
> type Seismogram;
> type PeakValue;
>
> type Station {
> string name;
> float lat;
> float lon;
> int erf;
> int variation_scenario;
> }
>
> type Sgt {
> SgtDim x;
> SgtDim y;
> }
>
> type Rupture {
> int source;
> int index;
> int size;
> }
>
> /* some constants used by the apps*/
> global int num_time_steps = 3000;
> global string spectra_period1 = "all";
> global float filter_highhz = 5.0;
> global float simulation_timeskip = 0.1;
>
> app (Sgt _ext) extract(Sgt _sgt, Station _stat, Variation _var) {
> jbsim3d @strcat("stat=", _stat.name) "extract_sgt=1"
> @strcat("slon=", _stat.lon) @strcat("slat=", _stat.lat)
>
> @strcat("rupmodfile=", @filename(_var))
> @strcat("sgt_xfile=", @filename(_sgt.x))
> @strcat("sgt_yfile=", @filename(_sgt.y))
> @strcat("extract_sgt_xfile=", @filename(_ext.x))
> @strcat("extract_sgt_yfile=", @filename(_ext.y));
> }
>
> app (Seismogram _seis) seismogram(Sgt _sgt, Variation _var, Station _stat) {
> jbsim3d @strcat("stat=", _stat.name) "extract_sgt=0"
> @strcat("slon=", _stat.lon) @strcat("slat=", _stat.lat)
> "outputBinary=1" "mergeOutput=1" @strcat("ntout=", num_time_steps)
>
> @strcat("rupmodfile=", @filename(_var))
> @strcat("sgt_xfile=", @filename(_sgt.x))
> @strcat("sgt_yfile=", @filename(_sgt.y))
> @strcat("seis_file=", @filename(_seis));
> }
>
> app (PeakValue _peak) peak_calc(Seismogram _seis, Variation _var) {
> // touch @_peak;
> surfeis_rspectra "simulation_out_pointsX=2" "simulation_out_pointsY=1"
> "surfseis_rspectra_seismogram_units=cmpersec"
> "surfseis_rspectra_output_units=cmpersec2"
> "surfseis_rspectra_output_type=aa"
> "surfseis_rspectra_apply_byteswap=no"
>
> @strcat("simulation_out_timesamples=", num_time_steps)
> @strcat("simulation_out_timeskip=", simulation_timeskip)
> @strcat("surfseis_rspectra_period=", spectra_period1)
> @strcat(" surfseis_rspectra_apply_filter_highHZ=", filter_highhz)
> @strcat("in=", @filename(_seis))
> @strcat("out=", @filename(_peak));
> }
>
> // Auxillary functions for the mappers
> type StationFile;
> app (StationFile _stat) getsite_file(int _run_id) {
> getsite _run_id stdout=@filename(_stat);
> }
> (Station _stat) get_site(int _run_id) {
> StationFile file<"site_tmp">;
> file = getsite_file(_run_id);
> _stat = readData(file);
> }
>
> type RuptureFile;
> app (RuptureFile _rup) getrupture_file(int _run_id) {
> getrupture _run_id stdout=@filename(_rup);
> }
> (Rupture _rup[]) get_ruptures(int _run_id) {
> RuptureFile file<"rup_tmp">;
> file = getrupture_file(_run_id);
> _rup = readData(file);
> }
>
> /* TODO: data management zip jobs */
>
> int run_id = 644;
> Station site = get_site(run_id);
>
> Sgt sgt_var[] <filesys_mapper; noauto="true",
> location="gsiftp://ff-grid.unl.edu//panfs/panasas/CMS/data/engage/scec/data/SgtFiles/TEST/",
> prefix=@strcat(site.name, "_f"), suffix=@strcat("_", run_id,".sgt")>;
>
> Rupture rups[] = get_ruptures(run_id);
>
> foreach rup in rups {
> /* TODO: insert location parameters */
> string datadir =
> "gsiftp://ff-grid.unl.edu//panfs/panasas/CMS/data/engage/swift";
> string loc_sub = @strcat(rup.index, "/", rup.source);
> Sgt sub[] <filesys_mapper; location=loc_sub, suffix=".sgt",
> prefix=@strcat(site.name, "_", rup.source, "_", rup.index, "_subf")>;
> Variation vars[] <ext; exec="variation_mapper.rb", e=site.erf,
> v=site.variation_scenario, s=rup.source, r=rup.index>;
>
> sub[0] = extract(sgt_var[0], site, vars[rup.size-1]);
>
> foreach var,i in vars {
> Seismogram seis[] <filesys_mapper; location=loc_sub,
> prefix=@strcat("Seismogram_", site.name,
> "_", rup.source, "_", rup.index, "_", i), suffix=".grm">;
> PeakValue peak[] <filesys_mapper; location=loc_sub,
> prefix=@strcat("PeakVals_", site.name,
> "_", rup.source, "_", rup.index, "_", i), suffix=".bsa">;
>
> seis[0] = seismogram(sub[0], var, site);
> peak[0] = peak_calc(seis[0], var);
> }
> }
>
>
> error message:
> swift-r3264 cog-r2728
>
> RunID: testing
> Progress:
> Progress: Active:1 Checking status:1
> Ex098
> java.lang.RuntimeException: Inconsistency between type declaration and
> handle for field 'y'
> at org.griphyn.vdl.mapping.AbstractDataNode.getFringePaths(AbstractDataNode.java:372)
> at org.griphyn.vdl.mapping.AbstractDataNode.getFringePaths(AbstractDataNode.java:353)
> at org.griphyn.vdl.karajan.lib.FringePaths.function(FringePaths.java:29)
> at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:67)
> at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
> at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:33)
> at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:334)
> at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:134)
> at org.globus.cog.karajan.workflow.events.EventBus.sendHooked(EventBus.java:108)
> at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:176)
> at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:298)
> at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
> at org.globus.cog.karajan.workflow.nodes.functions.AbstractFunction.post(AbstractFunction.java:28)
> at org.globus.cog.karajan.workflow.nodes.Sequential.startNext(Sequential.java:51)
> at org.globus.cog.karajan.workflow.nodes.Sequential.executeChildren(Sequential.java:27)
> at org.globus.cog.karajan.workflow.nodes.FlowContainer.execute(FlowContainer.java:63)
> at org.globus.cog.karajan.workflow.nodes.FlowNode.restart(FlowNode.java:233)
> at org.globus.cog.karajan.workflow.nodes.FlowNode.start(FlowNode.java:280)
> at org.globus.cog.karajan.workflow.nodes.FlowNode.controlEvent(FlowNode.java:393)
> at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:331)
> at org.globus.cog.karajan.workflow.FlowElementWrapper.event(FlowElementWrapper.java:229)
> at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:134)
> at org.globus.cog.karajan.workflow.events.EventBus.sendHooked(EventBus.java:108)
> at org.globus.cog.karajan.workflow.events.EventTargetPair.run(EventTargetPair.java:43)
> at edu.emory.mathcs.backport.java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:431)
> at edu.emory.mathcs.backport.java.util.concurrent.FutureTask.run(FutureTask.java:166)
> at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:643)
> at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:668)
> at java.lang.Thread.run(Thread.java:595)
> Execution failed:
> java.lang.RuntimeException: Data set initialization failed for
> org.griphyn.vdl.mapping.DataNode identifier
> tag:benc at ci.uchicago.edu,2008:swift:dataset:20100324-1959-n2vdgijf:720000000115
> type Sgt with no value at dataset=sgt_var path=[0] (closed). Missing
> required field: y
>
> I also attached the logfile.
>
> -Allan
>
> 2010/3/16 <bugzilla-daemon at mcs.anl.gov>:
> > https://bugzilla.mcs.anl.gov/swift/show_bug.cgi?id=223
> >
> > Summary: wildcard fails in the pattern parameter for
> > simple_mapper
> > Product: Swift
> > Version: unspecified
> > Platform: PC
> > OS/Version: Windows
> > Status: NEW
> > Severity: normal
> > Priority: P2
> > Component: SwiftScript language
> > AssignedTo: benc at hawaga.org.uk
> > ReportedBy: skenny at uchicago.edu
> >
> >
> > the simple_mapper is meant to take a 'pattern' parameter which can contain
> > wildcards, however...
> >
> > the following code:
> >
> > file physdata<simple_mapper;location="rawphys/",pattern="*017*",suffix=".txt">;
> > trace(@filename(physdata));
> >
> > produces this error:
> >
> > Execution failed:
> > java.lang.IllegalStateException: mapper.existing() returned a path
> > .r33subj017 that it cannot subsequently map
> >
> >
> > though, as i understand it, it should map a single file from this dir:
> >
> > $ ls rawphys/
> > r33subj003.txt r33subj017.txt
> >
> >
> > (workaround is to use filesys_mapper, so perhaps if this cannot be fixed it
> > could be altered in the documentation)
> >
> > --
> > Configure bugmail: https://bugzilla.mcs.anl.gov/swift/userprefs.cgi?tab=email
> > ------- You are receiving this mail because: -------
> > You are watching the assignee of the bug.
> _______________________________________________
> Swift-devel mailing list
> Swift-devel at ci.uchicago.edu
> http://mail.ci.uchicago.edu/mailman/listinfo/swift-devel
More information about the Swift-devel
mailing list