From noreply at svn.ci.uchicago.edu Tue Jan 5 21:31:49 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 5 Jan 2010 21:31:49 -0600 (CST) Subject: [Swift-commit] r3201 - branches/1.0/src/org/griphyn/vdl/mapping Message-ID: <20100106033149.EF2879CCAD@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-05 21:31:49 -0600 (Tue, 05 Jan 2010) New Revision: 3201 Modified: branches/1.0/src/org/griphyn/vdl/mapping/RootArrayDataNode.java branches/1.0/src/org/griphyn/vdl/mapping/RootDataNode.java Log: fixed the Justin bug Modified: branches/1.0/src/org/griphyn/vdl/mapping/RootArrayDataNode.java =================================================================== --- branches/1.0/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2009-12-17 20:31:27 UTC (rev 3200) +++ branches/1.0/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2010-01-06 03:31:49 UTC (rev 3201) @@ -55,7 +55,9 @@ catch (InvalidMapperException e) { throw new RuntimeException(e); } - notifyListeners(); + if (isClosed()) { + notifyListeners(); + } } private void checkInputs() { Modified: branches/1.0/src/org/griphyn/vdl/mapping/RootDataNode.java =================================================================== --- branches/1.0/src/org/griphyn/vdl/mapping/RootDataNode.java 2009-12-17 20:31:27 UTC (rev 3200) +++ branches/1.0/src/org/griphyn/vdl/mapping/RootDataNode.java 2010-01-06 03:31:49 UTC (rev 3201) @@ -65,7 +65,9 @@ catch (InvalidMapperException e) { throw new RuntimeException("InvalidMapperException caught in mapper initialization", e); } - notifyListeners(); + if (isClosed()) { + notifyListeners(); + } } private void checkInputs() { From noreply at svn.ci.uchicago.edu Tue Jan 5 21:36:22 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 5 Jan 2010 21:36:22 -0600 (CST) Subject: [Swift-commit] r3202 - trunk/src/org/griphyn/vdl/mapping Message-ID: <20100106033622.5081B9CCAD@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-05 21:36:22 -0600 (Tue, 05 Jan 2010) New Revision: 3202 Modified: trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/RootDataNode.java Log: merged fix from branch Modified: trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2010-01-06 03:31:49 UTC (rev 3201) +++ trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2010-01-06 03:36:22 UTC (rev 3202) @@ -55,7 +55,9 @@ catch (InvalidMapperException e) { throw new RuntimeException(e); } - notifyListeners(); + if (isClosed()) { + notifyListeners(); + } } private void checkInputs() { Modified: trunk/src/org/griphyn/vdl/mapping/RootDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2010-01-06 03:31:49 UTC (rev 3201) +++ trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2010-01-06 03:36:22 UTC (rev 3202) @@ -65,7 +65,9 @@ catch (InvalidMapperException e) { throw new RuntimeException("InvalidMapperException caught in mapper initialization", e); } - notifyListeners(); + if (isClosed()) { + notifyListeners(); + } } private void checkInputs() { From noreply at svn.ci.uchicago.edu Thu Jan 21 15:32:32 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 21 Jan 2010 15:32:32 -0600 (CST) Subject: [Swift-commit] r3203 - trunk/libexec Message-ID: <20100121213232.9AB1F9CC8C@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-21 15:32:32 -0600 (Thu, 21 Jan 2010) New Revision: 3203 Added: trunk/libexec/_swiftwrap.staging trunk/libexec/vdl-int-staging.k Modified: trunk/libexec/vdl.k Log: added provider staging specific files Added: trunk/libexec/_swiftwrap.staging =================================================================== --- trunk/libexec/_swiftwrap.staging (rev 0) +++ trunk/libexec/_swiftwrap.staging 2010-01-21 21:32:32 UTC (rev 3203) @@ -0,0 +1,284 @@ +# this script must be invoked inside of bash, not plain sh + +infosection() { + echo >& "$INFO" + echo "_____________________________________________________________________________" >& "$INFO" + echo >& "$INFO" + echo " $1" >& "$INFO" + echo "_____________________________________________________________________________" >& "$INFO" + echo >& "$INFO" +} + +info() { + infosection "uname -a" + uname -a 2>&1 >& "$INFO" + infosection "id" + id 2>&1 >& "$INFO" + infosection "env" + env 2>&1 >& "$INFO" + infosection "df" + df 2>&1 >& "$INFO" + if [ -e "/proc/cpuinfo" ]; then + infosection "/proc/cpuinfo" + cat /proc/cpuinfo 2>&1 >& "$INFO" + fi + if [ -e "/proc/meminfo" ]; then + infosection "/proc/meminfo" + cat /proc/meminfo 2>&1 >& "$INFO" + fi + infosection "command line" + echo $COMMANDLINE 2>&1 >& "$INFO" + if [ -f "$STDOUT" ] ; then + infosection "stdout" + cat $STDOUT >& "$INFO" + fi + if [ -f "$STDERR" ] ; then + infosection "stderr" + cat $STDERR >& "$INFO" + fi +} + +logstate() { + echo "Progress " `date +"%Y-%m-%d %H:%M:%S.%N%z"` " $@" >& "$INFO" +} + +log() { + echo "$@" >& "$INFO" +} + +fail() { + EC=$1 + shift + if [ "$STATUSMODE" = "files" ]; then + echo $@ >"$WFDIR/status/$JOBDIR/${ID}-error" + fi + log $@ + info + if [ "$STATUSMODE" = "files" ]; then + exit 0 + else + exit $EC + fi +} + +checkError() { + if [ "$?" != "0" ]; then + fail $@ + fi +} + +checkEmpty() { + if [ "$1" == "" ]; then + shift + fail 254 $@ + fi +} + +checkparamfile() { + log "checking for paramfile" + if [ "$1" == "-p" ]; then + JOBDIR=$2 + PARAMFILE=${WFDIR}/parameters/${JOBDIR}/param-${ID} + fi + log "paramfile is: $PARAMFILE" +} + +getarg() { + NAME=$1 + shift + VALUE="" + SHIFTCOUNT=0 + if [ "$PARAMFILE" == "" ] && [ "$1" == "$NAME" ]; then + shift + let "SHIFTCOUNT=$SHIFTCOUNT+1" + while [ "${1:0:1}" != "-" ] && [ "$#" != "0" ]; do + VALUE="$VALUE $1" + shift + let "SHIFTCOUNT=$SHIFTCOUNT+1" + done + VALUE="${VALUE:1}" + elif [ "$PARAMFILE" != "" ] && grep -E "^$NAME " $PARAMFILE ; then + VALUE=$(grep -E "^$NAME " $PARAMFILE | cut -d ' ' -f 2-) + else + fail 254 "Missing $NAME argument" + fi +} + +openinfo() { + exec 3<> $1 + INFO=3 +} + +closeinfo() { + exec 3>&- +} + +COMMANDLINE=$@ + +# make the WFDIR absolute +PARAMFILE= + +openinfo "wrapper.log" + +checkparamfile "$@" + +if [ "X$INFODIR" == "X" ]; then + INFODIR="." +fi + +logstate "LOG_START" +infosection "Wrapper" + +getarg "-e" "$@" +EXEC=$VALUE +shift $SHIFTCOUNT + +getarg "-out" "$@" +STDOUT=$VALUE +shift $SHIFTCOUNT + +getarg "-err" "$@" +STDERR=$VALUE +shift $SHIFTCOUNT + +getarg "-i" "$@" +STDIN=$VALUE +shift $SHIFTCOUNT + +getarg "-d" "$@" +DIRS=$VALUE +shift $SHIFTCOUNT + +getarg "-if" "$@" +INF=$VALUE +shift $SHIFTCOUNT + +getarg "-of" "$@" +OUTF=$VALUE +shift $SHIFTCOUNT + +getarg "-k" "$@" +KICKSTART=$VALUE +shift $SHIFTCOUNT + +getarg "-status" "$@" +STATUSMODE=$VALUE +shift $SHIFTCOUNT + +declare -a CMDARGS +if [ "$PARAMFILE" == "" ] && [ "$1" == "-a" ] ; then + shift + CMDARGS=("$@") +elif [ "$PARAMFILE" != "" ] ; then + CMDARGS=() + FIRST=1 + while read line ; do + if [ "$FIRST" == "1" ] ; then + CMDARGS=("$line") + FIRST=0 + else + CMDARGS=("${CMDARGS[*]}" "$line") + fi + done < <(grep -E "^-a " $PARAMFILE | cut -d " " -f 2-) +else + fail 254 "Missing arguments (-a option)" +fi + +PATH=$PATH:/bin:/usr/bin + +if [ "$PATHPREFIX" != "" ]; then + export PATH=$PATHPREFIX:$PATH +fi + +if [ "$SWIFT_EXTRA_INFO" != "" ]; then + log "EXTRAINFO=$($SWIFT_EXTRA_INFO)" +fi + +if [ "X${EXEC:0:1}" != "X/" ] ; then + export ORIGEXEC=$EXEC + export EXEC=$(which $EXEC) + if [ "X$EXEC" = "X" ] ; then + fail 254 "Cannot find executable $ORIGEXEC on site system path" + fi +fi + +log "EXEC=$EXEC" +log "STDIN=$STDIN" +log "STDOUT=$STDOUT" +log "STDERR=$STDERR" +log "DIRS=$DIRS" +log "INF=$INF" +log "OUTF=$OUTF" +log "KICKSTART=$KICKSTART" +log "ARGS=$@" +log "ARGC=$#" + +IFS="|" + +logstate "CREATE_INPUTDIR" +for D in $DIRS ; do + mkdir -p "$D" 2>&1 >>"$INFO" + checkError 254 "Failed to create input directory $D" + log "Created output directory: $D" +done + +logstate "EXECUTE" + +#ls >>$WRAPPERLOG +if [ ! -f "$EXEC" ]; then + fail 254 "The executable $EXEC does not exist" +fi +if [ ! -x "$EXEC" ]; then + fail 254 "The executable $EXEC does not have the executable bit set" +fi + +if [ "$STDIN" == "" ]; then + if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then + echo "#!/bin/bash" > run.sh + echo "\"$EXEC\" \"${CMDARGS[@]}\" 1>\"$STDOUT\" 2>\"$STDERR\"" >> run.sh + chmod +x run.sh + fi + "$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" +else + if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then + echo "#!/bin/bash" > run.sh + echo "\"$EXEC\" \"${CMDARGS[@]}\" 1>\"$STDOUT\" 2>\"$STDERR\" <\"$STDIN\"" >> run.sh + chmod +x run.sh + fi + "$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" <"$STDIN" +fi +checkError $? "Exit code $?" + +if [ ! -s "$STDOUT" ]; then + log "Removing empty stdout" + rm -f $STDOUT +fi +if [ ! -s "$STDERR" ]; then + log "Removing empty stderr" + rm -f $STDERR +fi + +logstate "EXECUTE_DONE" +log "Job ran successfully" + +MISSING= +for O in $OUTF ; do + if [ ! -f "$O" ]; then + if [ "$MISSING" == "" ]; then + MISSING=$O + else + MISSING="$MISSING, $O" + fi + fi +done +if [ "$MISSING" != "" ]; then + fail 254 "The following output files were not created by the application: $MISSING" +fi + +logstate "END" + +closeinfo + +# ensure we exit with a 0 after a successful execution +exit 0 + Property changes on: trunk/libexec/_swiftwrap.staging ___________________________________________________________________ Name: svn:executable + * Added: trunk/libexec/vdl-int-staging.k =================================================================== --- trunk/libexec/vdl-int-staging.k (rev 0) +++ trunk/libexec/vdl-int-staging.k 2010-01-21 21:32:32 UTC (rev 3203) @@ -0,0 +1,292 @@ +import("sys.k") +import("task.k") +import("vdl-lib.xml") +/* + * Things that are not exposed to the translated file + */ + +global(LOG:DEBUG, "debug") +global(LOG:INFO, "info") +global(LOG:WARN, "warn") +global(LOG:ERROR, "error") +global(LOG:FATAL, "fatal") + +namespace("vdl" + export( + element(isDone, [stageout] + sys:and( + for(pv, stageout + [path, var] := each(pv) + vdl:isLogged(var, path) + ) + sys:not(isEmpty(stageout)) + ) + ) + + element(mark, [restarts, err, optional(mapping)] + if( + err for(pv, restarts + [path, var] := each(pv) + vdl:setFutureFault(var, path=path, mapping=mapping) + ) + ) + ) + + element(flatten, [...] + if ( + isEmpty(...) "" + + concat( + for(i, butLast(...), if(isList(i) flatten(i) i), "|") last(...) + ) + ) + ) + + element(initDDir, [] + ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d" + once(ddir + if(sys:not(file:exists(ddir)) + task:dir:make(ddir) + ) + ) + ddir + ) + + element(inFileDirs, [stageins] + for(file, stageins + reldirname(file) + ) + ) + + element(outFileDirs, [stageouts] + for(pv, stageouts + [path, var] := each(pv) + + file := vdl:filename(vdl:getfield(var, path = path)) + + dirname(file) + ) + ) + + element(inFiles, [stageins] + pathnames(stageins) + ) + + + element(outFiles, [stageouts] + for(pv, stageouts + [path, var] := each(pv) + + file := vdl:filename(vdl:getfield(var, path = path)) + + file + ) + ) + + element(fileDirs, [stageins, stageouts] + list( + unique( + inFileDirs(stageins) + outFileDirs(stageouts) + ) + ) + ) + + element(appStageins, [jobid, files, dir] + for(file, files + provider := provider(file) + srchost := hostname(file) + srcdir := dirname(file) + destdir := dircat(dir, reldirname(file)) + filename := basename(file) + + stageIn( + "{provider}://{srchost}/{srcdir}/{filename}", + "{destdir}/{filename}" + ) + ) + ) + + element(appStageouts, [jobid, stageouts, dir] + for(pv, stageouts + [path, var] := each(pv) + file := vdl:absfilename(vdl:getfield(var, path = path)) + provider := vdl:provider(file) + dhost := vdl:hostname(file) + rdir := dircat(dir, reldirname(file)) + bname := basename(file) + ldir := dirname(file) + fullLocal := dircat(ldir, bname) + fullRemote := dircat(rdir, bname) + + stageOut( + "{rdir}/{bname}", + "{provider}://{dhost}/{ldir}/{bname}" + ) + ) + ) + + element(doRestartlog, [restartouts] + uParallelFor(f, restartouts, + [path, var] := each(f) + vdl:logvar(var, path) + ) + ) + + element(graphStuff, [tr, stagein, stageout, err, optional(args)] + if( + vdl:configProperty("pgraph") != "false" then( + errprops := if(err ",color=lightsalmon" ",color=lightsteelblue1") + tp := vdl:threadPrefix() + to(graph, + concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]") + ) + for(si, stagein + si := basename(si) + to(graph + concat(str:quote(si), " [shape=parallelogram]") + concat(str:quote(si), " -> ", str:quote(tp)) + ) + ) + for(pv, stageout + [path, var] := each(pv) + file := vdl:fileName(vdl:getfield(var, path=path)) + file := basename(file) + label := vdl:niceName(var, path = path) + to(graph + concat(str:quote(file), " [shape=parallelogram,label=", + str:quote(label), "]") + concat(str:quote(tp), " -> ", str:quote(file)) + ) + ) + ) + ) + ) + + element(fileSizes, [files] + math:sum( + for(f, files, file:size(file)) + ) + ) + + element(cleanups, [cleanup] + log(LOG:INFO, "START cleanups={cleanup}") + ) + + element(execute2, [tr, optional(arguments, stdin, stdout, stderr), stagein, stageout, restartout, + replicationGroup, replicationChannel] + stagein := list(unique(each(stagein))) + stageout := list(unique(each(stageout))) + allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein) + + ddir := initDDir() + + uid := uid() + jobdir := substring(uid, from=0, to=1) + jobid := concat(tr, "-", uid) + + fileDirs := fileDirs(stagein, stageout) + + log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") + + wrapper := "_swiftwrap.staging" + wrapfile := "{ddir}/param-{jobid}" + + stdout := try(stdout, "stdout.txt") + stderr := try(stderr, "stderr.txt") + + vdl:setprogress("Stage in") + wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" + tmpdir := dircat(concat(wfdir, "/jobs/", jobdir), jobid) + + try( + sequential( + log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}") + + vdl:setprogress("Submitting") + + vdl:execute( + vdl:siteprofile(rhost, "swift:wrapperInterpreter"), + list( + vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"), + wrapper, + "-e", vdl:executable(tr, rhost), + "-out", stdout, + "-err", stderr, + "-i", maybe(stdin), + "-d", flatten(each(fileDirs)), + "-if", flatten(infiles(stagein)), + "-of", flatten(outfiles(stageout)), + "-k", + "-status", "provider" + "-a", maybe(each(arguments)) + ) + directory = "{wfdir}-{jobdir}-{jobid}" + redirect = false + host = rhost + vdl:tcprofile(rhost, tr = tr) //this gets various app params from the tc, such as environment, walltime, etc + replicationGroup = replicationGroup + replicationChannel = replicationChannel + jobid = jobid + + /*stageIn("copy://localhost/{swift.home}/libexec/{wrapper}", wrapper) + appStageins(jobid, stagein, ".") + stageOut("wrapper.log", "copy://localhost/{ddir}/{jobid}.info") + stageOut("{stdout}", "copy://localhost/{ddir}/{stdout}") + stageOut("{stderr}", "copy://localhost/{ddir}/{stderr}")*/ + + stageIn("file://localhost/{swift.home}/libexec/{wrapper}", wrapper) + appStageins(jobid, stagein, ".") + stageOut("wrapper.log", "file://localhost/{ddir}/{jobid}.info") + stageOut("{stdout}", "file://localhost/{ddir}/{stdout}") + stageOut("{stderr}", "file://localhost/{ddir}/{stderr}") + + task:cleanUp("") //the whole job directory + appStageouts(jobid, stageout, ".") + ) + doRestartlog(restartout) + log(LOG:DEBUG, "JOB_END jobid={jobid}") + ) + catch("^Abort$" + log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") + throw(exception) + ) + catch("^(?!Abort$).*" + vdl:setprogress("Failed but can retry") + log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) + + throw( + exception( + concat( + "Exception in {tr}:", nl(), + maybe("Arguments: {arguments}", nl()), + "Host: {rhost}", nl(), + "Directory: {tmpdir}", + "TODO: outs", nl(), + "----", nl() + ) + exception + ) + ) + ) + ) + ) + ) + + element(generateProvenanceGraph, [gdata] + pgraph := vdl:configProperty("pgraph") + gname := if(pgraph == "true" "{VDL:SCRIPTNAME}-{VDL:RUNID}.dot" pgraph) + file:write(gname + "digraph SwiftProvenance {{", nl() + " graph [", vdl:configProperty("pgraph.graph.options"), "];", nl() + " node [", vdl:configProperty("pgraph.node.options"), "];", nl() + + for(i, gdata + " ", i, nl() + ) + "}", nl() + ) + log(LOG:INFO, "Provenance graph saved in ", gname) + ) + ) +) Modified: trunk/libexec/vdl.k =================================================================== --- trunk/libexec/vdl.k 2010-01-06 03:36:22 UTC (rev 3202) +++ trunk/libexec/vdl.k 2010-01-21 21:32:32 UTC (rev 3203) @@ -9,8 +9,11 @@ import("vdl-sc.k", export = true) import("vdl-lib.xml", export = true) + + pstaging := configProperty("use.provider.staging") + int := if (pstaging == "true", "vdl-int-staging.k", "vdl-int.k") - import("vdl-int.k") + import(int) import("java.k") once("vdl.k-print-version" From noreply at svn.ci.uchicago.edu Thu Jan 21 15:37:33 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 21 Jan 2010 15:37:33 -0600 (CST) Subject: [Swift-commit] r3204 - trunk/etc Message-ID: <20100121213733.6C58D9CC8C@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-21 15:37:33 -0600 (Thu, 21 Jan 2010) New Revision: 3204 Modified: trunk/etc/swift.properties Log: added staging setting Modified: trunk/etc/swift.properties =================================================================== --- trunk/etc/swift.properties 2010-01-21 21:32:32 UTC (rev 3203) +++ trunk/etc/swift.properties 2010-01-21 21:37:33 UTC (rev 3204) @@ -327,3 +327,11 @@ provenance.log=false +# Controls whether file staging is done by swift or by the execution +# provider. If set to false, the standard swift staging mechanism is +# used. If set to true, swift does not stage files. Instead, the +# execution provider is instructed to stage files in and out. +# Provider staging is experimental and only working with a limited +# number of providers (local and coaster). + +use.provider.staging=false \ No newline at end of file From noreply at svn.ci.uchicago.edu Thu Jan 21 15:38:32 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 21 Jan 2010 15:38:32 -0600 (CST) Subject: [Swift-commit] r3205 - trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui Message-ID: <20100121213832.E40F09CC8C@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-21 15:38:32 -0600 (Thu, 21 Jan 2010) New Revision: 3205 Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Table.java Log: fixed older bug with table scrolling Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Table.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Table.java 2010-01-21 21:37:33 UTC (rev 3204) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Table.java 2010-01-21 21:38:32 UTC (rev 3205) @@ -159,6 +159,12 @@ ((TableColumn) c).setSelectedRow(selectedRow); } } + if (selectedRow < firstRow) { + firstRow = Math.min(selectedRow - height + 4, 0); + } + if (selectedRow > firstRow + height - 4) { + firstRow = selectedRow; + } redraw(); } } From noreply at svn.ci.uchicago.edu Thu Jan 21 16:35:59 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 21 Jan 2010 16:35:59 -0600 (CST) Subject: [Swift-commit] r3206 - trunk/libexec Message-ID: <20100121223559.561999CC8C@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-21 16:35:59 -0600 (Thu, 21 Jan 2010) New Revision: 3206 Modified: trunk/libexec/vdl-int-staging.k Log: use coaster as provider for now Modified: trunk/libexec/vdl-int-staging.k =================================================================== --- trunk/libexec/vdl-int-staging.k 2010-01-21 21:38:32 UTC (rev 3205) +++ trunk/libexec/vdl-int-staging.k 2010-01-21 22:35:59 UTC (rev 3206) @@ -94,7 +94,8 @@ element(appStageins, [jobid, files, dir] for(file, files - provider := provider(file) + //provider := vdl:provider(file) + provider := "coaster" srchost := hostname(file) srcdir := dirname(file) destdir := dircat(dir, reldirname(file)) @@ -111,7 +112,8 @@ for(pv, stageouts [path, var] := each(pv) file := vdl:absfilename(vdl:getfield(var, path = path)) - provider := vdl:provider(file) + //provider := vdl:provider(file) + provider := "coaster" dhost := vdl:hostname(file) rdir := dircat(dir, reldirname(file)) bname := basename(file) From noreply at svn.ci.uchicago.edu Sat Jan 23 00:53:13 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 23 Jan 2010 00:53:13 -0600 (CST) Subject: [Swift-commit] r3207 - trunk/libexec Message-ID: <20100123065313.96D5C9CC9A@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-23 00:53:13 -0600 (Sat, 23 Jan 2010) New Revision: 3207 Modified: trunk/libexec/vdl-int.k Log: fixed ambiguity in cleanup() Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-01-21 22:35:59 UTC (rev 3206) +++ trunk/libexec/vdl-int.k 2010-01-23 06:53:13 UTC (rev 3207) @@ -208,7 +208,7 @@ parallelFor(i, cleanup [dir, host] := each(i) try( - cleanup(dir, host) + vdl:cleanup(dir, host) catch(".*", log(LOG:DEBUG, "EXCEPTION - Exception caught while cleaning up", exception) to(warnings, exception("Cleanup on {host} failed", exception)) From noreply at svn.ci.uchicago.edu Sun Jan 24 17:54:54 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 24 Jan 2010 17:54:54 -0600 (CST) Subject: [Swift-commit] r3208 - trunk/libexec Message-ID: <20100124235454.6C9F49CCA0@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-24 17:54:54 -0600 (Sun, 24 Jan 2010) New Revision: 3208 Modified: trunk/libexec/_swiftwrap.staging Log: put failure message in stderr if status mode is set to provider Modified: trunk/libexec/_swiftwrap.staging =================================================================== --- trunk/libexec/_swiftwrap.staging 2010-01-23 06:53:13 UTC (rev 3207) +++ trunk/libexec/_swiftwrap.staging 2010-01-24 23:54:54 UTC (rev 3208) @@ -57,6 +57,9 @@ if [ "$STATUSMODE" = "files" ]; then exit 0 else + if [ "X$STDERR" != "X" ]; then + echo "$@" >>$STDER + fi exit $EC fi } From noreply at svn.ci.uchicago.edu Sun Jan 24 17:55:28 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 24 Jan 2010 17:55:28 -0600 (CST) Subject: [Swift-commit] r3209 - trunk/libexec Message-ID: <20100124235528.2259F9CCA0@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-24 17:55:28 -0600 (Sun, 24 Jan 2010) New Revision: 3209 Modified: trunk/libexec/vdl-int-staging.k Log: better handling of staging methods Modified: trunk/libexec/vdl-int-staging.k =================================================================== --- trunk/libexec/vdl-int-staging.k 2010-01-24 23:54:54 UTC (rev 3208) +++ trunk/libexec/vdl-int-staging.k 2010-01-24 23:55:28 UTC (rev 3209) @@ -92,10 +92,10 @@ ) ) - element(appStageins, [jobid, files, dir] + element(appStageins, [jobid, files, dir, stagingMethod] for(file, files - //provider := vdl:provider(file) - provider := "coaster" + protocol := vdl:provider(file) + provider := if(protocol == "file", stagingMethod, protocol) srchost := hostname(file) srcdir := dirname(file) destdir := dircat(dir, reldirname(file)) @@ -108,12 +108,12 @@ ) ) - element(appStageouts, [jobid, stageouts, dir] + element(appStageouts, [jobid, stageouts, dir, stagingMethod] for(pv, stageouts [path, var] := each(pv) file := vdl:absfilename(vdl:getfield(var, path = path)) - //provider := vdl:provider(file) - provider := "coaster" + protocol := vdl:provider(file) + provider := if(protocol == "file", stagingMethod, protocol) dhost := vdl:hostname(file) rdir := dircat(dir, reldirname(file)) bname := basename(file) @@ -231,20 +231,17 @@ replicationChannel = replicationChannel jobid = jobid - /*stageIn("copy://localhost/{swift.home}/libexec/{wrapper}", wrapper) - appStageins(jobid, stagein, ".") - stageOut("wrapper.log", "copy://localhost/{ddir}/{jobid}.info") - stageOut("{stdout}", "copy://localhost/{ddir}/{stdout}") - stageOut("{stderr}", "copy://localhost/{ddir}/{stderr}")*/ + stagingMethod := vdl:siteProfile(rhost, "swift:stagingMethod", default="proxy") - stageIn("file://localhost/{swift.home}/libexec/{wrapper}", wrapper) - appStageins(jobid, stagein, ".") - stageOut("wrapper.log", "file://localhost/{ddir}/{jobid}.info") - stageOut("{stdout}", "file://localhost/{ddir}/{stdout}") - stageOut("{stderr}", "file://localhost/{ddir}/{stderr}") + stageIn("{stagingMethod}://localhost/{swift.home}/libexec/{wrapper}", wrapper) + appStageins(jobid, stagein, ".", stagingMethod) - task:cleanUp("") //the whole job directory - appStageouts(jobid, stageout, ".") + stageOut("wrapper.log", "{stagingMethod}://localhost/{ddir}/{jobid}.info") + stageOut("{stdout}", "{stagingMethod}://localhost/{ddir}/{stdout}") + stageOut("{stderr}", "{stagingMethod}://localhost/{ddir}/{stderr}") + appStageouts(jobid, stageout, ".", stagingMethod) + + task:cleanUp(".") //the whole job directory ) doRestartlog(restartout) log(LOG:DEBUG, "JOB_END jobid={jobid}") From noreply at svn.ci.uchicago.edu Sun Jan 24 17:56:14 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 24 Jan 2010 17:56:14 -0600 (CST) Subject: [Swift-commit] r3210 - trunk/etc Message-ID: <20100124235614.0D4A59CCA0@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-24 17:56:13 -0600 (Sun, 24 Jan 2010) New Revision: 3210 Modified: trunk/etc/swift.properties Log: added more explanation on provider staging Modified: trunk/etc/swift.properties =================================================================== --- trunk/etc/swift.properties 2010-01-24 23:55:28 UTC (rev 3209) +++ trunk/etc/swift.properties 2010-01-24 23:56:13 UTC (rev 3210) @@ -330,8 +330,23 @@ # Controls whether file staging is done by swift or by the execution # provider. If set to false, the standard swift staging mechanism is # used. If set to true, swift does not stage files. Instead, the -# execution provider is instructed to stage files in and out. -# Provider staging is experimental and only working with a limited -# number of providers (local and coaster). +# execution provider is instructed to stage files in and out. +# +# Provider staging is experimental. +# +# When enabled, and when coasters are used as an execution provider, +# a staging mechanism can be selected for each site +# using the swift:stagingMethod site profile in sites.xml. The +# following is a list of accepted mechanisms: +# +# * file: Staging is done from a filesystem accessible to the +# coaster service (typically running on the head node) +# * proxy: Staging is done from a filesystem accessible to the +# client machine that swift is running on, and is proxied +# through the coaster service +# * sfs: (short for "shared filesystem") Staging is done by +# copying files to and from a filesystem accessible +# by the compute node (such as an NFS or GPFS mount). + use.provider.staging=false \ No newline at end of file From noreply at svn.ci.uchicago.edu Sun Jan 24 17:57:07 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 24 Jan 2010 17:57:07 -0600 (CST) Subject: [Swift-commit] r3211 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100124235707.35A779CCA0@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-24 17:57:07 -0600 (Sun, 24 Jan 2010) New Revision: 3211 Modified: trunk/src/org/griphyn/vdl/karajan/lib/Execute.java Log: added staging parameters to execute Modified: trunk/src/org/griphyn/vdl/karajan/lib/Execute.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2010-01-24 23:56:13 UTC (rev 3210) +++ trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2010-01-24 23:57:07 UTC (rev 3211) @@ -34,7 +34,7 @@ A_ENVIRONMENT, A_QUEUE, A_PROJECT, A_MINMEMORY, A_MAXMEMORY, A_REDIRECT, A_SECURITY_CONTEXT, A_DIRECTORY, A_NATIVESPEC, A_DELEGATION, A_ATTRIBUTES, C_ENVIRONMENT, A_FAIL_ON_JOB_ERROR, A_BATCH, A_REPLICATION_GROUP, - A_REPLICATION_CHANNEL, A_JOBID }); + A_REPLICATION_CHANNEL, A_JOBID, C_STAGEIN, C_STAGEOUT, C_CLEANUP }); } public Execute() { From noreply at svn.ci.uchicago.edu Sun Jan 24 17:57:44 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 24 Jan 2010 17:57:44 -0600 (CST) Subject: [Swift-commit] r3212 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20100124235744.963309CCA0@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-24 17:57:44 -0600 (Sun, 24 Jan 2010) New Revision: 3212 Modified: trunk/src/org/griphyn/vdl/karajan/VDL2ErrorTranslator.java Log: fixed deprecated call Modified: trunk/src/org/griphyn/vdl/karajan/VDL2ErrorTranslator.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/VDL2ErrorTranslator.java 2010-01-24 23:57:07 UTC (rev 3211) +++ trunk/src/org/griphyn/vdl/karajan/VDL2ErrorTranslator.java 2010-01-24 23:57:44 UTC (rev 3212) @@ -225,6 +225,6 @@ public void windowIconified(WindowEvent e) {} public void windowOpened(WindowEvent e) {} }); - frame.show(); + frame.setVisible(true); } } From noreply at svn.ci.uchicago.edu Sun Jan 24 17:58:15 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 24 Jan 2010 17:58:15 -0600 (CST) Subject: [Swift-commit] r3213 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100124235815.945499CCA0@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-24 17:58:15 -0600 (Sun, 24 Jan 2010) New Revision: 3213 Modified: trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java Log: allow specifying default values Modified: trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java 2010-01-24 23:57:44 UTC (rev 3212) +++ trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java 2010-01-24 23:58:15 UTC (rev 3213) @@ -31,14 +31,15 @@ public static final Arg PA_HOST = new Arg.Positional("host"); public static final Arg PA_FQN = new Arg.Positional("fqn"); + public static final Arg OA_DEFAULT = new Arg.Optional("default", null); static { - setArguments(SiteProfile.class, new Arg[] { PA_HOST, PA_FQN }); + setArguments(SiteProfile.class, new Arg[] { PA_HOST, PA_FQN, OA_DEFAULT }); } public Object function(VariableStack stack) throws ExecutionException { BoundContact bc = (BoundContact) PA_HOST.getValue(stack); - return getSingle(stack, bc, new FQN(TypeUtil.toString(PA_FQN.getValue(stack)))); + return getSingle(stack, bc, new FQN(TypeUtil.toString(PA_FQN.getValue(stack))), OA_DEFAULT.getValue(stack)); } public static final FQN SWIFT_WRAPPER_INTERPRETER = new FQN("swift:wrapperInterpreter"); @@ -49,8 +50,10 @@ public static final FQN SYSINFO_OS = new FQN("SYSINFO:OS"); private static final Map DEFAULTS; + private static final Set DEFAULTS_NAMES; private static void addDefault(Os os, FQN fqn, Object value) { + DEFAULTS_NAMES.add(fqn); Map osm = (Map) DEFAULTS.get(os); if (osm == null) { osm = new HashMap(); @@ -79,6 +82,7 @@ static { DEFAULTS = new HashMap(); + DEFAULTS_NAMES = new HashSet(); addDefault(Os.WINDOWS, SWIFT_WRAPPER_INTERPRETER, "cscript.exe"); addDefault(Os.WINDOWS, SWIFT_WRAPPER_SCRIPT, "_swiftwrap.vbs"); addDefault(Os.WINDOWS, SWIFT_WRAPPER_INTERPRETER_OPTIONS, new String[] {"//Nologo"}); @@ -91,16 +95,20 @@ addDefault(null, SWIFT_CLEANUP_COMMAND_OPTIONS, new String[] {"-rf"}); } - private Object getSingle(VariableStack stack, BoundContact bc, FQN fqn) throws ExecutionException { + private Object getSingle(VariableStack stack, BoundContact bc, FQN fqn, Object defval) + throws ExecutionException { String value = getProfile(bc, fqn); if (value == null) { Os os = getOS(bc); - if ("swift".equals(fqn.getNamespace())) { + if (DEFAULTS_NAMES.contains(fqn)) { return getDefault(os, fqn); } else if (SYSINFO_OS.equals(fqn)) { return os; } + else if (defval != null) { + return defval; + } else { throw new ExecutionException(stack, "Missing profile: " + fqn); } From noreply at svn.ci.uchicago.edu Sun Jan 24 18:00:38 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 24 Jan 2010 18:00:38 -0600 (CST) Subject: [Swift-commit] r3214 - in trunk/src/org/globus/swift/catalog: . util Message-ID: <20100125000038.462729CCA0@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-24 18:00:38 -0600 (Sun, 24 Jan 2010) New Revision: 3214 Modified: trunk/src/org/globus/swift/catalog/Catalog.java trunk/src/org/globus/swift/catalog/CatalogEntry.java trunk/src/org/globus/swift/catalog/util/Escape.java trunk/src/org/globus/swift/catalog/util/ProfileParser.java trunk/src/org/globus/swift/catalog/util/ProfileParserException.java trunk/src/org/globus/swift/catalog/util/Separator.java Log: replaced funny characters in Jens' name; sorry Jens, there were warnings and I didn't want to change the encoding Modified: trunk/src/org/globus/swift/catalog/Catalog.java =================================================================== --- trunk/src/org/globus/swift/catalog/Catalog.java 2010-01-24 23:58:15 UTC (rev 3213) +++ trunk/src/org/globus/swift/catalog/Catalog.java 2010-01-25 00:00:38 UTC (rev 3214) @@ -21,7 +21,7 @@ * This interface create a common ancestor for all cataloging * interfaces. * - * @author Jens-S. V?ckler + * @author Jens-S. Voeckler * @author Yong Zhao * @version $Revision: 1.2 $ */ Modified: trunk/src/org/globus/swift/catalog/CatalogEntry.java =================================================================== --- trunk/src/org/globus/swift/catalog/CatalogEntry.java 2010-01-24 23:58:15 UTC (rev 3213) +++ trunk/src/org/globus/swift/catalog/CatalogEntry.java 2010-01-25 00:00:38 UTC (rev 3214) @@ -18,7 +18,7 @@ /** * This interface create a common ancestor for all catalog entries. * - * @author Jens-S. V?ckler + * @author Jens-S. Voeckler * @author Yong Zhao * @version $Revision: 1.1 $ */ Modified: trunk/src/org/globus/swift/catalog/util/Escape.java =================================================================== --- trunk/src/org/globus/swift/catalog/util/Escape.java 2010-01-24 23:58:15 UTC (rev 3213) +++ trunk/src/org/globus/swift/catalog/util/Escape.java 2010-01-25 00:00:38 UTC (rev 3214) @@ -33,7 +33,7 @@ * * @author Gaurang Mehta * @author Karan Vahi - * @author Jens-S. V?ckler + * @author Jens-S. Voeckler * @version $Revision: 1.1 $ */ public class Escape Modified: trunk/src/org/globus/swift/catalog/util/ProfileParser.java =================================================================== --- trunk/src/org/globus/swift/catalog/util/ProfileParser.java 2010-01-24 23:58:15 UTC (rev 3213) +++ trunk/src/org/globus/swift/catalog/util/ProfileParser.java 2010-01-25 00:00:38 UTC (rev 3214) @@ -26,7 +26,7 @@ * and the parsed triples and back again. * * @author Gaurang Mehta - * @author Jens-S. V???ckler + * @author Jens-S. Voeckler */ public class ProfileParser { Modified: trunk/src/org/globus/swift/catalog/util/ProfileParserException.java =================================================================== --- trunk/src/org/globus/swift/catalog/util/ProfileParserException.java 2010-01-24 23:58:15 UTC (rev 3213) +++ trunk/src/org/globus/swift/catalog/util/ProfileParserException.java 2010-01-25 00:00:38 UTC (rev 3214) @@ -20,7 +20,7 @@ * @see ProfileParser * * @author Gaurang Mehta - * @author Jens-S. V?ckler + * @author Jens-S. Voeckler * @version $Revision: 1.1 $ */ public class ProfileParserException Modified: trunk/src/org/globus/swift/catalog/util/Separator.java =================================================================== --- trunk/src/org/globus/swift/catalog/util/Separator.java 2010-01-24 23:58:15 UTC (rev 3213) +++ trunk/src/org/globus/swift/catalog/util/Separator.java 2010-01-25 00:00:38 UTC (rev 3214) @@ -22,7 +22,7 @@ * representation of a definition looks like ns::name:version, and * a textual representation of a uses like ns::name:min,max.

* - * @author Jens-S. V?ckler + * @author Jens-S. Voeckler * @author Yong Zhao * @version $Revision: 1.6 $ * From noreply at svn.ci.uchicago.edu Sun Jan 24 18:01:37 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 24 Jan 2010 18:01:37 -0600 (CST) Subject: [Swift-commit] r3215 - trunk/docs Message-ID: <20100125000137.A7DA59CCA0@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-24 18:01:37 -0600 (Sun, 24 Jan 2010) New Revision: 3215 Modified: trunk/docs/userguide.xml Log: added example on how to specify maxwalltime in tc.data Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2010-01-25 00:00:38 UTC (rev 3214) +++ trunk/docs/userguide.xml 2010-01-25 00:01:37 UTC (rev 3215) @@ -3234,6 +3234,10 @@ Hours:Minutes:Seconds + Example: + +localhost echo /bin/echo INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:20:00" + When replication is enabled (see replication), then walltime will also be enforced at the Swift client side: when a job has been active for more than twice the maxwalltime, Swift will kill the job and regard it as failed. From noreply at svn.ci.uchicago.edu Mon Jan 25 15:37:42 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 25 Jan 2010 15:37:42 -0600 (CST) Subject: [Swift-commit] r3216 - www/papers Message-ID: <20100125213742.7FDFB9CCA1@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-25 15:37:41 -0600 (Mon, 25 Jan 2010) New Revision: 3216 Added: www/papers/SwiftParallelScripting.pdf Modified: www/papers/index.php Log: added latest swift paper Added: www/papers/SwiftParallelScripting.pdf =================================================================== (Binary files differ) Property changes on: www/papers/SwiftParallelScripting.pdf ___________________________________________________________________ Name: svn:mime-type + application/octet-stream Modified: www/papers/index.php =================================================================== --- www/papers/index.php 2010-01-25 00:01:37 UTC (rev 3215) +++ www/papers/index.php 2010-01-25 21:37:41 UTC (rev 3216) @@ -29,6 +29,14 @@

Swift Technology and SwiftScript Application Papers

+ Mihcael Wilde, Ian Foster, Kamil Iskra, Pete Beckman, Zhao Zhang, Allan Espinosa, Mihael Hategan, Ben Clifford, Ioan Raicu + Parallel Scripting for Applications at the Petascale and Beyond + Computer, Vol. 42, No. 11 + 2009 + [ pdf ] +
+ +
Zhao, Y.,Dobson, J., Moreau, L., Foster, I. and Wilde, M A Notation and System for Expressing and Executing Cleanly Typed Workflows on Messy Scientific Data SIGMOD From noreply at svn.ci.uchicago.edu Mon Jan 25 15:50:52 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 25 Jan 2010 15:50:52 -0600 (CST) Subject: [Swift-commit] r3217 - in www: inc papers Message-ID: <20100125215052.7D2D99CC9F@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-25 15:50:52 -0600 (Mon, 25 Jan 2010) New Revision: 3217 Modified: www/inc/home_sidebar.php www/inc/papers_sidebar.php www/papers/index.php Log: re-arranged things a bit Modified: www/inc/home_sidebar.php =================================================================== --- www/inc/home_sidebar.php 2010-01-25 21:37:41 UTC (rev 3216) +++ www/inc/home_sidebar.php 2010-01-25 21:50:52 UTC (rev 3217) @@ -1,7 +1,9 @@

What's New?

-

SWIFT 0.8 RELEASE - 03 FEB 2009

+

Paper published

+

Parallel Scripting for Applications at the Petascale and Beyond [pdf]


+

SWIFT 0.9 RELEASE - 27 APR 2009

-The latest release of Swift, v0.8, is available from the +The latest release of Swift, v0.9, is available from the downloads page.


Modified: www/inc/papers_sidebar.php =================================================================== --- www/inc/papers_sidebar.php 2010-01-25 21:37:41 UTC (rev 3216) +++ www/inc/papers_sidebar.php 2010-01-25 21:50:52 UTC (rev 3217) @@ -1,5 +1,6 @@

HIGHLIGHTS

Technology

+

Parallel Scripting for Applications at the Petascale and Beyond [pdf]


A Notation and System for Expressing and Executing Cleanly Typed Workflows on Messy Scientific Data (2005) [pdf]


XDTM: XML Data Type and Mapping for Specifying Datasets (2005) [pdf]


The Virtual Data Grid: A New Model and Architecture for Data-Intensive Collaboration (2003) [pdf]


Modified: www/papers/index.php =================================================================== --- www/papers/index.php 2010-01-25 21:37:41 UTC (rev 3216) +++ www/papers/index.php 2010-01-25 21:50:52 UTC (rev 3217) @@ -29,14 +29,6 @@

Swift Technology and SwiftScript Application Papers

- Mihcael Wilde, Ian Foster, Kamil Iskra, Pete Beckman, Zhao Zhang, Allan Espinosa, Mihael Hategan, Ben Clifford, Ioan Raicu - Parallel Scripting for Applications at the Petascale and Beyond - Computer, Vol. 42, No. 11 - 2009 - [ pdf ] -
- -
Zhao, Y.,Dobson, J., Moreau, L., Foster, I. and Wilde, M A Notation and System for Expressing and Executing Cleanly Typed Workflows on Messy Scientific Data SIGMOD @@ -86,7 +78,15 @@ [ pdf ]
+
+ Mihcael Wilde, Ian Foster, Kamil Iskra, Pete Beckman, Zhao Zhang, Allan Espinosa, Mihael Hategan, Ben Clifford, Ioan Raicu + Parallel Scripting for Applications at the Petascale and Beyond + Computer, Vol. 42, No. 11 + 2009 + [ pdf ] +
+

Research leading to Swift and SwiftScript

From noreply at svn.ci.uchicago.edu Mon Jan 25 15:52:46 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 25 Jan 2010 15:52:46 -0600 (CST) Subject: [Swift-commit] r3218 - www/inc Message-ID: <20100125215246.8C46B9CC9F@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-25 15:52:46 -0600 (Mon, 25 Jan 2010) New Revision: 3218 Modified: www/inc/home_sidebar.php www/inc/papers_sidebar.php Log: fixes Modified: www/inc/home_sidebar.php =================================================================== --- www/inc/home_sidebar.php 2010-01-25 21:50:52 UTC (rev 3217) +++ www/inc/home_sidebar.php 2010-01-25 21:52:46 UTC (rev 3218) @@ -1,6 +1,6 @@

What's New?

Paper published

-

Parallel Scripting for Applications at the Petascale and Beyond [pdf]


+

Parallel Scripting for Applications at the Petascale and Beyond [pdf]


SWIFT 0.9 RELEASE - 27 APR 2009

The latest release of Swift, v0.9, is available from the Modified: www/inc/papers_sidebar.php =================================================================== --- www/inc/papers_sidebar.php 2010-01-25 21:50:52 UTC (rev 3217) +++ www/inc/papers_sidebar.php 2010-01-25 21:52:46 UTC (rev 3218) @@ -1,6 +1,6 @@

HIGHLIGHTS

Technology

-

Parallel Scripting for Applications at the Petascale and Beyond [pdf]


+

Parallel Scripting for Applications at the Petascale and Beyond (2009) [pdf]


A Notation and System for Expressing and Executing Cleanly Typed Workflows on Messy Scientific Data (2005) [pdf]


XDTM: XML Data Type and Mapping for Specifying Datasets (2005) [pdf]


The Virtual Data Grid: A New Model and Architecture for Data-Intensive Collaboration (2003) [pdf]


From noreply at svn.ci.uchicago.edu Mon Jan 25 15:57:43 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 25 Jan 2010 15:57:43 -0600 (CST) Subject: [Swift-commit] r3219 - www/papers Message-ID: <20100125215743.37A6C9CC9F@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-25 15:57:43 -0600 (Mon, 25 Jan 2010) New Revision: 3219 Modified: www/papers/index.php Log: inverse chronological order Modified: www/papers/index.php =================================================================== --- www/papers/index.php 2010-01-25 21:52:46 UTC (rev 3218) +++ www/papers/index.php 2010-01-25 21:57:43 UTC (rev 3219) @@ -29,27 +29,29 @@

Swift Technology and SwiftScript Application Papers

- Zhao, Y.,Dobson, J., Moreau, L., Foster, I. and Wilde, M - A Notation and System for Expressing and Executing Cleanly Typed Workflows on Messy Scientific Data - SIGMOD - 2005 - [ pdf ] + Mihcael Wilde, Ian Foster, Kamil Iskra, Pete Beckman, Zhao Zhang, Allan Espinosa, Mihael Hategan, Ben Clifford, Ioan Raicu + Parallel Scripting for Applications at the Petascale and Beyond + Computer, Vol. 42, No. 11 + 2009 + [ pdf ]
- Raicu, I., Zhao Y., Dumitrescu, C., Foster, I. and Wilde, M - Falkon: a Fast and Light-weight tasK executiON framework - Supercomputing Conference - 2007 - [ pdf ] + Andriy Fedorov, Benjamin Clifford, Simon K. War?eld, Ron Kikinis, Nikos Chrisochoides + + Non-Rigid Registration for Image-Guided Neurosurgery on the TeraGrid: A Case Study + + College of William and Mary Technical Report + 2009 + [ pdf ]
- Zhao Y., Hategan, M., Clifford, B., Foster, I., vonLaszewski, G., Raicu, I., Stef-Praun, T. and Wilde, M - Swift: Fast, Reliable, Loosely Coupled Parallel Computation - IEEE International Workshop on Scientific Workflows + Stef-Praun, T., Madeira, G., Foster, I., and Townsend, R. + Accelerating solution of a moral hazard problem with Swift + e-Social Science 2007 - [ pdf ] + [ pdf ]
@@ -61,29 +63,28 @@
- Stef-Praun, T., Madeira, G., Foster, I., and Townsend, R. - Accelerating solution of a moral hazard problem with Swift - e-Social Science + Zhao Y., Hategan, M., Clifford, B., Foster, I., vonLaszewski, G., Raicu, I., Stef-Praun, T. and Wilde, M + Swift: Fast, Reliable, Loosely Coupled Parallel Computation + IEEE International Workshop on Scientific Workflows 2007 - [ pdf ] + [ pdf ]
- Andriy Fedorov, Benjamin Clifford, Simon K. War?eld, Ron Kikinis, Nikos Chrisochoides - - Non-Rigid Registration for Image-Guided Neurosurgery on the TeraGrid: A Case Study - - College of William and Mary Technical Report - 2009 - [ pdf ] + Raicu, I., Zhao Y., Dumitrescu, C., Foster, I. and Wilde, M + Falkon: a Fast and Light-weight tasK executiON framework + Supercomputing Conference + 2007 + [ pdf ]
+
- Mihcael Wilde, Ian Foster, Kamil Iskra, Pete Beckman, Zhao Zhang, Allan Espinosa, Mihael Hategan, Ben Clifford, Ioan Raicu - Parallel Scripting for Applications at the Petascale and Beyond - Computer, Vol. 42, No. 11 - 2009 - [ pdf ] + Zhao, Y.,Dobson, J., Moreau, L., Foster, I. and Wilde, M + A Notation and System for Expressing and Executing Cleanly Typed Workflows on Messy Scientific Data + SIGMOD + 2005 + [ pdf ]
@@ -143,6 +144,23 @@
+ Vöckler, J.-S., Mehta, G., Zhao, Y., Deelman, E. and Wilde, M., + Kickstarting Remote Applications. + 2nd International Workshop on Grid Computing Environments, + 2006. + [ pdf ] +
+ +
+ Zhao, Y., Wilde, M., Foster, I., Voeckler, J., Jordan, T., Quigg, E. and Dobson, J., + Grid Middleware Services for Virtual Data Discovery, Composition, and Integration. + 2nd International Workshop on Middleware for Grid Computing, + 2004. + [ pdf ] +
+ + +
Foster, I., Voeckler, J., Wilde, M. and Zhao, Y., Chimera: A Virtual Data System for Representing, Querying, and Automating Data Derivation. 14th Intl. Conf. on Scientific and Statistical Database Management, Edinburgh, Scotland, @@ -151,14 +169,6 @@
- Vöckler, J.-S., Mehta, G., Zhao, Y., Deelman, E. and Wilde, M., - Kickstarting Remote Applications. - 2nd International Workshop on Grid Computing Environments, - 2006. - [ pdf ] -
- -
Vöckler, J.-S., Wilde, M. and Foster, I. The GriPhyN Virtual Data System. Technical Report GriPhyN-2002-02, @@ -173,23 +183,36 @@ [ pdf ]
+

VDL Applications - predecessors to Swift

+
- Zhao, Y., Wilde, M., Foster, I., Voeckler, J., Jordan, T., Quigg, E. and Dobson, J., - Grid Middleware Services for Virtual Data Discovery, Composition, and Integration. - 2nd International Workshop on Middleware for Grid Computing, - 2004. - [ pdf ] + Nefedova, V., Jacob, R., Foster, I., Liu, Y., Liu, Z., Deelman, E., Mehta, G. and Vahi, K., + Automating Climate Science: Large Ensemble Simulations on the TeraGrid with the GriPhyN Virtual Data System. + 2nd IEEE International Conference on eScience and Grid Computing, + 2006. + [ pdf ]
+
+ Horn, J.V., Dobson, J., Woodward, J., Wilde, M., Zhao, Y., Voeckler, J. and Foster, I. + Grid-Based Computing and the Future of Neuroscience Computation. + Methods in Mind, MIT Press, + 2006. +
-

VDL Applications - predecessors to Swift

+
+ Sulakhe, D., Rodriguez, A., Wilde, M., Foster, I. and Maltsev, N., + Using Multiple Grid Resources for Bioinformatics Applications in GADU. + IEEE/ACM International Symposium on Cluster Computing and Grid, + 2006. + [ pdf ] +
- Annis, J., Zhao, Y., Voeckler, J., Wilde, M., Kent, S. and Foster, I., - Applying Chimera Virtual Data Concepts to Cluster Finding in the Sloan Sky Survey. - SC2002, Baltimore, MD, - 2002. - [ pdf ] + Sulakhe, D., Rodriguez, A., D'Souza, M., Wilde, M., Nefedova, V., Foster, I. and Maltsev, N. + GNARE: An Environment for Grid-Based High-Throughput Genome Analysis. + Journal of Clinical Monitoring and Computing. + [ pdf ]
@@ -209,6 +232,14 @@
+ Annis, J., Zhao, Y., Voeckler, J., Wilde, M., Kent, S. and Foster, I., + Applying Chimera Virtual Data Concepts to Cluster Finding in the Sloan Sky Survey. + SC2002, Baltimore, MD, + 2002. + [ pdf ] +
+ +
Bardeen, M., Gilbert, E., Jordan, T., Nepywoda, P., Quigg, E., Wilde, M. and Zhao, Y. The QuarkNet/Grid Collaborative Learning e-Lab. Future Generation Computer Systems, 22 (6), @@ -217,36 +248,6 @@
- Horn, J.V., Dobson, J., Woodward, J., Wilde, M., Zhao, Y., Voeckler, J. and Foster, I. - Grid-Based Computing and the Future of Neuroscience Computation. - Methods in Mind, MIT Press, - 2006. -
- -
- Nefedova, V., Jacob, R., Foster, I., Liu, Y., Liu, Z., Deelman, E., Mehta, G. and Vahi, K., - Automating Climate Science: Large Ensemble Simulations on the TeraGrid with the GriPhyN Virtual Data System. - 2nd IEEE International Conference on eScience and Grid Computing, - 2006. - [ pdf ] -
- -
- Sulakhe, D., Rodriguez, A., D'Souza, M., Wilde, M., Nefedova, V., Foster, I. and Maltsev, N. - GNARE: An Environment for Grid-Based High-Throughput Genome Analysis. - Journal of Clinical Monitoring and Computing. - [ pdf ] -
- -
- Sulakhe, D., Rodriguez, A., Wilde, M., Foster, I. and Maltsev, N., - Using Multiple Grid Resources for Bioinformatics Applications in GADU. - IEEE/ACM International Symposium on Cluster Computing and Grid, - 2006. - [ pdf ] -
- -
Zhao, Y. Virtual Galaxy Clusters: An Application of the GriPhyN Virtual Data Toolkit to Sloan Digital Sky Survey Data. MS thesis, University of Chicago, GriPhyN-2002-06, From noreply at svn.ci.uchicago.edu Mon Jan 25 16:00:15 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 25 Jan 2010 16:00:15 -0600 (CST) Subject: [Swift-commit] r3220 - www/papers Message-ID: <20100125220015.3F15D9CC9F@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-01-25 16:00:15 -0600 (Mon, 25 Jan 2010) New Revision: 3220 Modified: www/papers/index.php Log: added oops paper Modified: www/papers/index.php =================================================================== --- www/papers/index.php 2010-01-25 21:57:43 UTC (rev 3219) +++ www/papers/index.php 2010-01-25 22:00:15 UTC (rev 3220) @@ -29,6 +29,13 @@

Swift Technology and SwiftScript Application Papers

+ Joe DeBartolo, Glen Hocky, Michael Wilde, Jinbo Xu, Karl F. Freed, and Tobin R. Sosnick + Protein Structure Prediction Enhanced with Evolutionary Diversity: SPEED + Protein Science Journal + Jan 2010 +
+ +
Mihcael Wilde, Ian Foster, Kamil Iskra, Pete Beckman, Zhao Zhang, Allan Espinosa, Mihael Hategan, Ben Clifford, Ioan Raicu Parallel Scripting for Applications at the Petascale and Beyond Computer, Vol. 42, No. 11