[Swift-commit] r6619 - in branches/faster: . bin docs docs/cookbook docs/documentation docs/merged docs/merged/refmanual docs/merged/userguide docs/quickstart docs/siteguide docs/tutorial docs/userguide docs/utils etc/sites libexec src src/org/globus/swift/data src/org/griphyn/vdl/engine src/org/griphyn/vdl/karajan src/org/griphyn/vdl/karajan/lib src/org/griphyn/vdl/karajan/lib/swiftscript tests tests/language-behaviour/mappers tests/sites/intrepid tests/sites/mcs
hategan at ci.uchicago.edu
hategan at ci.uchicago.edu
Mon Jul 8 03:20:06 CDT 2013
Author: hategan
Date: 2013-07-08 03:19:11 -0500 (Mon, 08 Jul 2013)
New Revision: 6619
Added:
branches/faster/docs/designs/
branches/faster/docs/merged/refmanual/build.sh
branches/faster/docs/merged/refmanual/grammar
branches/faster/docs/merged/refmanual/swift.css
branches/faster/docs/merged/refmanual/swiftlang
branches/faster/docs/siteguide/ec2
branches/faster/docs/siteguide/midway
branches/faster/docs/siteguide/stampede
branches/faster/docs/userguide/swift_monitor.png
branches/faster/etc/sites/local-coasters
branches/faster/etc/sites/stampede
branches/faster/etc/sites/stampede-ssh
branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java
branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b0.out.expected
branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b1.out.expected
branches/faster/tests/language-behaviour/mappers/077-regexpmapper-input.in
Removed:
branches/faster/etc/sites/stampede
branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b.out.expected
Modified:
branches/faster/
branches/faster/bin/gensites
branches/faster/bin/swiftrun
branches/faster/docs/build_docs.sh
branches/faster/docs/cookbook/cookbook.txt
branches/faster/docs/cookbook/log_processing
branches/faster/docs/documentation/documentation.txt
branches/faster/docs/merged/build_docs.sh
branches/faster/docs/merged/userguide/coasters
branches/faster/docs/quickstart/quickstart.txt
branches/faster/docs/siteguide/beagle
branches/faster/docs/siteguide/fusion
branches/faster/docs/siteguide/futuregrid
branches/faster/docs/siteguide/intrepid
branches/faster/docs/siteguide/prereqs
branches/faster/docs/siteguide/siteguide.txt
branches/faster/docs/tutorial/tutorial.txt
branches/faster/docs/userguide/app_procedures
branches/faster/docs/userguide/build_options
branches/faster/docs/userguide/clustering
branches/faster/docs/userguide/coasters
branches/faster/docs/userguide/commands
branches/faster/docs/userguide/configuration_properties
branches/faster/docs/userguide/kickstart
branches/faster/docs/userguide/log-processing
branches/faster/docs/userguide/mappers
branches/faster/docs/userguide/overview
branches/faster/docs/userguide/profiles
branches/faster/docs/userguide/site_catalog
branches/faster/docs/userguide/transformation_catalog
branches/faster/docs/userguide/userguide.txt
branches/faster/docs/utils/gensites.txt
branches/faster/etc/sites/beagle-ssh
branches/faster/libexec/_swiftwrap.wrapperstaging
branches/faster/libexec/swift-int.k
branches/faster/libexec/swift-lib.k
branches/faster/src/
branches/faster/src/org/globus/swift/data/Action.java
branches/faster/src/org/griphyn/vdl/engine/ProcedureSignature.java
branches/faster/src/org/griphyn/vdl/karajan/Loader.java
branches/faster/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java
branches/faster/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java
branches/faster/src/org/griphyn/vdl/karajan/lib/InFileDirs.java
branches/faster/src/org/griphyn/vdl/karajan/lib/PathUtils.java
branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java
branches/faster/tests/README
branches/faster/tests/USAGENOTES.txt
branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.swift
branches/faster/tests/language-behaviour/mappers/077-regexp-mapper.swift
branches/faster/tests/sites/intrepid/sites.template.xml
branches/faster/tests/sites/mcs/coaster-service.conf
Log:
merged changes from trunk
Property changes on: branches/faster
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/release-0.93:4761-5122
/trunk:6172,6177,6182,6189-6190,6202-6203,6206-6208,6215-6223,6231-6241,6255-6258,6263,6272,6274-6275
+ /branches/release-0.93:4761-5122
/trunk:6172,6177,6182,6189-6190,6202-6203,6206-6208,6215-6223,6231-6241,6255-6258,6263,6272,6274-6618
Modified: branches/faster/bin/gensites
===================================================================
--- branches/faster/bin/gensites 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/bin/gensites 2013-07-08 08:19:11 UTC (rev 6619)
@@ -123,10 +123,12 @@
if [ ! -f "$TEMPLATE_PATH" ]; then
if [ -f "$TEMPLATE" ]; then
TEMPLATE_PATH=$TEMPLATE
+ elif [ -f "$HOME/.swift/sites/$TEMPLATE" ]; then
+ TEMPLATE_PATH=$HOME/.swift/sites/$TEMPLATE
+ elif [ -f "$PWD/conf/$TEMPLATE" ]; then
+ TEMPLATE_PATH=$PWD/conf/$TEMPLATE
elif [ -f "$SWIFT_HOME/$TEMPLATE" ]; then
TEMPLATE_PATH=$SWIFT_HOME/$TEMPLATE
- elif [ -f "$HOME/.swift/sites/$TEMPLATE" ]; then
- TEMPLATE_PATH=$HOME/.swift/sites/$TEMPLATE
fi
if [ ! -f "$TEMPLATE_PATH" ]; then
crash "Cannot find template for $TEMPLATE"
@@ -151,9 +153,9 @@
# Setup for creating a TC file
if [ -f "$PROPERTIES_FILE" ]; then
if [ -n "`grep -e app $PROPERTIES_FILE`" ]; then
- if [ -f "tc.data" ]; then
- mv tc.data tc.data.old
- fi
+ #if [ -f "tc.data" ]; then
+ # mv tc.data tc.data.old
+ #fi
HOSTS=`grep -i "pool handle" $TEMPLATE_PATH|grep -v "^[[:space:]]*#"|cut -d'"' -f2`
fi
fi
Modified: branches/faster/bin/swiftrun
===================================================================
--- branches/faster/bin/swiftrun 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/bin/swiftrun 2013-07-08 08:19:11 UTC (rev 6619)
@@ -60,6 +60,14 @@
rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
mkdir $rundir
+# Link input_files
+if [ -d "input_files" ]; then
+ for input_file in input_files/*
+ do
+ ln -s $PWD/$input_file $rundir
+ done
+fi
+
# Swift script
if [ -f "$script" ]; then
cp $script $rundir
@@ -72,33 +80,37 @@
fi
# Create sites.xml
-cd $rundir
IFS=","
-echo "<config>" > sites.xml
+echo "<config>" > $rundir/sites.xml
for site in $SITES
do
# If config is not defined, but there is a config in conf/ that matches, use that
- if [ -z "$CONFIG" ] && [ -f "../conf/$site.cf" ]; then
- cp ../conf/$site.cf .
- CONFIG=$site.cf
- fi
+ if [ -z "$CONFIG" ] && [ -f "conf/$site.cf" ]; then
+ cp conf/$site.cf $rundir
+ config=$site.cf
+ fi
- if [ -f "$CONFIG" ]; then
- gensites -p $CONFIG $site >> sites.xml
+ if [ -f "conf/$config" ]; then
+ gensites -p conf/$config $site >> $rundir/sites.xml
else
- gensites $site >> sites.xml
+ gensites $site >> $rundir/sites.xml
fi
done
-echo "</config>" >> sites.xml
+echo "</config>" >> $rundir/sites.xml
+mv tc.data $rundir
+cd $rundir
+if [ -z "$CONFIG" ]; then
+ CONFIG=$config
+fi
# Run
timestamp=$( date +%s )
ln -s $PWD $HOME/.swift/runs/current/$rundir.$timestamp
if [ -n "$CONFIG" ]; then
- eval time swift -sites.file sites.xml -tc.file tc.data -config $CONFIG $script $args 2>&1 | tee swift.out
+ eval time swift -sites.file sites.xml -tc.file tc.data -config $CONFIG $( basename $script ) $args 2>&1 | tee swift.out
else
- eval time swift -sites.file sites.xml -tc.file tc.data $script $args 2>&1 | tee swift.out
+ eval time swift -sites.file sites.xml -tc.file tc.data $( basename $script ) $args 2>&1 | tee swift.out
fi
mv $HOME/.swift/runs/current/$rundir.$timestamp $HOME/.swift/runs/completed
Modified: branches/faster/docs/build_docs.sh
===================================================================
--- branches/faster/docs/build_docs.sh 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/build_docs.sh 2013-07-08 08:19:11 UTC (rev 6619)
@@ -48,8 +48,6 @@
exit 1
fi
-INSTALLATION_DIRECTORY=$1
-
# Create installation directory if needed
if [ ! -d "$INSTALLATION_DIRECTORY" ]; then
mkdir $INSTALLATION_DIRECTORY || crash "Unable to create directory $INSTALLATION_DIRECTORY"
Modified: branches/faster/docs/cookbook/cookbook.txt
===================================================================
--- branches/faster/docs/cookbook/cookbook.txt 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/cookbook/cookbook.txt 2013-07-08 08:19:11 UTC (rev 6619)
@@ -16,3 +16,6 @@
include::debugging[]
include::log_processing[]
+
+link:http://www.ci.uchicago.edu/swift/docs/index.php[home]
+
Modified: branches/faster/docs/cookbook/log_processing
===================================================================
--- branches/faster/docs/cookbook/log_processing 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/cookbook/log_processing 2013-07-08 08:19:11 UTC (rev 6619)
@@ -84,7 +84,7 @@
Problem Reporting
~~~~~~~~~~~~~~~~~
-When reporting problems to swift-user at ci.uchicago.edu, please attach the
+When reporting problems to swift-user at ci.uchicago.edu, attach the
following files and information:
. tc.data and sites.xml (or whatever you named these files)
Modified: branches/faster/docs/documentation/documentation.txt
===================================================================
--- branches/faster/docs/documentation/documentation.txt 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/documentation/documentation.txt 2013-07-08 08:19:11 UTC (rev 6619)
@@ -9,3 +9,5 @@
include::automation[]
+link:http://www.ci.uchicago.edu/swift/docs/index.php[home]
+
Modified: branches/faster/docs/merged/build_docs.sh
===================================================================
--- branches/faster/docs/merged/build_docs.sh 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/merged/build_docs.sh 2013-07-08 08:19:11 UTC (rev 6619)
@@ -113,3 +113,4 @@
find $INSTALLATION_DIRECTORY/$VERSION -type f -exec chgrp $GROUP {} \; -exec chmod $CHMOD_FILE_MODE {} \; > /dev/null 2>&1
find $INSTALLATION_DIRECTORY/$VERSION -type d -exec chgrp $GROUP {} \; -exec chmod $CHMOD_DIRECTORY_MODE {} \; > /dev/null 2>&1
+
Copied: branches/faster/docs/merged/refmanual/build.sh (from rev 6618, trunk/docs/merged/refmanual/build.sh)
===================================================================
--- branches/faster/docs/merged/refmanual/build.sh (rev 0)
+++ branches/faster/docs/merged/refmanual/build.sh 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+asciidoc --attribute stylesheet=${PWD}/swift.css refmanual.txt
Copied: branches/faster/docs/merged/refmanual/grammar (from rev 6618, trunk/docs/merged/refmanual/grammar)
===================================================================
--- branches/faster/docs/merged/refmanual/grammar (rev 0)
+++ branches/faster/docs/merged/refmanual/grammar 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1,1427 @@
+== Swift Language Grammar
+
+The specification for a SwiftScript program
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+----
+program:
+ (namespace_dec[code])*
+ (importStatement[code])*
+ (topLevelStatement[code])*
+ EOF
+ ;
+----
+
+Namespace Declaration
+~~~~~~~~~~~~~~~~~~~~~~
+----
+namespace_dec [StringTemplate code]
+{StringTemplate ns=template("nsDef");}:
+"namespace" (prefix:ID{ns.setAttribute("prefix", prefix.getText());})? uri:STRING_LITERAL SEMI {
+ ns.setAttribute("uri", uri.getText());
+ code.setAttribute("namespaces", ns);
+ if (ns.getAttribute("prefix") == null)
+ code.setAttribute("targetNS", ns.getAttribute("uri"));
+ }
+ ;
+----
+
+The import statement
+~~~~~~~~~~~~~~~~~~~~
+----
+importStatement [StringTemplate code] :
+"import" name:STRING_LITERAL SEMI {
+ StringTemplate i = template("import");
+ i.setAttribute("target", name.getText());
+ code.setAttribute("imports", i);
+ }
+ ;
+----
+
+Type Declaration
+~~~~~~~~~~~~~~~~~
+----
+typedecl [StringTemplate code]
+{StringTemplate r=template("typeDef");
+ StringTemplate t=null;} :
+ "type" id:ID {
+ r.setAttribute("name", id.getText()); }
+ (
+ SEMI
+ |
+ (t=type
+ {
+ r.setAttribute("type", t);
+ }
+ SEMI
+ )
+ | structdecl[r]
+ )
+ {code.setAttribute("types", r);}
+ ;
+----
+
+Structure declaration
+~~~~~~~~~~~~~~~~~~~~~~
+----
+structdecl [StringTemplate code]
+{StringTemplate e=null, e1=null, t=null; String thisType = null;} :
+ LCURLY (t=type id:ID {
+ thisType = (String) t.getAttribute("name");
+ e=template("memberdefinition");
+ e.setAttribute("name", id.getText());
+ }
+ (LBRACK RBRACK { thisType = thisType + "[]"; })* {
+ StringTemplate thisTypeTemplate;
+ thisTypeTemplate=template("type");
+ thisTypeTemplate.setAttribute("name", thisType);
+ e.setAttribute("type", thisTypeTemplate);
+ code.setAttribute("members", e);
+ }
+ ( COMMA
+ id1:ID {
+ thisType = (String) t.getAttribute("name");
+ e1=template("memberdefinition");
+ e1.setAttribute("name", id1.getText());
+ }
+ (LBRACK RBRACK { thisType = thisType + "[]"; })* {
+ StringTemplate thisTypeTemplate;
+ thisTypeTemplate=template("type");
+ thisTypeTemplate.setAttribute("name", thisType);
+ e1.setAttribute("type", thisTypeTemplate);
+ code.setAttribute("members", e1);
+ })*
+ SEMI)*
+ RCURLY (options {
+ warnWhenFollowAmbig = false;
+ } :SEMI)?
+ ;
+----
+
+Top level statement
+~~~~~~~~~~~~~~~~~~~
+----
+topLevelStatement[StringTemplate code]
+{StringTemplate d=null; } :
+
+// these are ll(1) and easy to predict
+
+ typedecl[code]
+ | d=ll1statement
+ {
+ code.setAttribute("statements",d);
+ }
+
+// these are non-declaration assign-like statements
+
+ | (predictAssignStat) => d=assignStat
+ {
+ code.setAttribute("statements",d);
+ }
+
+// these are non-declaration append-associative array statements
+
+ | (predictAppendStat) => d=appendStat
+ {
+ code.setAttribute("statements",d);
+ }
+
+// they all begin with (id name)
+ | (predictDeclaration) => declaration[code]
+
+// more complicated function invocations
+// note that function invocations can happen in above statements too
+// this section is just the remaining more specialised invocations
+
+ | (procedurecallCode) => d=procedurecallCode
+ {
+ code.setAttribute("statements",d);
+ }
+
+ | (procedurecallStatAssignManyReturnParam[code]) => procedurecallStatAssignManyReturnParam[code]
+
+// this is a declaration, but not sorted out the predications yet to
+// group it into a decl block
+ | ("app") => d=appproceduredecl {code.setAttribute("functions",d);}
+ | (predictProceduredecl) => d=proceduredecl {code.setAttribute("functions", d);}
+ ;
+----
+
+
+Predict Declaration
+~~~~~~~~~~~~~~~~~~~
+----
+predictDeclaration {StringTemplate x,y;} : ("global") | (x=type y=declarator) ;
+
+declaration [StringTemplate code]
+{StringTemplate t=null;
+ boolean isGlobal = false;}
+ : ("global" {isGlobal = true;})?
+ t=type
+ declpart[code, t, isGlobal]
+ (COMMA declpart[code, t, isGlobal])*
+ SEMI
+ ;
+----
+
+Declaration part
+~~~~~~~~~~~~~~~~
+----
+declpart [StringTemplate code, StringTemplate t, boolean isGlobal]
+ {
+ StringTemplate n=null;
+ StringTemplate thisTypeTemplate=null;
+ String thisType = (String) t.getAttribute("name");
+ StringTemplate variable=null;
+ StringTemplate m = null;
+ StringTemplate sTemp = null;
+ String sType = "";
+ } :
+ n=declarator
+ (LBRACK
+ (sTemp=type {sType = (String) sTemp.getAttribute("name") ;} )?
+ RBRACK {thisType = thisType + "[" + sType + "]" ; sType = ""; } )*
+ {
+ thisTypeTemplate=template("type");
+ thisTypeTemplate.setAttribute("name", thisType);
+ variable = template("variable");
+ variable.setAttribute("name", n);
+ variable.setAttribute("type", thisTypeTemplate);
+ variable.setAttribute("global", ""+isGlobal);
+ code.setAttribute("statements", variable);
+ }
+
+ (LT (m=mappingdecl | f:STRING_LITERAL) GT
+ {
+ if (m!=null)
+ variable.setAttribute("mapping", m);
+ else
+ variable.setAttribute("lfn", quote(f.getText()));
+ })?
+
+// TODO: mapping does here...
+// which means construction of the variable template goes here, rather than
+// in procedurecallDecl/variableDecl
+
+ (
+ (predictProcedurecallDecl) => procedurecallDecl[code, thisTypeTemplate, n, variable]
+ | variableDecl[code, thisTypeTemplate, n, variable]
+// nice to lose this distinction entirely...
+// | (predictDatasetdecl) => datasetdecl[code, thisTypeTemplate, n]
+// TODO can shorten variableDecl predictor now we dont' need to
+// distinguish it from datasetdecl?
+ )
+ ;
+----
+
+Variable Declarator
+~~~~~~~~~~~~~~~~~~~
+----
+variableDecl [StringTemplate code, StringTemplate t, StringTemplate d, StringTemplate v1]
+{StringTemplate i1=null, m=null;
+
+} :
+ (i1=varInitializer
+ {
+ if (i1 != null) {
+ StringTemplate valueAssignment = template("assign");
+ StringTemplate vr = template("variableReference");
+ vr.setAttribute("name",d);
+ valueAssignment.setAttribute("lhs",vr);
+ valueAssignment.setAttribute("rhs",i1);
+ code.setAttribute("statements", valueAssignment);
+ }
+ })?
+ ;
+----
+
+Declarator
+~~~~~~~~~~
+----
+declarator returns [StringTemplate code=null] :
+id:ID {code=text(id.getText());}
+ ;
+----
+
+
+Variable initializer
+~~~~~~~~~~~~~~~~~~~~
+----
+varInitializer returns [StringTemplate code=null] :
+ASSIGN code=expression
+ ;
+----
+
+This is an initializer used to set up an array.
+ currently does not support nested array.
+----
+arrayInitializer returns [StringTemplate code=template("arrayInit")]
+{StringTemplate e=null,from=null,to=null,step=null;} :
+LBRACK
+ (
+ (expression COLON) =>
+ (
+ from=expression COLON to=expression (COLON step=expression)?
+ {
+ StringTemplate range=template("range");
+ range.setAttribute("from", from);
+ range.setAttribute("to", to);
+ if (step != null)
+ range.setAttribute("step", step);
+ code.setAttribute("range", range);
+ }
+ )
+ |
+ (
+ e=expression {code.setAttribute("elements", e);}
+ (
+ // CONFLICT: does a COMMA after an initializer start a new
+ // initializer or start the option ',' at end?
+ // ANTLR generates proper code by matching
+ // the comma as soon as possible.
+ options {
+ warnWhenFollowAmbig = false;
+ } : COMMA e=expression {code.setAttribute("elements", e);}
+ )*
+ (COMMA)?
+ )
+ )?
+ RBRACK
+ ;
+----
+
+Mapping Declaration
+~~~~~~~~~~~~~~~~~~~
+----
+mappingdecl returns [StringTemplate code=template("mapping")]
+{StringTemplate p=null, d=null;}
+ : d=declarator {code.setAttribute("descriptor",d);} SEMI
+ mapparamdecl[code]
+ ;
+----
+
+Mapping parameters declaration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+----
+mapparamdecl [StringTemplate code]
+{StringTemplate p=null;} :
+( p=mapparam {code.setAttribute("params", p);}
+ ( COMMA p=mapparam {code.setAttribute("params", p);} )*
+ )?
+ ;
+
+mapparam returns [StringTemplate code=template("mapParam")]
+{StringTemplate n=null, v=null;} :
+n=declarator ASSIGN v=mappingExpr
+ {
+ code.setAttribute("name", n);
+ code.setAttribute("value", v);
+ }
+ ;
+----
+
+Predict Procedure Declaration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This predicts in two different ways.
+The first choice is procedures with no return parameters. For these,
+we must predict as far as the opening { in order to distinguish
+from procedure calls to procedures with no return parameters.
+The second choice is for procedures with return parameters. Here we
+predict as far as the bracket after the procedure name. We have to
+predict on the return parameters, which means we won't get good
+error reporting when there is a syntax error in there.
+
+----
+predictProceduredecl
+{StringTemplate f=null;} :
+( id:ID LPAREN
+ ( f=formalParameter (COMMA f=formalParameter)* )?
+ RPAREN
+ LCURLY
+ )
+ |
+ (
+ LPAREN
+ f=formalParameter
+ ( COMMA f=formalParameter
+ )*
+ RPAREN ID LPAREN
+ )
+ ;
+----
+
+Procedure declaration
+~~~~~~~~~~~~~~~~~~~~~
+----
+proceduredecl returns [StringTemplate code=template("function")]
+{StringTemplate f=null;} :
+( LPAREN
+ f=formalParameter
+ {
+ f.setAttribute("outlink", "true");
+ code.setAttribute("outputs", f);
+ }
+ ( COMMA f=formalParameter
+ {
+ f.setAttribute("outlink", "true");
+ code.setAttribute("outputs", f);
+ }
+ )*
+ RPAREN )?
+ id:ID {currentFunctionName=id.getText();} LPAREN
+ ( f=formalParameter
+ {
+ code.setAttribute("inputs", f);
+ }
+ ( COMMA f=formalParameter
+ {
+ code.setAttribute("inputs", f);
+ }
+ )*
+ )?
+ RPAREN
+ LCURLY
+ (
+ atomicBody[code]
+ |
+ compoundBody[code]
+ )
+ RCURLY
+ {
+ code.setAttribute("name", id.getText());
+ currentFunctionName=null;
+ }
+ ;
+----
+
+App declaration
+~~~~~~~~~~~~~~~~~~~~~~~~~
+----
+appproceduredecl returns [StringTemplate code=template("function")]
+{StringTemplate f=null;
+ StringTemplate app=template("app");
+ StringTemplate exec=null; } :
+ "app"
+ ( LPAREN
+ f=formalParameter
+ {
+ f.setAttribute("outlink", "true");
+ code.setAttribute("outputs", f);
+ }
+ ( COMMA f=formalParameter
+ {
+ f.setAttribute("outlink", "true");
+ code.setAttribute("outputs", f);
+ }
+ )*
+ RPAREN )?
+ id:ID {currentFunctionName=id.getText();} LPAREN
+ ( f=formalParameter
+ {
+ code.setAttribute("inputs", f);
+ }
+ ( COMMA f=formalParameter
+ {
+ code.setAttribute("inputs", f);
+ }
+ )*
+ )?
+ RPAREN
+ LCURLY
+ ( appProfile[app] )*
+ exec=declarator
+ {app.setAttribute("exec",exec);}
+ ( appArg[app] )* SEMI
+ {code.setAttribute("config",app);}
+ RCURLY
+ {
+ code.setAttribute("name", id.getText());
+ currentFunctionName=null;
+ }
+ ;
+----
+
+App profile
+~~~~~~~~~~~
+----
+appProfile [StringTemplate code]
+{ StringTemplate p=null;
+ StringTemplate k=null;
+ StringTemplate v=null;} :
+ "profile" k=expression ASSIGN v=expression SEMI {
+ p=template("app_profile");
+ p.setAttribute("key", k);
+ p.setAttribute("value", v);
+ code.setAttribute("profiles", p);
+ }
+ ;
+----
+
+App Formal parameter
+~~~~~~~~~~~~~~~~~~~~
+
+----
+formalParameter returns [StringTemplate code=template("parameter")]
+{StringTemplate t=null,d=null,v=null; String thisType = null; } :
+(t=type d=declarator
+ {
+ thisType = (String) t.getAttribute("name");
+ code.setAttribute("name", d);
+ }
+ (LBRACK RBRACK {thisType = thisType + "[]"; })*
+ (ASSIGN v=constant
+ {
+ String value = (String)v.getAttribute("value");
+ if (v.getName().equals("sConst")) {
+ v.removeAttribute("value");
+ v.setAttribute("value", quote(value));
+ }
+ code.setAttribute("defaultv", v);
+ }
+ )?) {
+ StringTemplate thisTypeTemplate;
+ thisTypeTemplate=template("type");
+ thisTypeTemplate.setAttribute("name", thisType);
+ code.setAttribute("type", thisTypeTemplate);
+ }
+ ;
+----
+
+Type declaration
+~~~~~~~~~~~~~~~~
+----
+type returns [ StringTemplate code = null ]
+ { StringBuilder buf = new StringBuilder(); } :
+ id:ID {
+ code = template("type");
+ buf.append(id.getText());
+ }
+ (typeSubscript[buf]) * {
+ code.setAttribute("name", buf.toString());
+ }
+;
+----
+
+Type Subscript
+~~~~~~~~~~~~~~
+----
+typeSubscript[StringBuilder buf] :
+ LBRACK { buf.append('['); }
+ (id:ID { buf.append(id.getText()); })?
+ RBRACK { buf.append(']'); }
+;
+----
+
+Compound Statement
+~~~~~~~~~~~~~~~~~~
+----
+compoundStat[StringTemplate code]
+ : LCURLY
+ ( innerStatement[code] )*
+ RCURLY
+ ;
+----
+
+Compound body
+~~~~~~~~~~~~~
+----
+compoundBody[StringTemplate code]
+ : ( innerStatement[code] )*
+ ;
+
+innerStatement[StringTemplate code]
+{StringTemplate s=null;}
+ : (predictDeclaration) => declaration[code]
+ |
+ ((
+ s=ll1statement
+ | (procedurecallCode) => s=procedurecallCode
+ | (predictAssignStat) => s=assignStat
+ | (predictAppendStat) => s=appendStat
+ )
+ {
+ code.setAttribute("statements",s);
+ })
+ | (procedurecallStatAssignManyReturnParam[code]) => procedurecallStatAssignManyReturnParam[code]
+ ;
+----
+
+Case Inner Statement
+~~~~~~~~~~~~~~~~~~~~
+----
+caseInnerStatement [StringTemplate statements]
+{ StringTemplate code = null; }
+ :
+ ( code=ll1statement
+ | (procedurecallCode) => code=procedurecallCode
+ | (predictAssignStat) => code=assignStat
+ | (predictAppendStat) => code=appendStat
+ ) {statements.setAttribute("statements",code);}
+ | (procedurecallStatAssignManyReturnParam[statements]) => procedurecallStatAssignManyReturnParam[statements]
+ ;
+----
+
+These are the statements that can be predicted with ll(1) grammer
+i.e. with one token of lookahead
+
+----
+ll1statement returns [StringTemplate code=null]
+ :
+ code=ifStat
+ | code=foreachStat
+ | code=switchStat
+ | code=iterateStat
+ ;
+----
+
+If statement
+~~~~~~~~~~~~
+----
+ifStat returns [StringTemplate code=template("if")]
+{
+ StringTemplate cond=null;
+ StringTemplate body=template("statementList");
+ StringTemplate els=template("statementList");
+}
+ : "if" LPAREN cond=expression RPAREN {
+ code.setAttribute("cond", cond);
+ }
+ compoundStat[body] {code.setAttribute("body", body);}
+ (
+ options {
+ warnWhenFollowAmbig = false;
+ }
+ : "else"
+ compoundStat[els] {code.setAttribute("els", els);}
+ )?
+ ;
+----
+
+Foreach statement
+~~~~~~~~~~~~~~~~~
+----
+foreachStat returns [StringTemplate code=template("foreach")] {
+ StringTemplate ds=null;
+ StringTemplate body=template("statementList");
+}
+ : "foreach" id:ID (COMMA indexId:ID)? "in" ds=expression {
+ code.setAttribute("var", id.getText());
+ code.setAttribute("in", ds);
+ if (indexId != null) {
+ code.setAttribute("index", indexId.getText());
+ }
+ }
+ compoundStat[body] {code.setAttribute("body", body);}
+ ;
+----
+
+Iterate statement
+~~~~~~~~~~~~~~~~~
+----
+iterateStat returns [StringTemplate code=template("iterate")]
+{
+ StringTemplate cond=null;
+ StringTemplate body=template("statementList");
+}
+ : "iterate" id:ID
+ compoundStat[body] {code.setAttribute("body", body);}
+ "until" LPAREN cond=expression RPAREN SEMI
+ {
+ code.setAttribute("var", id.getText());
+ code.setAttribute("cond", cond);
+ }
+ ;
+----
+
+Switch statement
+~~~~~~~~~~~~~~~~
+
+----
+switchStat returns [StringTemplate code=template("switch")]
+{
+ StringTemplate cond=null, b=null;
+}
+ : "switch" LPAREN cond=expression RPAREN
+ {code.setAttribute("cond", cond);}
+ LCURLY
+ ( b = casesGroup {code.setAttribute("cases", b);} )*
+ RCURLY
+ ;
+----
+
+Cases Group
+~~~~~~~~~~~
+----
+casesGroup returns [StringTemplate code=template("case")]
+{StringTemplate b=null;}
+ : ( // CONFLICT: to which case group do the statements bind?
+ // ANTLR generates proper code: it groups the
+ // many "case"/"default" labels together then
+ // follows them with the statements
+ options {
+ greedy = true;
+ }
+ :
+ aCase[code]
+ )
+ caseSList[code]
+ ;
+
+aCase [StringTemplate code]
+{StringTemplate v=null;}
+ : (
+ "case" v=expression {code.setAttribute("value", v);}
+ | "default"
+ )
+ COLON
+ ;
+----
+
+Case Slist
+~~~~~~~~~~
+----
+caseSList [StringTemplate code]
+{StringTemplate s=null;}
+ : ( caseInnerStatement[code] )*
+ ;
+----
+
+Predict assignment statement
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+----
+predictAssignStat
+{StringTemplate x=null;}
+ : x=identifier ASSIGN ;
+----
+
+Predict append statement
+~~~~~~~~~~~~~~~~~~~~~~~~
+----
+predictAppendStat
+{StringTemplate x=null;}
+ : x=identifier APPEND ;
+----
+
+Assignment statement
+~~~~~~~~~~~~~~~~~~~~
+----
+assignStat returns [StringTemplate code=null]
+{StringTemplate id=null;}
+ :
+ id=identifier
+ ASSIGN
+ (
+ (predictProcedurecallAssign) => code=procedurecallCode
+ { StringTemplate o = template("returnParam");
+ o.setAttribute("name",id);
+ code.setAttribute("outputs",o);
+ }
+ |
+ code=variableAssign
+ {
+ code.setAttribute("lhs",id);
+ }
+ )
+ ;
+----
+
+Append statement
+~~~~~~~~~~~~~~~~
+----
+appendStat returns [ StringTemplate code = null ]
+ { StringTemplate id=null; }
+:
+ id=identifier
+ APPEND
+ (
+ (predictProcedurecallAssign) => code=procedurecallCode {
+ StringTemplate o = template("returnParam");
+ o.setAttribute("name",id);
+ code.setAttribute("outputs",o);
+ }
+ |
+ code=arrayAppend {
+ code.setAttribute("array", id);
+ }
+ )
+;
+----
+
+Array append statement
+~~~~~~~~~~~~~~~~~~~~~~
+----
+arrayAppend returns [ StringTemplate code = null ]
+ { StringTemplate a = null, e = null, id = null; }
+:
+ e = expression SEMI {
+ code = template("append");
+ code.setAttribute("value", e);
+ }
+;
+----
+
+Variable Assign statement
+~~~~~~~~~~~~~~~~~~~~~~~~~
+----
+variableAssign returns [StringTemplate code=null]
+{StringTemplate a=null, e=null, id=null;}
+ :
+ e=expression SEMI {
+ code=template("assign");
+ code.setAttribute("rhs", e);
+ }
+ ;
+----
+
+Procedure call
+~~~~~~~~~~~~~~
+----
+procedurecallCode returns [StringTemplate code=template("call")]
+{StringTemplate f=null;}
+ :
+ procedureInvocation[code]
+ ;
+----
+
+procedure invocation
+~~~~~~~~~~~~~~~~~~~~
+----
+procedureInvocation [StringTemplate code]
+ :
+ procedureInvocationWithoutSemi[code]
+ SEMI
+ ;
+----
+
+procedure invocation without semicolon
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+----
+procedureInvocationWithoutSemi [StringTemplate code]
+{StringTemplate f=null;}
+ :
+ id:ID {code.setAttribute("func", id.getText());}
+ LPAREN
+ ( f=actualParameter {
+ code.setAttribute("inputs", f);
+ }
+ ( COMMA f=actualParameter {
+ code.setAttribute("inputs", f);
+ }
+ )*
+ )?
+ RPAREN
+ ;
+----
+
+procedure invocation expression
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+----
+procedureInvocationExpr [StringTemplate code]
+{StringTemplate f=null;}
+ :
+ id:ID {code.setAttribute("func", id.getText());}
+ LPAREN (
+ f=actualParameter {
+ code.setAttribute("inputs", f);
+ }
+ (
+ COMMA f=actualParameter {
+ code.setAttribute("inputs", f);
+ }
+ )*
+ )?
+ RPAREN
+ ;
+----
+
+procedure call expression
+~~~~~~~~~~~~~~~~~~~~~~~~~
+----
+procedureCallExpr returns [StringTemplate code=template("call")]
+{StringTemplate f=null;}
+ :
+ procedureInvocationExpr[code]
+ ;
+----
+
+predict procedure call
+~~~~~~~~~~~~~~~~~~~~~~
+----
+predictProcedurecallDecl : ASSIGN ID LPAREN ;
+
+procedurecallDecl [StringTemplate container, StringTemplate type, StringTemplate decl, StringTemplate var]
+{
+StringTemplate code=template("call");
+StringTemplate f=template("returnParam");
+
+StringTemplate declref=template("variableReference");
+declref.setAttribute("name",decl);
+f.setAttribute("name", declref);
+code.setAttribute("outputs", f);
+container.setAttribute("statements",code);
+
+}
+ :
+ ASSIGN
+ procedureInvocationWithoutSemi[code]
+ ;
+----
+
+
+----
+procedurecallStatAssignManyReturnParam [StringTemplate s]
+{ StringTemplate code=template("call"); }
+ :
+ LPAREN
+ procedurecallStatAssignManyReturnOutput[s,code]
+ ( COMMA procedurecallStatAssignManyReturnOutput[s,code] )*
+ RPAREN
+ ASSIGN
+ procedureInvocation[code]
+ {
+ s.setAttribute("statements",code);
+ }
+ ;
+----
+
+----
+procedurecallStatAssignManyReturnOutput [StringTemplate s, StringTemplate code]
+{StringTemplate var = null, f = null; }
+ :
+ f=returnParameter
+ {
+ code.setAttribute("outputs", f);
+ var = template("variable");
+ if(f.getAttribute("type") != null) {
+ StringTemplate nameST = (StringTemplate)f.getAttribute("name");
+ var.setAttribute("name",nameST.getAttribute("name"));
+ var.setAttribute("type",f.getAttribute("type"));
+ var.setAttribute("global", Boolean.FALSE);
+
+ s.setAttribute("statements",var);
+ }
+ }
+ ;
+----
+
+----
+returnParameter returns [ StringTemplate code = template("returnParam") ]
+ { StringTemplate t = null, id = null, d = null; StringBuilder buf = new StringBuilder(); }
+:
+ t = identifier { buf.append(t.getAttribute("name")); }
+ (typeSubscript[buf])*
+ ( id = identifier )?
+ {
+ if (id == null) {
+ code.setAttribute("name", t);
+ }
+ else {
+ t = template("type");
+ t.setAttribute("name", buf.toString());
+ code.setAttribute("name", id);
+ code.setAttribute("type", t);
+ }
+ }
+ ((ASSIGN declarator) => (ASSIGN d=declarator) {
+ code.setAttribute("bind", d);
+ })?
+;
+----
+
+----
+actualParameter returns [StringTemplate code=template("actualParam")]
+{StringTemplate d=null, id=null, ai=null;}
+ :
+ (
+ (declarator ASSIGN)=> (d=declarator ASSIGN)
+ {
+ code.setAttribute("bind", d);
+ }
+ )?
+ id=expression {
+ code.setAttribute("value", id);
+ }
+ ;
+----
+
+Atomic Procedure Body
+~~~~~~~~~~~~~~~~~~~~~
+----
+atomicBody [StringTemplate code]
+{StringTemplate app=null, svc=null;}
+ : app=appSpec
+ {code.setAttribute("config",app);}
+ ;
+----
+
+/* This is the deprecated format for app { } blocks */
+----
+appSpec returns [StringTemplate code=template("app")]
+{StringTemplate exec=null;}
+ : "app" LCURLY
+ exec=declarator
+ { code.setAttribute("exec", exec);}
+ (
+ appArg[code]
+ )*
+ SEMI RCURLY
+ ;
+----
+
+App Args
+~~~~~~~~
+----
+appArg [StringTemplate code]
+{StringTemplate arg=null;}
+ : arg=mappingExpr
+ {code.setAttribute("arguments", arg);}
+ |
+ stdioArg[code]
+ ;
+----
+
+Mapping Expression
+~~~~~~~~~~~~~~~~~~
+----
+mappingExpr returns [StringTemplate code=null]
+{StringTemplate e=null;}
+ :
+ e = expression
+ {
+ code=template("mappingExpr");
+ code.setAttribute("expr", e);
+ }
+ ;
+----
+
+Function Invocation
+~~~~~~~~~~~~~~~~~~~
+----
+functionInvocation returns [StringTemplate code=template("functionInvocation")]
+{StringTemplate func=null, e=null;}
+ : AT (
+ (declarator LPAREN) =>
+ (func=declarator
+ {
+ code.setAttribute("name", func);
+ }
+ LPAREN
+ (
+ functionInvocationArgument[code]
+ (
+ COMMA
+ functionInvocationArgument[code]
+ )*)?
+ RPAREN
+ )
+ |
+ (e=identifier | (LPAREN e=identifier RPAREN) )
+ {
+ code.setAttribute("name", "filename");
+ code.setAttribute("args", e);
+ })
+ ;
+----
+
+----
+functionInvocationArgument [StringTemplate code]
+{StringTemplate e = null;}
+ :
+ e=expression
+ {
+ code.setAttribute("args", e);
+ }
+ ;
+----
+
+Standard IO Arg
+~~~~~~~~~~~~~~~
+----
+stdioArg [StringTemplate code]
+{StringTemplate t=null,m=null; String name=null;}
+ : ("stdin" {t=template("stdin"); name="stdin";}
+ |
+ "stdout" {t=template("stdout"); name="stdout";}
+ |
+ "stderr" {t=template("stderr"); name="stderr";}
+ )
+ ASSIGN
+ m=mappingExpr
+ {
+ t.setAttribute("content", m);
+ code.setAttribute(name, t);
+ }
+ ;
+----
+
+Expression
+~~~~~~~~~~
+----
+expression returns [StringTemplate code=null]
+ : code=orExpr
+ ;
+----
+
+----
+orExpr returns [StringTemplate code=null]
+{StringTemplate a,b;}
+ : code=andExpr
+ ( OR b=andExpr
+ {
+ a = code;
+ code=template("or");
+ code.setAttribute("left", a);
+ code.setAttribute("right", b);
+ }
+ )*
+ ;
+----
+
+----
+andExpr returns [StringTemplate code=null]
+{StringTemplate a,b;}
+ : code=equalExpr
+ ( AND b=equalExpr
+ {
+ a = code;
+ code=template("and");
+ code.setAttribute("left", a);
+ code.setAttribute("right", b);
+ }
+ )*
+ ;
+----
+
+----
+equalExpr returns [StringTemplate code=null]
+{
+StringTemplate a,b=null;
+Token op=null;
+}
+ : code=condExpr
+ (
+ {op=LT(1);}
+ ( EQ | NE ) b=condExpr
+ {
+ a = code;
+ code=template("cond");
+ code.setAttribute("op", escape(op.getText()));
+ code.setAttribute("left", a);
+ code.setAttribute("right", b);
+ }
+ )?
+ ;
+----
+
+Conditional Expression
+~~~~~~~~~~~~~~~~~~~~~~
+----
+condExpr returns [StringTemplate code=null] {
+StringTemplate a,b=null;
+Token op=null;
+} : code=additiveExpr
+ (
+ options {
+ greedy = true;
+ //warnWhenFollowAmbig = false;
+ }
+ :
+ {op=LT(1);}
+ ( LT | LE | GT | GE ) b=additiveExpr
+ {
+ a = code;
+ code=template("cond");
+ code.setAttribute("op", escape(op.getText()));
+ code.setAttribute("left", a);
+ code.setAttribute("right", b);
+ }
+ )?
+ ;
+----
+
+
+----
+additiveExpr returns [StringTemplate code=null]
+{
+StringTemplate a,b=null;
+Token op=null;
+}
+ : code=multiExpr
+ (
+ options {
+ greedy = true;
+ //warnWhenFollowAmbig = false;
+ }
+ :
+ {op=LT(1);}
+ ( PLUS | MINUS ) b=multiExpr
+ {
+ a = code;
+ code=template("arith");
+ code.setAttribute("op", escape(op.getText()));
+ code.setAttribute("left", a);
+ code.setAttribute("right", b);
+ }
+ )*
+ ;
+----
+
+----
+multiExpr returns [StringTemplate code=null]
+{
+StringTemplate a,b=null;
+Token op=null;
+}
+ : code=unaryExpr
+ (
+ options {
+ greedy = true;
+ //warnWhenFollowAmbig = false;
+ }
+ :
+ {op=LT(1);}
+ ( STAR | IDIV | FDIV | MOD ) b=unaryExpr
+ {
+ a = code;
+ code=template("arith");
+ code.setAttribute("op", escape(op.getText()));
+ code.setAttribute("left", a);
+ code.setAttribute("right", b);
+ }
+ )*
+ ;
+----
+
+----
+unaryExpr returns [StringTemplate code=null]
+{StringTemplate u=null;}
+ : MINUS u=unaryExpr
+ {code=template("unaryNegation"); code.setAttribute("exp", u);}
+ | PLUS u=unaryExpr // unary plus has no effect
+ {code=u;}
+ | NOT u=unaryExpr
+ {code=template("not"); code.setAttribute("exp", u);}
+ | code=primExpr
+ ;
+----
+
+----
+primExpr returns [StringTemplate code=null]
+{StringTemplate id=null, exp=null;}
+ : (predictProcedureCallExpr) => code=procedureCallExpr
+ | code=identifier
+ | LPAREN exp=orExpr RPAREN { code=template("paren");
+ code.setAttribute("exp", exp);}
+ | code=constant
+ | code=functionInvocation
+ ;
+----
+
+----
+predictProcedureCallExpr
+ : ID LPAREN ;
+
+----
+
+Identifier
+~~~~~~~~~~
+----
+identifier returns [StringTemplate code=null]
+{
+ StringTemplate c=null;
+ code=template("variableReference");
+}
+ :
+ base:ID {code.setAttribute("name",base.getText());}
+
+ ( ( c=arrayIndex { c.setAttribute("array",code); code=c; } )
+ |
+ ( c=memberName {c.setAttribute("structure",code); code=c;} )
+ )*
+ ;
+----
+
+Array Index
+~~~~~~~~~~~
+----
+arrayIndex returns [StringTemplate code=null]
+{StringTemplate e=null;}
+ :
+ LBRACK
+ (e=expression | s:STAR)
+ RBRACK
+ {
+ code=template("arraySubscript");
+ if(e != null) code.setAttribute("subscript",e);
+ if(s != null) {
+ StringTemplate st = template("sConst");
+ st.setAttribute("value","*");
+ code.setAttribute("subscript",st);
+ }
+ }
+ ;
+----
+
+Array Member
+~~~~~~~~~~~~
+----
+memberName returns [StringTemplate code=null] :
+ d:DOT (member:ID | s:STAR) {
+ code=template("memberAccess");
+ if(member != null) code.setAttribute("name",member.getText());
+ if(s != null) code.setAttribute("name","*");
+ }
+ ;
+----
+
+Constant
+~~~~~~~~
+----
+constant returns [StringTemplate code=null]
+ : i:INT_LITERAL
+ {
+ code=template("iConst");
+ code.setAttribute("value",i.getText());
+ }
+ | d:FLOAT_LITERAL
+ {
+ code=template("fConst");
+ code.setAttribute("value",d.getText());
+ }
+ | s:STRING_LITERAL
+ {
+ code=template("sConst");
+ code.setAttribute("value",quote(escape(s.getText())));
+ }
+ | t:"true"
+ {
+ code=template("bConst");
+ code.setAttribute("value", t.getText());
+ }
+ | f:"false"
+ {
+ code=template("bConst");
+ code.setAttribute("value", f.getText());
+ }
+ | code=arrayInitializer
+ ;
+----
+
+Literal Symbols
+~~~~~~~~~~~~~~~
+
+----
+AT : "@" ;
+PLUS : "+" ;
+MINUS : '-' ;
+FDIV : '/' ;
+IDIV : "%/" ;
+MOD : "%%" ;
+EQ : "==" ;
+NE : "!=" ;
+LT : '<' ;
+LE : "<=" ;
+GT : ">" ;
+GE : ">=";
+APPEND : "<<";
+ASSIGN : '=' ;
+AND : "&&";
+OR : "||";
+NOT : "!";
+LBRACK options { paraphrase = "'['"; } : '[' ;
+RBRACK options { paraphrase = "']'"; } : ']' ;
+LPAREN options { paraphrase = "'('"; } : '(' ;
+RPAREN options { paraphrase = "')'"; } : ')' ;
+LCURLY options { paraphrase = "'{'"; } : '{' ;
+RCURLY options { paraphrase = "'}'"; } : '}' ;
+SEMI options { paraphrase = "a semicolon"; } : ';' ;
+
+----
+
+Id
+
+----
+ID options
+ {
+ paraphrase = "an identifier";
+ testLiterals = true;
+ }
+ :
+ ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*
+ ;
+----
+
+----
+// string literals
+STRING_LITERAL
+// : '"'! (~('"'|'\n'|'\r'))* '"'!
+ : '"'! (escape_sequence|~('"'|'\\'|'\n'|'\r'))* '"'!
+ ;
+
+NUMBER
+ :
+ ( INTPART {_ttype=INT_LITERAL; }
+ ('.' FLOATPART {_ttype=FLOAT_LITERAL; })?
+ (EXPONENT {_ttype=FLOAT_LITERAL; })?
+ )
+ |
+ ( '.' { _ttype=DOT; }
+ ((FLOATPART {_ttype=FLOAT_LITERAL; })
+ (EXPONENT)?)?
+ )
+ ;
+
+whitespace : ( ' '
+ | '\t'
+ | '\r'
+ | '\n' {newline();}
+ )+
+ { $setType(Token.SKIP); }
+ ;
+
+//Single line comment C/C++ style
+singleline_ccomment:
+ "//"
+ (~('\n'|'\r'))* ('\n'|'\r'('\n')?)
+ {$setType(Token.SKIP); newline();}
+ ;
+
+//Single line comment shell style
+singleline_scomment:
+ "#"
+ (~('\n'|'\r'))* ('\n'|'\r'('\n')?)
+ {$setType(Token.SKIP); newline();}
+ ;
+
+//Multiline comment C style
+multiline_comment :
+ "/*"
+ (
+ options {
+ generateAmbigWarnings=false;
+ }
+ :
+ { LA(2)!='/' }? '*'
+ | '\r' '\n' {newline();}
+ | '\r' {newline();}
+ | '\n' {newline();}
+ | ~('*'|'\n'|'\r')
+ )*
+ "*/"
+ {$setType(Token.SKIP);}
+ ;
+
+escape_sequence :
+ '\\'
+ ( 'n'
+ | 'r'
+ | 't'
+ | 'b'
+ | 'f'
+ | '"'
+ | '\''
+ | '\\'
+ )
+ ;
+
+----
Copied: branches/faster/docs/merged/refmanual/swift.css (from rev 6618, trunk/docs/merged/refmanual/swift.css)
===================================================================
--- branches/faster/docs/merged/refmanual/swift.css (rev 0)
+++ branches/faster/docs/merged/refmanual/swift.css 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1,6 @@
+
+/* SWIFT/T GUIDE CUSTOMIZATIONS */
+
+a:visited {
+ color: black;
+}
Copied: branches/faster/docs/merged/refmanual/swiftlang (from rev 6618, trunk/docs/merged/refmanual/swiftlang)
===================================================================
--- branches/faster/docs/merged/refmanual/swiftlang (rev 0)
+++ branches/faster/docs/merged/refmanual/swiftlang 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1,486 @@
+
+== Swift Language Reference Manual
+This reference manual semi-formally outlines the Swift language conventions
+
+== Support
+
+An overview of Swift may be found at:
+
+http://www.ci.uchicago.edu/swift/guides/trunk/userguide/userguide.html
+
+The Swift user discussion mailing list is found here:
+
+https://lists.ci.uchicago.edu/cgi-bin/mailman/listinfo/swift-user
+
+== Usage
+
+Swift code is conventionally written in +\*.swift+ files.
+After writing the Swift program
++program.swift+, run:
+
+----
+swift -sites.file <sites.xml> -config <cf> -tc.file <tc.data> <program.swift>
+----
+
+Swift accepts many commandline arguments. Following are the most frequently used:
+
++-config <file>+ :: Indicates the Swift configuration file to be used for this
+run. Properties in this configuration file will override the default
+properties. If individual command line arguments are used for properties, they
+will override the contents of this file.
+
++-sites.file <file>+ :: Points to the location of the sites.xml file
+
++-tc.file <file>+ :: Points to the location of the tc.data file
+
++-resume <file>+ :: Resumes the execution using a log file
+
+== Program Structure
+
+Swift programs are composed of declaration of and calls to _composite_
+functions. These share syntax with C-like languages. The following is a
+complete Swift program:
+
+----
+tracef("%s: %i\n", "The answer is", 42);
+----
+
+== Comments
+
+Swift supports C/C++ comments:
+
+----
+# This is a comment
+// This is a comment
+/* This is a
+comment */
+/** Also a
+comment */
+----
+
+== Dataflow Evaluation
+
+Swift expressions are evaluated in _dataflow_ order:
+
+----
+int z1,z2;
+int y;
+int x = f(y);
+y = g(2);
+z1 = h(x,y,1);
+z2 = h(x,y,2);
+int output = r(z1,z2);
+----
+
+This allows code to execute as concurrently as possible, limited
+only by data availability. In this example, +g()+ runs first, because it
+is dependent only on a literal. When +y+ is set, +f()+ runs, setting
++x+. Then, two invocations of +h()+ execute. Finally, +z1+ and +z2+
+are set, allowing +r()+ to run.
+
+Variables may be assigned only once. Multiple assignment is typically
+detected at compile time, although in some cases it will result in a
+run time error. Unassigned variables that are inputs to functions
+will lead to a stall in progress.
+
+== Composite functions
+
+Swift code is written in composite functions.
+
+----
+[(<output list>)] function_name [(<input list>)]
+{
+ statement;
+ statement;
+ ...
+}
+----
+
+An example of a Swift composite function is shown below:
+
+----
+(Station _stat) get_site(int _run_id) {
+ StationFile file<"/tmp/site_tmp">;
+ file = getsite_file(_run_id);
+ _stat = readData(file);
+}
+----
+
+== App functions
+Swift app functions are used to integrate external apps into Swift and made
+callable as Swift functions. App functions have the form:
+
+----
+[(<output list>)] app function_name [(<input list>)]
+{
+ statement;
+}
+----
+
+An example of an app function definition is as follows:
+
+----
+(RuptureFile _rup) app getrupture_file(int _run_id) {
+ getrupture _run_id stdout=@filename(_rup);
+}
+----
+
+Note that an app function may have only one commandline statement while a
+composite function may have multiple statements.
+
+An empty input or output list may be omitted or written as +()+.
+
+The output list may have more than one entry. Thus, assignments
+may be written as:
+----
+x1, x2 = f(i1, i2);
+// or equivalently:
+(x1, x2) = f(i1, i2);
+----
+
+=== Swift keywords
+
+Following is a list of Swift keywords:
+
+----
+ app type float int boolean string
+ global foreach if else iterate
+ switch case default file while import
+----
+
+== Types
+
+Swift provides a similar range of primitive types to many other programming
+languages. Files are a primitive type in Swift, unlike in many other languages,
+and have a number of special characteristics that merit special mention. Two
+basic kinds of data structure are provided: arrays and structs.
+
+=== Primitive Types
+
+Swift has the conventional types:
+
++string+:: A complete string (not an array of characters).
++int+:: A 64-bit integer.
++float+:: A 64-bit (double-precision) floating point number.
++boolean+:: A boolean (true/false).
++file+:: A file (see Section Files).
+
+Literals for these types use conventional syntax:
+
+* +int+ literals are written as decimal numbers, e.g. +-1234+
+* +float+ literals are written as decimal numbers with a decimal point,
+ e.g +5493.352+ or +1.0+.
+ The literals +NaN+ and +inf+ may be used. In some contexts +int+
+ literals are promoted automatically to +float+.
+* +boolean+ literals
+* +string+ literals are enclosed in double quotes, with a range of escape
+ sequences supported:
+** \\ for a single backslash
+** \" for a quote
+** \n for newline
+** \t for tab
+** \a
+** \b
+** \f
+** \r
+** \v
+** hexadecimal escape codes, e.g. \xf2
+
+The literals +true+ and +false+ may be used for boolean.
+
+----
+int four = 2+2;
+string s = "hello ";
+string hw = s + "world";
+----
+
+=== Files
+
+A file is a first-class entity in Swift that in many ways can be treated
+as any other variable. The main difference is that a file can be
+*mapped* to path in a filesystem. Assigning to a mapped file variable
+results in a file being created in the file system at the specified path.
+File variables can also be initialized with data from a pre-existing
+file using the +input_file+ function. File paths are relative to the
+working directory.
+
+For example, if +/home/user/in.txt+ is a file with some data in it,
+the following Swift program will copy the file to +/home/user/out.txt+.
+----
+ type file;
+
+ app (file o) cp_app(file i){
+ cp @i @o;
+ }
+
+ file x <"/home/user/in.txt">;
+ file y <"/home/user/out.txt">;
+
+ y = cp_app(x);
+----
+
+=== Arrays
+
+Arrays are declared with empty square brackets:
+
+----
+int A[];
+----
+
+Arrays are indexed using square brackets.
+
+Each array index can only be assigned to once.
+
+Arrays may be used as inputs or outputs of functions.
+
+Arrays are part of Swift dataflow semantics. An array is closed
+when all possible insertions to it are complete.
+
+----
+int A[];
+
+foreach i in [1:10] {
+ A[i] = i;
+ tracef("Element %i is %i\n", i, A[i]);
+}
+----
+
+Array literals may be expressed with list syntax:
+----
+int C[] = [4,5,6];
+----
+
+=== Structs
+
+In Swift, structs are defined with the +type+ keyword. They define
+a new type.
+
+----
+type person
+{
+ string name;
+ int age;
+ int events[];
+}
+----
+
+Structs are accessed with the +.+ syntax:
+
+----
+person p;
+p.name = "Abe";
+p.age = 90;
+----
+
+=== Defining new types
+
+New types can be defined with +type+, which creates a new type that is a
+specialization of an existing type. That is, it is a distinct type that is not
+interchangeable.
+
+----
+type messagefile;
+----
+
+== Control structures
+
+Swift provides control structures that may be placed as statements
+inside a composite function.
+
+=== Conditionals
+
+==== If statement
+
+If statements have the form:
+
+----
+if (<condition>)
+{
+ statement;
+ ...
+}
+else
+{
+ statement;
+ ...
+}
+----
+
+As required by dataflow processing, progress blocks on the availability
+of the condition value.
+
+==== Switch statement
+
+----
+int a = 20;
+switch (a)
+{
+ case 1:
+ int c;
+ c = a + a;
+ b = a + 1;
+ case 20:
+ b = 1;
+ case 2000:
+ b = 2;
+ default:
+ b = 2102 + 2420;
+}
+tracef("b: %i\n", b);
+----
+
+=== Iteration
+
+Iteration in parallel is performed with the +foreach+ and +iterate+ statement.
+
+==== Foreach loop
+
+The +foreach+ loop allows for parallel iteration over an array:
+
+----
+foreach value[, index] in [expression]{
+ statement;
+ statement;
+ ...
+}
+----
+
+The +index+ and +value+ variables are automatically declared. The
++index+ variable may be omitted from the syntax.
+
+A special case of the foreach loop occurs when combined with the
+array range operator. This is the idiomatic way to iterate over
+a range of integer values in Swift. The Swift compiler has special
+handling for this case that avoids constructing an array.
+
+----
+foreach i in [start:stop:step] {
+ ...
+}
+----
+
+Examples of +foreach+ loop are as follows:
+
+----
+string A[];
+foreach value, index in A
+{
+ tracef("A[%i] = %s\n", index, value);
+}
+----
+
+----
+trace(" First five odd numbers");
+foreach idx in [1:10:2]{
+ tracef("%i\n",idx);
+}
+----
+
+==== Iterate loop
+
+The +iterate+ loop allows for sequential iteration.
+
+----
+iterate months {
+ tracef("%i\n", months);
+ } until (months==12);
+----
+
+The general form is:
+----
+iterate var
+{
+ statement;
+ ...
+} until (expression);
+----
+
+*Performance Tip:* use the +foreach+ loop instead of +iterate+ if your
+loop iterations are independent and can be executed in parallel.
+
+== Operators
+
+The following binary arithmetic operators on numbers are defined:
+
++++ (plus), +-+ (minus), +\*+ (times), +/+ (divide),
++%/+ (integer divide), +%%+ (modulus), +**+ (power)
+
++&&+ (boolean and), +||+ (boolean or),
++==+ (equals), +!=+ (not equals), +>+ (greater than), +<+ (less than),
++>=+ (greater than or equal to), +<=+ (less than or equal to)
+
+The following unary operators are defined:
+
++-+ (negate), +!+ (boolean not)
+
+String concatenation is also performed with +++ (plus).
++==+ and +!=+ may also be used on strings.
+
+== Defining Apps
+
+In typical Swift applications, the computationally intensive parts of the
+application are not written in the Swift language. Rather,
+the work is done by _app functions_ that are _composed_ together with
+Swift code. The builtin functions mentioned above are implemented as extension
+functions in Tcl.
+
+=== App functions
+
+App functions are functions that are implemented as command-line
+programs. These command-line programs can be brought into a Swift
+program as functions with typed inputs and outputs.
+
+An app function definition comprises:
+
+* The standard components of a Swift function declaration: input and
+ output arguments and the function name. Note that the output
+ variable types are restricted to individual +file+#s#.
+* The command line, which comprises an initial string which is the
+ executable to run, and then a series of arguments which are
+ the command-line arguments to pass to the program.
+
+App arguments can be:
+
+* Literals such as numbers or strings.
+* File variables (passed as file paths).
+* Other variables, which are converted to string arguments.
+ Arrays (including multi-dimensional arrays) are expanded to
+ multiple arguments.
+* Arbitrary expressions surrounded by parentheses.
+
+Standard input, output and error can be redirected to files.
+
+Join files together:
+----
+type file;
+
+app (file out) cat (file inputs[]) {
+ "/bin/cat" inputs @stdout=out
+}
+
+main {
+ file joined <"joined.txt"> = cat(glob("*.txt"));
+}
+----
+
+Sleep an arbitrary amount of time:
+
+----
+app (void signal) sleep (int secs) {
+ "/bin/sleep" secs
+}
+
+ foreach time in [1:5] {
+ void signal = sleep(time);
+ // Wait on output signal so that trace occurs after sleep
+ wait(signal) {
+ trace("Slept " + fromint(time));
+ }
+ }
+----
+
+////
+Local Variables:
+mode: doc
+End:
+////
Modified: branches/faster/docs/merged/userguide/coasters
===================================================================
--- branches/faster/docs/merged/userguide/coasters 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/merged/userguide/coasters 2013-07-08 08:19:11 UTC (rev 6619)
@@ -3,7 +3,7 @@
Introduction
~~~~~~~~~~~~
-Coasters are the Swift's implementation of _pilot job abstraction_.
+Coasters are the Swift's implementation of the _pilot job abstraction_.
In many applications, Swift performance can be greatly enhanced by the use of
coasters. Coasters provide a low-overhead job submission and file transfer
@@ -39,7 +39,7 @@
To use for job execution, specify a sites.xml execution element like this:
----
-<execution provider="coaster" jobmanager="gt2:gt2:pbs" url="grid.myhost.org">
+<execution provider="coaster" jobmanager="local:pbs" url="grid.myhost.org">
----
The jobmanager string contains more detail than with other providers. It
@@ -55,7 +55,7 @@
To use for file transfer, specify a sites.xml filesystem element like this:
----
-<filesystem provider="coaster" url="gt2://grid.myhost.org" />
+<filesystem provider="coaster" url="gftp://grid.myhost.org" />
----
The url parameter should be a pseudo-URI formed with the URI scheme
@@ -93,10 +93,11 @@
Coasters
--------
Coasters were introduced in Swift v0.6 as an experimental feature. In many
-applications, Swift performance can be greatly enhanced by the use of
-coasters. Coasters provide a low-overhead job submission and file transfer
-mechanism suited for the execution of short jobs (on the order of a few
-seconds). A detailed information on coasters can be found at http://www.ci.uchicago.edu/swift/guides/userguide.php#coasters.
+applications, Swift performance can be greatly enhanced by the use of coasters.
+Coasters provide a low-overhead job submission and file transfer mechanism
+suited for the execution of short jobs (on the order of a few seconds). A
+detailed information on coasters can be found at
+http://www.ci.uchicago.edu/swift/guides/userguide.php#coasters.
//**Include neat diagrams.**
Following is a coasters setup case-study for a PBS underlying provider where sites.xml coaster settings were:
@@ -129,7 +130,7 @@
<profile namespace="karajan" key="initialScore">10000</profile>
source~~~~
-The following table briefly describes the elements on the coasters setup:
+The following table briefly describes the elements of the coasters setup:
[width="70%", cols="^3,10", options="header"]
|=============================================================================================
|profile key | brief description
@@ -144,59 +145,22 @@
|jobThrottle |the number of concurrent jobs allowed on a site
|==============================================================================================
-//
-//For Beginners
-//~~~~~~~~~~~~~~
-//Coasters for beginners. Usage of existing, prebuilt templates.
-//
-//For Intermediate Users
-//~~~~~~~~~~~~~~~~~~~~~~~
-//Coasters for intermediate users.
-//
-//Using gensites
-//^^^^^^^^^^^^^^^
-//Usage of gensites to generate your own sites
-//configurations.
-//
-//
-//
+Advanced User-Guide
+~~~~~~~~~~~~~~~~~~~
-For Advanced Users
-~~~~~~~~~~~~~~~~~~
-
//Coasters for advanced users. Getting your hands dirty.
-One of the main reason that one would initially deviate from coaster
-defaults into more complex pool entries is to force jobs to fit into some
-site-imposed constraint. For instance a typical submission to the experimental
-queue requires a user to request upto 3 nodes for under 1 hour. This setup
-could be achieved with a careful tuning of coaters parameters.
+One of the main reason that one would initially deviate from coaster defaults
+into more complex pool entries is to force jobs to fit into some site-imposed
+constraint. For instance a typical submission to a fast PBS queue requires a
+user to request upto 3 nodes for under 1 hour. This setup could be achieved
+with a careful tuning of coaters parameters.
-//How to run Swift under different Coasters configurations
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-//
-//Manual
-//^^^^^^^
-//Todo
-//
-//Passive
-//^^^^^^^^
-//Todo
-//
-//Persistent
-//^^^^^^^^^^^
-//Todo
-//
-//Summary of Differences Between different Coaster types
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-//A tabular representations of highlights of different coaster setups
-//
-//
-//Data Management
-//
+Coasters configurations
+~~~~~~~~~~~~~~~~~~~~~~~
-Coaster providers: local, ssh, pbs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Coaster providers
+~~~~~~~~~~~~~~~~~
Settings and examples for different coaster providers mechanisms.
Modified: branches/faster/docs/quickstart/quickstart.txt
===================================================================
--- branches/faster/docs/quickstart/quickstart.txt 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/quickstart/quickstart.txt 2013-07-08 08:19:11 UTC (rev 6619)
@@ -14,9 +14,9 @@
In general, they are more stable, have fewer bugs, and have been tested on a variety of
systems.
-The development version of Swift is aimed at developers and testers. The development
-code has the highest chance of containing buggy and untested code. If you need stability
-please use the latest stable release.
+The development version of Swift is aimed at developers and testers. The
+development code has the highest chance of containing buggy and/or untested
+code. If you need stability, use the latest stable release.
Downloading a Swift Distribution
--------------------------------
@@ -64,7 +64,7 @@
directory is located. If you installed Swift from a binary release, it will
be in the swift-0.93/bin directory where you installed it. If you followed
the instructions above for installing Swift from a source repository, it
-will be located in swift-0.93/cog/modules/swift/dist/swift-svn/bin.
+will be located in swift-<version>/cog/modules/swift/dist/swift-svn/bin.
Add the following line to the bottom of ~/.bashrc:
@@ -82,11 +82,11 @@
Running Swift Examples
----------------------
The Swift examples can be found in the examples directory in the Swift
-distribution. The examples are written in the SwiftScript language, and
+distribution. The examples are written in the Swift scripting language, and
have .swift as a file extension.
-Execution of a Swift workflow is done using the swift command, which
-takes the Swift workflow file name as an argument:
+Execution of a Swift script is done using the swift command, which
+takes the Swift script file name as an argument:
-----
cd examples/tutorial
@@ -98,4 +98,4 @@
Swift script!
More documentation on how to run Swift can be found at
-http://www.ci.uchicago.edu/swift/docs/index.php,
+http://www.ci.uchicago.edu/swift/docs/index.php
Modified: branches/faster/docs/siteguide/beagle
===================================================================
--- branches/faster/docs/siteguide/beagle 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/siteguide/beagle 2013-07-08 08:19:11 UTC (rev 6619)
@@ -3,24 +3,34 @@
Beagle is a Cray XE6 supercomputer at UChicago. It employs a batch-oriented
computational model where-in a PBS schedular accepts user's jobs and queues
-them in the queueing system for execution. The computational model requires
-a user to prepare the submit files, track job submissions, chackpointing,
-managing input/output data and handling exceptional conditions manually.
-Running Swift under Beagle can accomplish the above tasks with least manual
-user intervention and maximal oppurtunistic computation time on Beagle
-queues. In the following sections, we discuss more about specifics of
-running Swift on Beagle. A more detailed information about Swift and its
+them in the queueing system for execution.
+
+The computational model requires a user to prepare the submit files, track job
+submissions, chackpointing, managing input/output data and handling exceptional
+conditions manually. Running Swift under Beagle can accomplish the above tasks
+with least manual user intervention and maximal oppurtunistic computation time
+on Beagle queues. In the following sections, we discuss more about specifics
+of running Swift on Beagle. A more detailed information about Swift and its
workings can be found on Swift documentation page here:
+
+----
http://www.ci.uchicago.edu/swift/wwwdev/docs/index.php
+----
+
More information on Beagle can be found on UChicago Beagle website here:
+
+----
http://beagle.ci.uchicago.edu
+----
Requesting Access
~~~~~~~~~~~~~~~~~
-If you do not already have a Computation Institute account, you can request
-one at https://www.ci.uchicago.edu/accounts/. This page will give you a list
-of resources you can request access to.
-You already have an existing CI account, but do not have access to Beagle,
+
+If you do not already have a Computation Institute (CI) account, you can
+request one at https://www.ci.uchicago.edu/accounts/. This page will give you a
+list of resources you can request access to.
+
+If you already have an existing CI account, but do not have access to Beagle,
send an email to support at ci.uchicago.edu to request access.
Connecting to a login node
@@ -32,9 +42,11 @@
ssh yourusername at login.beagle.ci.uchicago.edu
-----
+Getting Started with Swift
+~~~~~~~~~~~~~~~~~~~~~~~~~~
Follow the steps outlined below to get started with Swift on Beagle:
-*step 1.* Load the Swift module on Beagle as follows: +module load swift+
+*step 1.* Load the Swift and Sun-java module on Beagle as follows: +module load swift sun-java+
*step 2.* Create and change to a directory where your Swift related work will
stay. (say, +mkdir swift-lab+, followed by, +cd swift-lab+)
@@ -45,7 +57,7 @@
-----
type file;
-/* App definitio */
+/* App definition */
app (file o) cat (file i)
{
cat @i stdout=@o;
@@ -107,7 +119,8 @@
pbs cat /bin/cat null null null
-----
-More about config and tc file options can be found in the swift userguide here: http://www.ci.uchicago.edu/swift/wwwdev/guides/release-0.93/userguide/userguide.html#_swift_configuration_properties.
+More about config and tc file options can be found in the swift userguide here:
+http://www.ci.uchicago.edu/swift/guides/trunk/userguide/userguide.html#_swift_configuration_properties
*step 6.* Run the example using following commandline:
@@ -129,9 +142,6 @@
Final status: time: Sun, 18 Dec 2011 02:46:43 +0000 Finished successfully:1
----
-Note: Running from sandbox node or requesting 30 minutes walltime for upto 3 nodes
-will get fast prioritized execution. Suitable for small tests.
-
Larger Runs on Beagle
~~~~~~~~~~~~~~~~~~~~~
A key factor in scaling up Swift runs on Beagle is to setup the sites.xml parameters.
@@ -192,8 +202,9 @@
The most likely cause is the module is not loaded. Do the following to load the Swift module:
-----
-$ module load swift
-Swift version swift-0.93RC5 loaded
+$ module load swift sun-java
+Swift version swift-0.93 loaded
+sun-java version jdk1.7.0_02 loaded
-----
* Failed to transfer wrapperlog for job cat-nmobtbkk and/or Job failed with an exit code of 254. Check the <workdirectory> element on the sites.xml file.
Copied: branches/faster/docs/siteguide/ec2 (from rev 6618, trunk/docs/siteguide/ec2)
===================================================================
--- branches/faster/docs/siteguide/ec2 (rev 0)
+++ branches/faster/docs/siteguide/ec2 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1,177 @@
+Amazon EC2 Cloud
+----------------
+Amazon EC2 offers virtualized resources on demand from its cloud data centers.
+This section describes how to use Swift to run applications on EC2 resources.
+More information on EC2 can be found link:https://aws.amazon.com/ec2[here].
+
+Getting Access
+~~~~~~~~~~~~~~
+
+Users with EC2 Allocation
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you already have an EC2 allocation, you can use either the web console or command-line interface to manage resources.
+
+Before using EC2 for Swift you must create a security group in order to allow
+Swift communicate with EC2 instances via open ports. In order to create a
+security group, follow instructions
+link:http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html#creating-security-group[here].
+Preferably, open ports between 50000 and 51000.
+
+Paste your credentials from the EC2 web console into AWS_ACCESS_KEY and AWS_SECRET_KEY and export them in your .bashrc as follows:
+
+----
+export AWS_ACCESS_KEY=<your access key>
+export AWS_SECRET_KEY=<your secret key>
+----
+
+Command-line tools are pre-installed on +swift.rcc.uchicago.edu+. In order to use them add the following lines to your .bashrc:
+
+----
+export EC2_HOME=/home/maheshwari/.ec2
+export PATH=$PATH:$EC2_HOME/bin
+----
+
+A public AMI on EC2 is available to create instance. Everything needed to run
+Swift is preinstalled on this image. The id of this AMI is: ami-332e585a.
+
+Create instances from the above AMI as follows:
+
+----
+ec2run -k <keypair_name> -g <sec_group_name> -n <count> ami-332e585a
+----
+
+Users without EC2 Allocation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Please request access to prepared instances and PKI credentials by mailing: email at mcs.anl.gov.
+
+
+Configuring coaster-service.conf
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To run on EC2, you will need a file called coaster-service.conf.
+This file contains many options to control how things run. Here is
+an example of a working coaster-service.conf for EC2.
+
+-----
+#Location of SWIFT. If empty, PATH is referenced
+export SWIFT=
+
+# Where to place/launch worker.pl on the remote machine for sites.xml
+export WORKER_LOCATION=/root
+
+# How to launch workers: local, ssh, or cobalt
+export WORKER_MODE=ssh
+
+# Worker logging setting passed to worker.pl for sites.xml
+export WORKER_LOGGING_LEVEL=DEBUG
+
+export SSH_TUNNELING=yes
+
+# User name to use for all systems
+export WORKER_USERNAME=root
+
+# Worker host names for ssh
+# export WORKER_HOSTS="crush thwomp"
+
+# Directory to keep log files,
+#relative to working directory when launching start-coaster-service
+export LOG_DIR=logs
+export WORKER_LOG_DIR=/root
+
+# Manually define ports. If not specified, ports will be automatically generated
+export LOCAL_PORT=50100
+export SERVICE_PORT=50200
+
+# Set shared filesystem to no since work will be done in local /sandbox directory
+export SHARED_FILESYSTEM=no
+
+# Below are various settings to give information about how to create sites.xml
+export WORK=/tmp/swift.work
+export JOBSPERNODE=4
+export JOBTHROTTLE=$( echo "scale=5; ($JOBS_PER_NODE * $( echo $WORKER_HOSTS|wc -w ))/100 - 0.00001"|bc )
+
+# Swift applications
+#app cat=/bin/cat
+#app bash=/bin/bash
+-----
+Add a list of any applications you want to run in the format "#app myapp=/path/to/app".
+
+Starting the Coaster Service Script
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Log in to the +swift.rcc.uchicago.edu+ and copy the swift_cloud directory to your home:
+
+----
+cp -r ~maheshwari/swift_cloud $HOME/
+----
+
+Make sure Swift module is loaded and your credentials are added as follows:
+
+----
+module load swift
+----
+
+----
+ssh-add
+----
+
+Replace the hosts.txt found in +swift_cloud+ directory by adding the address/DNS of instances.
+
+With the ec2 commandline, the list of ip addresses of running instances can be obtained and added to hosts.txt as follows:
+
+----
+ec2din | grep running | awk '{print $15}' > hosts.txt
+----
+
+Now that everything is configured, run this command to start the coaster service:
+
+-----
+source setup
+-----
+
+This command will add the instances to coaster configuration, start the
+required processes on the worker nodes, and generate Swift configuration files
+for you to use. The configuration files will be generated in your current
+directory. These files are sites.xml, tc.data, and cf.
+
+Running Swift
+~~~~~~~~~~~~~
+
+
+
+Now that you have all of your configuration files generated, run the following command:
+
+-----
+$ swift -sites.file sites.xml -tc.file tc.data -config cf catsn.swift
+-----
+
+If you like to create a custom tc and/or cf file for repeated use, rename it to something other
+than tc.data/cf to prevent it from being overwritten. The sites.xml however will need to be
+regenerated every time you start the coaster service. If you need to repeatedly modify some
+sites.xml options, you may edit the template in Swift's etc/sites/persistent-coasters. You
+may also create your own custom tc files with the hostname of persistent-coasters. More
+information about this can be found in the Swift userguide at
+http://www.ci.uchicago.edu/swift/guides/trunk/userguide/userguide.html.
+
+Stopping the Coaster Service Script
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To stop the coaster service, run the following command:
+-----
+$ stop-coaster-service
+-----
+
+This will kill the coaster service, kill the worker scripts on remote systems.
+
+Terminate the virtual machines that were created for this work as follows:
+
+----
+ec2din|grep running|awk '{print $2}'|xargs ec2kill
+----
+
+More Help
+~~~~~~~~~
+The best place for additional help is the Swift user mailing list. You can subscribe to this list at
+http://mail.ci.uchicago.edu/mailman/listinfo/swift-user. When submitting information, send
+your sites.xml file, your tc.data, and any error messages you run into.
+
Modified: branches/faster/docs/siteguide/fusion
===================================================================
--- branches/faster/docs/siteguide/fusion 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/siteguide/fusion 2013-07-08 08:19:11 UTC (rev 6619)
@@ -1,10 +1,10 @@
Fusion (x86 cluster)
-------------------
-Fusion is a 320-node computing cluster for the Argonne
-National Laboratory community. The primary goal of the LCRC is to
-facilitate mid-range computing in all of the scientific programs of
-Argonne and the University of Chicago.
+Fusion is a 320-node computing cluster for the Argonne's Laboratory Computing
+Resource Center (LCRC). The primary goal of the LCRC is to facilitate mid-range
+computing in all of the scientific programs of Argonne and the University of
+Chicago.
This section will walk you through running a simple Swift script
on Fusion.
@@ -30,9 +30,10 @@
SSH Keys
~~~~~~~~
Before accessing Fusion, be sure to have your SSH keys configured correctly.
-SSH keys are required to access fusion. You should see information about
-this when you request your account. Email support at lcrc.anl.gov for
-additional help.
+SSH keys are required to access fusion. You should see information about this
+when you request your account. Check
+link:http://wiki.mcs.anl.gov/IT/index.php/SSH_keys[ssh FAQ] or email
+support at lcrc.anl.gov for additional help.
Connecting to a login node
~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -40,17 +41,17 @@
node with the following command:
-----
-ssh yourusername at fusion.lcrc.anl.gov
+ssh <yourusername>@fusion.lcrc.anl.gov
-----
Creating sites.xml
~~~~~~~~~~~~~~~~~~
-Swift relies on various configuration files to determine how to
-run. This section will provide a working configuration file which
+Swift uses various configuration files to determine how to
+run an application. This section will provide a working configuration file which
you can copy and paste to get running quickly. The sites.xml file
tells Swift how to submit jobs, where working directories are
located, and various other configuration information. More
-information on sites.xml can be found in the Swift User's Guide.
+information on sites.xml can be found in the Swift link:http://www.ci.uchicago.edu/swift/guides/trunk/userguide/userguide.html[user guide].
The first step is to paste the text below into a file named sites.xml.
@@ -60,18 +61,19 @@
This file will require one customization. Create a
directory called swiftwork. Modify \_WORK_ in sites.xml
-to point to this new directory. For example
+to point to a new directory. For example
+
-----
-<workdirectory>/home/myhome/swiftwork</workdirectory>
+<workdirectory>/tmp/swiftwork</workdirectory>
-----
-Creating tc.data
-~~~~~~~~~~~~~~~~
-The tc.data configuration file gives information about the applications
+Creating tc
+~~~~~~~~~~~
+The tc configuration file gives information about the applications
that will be called by Swift. More information about the format
-of tc.data can be found in the Swift User's guide.
+of tc can be found in the Swift user guide.
-Paste the following example into a file named tc.data
+Paste the following example into a file named tc
-----
include::../../tests/providers/fusion/coasters/tc.template.data[]
@@ -79,22 +81,26 @@
Copy a Swift Script
~~~~~~~~~~~~~~~~~~~~
-Within the Swift directory is an examples directory which contains
-several introductory Swift scripts. The example we will use in this
-section is called catsn.swift. Copy this script to the same directory
-that your sites.xml and tc.data files are located.
+Within the Swift directory is an examples directory which contains several
+introductory Swift scripts. The example we will use in this section is called
+catsn.swift. The script copies input file's content to another file using the
+Unix cat utility. Copy this script to the same directory that your sites.xml
+and tc.data files are located.
-----
$ cp ~/swift/examples/misc/catsn.swift .
$ cp ~/swift/examples/misc/data.txt .
-----
-TIP: The location of your swift directory may vary depending on how you installed it. Change this to the examples/misc directory of your installation as needed.
+TIP: The location of your swift directory may vary depending on how you
+installed it. Change this to the examples/misc directory of your installation
+as needed.
+
Run Swift
~~~~~~~~~
Finally, run the script
-----
-$ swift -sites.file sites.xml -tc.file tc.data catsn.swift
+$ swift -sites.file sites.xml -tc.file tc catsn.swift
-----
You should see 10 new text files get created, named catsn*.out. If
@@ -118,7 +124,10 @@
More Help
~~~~~~~~~
-The best place for additional help is the Swift user mailing list. You can subscribe to this list at
-https://lists.ci.uchicago.edu/cgi-bin/mailman/listinfo/swift-user. When submitting information, please send your sites.xml file, your tc.data, and any Swift log files that were created during your attempt.
+The best place for additional help is the Swift user mailing list. You can
+subscribe to this list at
+link:https://lists.ci.uchicago.edu/cgi-bin/mailman/listinfo/swift-user[swift-user].
+When submitting information, send your sites.xml file, your tc.data, and
+any Swift log files that were created during your attempt.
Modified: branches/faster/docs/siteguide/futuregrid
===================================================================
--- branches/faster/docs/siteguide/futuregrid 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/siteguide/futuregrid 2013-07-08 08:19:11 UTC (rev 6619)
@@ -9,12 +9,12 @@
physical clusters. We use the resources offered by one such cluster via the
Nebula middleware.
-More information on futuregrid can be found at https://portal.futuregrid.org/.
+More information on futuregrid can be found link:https://portal.futuregrid.org/[here].
Requesting Futuregrid Access
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
If you do not already have a futuregrid account, you can follow the
-instructions at https://portal.futuregrid.org/gettingstarted to get started.
+instructions link:https://portal.futuregrid.org/gettingstarted[here] to get started.
This page provides information on how to create an account, how to join
a project, how to set up your SSH keys, and how to create a new project.
@@ -31,11 +31,13 @@
Download your Credentials
~~~~~~~~~~~~~~~~~~~~~~~~~
Run the following commands to retrieve your credentials:
+
-----
$ cd swift-vm-boot
$ scp yourusername at hotel.futuregrid.org:nimbus_creds.tar.gz .
$ tar xvfz nimbus_creds.tar.gz
-----
+
When you extract your credential file, look at the file called
hotel.conf. Near the bottom of this file will be two settings
called vws.repository.s3id and vws.repository.s3key. Copy these
@@ -111,8 +113,8 @@
$ swift -sites.file sites.xml -tc.file tc.data -config cf <yourscript.swift>
-----
-If you would like to create a custom tc file for repeated use, rename it to something other
-than tc.data to prevent it from being overwritten. The sites.xml however will need to be
+If you like to create a custom tc and/or cf file for repeated use, rename it to something other
+than tc.data/cf to prevent it from being overwritten. The sites.xml however will need to be
regenerated every time you start the coaster service. If you need to repeatedly modify some
sites.xml options, you may edit the template in Swift's etc/sites/persistent-coasters. You
may also create your own custom tc files with the hostname of persistent-coasters. More
@@ -132,6 +134,6 @@
More Help
~~~~~~~~~
The best place for additional help is the Swift user mailing list. You can subscribe to this list at
-http://mail.ci.uchicago.edu/mailman/listinfo/swift-user. When submitting information, please send
+http://mail.ci.uchicago.edu/mailman/listinfo/swift-user. When submitting information, send
your sites.xml file, your tc.data, and any error messages you run into.
Modified: branches/faster/docs/siteguide/intrepid
===================================================================
--- branches/faster/docs/siteguide/intrepid 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/siteguide/intrepid 2013-07-08 08:19:11 UTC (rev 6619)
@@ -1,17 +1,16 @@
Intrepid (Blue Gene/P)
---------------------
+Intrepid is an IBM Blue Gene/P (BG/p) supercomputer located at the Argonne
+Leadership Computing Facility. More information on Intrepid can be found at
+http://www.alcf.anl.gov. Surveyor and Challenger are similar, smaller
+machines.
-Intrepid is an IBM Blue Gene/P supercomputer located at the Argonne
-Leadership Computing Facility. More information on Intrepid can be
-found at http://www.alcf.anl.gov. Surveyor and Challenger are
-similar, smaller machines.
-
Requesting Access
~~~~~~~~~~~~~~~~~
-If you do not already have an account on Intrepid, you can request
-one at https://accounts.alcf.anl.gov/accounts/request.php. More information about
-this process and requesting allocations for your project can be found at
-http://www.alcf.anl.gov/support/gettingstarted/index.php.
+If you do not already have an account on Intrepid, you can request one
+link:https://accounts.alcf.anl.gov/accounts/request.php[here]. More information
+about this process and requesting allocations for your project can be found
+link:http://www.alcf.anl.gov/support/gettingstarted/index.php[here].
SSH Keys
~~~~~~~~
@@ -69,6 +68,13 @@
What You Need To Know Before Running Swift
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Note that on Intrepid, the compute nodes can not create or write to a /home filesystem. Consequently, in order for Swift to interface correctly from login node to the compute nodes, Swift must write all internal and intermediate files to /intrepid-fs0, which is writable by the compute nodes. In order to accomplish this, export the environment variable SWIFT_USERHOME as follows:
+
+----
+export SWIFT_USERHOME=/intrepid-fs0/users/`whoami`/scratch
+----
+
Before you can create a Swift configuration file, there are some things
you will need to know.
@@ -80,7 +86,7 @@
-----
/home/username/swiftwork
/home/username/work
-/tmp
+/tmp/swift.work
-----
Which project(s) are you a member of?
@@ -122,7 +128,7 @@
Now that you know what queue to use, your project, and your work
directory, it is time to set up Swift. Swift uses a configuration file
-called sites.xml to determine how it should run. There are two
+called sites.xml to determine how it should run. There are two
methods you can use for creating this file. You can manually edit the
configuration file, or generate it with a utility called +gensites+.
@@ -155,7 +161,7 @@
Manually Editing tc.data
~~~~~~~~~~~~~~~~~~~~~~~~
-Below is the tc.data file used by Swift's test suite for running on PADS.
+Below is the tc.data file used by Swift's test suite.
-----
include::../../tests/providers/intrepid/tc.template.data[]
@@ -165,7 +171,10 @@
Catsn.swift
~~~~~~~~~~~
-The swift script we will run is called catsn.swift. It simply cats a file and saves the result. This is a nice simple test to ensure jobs are running correctly. Create a file called data.txt which contains some simple input - a "hello world" will do the trick.
+The swift script we will run is called catsn.swift. It simply cats a file and
+saves the result. This is a nice simple test to ensure jobs are running
+correctly. Create a file called data.txt which contains some simple input - a
+"hello world" will do the trick.
-----
include::../../examples/misc/catsn.swift[]
@@ -179,13 +188,16 @@
swift -sites.file sites.xml -tc.file tc.data catsn.swift -n=10
-----
-You should see several new files being created, called catsn.0001.out, catsn.0002.out, etc. Each of these
-files should contain the contents of what you placed into data.txt. If this happens, your job has run
-successfully on PADS!
+You should see several new files being created, called catsn.0001.out,
+catsn.0002.out, etc. Each of these files should contain the contents of what
+you placed into data.txt. If this happens, your job has run successfully!
More Help
~~~~~~~~~
-The best place for additional help is the Swift user mailing list. You can subscribe to this list at
-https://lists.ci.uchicago.edu/cgi-bin/mailman/listinfo/swift-user. When submitting information, please send your sites.xml file, your tc.data, and any Swift log files that were created during your attempt.
+The best place for additional help is the Swift user mailing list. You can
+subscribe to this
+link:https://lists.ci.uchicago.edu/cgi-bin/mailman/listinfo/swift-user[list].
+When submitting information, send your sites.xml file, your tc.data, and any
+Swift log files that were created during your attempt.
Copied: branches/faster/docs/siteguide/midway (from rev 6618, trunk/docs/siteguide/midway)
===================================================================
--- branches/faster/docs/siteguide/midway (rev 0)
+++ branches/faster/docs/siteguide/midway 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1,92 @@
+Midway (x86 cluster)
+--------------------
+Midway is a cluster maintained by the Research Computing Center
+at the University of Chicago. Midway uses Slurm to handle job
+scheduling. For more details about Midway, and to request
+an account, visit http://rcc.uchicago.edu.
+
+Connecting to a login node
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+Once you have access to Midway, connect to
+a Midway login node.
+
+-----
+$ ssh userid at midway.rcc.uchicago.edu
+-----
+
+Loading Swift
+~~~~~~~~~~~~~
+Swift is available on Midway as a module. To load the
+Swift, run:
+
+-----
+$ module load swift
+-----
+
+Example sites.xml
+~~~~~~~~~~~~~~~~~
+Below is an example that uses two of the queues available
+on Midway, sandyb and westmere. Be sure to adjust walltime,
+work directory, and other options as needed.
+
+-----
+<config>
+ <pool handle="midway-sandyb">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">00:05:00</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>/scratch/midway/{env.USER}/work</workdirectory>
+ </pool>
+
+ <pool handle="midway-westmere">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <profile namespace="globus" key="jobsPerNode">12</profile>
+ <profile namespace="globus" key="maxWalltime">00:05:00</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="queue">westmere</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>/scratch/midway/{env.USER}/work</workdirectory>
+ </pool>
+</config>
+-----
+
+Example sites.xml for use with MPI
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Below is an example sites.xml that is suitable for running with MPI.
+Jobtype must be set to single. The value you set for ppn
+will determine the number of cores/slots your application uses per node.
+The value of count will set the number of nodes to request. The example
+below requests 2 nodes with 12 slots per node.
+
+-----
+<config>
+ <pool handle="midway-westmere">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="ppn">12</profile>
+ <profile namespace="globus" key="maxWalltime">_WALLTIME_</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="queue">westmere</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="jobtype">single</profile>
+ <profile namespace="globus" key="count">2</profile>
+ <filesystem provider="local"/>
+ <workdirectory>/scratch/midway/{env.USER}/work</workdirectory>
+ </pool>
+</config>
+-----
+
+Various tips for running MPI jobs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* You'll need to load an MPI module. Run "module load openmpi" to add to your path.
+* The app that Swift runs should be a wrapper script that invokes your
+ MPI application by running "mpiexec /path/to/yourMPIApp"
+
Modified: branches/faster/docs/siteguide/prereqs
===================================================================
--- branches/faster/docs/siteguide/prereqs 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/siteguide/prereqs 2013-07-08 08:19:11 UTC (rev 6619)
@@ -4,5 +4,5 @@
This guide assumes that you have already downloaded and installed Swift.
It assumes that Swift is in your PATH and that you have a working
version of Sun Java 1.5+. For more information on downloading and
-installing Swift, please see the
+installing Swift, see the
http://www.ci.uchicago.edu/swift/guides/release-0.93/quickstart/quickstart.html[Swift Quickstart Guide].
Modified: branches/faster/docs/siteguide/siteguide.txt
===================================================================
--- branches/faster/docs/siteguide/siteguide.txt 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/siteguide/siteguide.txt 2013-07-08 08:19:11 UTC (rev 6619)
@@ -22,4 +22,13 @@
include::mcs[]
+include::midway[]
+
include::uc3[]
+
+include::stampede[]
+
+include::ec2[]
+
+link:http://www.ci.uchicago.edu/swift/docs/index.php[home]
+
Copied: branches/faster/docs/siteguide/stampede (from rev 6618, trunk/docs/siteguide/stampede)
===================================================================
--- branches/faster/docs/siteguide/stampede (rev 0)
+++ branches/faster/docs/siteguide/stampede 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1,167 @@
+Stampede
+---------
+
+Stampede is a 10 petaflop supercomputer available as part of XSEDE resources.
+It employs a batch-oriented computational model where-in a SLURM schedular
+accepts user's jobs and queues them in the queueing system for execution. The
+computational model requires a user to prepare the submit files, track job
+submissions, chackpointing, managing input/output data and handling exceptional
+conditions manually.
+
+Running Swift under Stampede can accomplish the above tasks with least manual
+user intervention. In the following sections, we discuss more about specifics of
+running Swift on Stampede. A more detailed information about Swift and its
+workings can be found on Swift documentation page here:
+http://www.ci.uchicago.edu/swift/wwwdev/docs/index.php
+
+More information on Stampede can be found on XSEDE Stampede website here:
+https://www.xsede.org/stampede
+
+Requesting Access
+~~~~~~~~~~~~~~~~~
+Initial access to XSEDE resources could be obtained by submitting a startup proposal. Advanced users could submit a proposal for research allocation. An educational allocation is available for teaching and/or training purposes. More on XSEDE allocations can be found here:
+https://www.xsede.org/allocations
+
+Connecting to a login node
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+Once you have an account, you should be able to access a Stampede login
+node with the following command:
+
+-----
+ssh yourusername at stampede.tacc.utexas.edu
+-----
+
+Follow the steps outlined below to get started with Swift on Stampede:
+
+*step 1.* Install Swift using one of the installation methods documented on Swift home: http://www.ci.uchicago.edu/swift/downloads/index.php,
+
+if installing from source, java can be loaded on Stampede using +module load jdk32+ and apache ant could be downloaded from here: http://ant.apache.org
+
+*step 2.* Create and change to a directory where your Swift related work will
+stay. (say, +mkdir swift-work+, followed by, +cd swift-work+)
+
+*step 3.* To get started with a simple example running the Linux +/bin/cat+ command to read an
+input file +data.txt+ and write it to an output file, start with writing a simple Swift source script as follows:
+
+-----
+type file;
+
+/* App definitio */
+app (file o) cat (file i)
+{
+ cat @i stdout=@o;
+}
+
+file out[]<simple_mapper; location="outdir", prefix="f.",suffix=".out">;
+file data<"data.txt">;
+
+/* App invocation: n times */
+foreach j in [1:@toint(@arg("n","1"))] {
+ out[j] = cat(data);
+}
+-----
+
+Make sure a file named +data.txt+ is available in the current directory where the above Swift source file will be saved.
+
+*step 4.* The next step is to create a sites file. An example sites file (sites.xml) is shown as follows:
+
+-----
+<config>
+ <pool handle="stampede">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+
+ <!-- **replace with your project** -->
+ <profile namespace="globus" key="project">TG-EAR130015</profile>
+
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">00:11:00</profile>
+ <profile namespace="globus" key="maxtime">800</profile>
+
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+
+ <!-- queues on stampede: development, normal, large, etc. -->
+ <profile namespace="globus" key="queue">development</profile>
+
+ <!-- for mail notification -->
+ <profile namespace="globus" key="slurm.mail-user">me at dept.org</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+
+ <filesystem provider="local"/>
+ <workdirectory>/path/to/workdir</workdirectory>
+ </pool>
+</config>
+-----
+
+*step 5.* In this step, we will see the config and tc files. The config file (cf) is as follows:
+
+-----
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=0
+lazy.errors=false
+status.mode=provider
+use.provider.staging=false
+provider.staging.pin.swiftfiles=false
+use.wrapper.staging=false
+-----
+
+The tc file (tc) is as follows:
+
+-----
+stampede cat /bin/cat null null null
+-----
+
+More about config and tc file options can be found in the Swift userguide here:
+http://www.ci.uchicago.edu/swift/guides/trunk/userguide/userguide.html#_swift_configuration_properties.
+
+*step 6.* Run the example using following commandline:
+
+-----
+swift -config cf -tc.file tc -sites.file sites.xml catsn.swift -n=1
+-----
+
+You can further change the value of +-n+ to any arbitrary number to run that
+many number of +cat+ in parallel
+
+*step 7.* Swift will show a status message as "done" after the job has completed its run in the queue. Check the output in the generated +outdir+ directory (+ls outdir+)
+
+----
+login3$ swift -sites.file sites.stampede.xml -config cf -tc.file tc catsn.swift
+Swift trunk swift-r6290 cog-r3609
+
+RunID: 20130221-1030-faapk389
+Progress: time: Thu, 21 Feb 2013 10:30:21 -0600
+Progress: time: Thu, 21 Feb 2013 10:30:22 -0600 Submitting:1
+Progress: time: Thu, 21 Feb 2013 10:30:29 -0600 Submitted:1
+Progress: time: Thu, 21 Feb 2013 10:30:51 -0600 Active:1
+Progress: time: Thu, 21 Feb 2013 10:30:54 -0600 Finished successfully:1
+Final status: Thu, 21 Feb 2013 10:30:54 -0600 Finished successfully:1
+----
+
+Troubleshooting
+~~~~~~~~~~~~~~~
+
+In this section we will discuss some of the common issues and remedies while using Swift on Stampede. The origin of these issues can be Swift or Stampede's configuration, state and usage load among other factors. We try to identify maximum known issues and address them here:
+
+* Command not found: Make sure the +bin+ directory of Swift installation is in +PATH+.
+
+
+* Failed to transfer wrapperlog for job cat-nmobtbkk and/or Job failed with an exit code of 254. Check the <workdirectory> element on the sites.xml file.
+
+-----
+<workdirectory >/work/your/path/swift.workdir</workdirectory>
+-----
+
+It is likely that it is set to a path where the compute nodes can not write or no space available, e.g. your /home directory. The remedy for this error is to set your workdirectory to the path where Swift could write from compute nodes and there is enough space, e.g. /scratch directory.
+
+* If the jobs are not getting to active state for a long time, check the job status using the slurm squeue command:
+----
+$ squeue -u `whoami`
+----
+
+The output will give an indication of the status of jobs. See the slurm manual for more information on job management commands:
+
+----
+$ man slurm
+----
Modified: branches/faster/docs/tutorial/tutorial.txt
===================================================================
--- branches/faster/docs/tutorial/tutorial.txt 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/tutorial/tutorial.txt 2013-07-08 08:19:11 UTC (rev 6619)
@@ -15,3 +15,6 @@
include::runtime_features[]
include::bits[]
+
+link:http://www.ci.uchicago.edu/swift/docs/index.php[home]
+
Modified: branches/faster/docs/userguide/app_procedures
===================================================================
--- branches/faster/docs/userguide/app_procedures 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/app_procedures 2013-07-08 08:19:11 UTC (rev 6619)
@@ -292,11 +292,15 @@
Swift trace: hello
----
- at extractint
+ at extractInt
~~~~~~~~~~~
- at extractint(file) will read the specified file, parse an integer from
+ at extractInt(file) will read the specified file, parse an integer from
the file contents and return that integer.
+ at extractFloat
+~~~~~~~~~~~~~
+Similar to @extractInt, @extractFloat(file) will read the specified file, parse a float from
+the file contents and return that float.
@filename
~~~~~~~~~
@@ -406,18 +410,24 @@
parallel).
- at toint
+ at toInt
~~~~~~
- at toint(input) will parse its input string into an integer. This can be
+ at toInt(input) will parse its input string into an integer. This can be
used with @arg to pass input parameters to a Swift script as
integers.
- at tofloat
-~~~~~~
- at tofloat(input) will parse its input string into a floating point number. This can be
+ at toFloat
+~~~~~~~~
+ at toFloat(input) will parse its input string into a floating point number. This can be
used with @arg to pass input parameters to a Swift script as
floating point numbers.
+ at toString
+~~~~~~~~
+ at toString(input) will parse its input into a string. This can be
+used with @arg to pass input parameters to a Swift script as
+strings.
+
@length
~~~~~~
@length(array) will return the length of an array in Swift. This function will wait for all elements in the array to be written before returning the length.
@@ -458,7 +468,7 @@
Where the contents of the "emps.txt" file are:
----
-name id address
+name id loc
Thomas 2222 Chicago
Gina 3333 Boston
Anne 4444 Houston
@@ -514,7 +524,6 @@
trace will log its parameters. By default these will appear on both
stdout and in the run log file. Some formatting occurs to produce the
log message. The particular output format should not be relied upon.
-(since Swift 0.4)
tracef
~~~~~~
@@ -533,6 +542,7 @@
Specifiers:
+%s+:: Format a string.
++%b+:: Format a boolean.
+%i+:: Format a number as an integer.
+%f+:: Format a number as a floating point number.
+%q+:: Format an array.
@@ -551,4 +561,9 @@
writeData
~~~~~~~~~
writeData will write out data structures in the format described for
-readData
+readData. The following example demonstrates how one can write a string "foo" into a file "writeDataPrimitive.out":
+
+----
+include::../../tests/language-behaviour/IO/writeDataPrimitive.swift[]
+----
+
Modified: branches/faster/docs/userguide/build_options
===================================================================
--- branches/faster/docs/userguide/build_options 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/build_options 2013-07-08 08:19:11 UTC (rev 6619)
@@ -1,50 +1,24 @@
Build options
-------------
-See the Swift download page
-<http://www.ci.uchicago.edu/swift/downloads/> for instructions on
-downloading and building Swift from source. When building, various build
-options can be supplied on the ant commandline. These are summarised here:
+See the Swift download page <http://www.ci.uchicago.edu/swift/downloads/> for
+instructions on downloading and building Swift from source. When building,
+various build options can be supplied on the ant commandline. These are
+summarised here:
with-provider-condor - build with CoG condor provider
-with-provider-coaster - build with CoG coaster provider (see the
-section on coasters). Since 0.8, coasters are always built,
-and this option has no effect.
-
-with-provider-deef - build with Falkon provider deef. In order for
-this option to work, it is necessary to check out the provider-deef code
-in the cog/modules directory alongside swift:
-
-----
-$ cd cog/modules
-$ svn co https://svn.ci.uchicago.edu/svn/vdl2/provider-deef
-$ cd ../swift
-$ ant -Dwith-provider-deef=true redist
-----
+with-provider-coaster - build with CoG coaster provider (see the section on
+coasters). Since 0.8, coasters are always built, and this option has no effect.
-with-provider-wonky - build with provider-wonky, an execution provider
-that provides delays and unreliability for the purposes of testing
-Swift's fault tolerance mechanisms. In order for this option to work, it
-is necessary to check out the provider-wonky code in the |cog/modules|
-directory alongside swift:
+no-supporting - produces a distribution without supporting commands such as
+grid-proxy-init. This is intended for when the Swift distribution will be used
+in an environment where those commands are already provided by other packages,
+where the Swift package should be providing only Swift commands, and where the
+presence of commands such as grid-proxy-init from the Swift distribution in the
+path will mask the presence of those commands from their true distribution
+package such as a Globus Toolkit package.
----
-$ cd cog/modules
-$ svn co https://svn.ci.uchicago.edu/svn/vdl2/provider-wonky
-$ cd ../swift
-$ ant -Dwith-provider-wonky=true redist
-----
-
-no-supporting - produces a distribution without supporting commands
-such as grid-proxy-init. This is intended for when the Swift
-distribution will be used in an environment where those commands are
-already provided by other packages, where the Swift package should be
-providing only Swift commands, and where the presence of commands such
-as grid-proxy-init from the Swift distribution in the path will mask the
-presence of those commands from their true distribution package such as
-a Globus Toolkit package.
-
-----
$ ant -Dno-supporting=true redist
----
Modified: branches/faster/docs/userguide/clustering
===================================================================
--- branches/faster/docs/userguide/clustering 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/clustering 2013-07-08 08:19:11 UTC (rev 6619)
@@ -1,26 +1,24 @@
Clustering
----------
-Swift can group a number of short job submissions into a single larger
-job submission to minimize overhead involved in launching jobs (for
-example, caused by security negotiation and queuing delay). In general,
-CoG coasters should be used in preference to the clustering
-mechanism documented in this section.
+Swift can group a number of short job submissions into a single larger job
+submission to minimize overhead involved in launching jobs (for example, caused
+by security negotiation and queuing delay). In general, coasters should be used
+in preference to the clustering mechanism documented in this section.
By default, clustering is disabled. It can be activated by setting the
clustering.enabled property to true.
-A job is eligible for clustering if the GLOBUS::maxwalltime profile
-is specified in the tc.data entry for that job, and its value is
-less than the value of the clustering.min.time property.
+A job is eligible for clustering if the GLOBUS::maxwalltime profile is
+specified in the tc.data entry for that job, and its value is less than the
+value of the clustering.min.time property.
-Two or more jobs are considered compatible if they share the same site
-and do not have conflicting profiles (e.g. different values for the same
-environment variable).
+Two or more jobs are considered compatible if they share the same site and do
+not have conflicting profiles (e.g. different values for the same environment
+variable).
-When a submitted job is eligible for clustering, it will be put in a
-clustering queue rather than being submitted to a remote site. The
-clustering queue is processed at intervals specified by the
-clustering.queue.delay property. The
-processing of the clustering queue consists of selecting compatible jobs
-and grouping them into clusters whose maximum wall time does not exceed
-twice the value of the clustering.min.time property.
+When a submitted job is eligible for clustering, it will be put in a clustering
+queue rather than being submitted to a remote site. The queue is processed at
+intervals specified by the clustering.queue.delay property. The processing of
+the clustering queue consists of selecting compatible jobs and grouping them
+into clusters whose maximum wall time does not exceed twice the value of the
+clustering.min.time property.
Modified: branches/faster/docs/userguide/coasters
===================================================================
--- branches/faster/docs/userguide/coasters 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/coasters 2013-07-08 08:19:11 UTC (rev 6619)
@@ -90,3 +90,5 @@
|remoteMonitorEnabled|If true, show a graphical display of the status of
the coaster service
|==================
+
+
Modified: branches/faster/docs/userguide/commands
===================================================================
--- branches/faster/docs/userguide/commands 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/commands 2013-07-08 08:19:11 UTC (rev 6619)
@@ -74,10 +74,34 @@
unique with respect to all other run IDs that might be used,
irrespective of (at least) expected execution sites, program or user.
+-version
+
+ Display Swift version and exit
+
+-recompile
+
+ Forces Swift to re-compile the invoked Swift script. While Swift
+ is meant to detect when recompilation is necessary, in some
+ special cases it fails to do so. This flag helps with those
+ special cases.
+
+-cdm.file
+
+ Specifies a CDM policy file.
+
+-reduced.logging
+
+ Makes logging more terse by disabling provenance information and
+ low-level task messages
+
+-minimal.logging
+
+ Makes logging much more terse: reports warnings only
+
-tui
Displays an interactive text mode monitor during a run. (since Swift
- 0.9)
+ 0.9)
In addition, the following Swift properties can be set on the command line:
@@ -118,20 +142,28 @@
The swift is influenced by the following environment variables:
-GLOBUS_HOSTNAME, GLOBUS_TCP_PORT_RANGE - set in the environment
-before running Swift. These can be set to inform Swift of the
-configuration of your local firewall. More information can be found in
-the Globus firewall How-to <http://dev.globus.org/wiki/FirewallHowTo>.
+GLOBUS_HOSTNAME, GLOBUS_TCP_PORT_RANGE
-COG_OPTS - set in the environment before running Swift. Options set in
-this variable will be passed as parameters to the Java Virtual Machine
-which will run Swift. The parameters vary between virtual machine
-imlementations, but can usually be used to alter settings such as
-maximum heap size. Typing 'java -help' will sometimes give a list of
-commands. The Sun Java 1.4.2 command line options are documented here
-<http://java.sun.com/j2se/1.4.2/docs/tooldocs/windows/java.html>.
+ Set in the environment before running Swift. These can be set to inform Swift
+ of the configuration of your local firewall. More information can be found in
+ the Globus firewall How-to <http://dev.globus.org/wiki/FirewallHowTo>.
+SWIFT_HEAP_MAX
+ Sets the java heap size. Use this if Swift runs out of memory.
+ Uses the format set by java -Xmx, which is how this is implemented.
+ The default setting is 1024M.
+
+COG_OPTS
+
+ Set in the environment before running Swift. Options set in
+ this variable will be passed as parameters to the Java Virtual Machine
+ which will run Swift. The parameters vary between virtual machine
+ imlementations, but can usually be used to alter settings such as
+ maximum heap size. Typing 'java -help' will sometimes give a list of
+ commands.
+
+
swift-osg-ress-site-catalog
~~~~~~~~~~~~~~~~~~~~~~~~~~~
The swift-osg-ress-site-catalog command generates a site catalog based
Modified: branches/faster/docs/userguide/configuration_properties
===================================================================
--- branches/faster/docs/userguide/configuration_properties 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/configuration_properties 2013-07-08 08:19:11 UTC (rev 6619)
@@ -1,17 +1,16 @@
Swift configuration properties
------------------------------
-Various aspects of the behavior of the Swift Engine can be configured
-through properties. The Swift Engine recognizes a global, per
-installation properties file which can found in etc/swift.properties
-in the Swift installation directory and a user properties file which can
-be created by each user in ~/.swift/swift.properties. The Swift Engine
-will first load the global properties file. It will then try to load the
-user properties file. If a user properties file is found, individual
-properties explicitly set in that file will override the respective
-properties in the global properties file. Furthermore, some of the
-properties can be overridden directly using command line arguments to
-the *swift* command.
+Various aspects of the behavior of Swift can be configured through properties.
+Swift recognizes a global, per installation properties file which can found in
+etc/swift.properties in the Swift installation directory and a user properties
+file which can be created by each user in ~/.swift/swift.properties.
+Swift will first load the global properties file. It will then try to load the
+user properties file. If a user properties file is found, individual properties
+explicitly set in that file will override the respective properties in the
+global properties file. Furthermore, some of the properties can be overridden
+directly using command line arguments to the *swift* command.
+
Swift properties are specified in the following format:
<name>=<value>
@@ -416,6 +415,13 @@
directory. In such cases, relative mode must be used. (since Swift
0.9)
+use.wrapper.staging
+
+ Valid values: true, false
+ Default value: false
+
+ Determines if the Swift wrapper should do file staging.
+
wrapper.parameter.mode
Controls how Swift will supply parameters to the remote wrapper
@@ -445,3 +451,43 @@
ip.address=192.168.0.1
----
+Monitoring Swift
+~~~~~~~~~~~~~~~~
+
+A Swift run can be monitored for progress and resource usage. To monitor the resource usage, use the +-monitor+ option with the Swift commandline. For example:
+
+----
+swift -tc.file tc -sites.file sites.xml -config cf modis04.swift -monitor
+----
+
+This will produce a gui/X window consisting of the following quantities:
+
+* Allocated memory
+* Heap Size
+* Total Threads
+* Total Workers
+
+.Figure 4. Resource Monitor
+image:swift_monitor.png[]
+
+Figure 4 shows a snapshot of a Swift resource monitor.
+
+
+The progress of a Swift run can be monitored using the +-tui+ option. For example:
+
+----
+swift -tc.file tc -sites.file sites.xml -config cf modis04.swift -tui
+----
+
+This will produce a textual user interface with multiple tabs, each showing the following features of the current Swift run:
+
+* A summary view showing task status
+* An apps tab
+* A jobs tab
+* A transfer tab
+* A scheduler tab
+* A Tast statistics tab
+* A customized tab called 'Ben's View'
+
+Navigation between these tabs can be done using the function keys f2 through f8.
+
Modified: branches/faster/docs/userguide/kickstart
===================================================================
--- branches/faster/docs/userguide/kickstart 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/kickstart 2013-07-08 08:19:11 UTC (rev 6619)
@@ -27,5 +27,5 @@
</pool>
----
-There are various kickstat.* properties, which have sensible default
+There are various kickstart.* properties, which have sensible default
values. These are documented in the properties section.
Modified: branches/faster/docs/userguide/log-processing
===================================================================
--- branches/faster/docs/userguide/log-processing 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/log-processing 2013-07-08 08:19:11 UTC (rev 6619)
@@ -112,18 +112,22 @@
the "-resume" option. The .rlog file is only for Swift's internal purpose and
not to be interpreted by the user.
-Each line in the log file is typically made up of three parts. The first part
+Each line in the log file typically consists of three parts. The first part
is the timestamp, the second is the type of log message and the third is the
message itself. The types of log messages follows the java log4j standard types
-of TRACE, DEBUG, INFO, WARN, ERROR and FATAL. This section lists the various
-Swift log messages and explains the meaning and likely interpretation of those
-messages. Please note that the list is not comprehensive at this time. Also
-note that we will ignore the timestamps here.
+of TRACE, DEBUG, INFO, WARN, ERROR and FATAL.
+////
+This section lists the various Swift log messages and explains the meaning and
+likely interpretation of those messages. Note that the list is not
+comprehensive at this time. Also note that we will ignore the timestamps here.
+
. _DEBUG Loader arguments: [-sites.file, sites.xml, -config, cf, -tc.file, tc, postproc-gridftp.swift]_
+ Swift commandline arguments
. _DEBUG Loader Max heap: 5592449024_
+ The java runtime heap size
. _DEBUG textfiles BEGIN_
-. _DEBUG Loader kmlversion is_
+ A dump of config and source files associated with this run
. _DEBUG VDL2ExecutionContext Stack dump_
. _INFO SetFieldValue Set_
. _INFO get__site STARTCOMPOUND thread=0-8 name=get__site_
@@ -142,7 +146,9 @@
. _INFO ThrottleManager mem=113.54 MB, heap=482.88 MB, maxHeap=5.21 GB_
. _INFO ThrottleManager I maxBuffers=512, crtBuffers=0, allowedTransfers=256, active=0, suspended=0_
. _INFO PerformanceDiagnosticInputStream [MEM] Heap total: 482.88 MB, Heap used: 118.58 MB_
+Heap sizes for performance
. _INFO vdl:execute END_SUCCESS thread=0-8-0 tr=getsite_
+The job ended successfully
. _INFO WeightedHostScoreScheduler CONTACT_SELECTED host=localhost, score=99.854_
. _
-
+////
Modified: branches/faster/docs/userguide/mappers
===================================================================
--- branches/faster/docs/userguide/mappers 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/mappers 2013-07-08 08:19:11 UTC (rev 6619)
@@ -244,7 +244,24 @@
subsequent case blocks, and no break statement is necessary at the
end of each block.
+Following is an example of a switch expression in Swift:
+----
+int score=60;
+switch (score){
+case 100:
+ tracef("%s\n", "Bravo!");
+case 90:
+ tracef("%s\n", "very good");
+case 80:
+ tracef("%s\n", "good");
+case 70:
+ tracef("%s\n", "fair");
+default:
+ tracef("%s\n", "unknown grade");
+ }
+----
+
iterate
^^^^^^^
iterate expressions allow a block of code to be evaluated repeatedly,
@@ -263,6 +280,14 @@
iteration. That variable is in scope in the statements block and when
evaluating the termination expression.
+For example, the following block will iterate over the +months+ variable 12 times:
+
+----
+iterate month {
+ tracef("%i\n", month);
+ } until (month==12);
+----
+
Operators
~~~~~~~~~
The following infix operators are available for use in Swift script
@@ -631,14 +656,12 @@
Example:
----
-string s = "picture.gif";
-file f <regexp_mapper;
- source=s,
- match="(.*)gif",
- transform="\\1jpg">;
+file s <"picture.gif">;
+file f <regexp_mapper; source=s,
+ match="(.*)gif", transform="\\1jpg">;
----
-This example transforms a string ending gif into one ending jpg and
+This example transforms a file ending gif into one ending jpg and
maps that to a file.
[options="header, autowidth"]
@@ -649,9 +672,10 @@
structured regular expression mapper
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-The structured_regexp_mapper is similar to the regexp_mapper. The
-structured_regexp_mapper can be applied to lists while the regexp_mapper cannot.
+The structured_regexp_mapper is similar to the regexp_mapper with the only
+difference that it can be applied to arrays while the regexp_mapper cannot.
+
[options="header, autowidth"]
|==========
|parameter|meaning
@@ -667,14 +691,13 @@
Example:
----
-string s[] = ["picture.gif", "hello.gif", "world.gif"];
-file f[] <structured_regexp_mapper;
- source=s,
- match="(.*)gif",
- transform="\\1jpg">;
+file s[] <filesys_mapper; pattern="*.gif">;
+
+file f[] <structured_regexp_mapper; source=s,
+ match="(.*)gif", transform="\\1jpg">;
----
-This example transforms all strings in a list that end in gif to end in jpg and maps
+This example transforms all files in a list that end in gif to end in jpg and maps
the list to those files.
csv mapper
Modified: branches/faster/docs/userguide/overview
===================================================================
--- branches/faster/docs/userguide/overview 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/overview 2013-07-08 08:19:11 UTC (rev 6619)
@@ -3,18 +3,17 @@
This manual provides reference material for Swift: the Swift
language and the Swift runtime system. For introductory material,
consult the Swift tutorial
-<http://www.ci.uchicago.edu/swift/guides/tutorial.php>.
+http://www.ci.uchicago.edu/swift/guides/trunk/tutorial/tutorial.html
-Swift is a data-oriented coarse grained scripting language that supports
+Swift is a data-flow oriented coarse grained scripting language that supports
dataset typing and mapping, dataset iteration, conditional branching,
and procedural composition.
Swift programs (or workflows) are written in a language called
-Swift scripts.
+Swift script.
-Swift scripts are dataflow oriented - they are primarily
-concerned with processing (possibly large) collections of data files, by
-invoking programs to do that processing. Swift handles execution of such
-programs on remote sites by choosing sites, handling the staging of
-input and output files to and from the chosen sites and remote execution
-of program code.
+Swift scripts are primarily concerned with processing (possibly large)
+collections of data files, by invoking programs to do that processing. Swift
+handles execution of such programs on remote sites by choosing sites, handling
+the staging of input and output files to and from the chosen sites and remote
+execution of program code.
Modified: branches/faster/docs/userguide/profiles
===================================================================
--- branches/faster/docs/userguide/profiles 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/profiles 2013-07-08 08:19:11 UTC (rev 6619)
@@ -39,35 +39,47 @@
swift namespace
~~~~~~~~~~~~~~~
-storagesize limits the amount of space that will be used on the remote
-site for temporary files. When more than that amount of space is used,
-the remote temporary file cache will be cleared using the algorithm
-specified in the caching.algorithm property.
-wrapperInterpreter - The wrapper interpreter indicates the command
-(executable) to be used to run the Swift wrapper script. The default is
-"/bin/bash" on Unix sites and "cscript.exe" on Windows sites.
+storagesize
-wrapperInterpreterOptions - Allows specifying additional options to the
-executable used to run the Swift wrapper. The defaults are no options on
-Unix sites and "Nologo" on Windows sites.
+ limits the amount of space that will be used on the remote site for temporary
+ files. When more than that amount of space is used, the remote temporary file
+ cache will be cleared using the algorithm specified in the caching.algorithm
+ property.
-wrapperScript - Specifies the name of the wrapper script to be used on a
-site. The defaults are "_swiftwrap" on Unix sites and "_swiftwrap.vbs"
-on Windows sites. If you specify a custom wrapper script, it must be
-present in the "libexec" directory of the Swift installation.
+wrapperInterpreter
-cleanupCommand Indicates the command to be run at the end of a Swift
-run to clean up the run directories on a remote site. Defaults are
-"/bin/rm" on Unix sites and "cmd.exe" on Windows sites
+ The wrapper interpreter indicates the command (executable) to be used to run
+ the Swift wrapper script. The default is "/bin/bash" on Unix sites and
+ "cscript.exe" on Windows sites.
-cleanupCommandOptions Specifies the options to be passed to the
-cleanup command above. The options are passed in the argument list to
-the cleanup command. After the options, the last argument is the
-directory to be deleted. The default on Unix sites is "-rf". The default
-on Windows sites is ["/C", "del", "/Q"].
+wrapperInterpreterOptions
+ Allows specifying additional options to the executable used to run the Swift
+ wrapper. The defaults are no options on Unix sites and "Nologo" on Windows
+ sites.
+wrapperScript
+
+ Specifies the name of the wrapper script to be used on a site. The defaults are
+ "_swiftwrap" on Unix sites and "_swiftwrap.vbs" on Windows sites. If you
+ specify a custom wrapper script, it must be present in the "libexec" directory
+ of the Swift installation.
+
+cleanupCommand
+
+ Indicates the command to be run at the end of a Swift run to clean up the run
+ directories on a remote site. Defaults are "/bin/rm" on Unix sites and
+ "cmd.exe" on Windows sites
+
+cleanupCommandOptions
+
+ Specifies the options to be passed to the cleanup command above. The options
+ are passed in the argument list to the cleanup command. After the options, the
+ last argument is the directory to be deleted. The default on Unix sites is
+ "-rf". The default on Windows sites is ["/C", "del", "/Q"].
+
+
Globus namespace
~~~~~~~~~~~~~~~~
maxwalltime specifies a walltime limit for each job, in minutes.
@@ -80,29 +92,28 @@
Example:
----
-localhost echo /bin/echo INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:20:00"
+localhost echo /bin/echo INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:20:00"
----
-When replication is enabled (see replication), then
-walltime will also be enforced at the Swift client side: when a job has
-been active for more than twice the maxwalltime, Swift will kill the job
-and regard it as failed.
+When replication is enabled (see replication), then walltime will also be
+enforced at the Swift client side: when a job has been active for more than
+twice the maxwalltime, Swift will kill the job and regard it as failed.
-When clustering is used, maxwalltime will be used to select which jobs
-will be clustered together. More information on this is available in the
-clustering section.
+When clustering is used, maxwalltime will be used to select which jobs will be
+clustered together. More information on this is available in the clustering
+section.
-When coasters as used, maxwalltime influences the default coaster
-worker maxwalltime, and which jobs will be sent to which workers. More
-information on this is available in the coasters section.
+When coasters as used, maxwalltime influences the default coaster worker
+maxwalltime, and which jobs will be sent to which workers. More information on
+this is available in the coasters section.
-queue is used by the PBS, GRAM2 and GRAM4 providers. This profile
-entry specifies which queue jobs will be submitted to. The valid queue
-names are site-specific.
+queue is used by the PBS, GRAM2 and GRAM4 providers. This profile entry
+specifies which queue jobs will be submitted to. The valid queue names are
+site-specific.
-host_types specifies the types of host that are permissible for a job
-to run on. The valid values are site-specific. This profile entry is
-used by the GRAM2 and GRAM4 providers.
+host_types specifies the types of host that are permissible for a job to run
+on. The valid values are site-specific. This profile entry is used by the GRAM2
+and GRAM4 providers.
condor_requirements allows a requirements string to be specified when
Condor is used as an LRM behind GRAM2. Example:
@@ -111,113 +122,144 @@
<profile namespace="globus" key="condor_requirements">Arch == "X86_64" || Arch="INTEL"</profile>
----
-slots When using coasters, this parameter specifies the
-maximum number of jobs/blocks that the coaster scheduler will have
-running at any given time. The default is 20.
+slots
-jobsPerNode - This parameter determines how many coaster workers are
-started one each compute node. The default value is 1.
+ When using coasters, this parameter specifies the maximum number of jobs/blocks
+ that the coaster scheduler will have running at any given time. The default is
+ 20.
-nodeGranularity - When allocating a coaster worker block, this parameter
-restricts the number of nodes in a block to a multiple of this value.
-The total number of workers will then be a multiple of workersPerNode *
-nodeGranularity. The default value is 1.
+jobsPerNode
-allocationStepSize - Each time the coaster block scheduler computes a
-schedule, it will attempt to allocate a number of slots from the number
-of available slots (limited using the above slots profile). This
-parameter specifies the maximum fraction of slots that are allocated in
-one schedule. Default is 0.1.
+ This parameter determines how many coaster workers are started one each compute
+ node. The default value is 1.
-lowOverallocation - Overallocation is a function of the walltime of a
-job which determines how long (time-wise) a worker job will be. For
-example, if a number of 10s jobs are submitted to the coaster service,
-and the overallocation for 10s jobs is 10, the coaster scheduler will
-attempt to start worker jobs that have a walltime of 100s. The
-overallocation is controlled by manipulating the end-points of an
-overallocation function. The low endpoint, specified by this parameter,
-is the overallocation for a 1s job. The high endpoint is the
-overallocation for a (theoretical) job of infinite length. The
-overallocation for job sizes in the [1s, +inf) interval is determined
-using an exponential decay function: overallocation(walltime) = walltime
-* (lowOverallocation - highOverallocation) * exp(-walltime *
-overallocationDecayFactor) + highOverallocation The default value of
-lowOverallocation is 10.
+nodeGranularity
-highOverallocation - The high overallocation endpoint (as described
-above). Default: 1
+ When allocating a coaster worker block, this parameter restricts the number of
+ nodes in a block to a multiple of this value. The total number of workers will
+ then be a multiple of workersPerNode * nodeGranularity. The default value is 1.
-overallocationDecayFactor - The decay factor for the overallocation
-curve. Default 0.001 (1e-3).
+allocationStepSize
-spread - When a large number of jobs is submitted to the a coaster
-service, the work is divided into blocks. This parameter allows a rough
-control of the relative sizes of those blocks. A value of 0 indicates
-that all work should be divided equally between the blocks (and blocks
-will therefore have equal sizes). A value of 1 indicates the largest
-possible spread. The existence of the spread parameter is based on the
-assumption that smaller overall jobs will generally spend less time in
-the queue than larger jobs. By submitting blocks of different sizes,
-submitted jobs may be finished quicker by smaller blocks. Default: 0.9.
+ Each time the coaster block scheduler computes a schedule, it will attempt to
+ allocate a number of slots from the number of available slots (limited using
+ the above slots profile). This parameter specifies the maximum fraction of
+ slots that are allocated in one schedule. Default is 0.1.
-reserve - Reserve time is a time in the allocation of a worker that sits
-at the end of the worker time and is useable only for critical
-operations. For example, a job will not be submitted to a worker if it
-overlaps its reserve time, but a job that (due to inaccurate walltime
-specification) runs into the reserve time will not be killed (note that
-once the worker exceeds its walltime, the queuing system will kill the
-job anyway). Default 10 (s).
+lowOverallocation
-maxnodes - Determines the maximum number of nodes that can be allocated
-in one coaster block. Default: unlimited.
+ Overallocation is a function of the walltime of a job which determines how
+ long (time-wise) a worker job will be. For example, if a number of 10s jobs
+ are submitted to the coaster service, and the overallocation for 10s jobs
+ is 10, the coaster scheduler will attempt to start worker jobs that have a
+ walltime of 100s. The overallocation is controlled by manipulating the
+ end-points of an overallocation function. The low endpoint, specified by
+ this parameter, is the overallocation for a 1s job. The high endpoint is
+ the overallocation for a (theoretical) job of infinite length. The
+ overallocation for job sizes in the [1s, +inf) interval is determined using
+ an exponential decay function:
+
+ overallocation(walltime) = walltime * (lowOverallocation -
+ highOverallocation) * exp(-walltime * overallocationDecayFactor) +
+ highOverallocation
-maxtime - Indicates the maximum walltime, in seconds, that a coaster
-block can have.
-Default: unlimited.
+ The default value of lowOverallocation is 10.
-remoteMonitorEnabled - If set to "true", the client side will get a
-Swing window showing, graphically, the state of the coaster scheduler
-(blocks, jobs, etc.). Default: false
+highOverallocation
-internalhostname - If the head node has multiple network interfaces,
-only one of which is visible from the worker nodes. The choice of
-which interface is the one that worker nodes can connect to is a
-matter of the particular cluster. This must be set in the your
-sites file to clarify to the workers which exact interface on the
-head node they are to try to connect to.
+ The high overallocation endpoint (as described above). Default: 1
+overallocationDecayFactor
+
+ The decay factor for the overallocation curve. Default 0.001 (1e-3).
+
+spread
+
+ When a large number of jobs is submitted to coaster service, the work is
+ divided into blocks. This parameter allows a rough control of the relative
+ sizes of those blocks. A value of 0 indicates that all work should be divided
+ equally between the blocks (and blocks will therefore have equal sizes). A
+ value of 1 indicates the largest possible spread. The existence of the spread
+ parameter is based on the assumption that smaller overall jobs will generally
+ spend less time in the queue than larger jobs. By submitting blocks of
+ different sizes, submitted jobs may be finished quicker by smaller blocks.
+ Default: 0.9.
+
+reserve
+
+ Reserve time is a time in the allocation of a worker that sits
+ at the end of the worker time and is useable only for critical
+ operations. For example, a job will not be submitted to a worker if it
+ overlaps its reserve time, but a job that (due to inaccurate walltime
+ specification) runs into the reserve time will not be killed (note that
+ once the worker exceeds its walltime, the queuing system will kill the
+ job anyway). Default 10 (s).
+
+maxnodes
+
+ Determines the maximum number of nodes that can be allocated
+ in one coaster block. Default: unlimited.
+
+maxtime
+
+ Indicates the maximum walltime, in seconds, that a coaster
+ block can have.
+ Default: unlimited.
+
+remoteMonitorEnabled
+
+ If set to "true", the client side will get a Swing window showing, graphically,
+ the state of the coaster scheduler (blocks, jobs, etc.). Default: false
+
+internalhostname
+
+ If the head node has multiple network interfaces,
+ only one of which is visible from the worker nodes. The choice of
+ which interface is the one that worker nodes can connect to is a
+ matter of the particular cluster. This must be set in the your
+ sites file to clarify to the workers which exact interface on the
+ head node they are to try to connect to.
+
env namespace
~~~~~~~~~~~~~
+
Profile keys set in the env namespace will be set in the unix
environment of the executed job. Some environment variables influence
the worker-side behaviour of Swift:
-PATHPREFIX - set in env namespace profiles. This path is prefixed onto
-the start of the PATH when jobs are executed. It can be more useful
-than setting the PATH environment variable directly, because setting
-PATH will cause the execution site's default path to be lost.
+PATHPREFIX
-SWIFT_JOBDIR_PATH - set in env namespace profiles. If set, then Swift
-will use the path specified here as a worker-node local temporary
-directory to copy input files to before running a job. If unset, Swift
-will keep input files on the site-shared filesystem. In some cases,
-copying to a worker-node local directory can be much faster than having
-applications access the site-shared filesystem directly.
+ set in env namespace profiles. This path is prefixed onto the start of the
+ PATH when jobs are executed. It can be more useful than setting the PATH
+ environment variable directly, because setting PATH will cause the
+ execution site's default path to be lost.
-SWIFT_EXTRA_INFO - set in env namespace profiles. If set, then Swift
-will execute the command specified in SWIFT_EXTRA_INFO on execution
-sites immediately before each application execution, and will record the
-stdout of that command in the wrapper info log file for that job. This
-is intended to allow software version and other arbitrary information
-about the remote site to be gathered and returned to the submit side.
-(since Swift 0.9)
+SWIFT_JOBDIR_PATH
-SWIFT_GEN_SCRIPTS - set in the env namespace profiles. This variable
-just needs to be set, it doesn't matter what it is set to. If set, then Swift
-will keep the script that was used to execute the job in the job directory.
-The script will be called run.sh and will have the command line that Swift
-tried to execute with.
+ set in env namespace profiles. If set, then Swift will use the path
+ specified here as a worker-node local temporary directory to copy input
+ files to before running a job. If unset, Swift will keep input files on the
+ site-shared filesystem. In some cases, copying to a worker-node local
+ directory can be much faster than having applications access the
+ site-shared filesystem directly.
+SWIFT_EXTRA_INFO
+
+ set in env namespace profiles. If set, then Swift will execute the command
+ specified in SWIFT_EXTRA_INFO on execution sites immediately before each
+ application execution, and will record the stdout of that command in the
+ wrapper info log file for that job. This is intended to allow software
+ version and other arbitrary information about the remote site to be
+ gathered and returned to the submit side. (since Swift 0.9)
+
+SWIFT_GEN_SCRIPTS
+
+ set in the env namespace profiles. This variable just needs to be set, it
+ doesn't matter what it is set to. If set, then Swift will keep the script
+ that was used to execute the job in the job directory. The script will be
+ called run.sh and will have the command line that Swift tried to execute
+ with.
+
=== Dynamic profiles
To set a profile setting based on the value of a Swift variable, you
Modified: branches/faster/docs/userguide/site_catalog
===================================================================
--- branches/faster/docs/userguide/site_catalog 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/site_catalog 2013-07-08 08:19:11 UTC (rev 6619)
@@ -11,7 +11,6 @@
The sites file is formatted as XML. It consists of <pool> elements,
one for each site that Swift will use.
-
Pool element
~~~~~~~~~~~~
Each pool element must have a handle attribute, giving a symbolic
@@ -25,7 +24,6 @@
a remote working directory. Optionally, profile settings can
be specified.
-
File transfer method
~~~~~~~~~~~~~~~~~~~~
Transfer methods are specified with either the <gridftp> element or
@@ -150,14 +148,29 @@
stored.
----
-<workdirectory>/home/benc</workdirectory>
+<workdirectory>/tmp/swift.workdir</workdirectory>
----
-This file must be accessible through the transfer mechanism specified in
-the <gridftp> element and also mounted on all worker nodes that will
-be used for execution. A shared cluster scratch filesystem is
-appropriate for this.
+This directory must be accessible through the specified transfer mechanism and
+also mounted on all worker nodes that will be used for execution. A shared
+cluster scratch filesystem is appropriate for this. Note that you need to
+specify _absolute pathname_ for this field.
+
+Scratch
+~~~~~~~
+
+The scratch element takes in a value of a directory on a shared filesystem. For example:
+
+----
+<scratch>/work/01739/ketan/lab/swift/myscratch</scratch>
+----
+
+The scratch element specifies that the underlying Swift wrapper will copy the
+input files into the scratch directory and the job will be run from that
+directory. In the absence of scratch tag, Swift will run the job from
+workdirectory by creating symlinks to input files in a shared directory.
+
Profiles
~~~~~~~~
Profile keys can be specified using the <profile> element.
Copied: branches/faster/docs/userguide/swift_monitor.png (from rev 6618, trunk/docs/userguide/swift_monitor.png)
===================================================================
(Binary files differ)
Modified: branches/faster/docs/userguide/transformation_catalog
===================================================================
--- branches/faster/docs/userguide/transformation_catalog 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/transformation_catalog 2013-07-08 08:19:11 UTC (rev 6619)
@@ -1,19 +1,20 @@
-The Transformation Catalog - tc.data
-------------------------------------
+The Transformation Catalog - tc
+-------------------------------
The transformation catalog lists where application executables are
-located on remote sites.
+located on execution sites.
By default, the site catalog is stored in etc/tc.data. This path can
be overridden with the tc.file configuration property, either in the
Swift configuration file or on the command line.
-The format is one line per executable per site, with fields separated by
-tabs.
+The format is one line per executable per site, with fields separated by space or tab.
Some example entries:
----
-localhost echo /bin/echo INSTALLED INTEL32::LINUX null
-TGUC touch /usr/bin/touch INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="0:1"
+localhost cat /bin/cat null null null
+localhost vasp /home/ketan/runvasp.sh null null null
+fusion echo /bin/echo INSTALLED INTEL32::LINUX null
+TGUC touch /usr/bin/touch INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="0:1"
----
The fields are: site, transformation name, executable path, installation
@@ -24,11 +25,11 @@
The transformation name should correspond to the transformation name
used in a Swift script app procedure.
-The executable path should specify where the particular executable is
+The executable path should specify where the executable is
located on that site.
The installation status and platform fields are not used. Set them to
-INSTALLED and INTEL32::LINUX respectively.
+INSTALLED and INTEL32::LINUX respectively. Alternatively, they could be set to null.
The profiles field should be set to null if no profile entries are to
be specified.
@@ -39,8 +40,9 @@
This can be accomplished using *env* in the profile entry. For example,
the following application sets an environment variable called R_LIBS to
/home/user/R_libs.
+
-----
-localhost R /usr/bin/R INSTALLED INTEL32::LINUX env::R_LIBS=/home/user/r_libs
+localhost R /usr/bin/R INSTALLED INTEL32::LINUX env::R_LIBS=/home/user/r_libs
-----
Setting Multiple Profiles
@@ -49,5 +51,5 @@
sets two environment variables: R_LIBS and R_HOME.
-----
-localhost R /usr/bin/R INSTALLED INTEL32::LINUX env::R_LIBS=/home/user/r_libs;env::R_HOME=/home/user/r
+localhost R /usr/bin/R INSTALLED INTEL32::LINUX env::R_LIBS=/home/user/r_libs;env::R_HOME=/home/user/r
-----
Modified: branches/faster/docs/userguide/userguide.txt
===================================================================
--- branches/faster/docs/userguide/userguide.txt 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/userguide/userguide.txt 2013-07-08 08:19:11 UTC (rev 6619)
@@ -39,3 +39,5 @@
include::cdm[]
include::log-processing[]
+
+link:http://www.ci.uchicago.edu/swift/docs/index.php[home]
Modified: branches/faster/docs/utils/gensites.txt
===================================================================
--- branches/faster/docs/utils/gensites.txt 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/docs/utils/gensites.txt 2013-07-08 08:19:11 UTC (rev 6619)
@@ -9,8 +9,8 @@
--------
To simplify this configuration process, versions of Swift starting with
0.92 include a utility called gensites. The gensites command is used
-to generate a sites.xml file for running a swift workflow on a given
-site. It accomplishes this by using a series of templates. The templates
+to generate a sites.xml file for running a Swift script on a given
+site. It accomplishes this by using site templates. The templates
used by gensites are the same templates used for internal testing, so
they are likely up to date and known to work on a given site.
@@ -25,21 +25,25 @@
You should see output similar to this:
-----
+beagle
+beagle-ssh
intrepid
local
-local-pbs-coasters
+midway
pads
+persistent-coasters
queenbee
sge-local
-ssh
ssh-pbs-coasters
+stampede
surveyor
+uc3
-----
You will notice that the templates can be specific to a particular set
of machines like Intrepid and Queenbee, or they may be more general and
aim to work across a variety of machines, as in the case of
-local-pbs-coasters. Gensites will look in three directories for available
+ssh-pbs-coasters. Gensites will look in three directories for available
templates: your current directory, $SWIFT_HOME/etc/sites and $HOME/.swift/sites.
Listing the Template
@@ -50,19 +54,19 @@
$ gensites -l templatename
-----
-Running this command will print some information about the template and give
-you an idea of what settings you will need to specify.
+Running this command will print the contents of sites.xml file corresponding to
+the template and give you an idea of what settings you will need to specify.
-The required tokens are required to properly use the templates. These are
-placeholder values you will need to specify in the following steps.
+The tokens are required to properly use the templates. These are placeholder
+values you will need to specify in the following steps.
Providing Site Specific Values
------------------------------
-The gensites script needs to know how to replace the placeholder values
-in the template. This is done by configuring the swift.properties file.
-Gensites will first look for a swift.properties file in the current
-directory. If it does not exist, it will next look in $HOME/.swift.
+The gensites script needs to know how to replace the placeholder values in the
+template. This is done by configuring the swift.properties file. Gensites will
+first look for a swift.properties file in the current directory. If it does not
+exist, it will next look in $HOME/.swift.
To add site specific values to swift.properties, add a line in the
follow format:
@@ -71,9 +75,8 @@
#site templatename setting=value
-----
-Continuing the previous example, here is what you could add to
-swift.properties to replace the values of project, queue and work for
-the surveyor template:
+For example, here is what you could add to swift.properties to replace the
+values of project, queue and work for the surveyor template:
-----
#site surveyor project=MyProject
@@ -119,7 +122,7 @@
This first example shows a site specific application. The #app definition
tells gensites this is related to an application rather than a #site
definition. In the second part, echo=/usr/bin/echo, the left hand side
-is the name of the application that will be called from within swift. The
+is the name of the application that will be called from within Swift. The
right hand site is the path name which points to the binary.
-----
@@ -139,7 +142,7 @@
WARNING: Running gensites with #app definitions will replace any file
called tc.data in your current directory. If a file called tc.data exists,
it will be renamed to tc.data.old. If you run gensites twice, the original
-contents of your tc.data will be lost. Please either rename your tc file
+contents of your tc.data will be lost. Either rename your tc file
or copy to a different location.
Running Swift With the New Configuration
@@ -159,18 +162,19 @@
to the swift.properties, use:
-----
-$ gensites surveyor -p myswift.properties > myconfig.xml
+$ gensites surveyor -p myswift.properties > sites.surveyor.xml
-----
-Next, provide the configuration filename to swift:
+Next, provide the configuration filename to Swift:
-----
-$ swift -sites.file myconfig.xml myscript.swift
+$ swift -sites.file sites.surveyor.xml myscript.swift
-----
Alternatively, if you have specified applications, be sure to load that into
-Swift
+Swift:
+
-----
-$ swift -sites.file myconfig.xml -tc.file tc.data mycript.swift
+$ swift -sites.file sites.surveyor.xml -tc.file tc.data mycript.swift
-----
Providing Default Values for All Templates
Modified: branches/faster/etc/sites/beagle-ssh
===================================================================
--- branches/faster/etc/sites/beagle-ssh 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/etc/sites/beagle-ssh 2013-07-08 08:19:11 UTC (rev 6619)
@@ -1,5 +1,5 @@
<pool handle="beagle">
- <execution provider="coaster" jobmanager="local:pbs" url="login4.beagle.ci.uchicago.edu"/>
+ <execution provider="coaster" jobmanager="ssh-cl:pbs" url="login4.beagle.ci.uchicago.edu"/>
<profile namespace="globus" key="jobsPerNode">24</profile>
<profile namespace="globus" key="lowOverAllocation">100</profile>
<profile namespace="globus" key="highOverAllocation">100</profile>
Copied: branches/faster/etc/sites/local-coasters (from rev 6618, trunk/etc/sites/local-coasters)
===================================================================
--- branches/faster/etc/sites/local-coasters (rev 0)
+++ branches/faster/etc/sites/local-coasters 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1,12 @@
+
+ <pool handle="local-coasters">
+ <filesystem provider="local" />
+ <execution provider="coaster" jobmanager="local:local"/>
+ <profile namespace="karajan" key="jobthrottle">0</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="maxTime">1000</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <workdirectory>_WORK_</workdirectory>
+ <scratch>_SCRATCH_</scratch>
+ </pool>
+
Deleted: branches/faster/etc/sites/stampede
===================================================================
--- branches/faster/etc/sites/stampede 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/etc/sites/stampede 2013-07-08 08:19:11 UTC (rev 6619)
@@ -1,18 +0,0 @@
-
- <pool handle="stampede">
- <execution provider="coaster" url="stampede.tacc.utexas.edu" jobmanager="ssh-cl:slurm"/>
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="maxTime">43200</profile>
- <profile namespace="globus" key="maxwalltime">00:05:00</profile>
- <profile namespace="globus" key="lowOverallocation">100</profile>
- <profile namespace="globus" key="highOverallocation">100</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="project">TG-ASC090068</profile>
- <profile namespace="karajan" key="jobThrottle">100</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>{env.SCRATCH}</workdirectory>
- </pool>
-
Copied: branches/faster/etc/sites/stampede (from rev 6618, trunk/etc/sites/stampede)
===================================================================
--- branches/faster/etc/sites/stampede (rev 0)
+++ branches/faster/etc/sites/stampede 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1,17 @@
+ <pool handle="stampede">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <profile namespace="globus" key="project">TG-EAR130015</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="maxWalltime">01:00:00</profile>
+ <profile namespace="globus" key="maxtime">3700</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="queue">development</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>{env.SCRATCH}</workdirectory>
+ </pool>
Copied: branches/faster/etc/sites/stampede-ssh (from rev 6618, trunk/etc/sites/stampede-ssh)
===================================================================
--- branches/faster/etc/sites/stampede-ssh (rev 0)
+++ branches/faster/etc/sites/stampede-ssh 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1,18 @@
+
+ <pool handle="stampede">
+ <execution provider="coaster" url="stampede.tacc.utexas.edu" jobmanager="ssh-cl:slurm"/>
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="maxTime">43200</profile>
+ <profile namespace="globus" key="maxwalltime">00:05:00</profile>
+ <profile namespace="globus" key="lowOverallocation">100</profile>
+ <profile namespace="globus" key="highOverallocation">100</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="project">TG-ASC090068</profile>
+ <profile namespace="karajan" key="jobThrottle">100</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>{env.SCRATCH}</workdirectory>
+ </pool>
+
Modified: branches/faster/libexec/_swiftwrap.wrapperstaging
===================================================================
--- branches/faster/libexec/_swiftwrap.wrapperstaging 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/libexec/_swiftwrap.wrapperstaging 2013-07-08 08:19:11 UTC (rev 6619)
@@ -58,6 +58,8 @@
localPath() {
# remove protocol://
PATH="${1#*://}"
+ # Remove hostname
+ PATH="${PATH#*/}"
# remove leading "/" if present
PATH="${PATH#/}"
log "Transformed $1 to $PATH"
@@ -67,7 +69,6 @@
stagein() {
SRC=$1
DST=$2
-
case $SRC in
*://*)
;;
@@ -80,12 +81,12 @@
case $SRC in
file://*)
- SRC=${SRC#file://}
+ SRC=${SRC#file://*/}
log "Copying $SRC to $DST"
if [ ! -f $SRC ]; then
fail 254 "Cannot stage in $SRC. File not found."
fi
- cp $SRC $DST 2>&1 >& "$INFO"
+ dd if=$SRC of=$DST bs=16M 2>&1 >& "$INFO"
;;
http://*)
log "Fetching $SRC using wget"
@@ -121,7 +122,7 @@
DIR=`dirname $DST`
mkdir -p $DIR
log "Copying $SRC to $DST"
- cp $SRC $DST 2>&1 >& "$INFO"
+ dd if=$SRC of=$DST bs=16M 2>&1 >& "$INFO"
;;
*://*)
fail 254 "Cannot handle protocol ${DST%%://*}"
@@ -268,6 +269,11 @@
cdm_gather_action $GATHER_MAX $GATHER_OUTPUT
}
+strip_prefix()
+{
+ echo $@ | sed -e s/__root__//g -e s/__parent__//g
+}
+
COMMANDLINE=$@
PARAMFILE=
@@ -329,7 +335,7 @@
shift $SHIFTCOUNT
getarg "-d" "$@"
-DIRS=$VALUE
+DIRS=$( strip_prefix $VALUE )
shift $SHIFTCOUNT
getarg "-if" "$@"
@@ -362,7 +368,7 @@
declare -a CMDARGS
if [ "$PARAMFILE" == "" ] && [ "$1" == "-a" ] ; then
shift
- CMDARGS=("$@")
+ CMDARGS=( $( strip_prefix "$@" ) )
elif [ "$PARAMFILE" != "" ] ; then
CMDARGS=()
FIRST=1
@@ -425,6 +431,7 @@
log "CDM_FILE=$CDM_FILE"
log "ARGS=$@"
log "ARGC=$#"
+
[ -n $MPI_RANK ] && log "MPI_RANK=$MPI_RANK" && log "PMI_RANK=$PMI_RANK"
IFS="|"
@@ -453,8 +460,8 @@
eval cdm_action $DIR "INPUT" $L $CDM_POLICY
continue
fi
-
- stagein $L "$DIR/`localPath $L`"
+
+ stagein $L "$DIR/`localPath $L`"
done
if [[ $CDM_FILE != "" ]]; then
Modified: branches/faster/libexec/swift-int.k
===================================================================
--- branches/faster/libexec/swift-int.k 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/libexec/swift-int.k 2013-07-08 08:19:11 UTC (rev 6619)
@@ -57,17 +57,15 @@
log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found")
}
else {
- try {
- msg := checkErrorFile(rhost, wfdir, jobid, jobdir)
+ msg := try {
+ checkErrorFile(rhost, wfdir, jobid, jobdir)
}
else {
log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing")
throw("No status file was found. Check the shared filesystem on {rhost}")
}
+ throw(msg)
}
- else {
- throw(checkErrorFile(rhost, wfdir, jobid, jobdir))
- }
}
initSharedDir := function(progress, rhost) {
Modified: branches/faster/libexec/swift-lib.k
===================================================================
--- branches/faster/libexec/swift-lib.k 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/libexec/swift-lib.k 2013-07-08 08:19:11 UTC (rev 6619)
@@ -1,5 +1,6 @@
namespace(swiftscript) {
export(extractInt, def("org.griphyn.vdl.karajan.lib.swiftscript.ExtractInt"))
+ export(extractFloat, def("org.griphyn.vdl.karajan.lib.swiftscript.ExtractFloat"))
export(readData, def("org.griphyn.vdl.karajan.lib.swiftscript.ReadData"))
export(readData2, def("org.griphyn.vdl.karajan.lib.swiftscript.ReadStructured"))
export(readStructured, def("org.griphyn.vdl.karajan.lib.swiftscript.ReadStructured"))
Property changes on: branches/faster/src
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/release-0.94/src:6387
/trunk/src:6214,6255
+ /branches/release-0.94/src:6387
/trunk/src:6214,6255,6275-6618
Modified: branches/faster/src/org/globus/swift/data/Action.java
===================================================================
--- branches/faster/src/org/globus/swift/data/Action.java 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/src/org/globus/swift/data/Action.java 2013-07-08 08:19:11 UTC (rev 6619)
@@ -94,6 +94,10 @@
String desthost = bc.getHost();
String workdir = (String) bc.getProperty("workdir");
+ if (workdir != null && !workdir.startsWith("/")) {
+ workdir = System.getProperty("user.dir") + "/" + workdir;
+ }
+
org.globus.swift.data.policy.External.doExternal(srcfile, srcdir,
desthost, workdir + "/" + destdir);
}
Modified: branches/faster/src/org/griphyn/vdl/engine/ProcedureSignature.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/engine/ProcedureSignature.java 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/src/org/griphyn/vdl/engine/ProcedureSignature.java 2013-07-08 08:19:11 UTC (rev 6619)
@@ -185,6 +185,7 @@
add(functionsMap, "arg", returns(STRING), args(STRING, optional(STRING)));
add(functionsMap, "extractInt", returns(INT), args(ANY));
+ add(functionsMap, "extractFloat", returns(FLOAT), args(ANY));
add(functionsMap, "filename", returns(STRING), args(ANY));
add(functionsMap, "filenames", returns(STRING_ARRAY), args(ANY));
add(functionsMap, "dirname", returns(STRING), args(ANY));
Modified: branches/faster/src/org/griphyn/vdl/karajan/Loader.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/karajan/Loader.java 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/src/org/griphyn/vdl/karajan/Loader.java 2013-07-08 08:19:11 UTC (rev 6619)
@@ -111,7 +111,8 @@
System.exit(0);
}
if (!ap.hasValue(ArgumentParser.DEFAULT)) {
- error("No SwiftScript program specified");
+ version();
+ error("No Swift script specified");
}
project = ap.getStringValue(ArgumentParser.DEFAULT);
}
Modified: branches/faster/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2013-07-08 08:19:11 UTC (rev 6619)
@@ -124,6 +124,14 @@
String dir = spec.getDirectory();
BoundContact bc = (BoundContact) contacts[0];
String workdir = (String) bc.getProperty("workdir");
+
+ if (workdir==null){
+ workdir=System.getProperty("user.dir");
+ }
+
+ if (workdir!=null && !workdir.startsWith("/")){
+ workdir=System.getProperty("user.dir")+"/"+workdir;
+ }
if (dir == null || !dir.startsWith("/")) {
if (workdir != null) {
if (dir == null) {
Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java 2013-07-08 08:19:11 UTC (rev 6619)
@@ -87,7 +87,7 @@
for (Object o : pairs) {
String file = (String) o;
- File f = new File(new AbsFile(file).getPath(), dir, host, 0);
+ File f = new File(PathUtils.remotePathName(new AbsFile(file).getPath()), dir, host, 0);
CacheReturn cr = cache.unlockEntry(f, force);
rem.addAll(cr.remove);
}
Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/InFileDirs.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2013-07-08 08:19:11 UTC (rev 6619)
@@ -50,22 +50,13 @@
Channel<Object> ret = cr_vargs.get(stack);
for (String path : files) {
AbsFile af = new AbsFile(path);
+ String dir = af.getDir();
if ("file".equals(af.getProtocol())) {
- String dir = af.getDir();
- // there could be a clash here since
- // "/a/b/c.txt" would be remotely the same
- // as "a/b/c.txt". Perhaps absolute paths
- // should have a unique prefix.
- if (dir.startsWith("/") && dir.length() != 1) {
- ret.add(dir.substring(1));
- }
- else if (dir.length() != 0) {
- ret.add(dir);
- }
+ ret.add(PathUtils.remotePathName(dir));
}
else {
// also prepend host name to the path
- ret.add(af.getHost() + "/" + af.getDir());
+ ret.add(af.getHost() + "/" + PathUtils.remotePathName(dir));
}
}
}
Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/PathUtils.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2013-07-08 08:19:11 UTC (rev 6619)
@@ -55,15 +55,154 @@
@Override
public Object function(Stack stack) {
String dir = new AbsFile(path.getValue(stack)).getDir();
- if (dir.startsWith("/")) {
- return dir.substring(1);
+ return remotePathName(dir);
+ }
+ }
+
+ private static final char EOL = '\0';
+ /**
+ * Replace leading slash if present with "__root__" and replace
+ * parent dir references with "__parent__"
+ */
+
+ public static String remotePathName(String dir) {
+ if (dir.length() == 0) {
+ return dir;
+ StringBuilder sb = null;
+
+ // parse it by hand to avoid creating too much string object garbage
+ boolean modified = false;
+ boolean nondot = false;
+ int dotcount = 0;
+ int start = 0;
+ for (int i = 0; i <= dir.length(); i++) {
+ boolean skip = false;
+ char c;
+ if (i == dir.length()) {
+ c = EOL;
}
else {
- return dir;
+ c = dir.charAt(i);
}
+ switch (c) {
+ case EOL:
+ case '/':
+ if (i == 0) {
+ sb = new StringBuilder();
+ sb.append("__root__/");
+ skip = true;
+ modified = true;
+ }
+ else if (nondot) {
+ // do nothing
+ }
+ else {
+ // only dots. If zero or one, remove completely ("//", "/./")
+ switch (dotcount) {
+ case 0:
+ case 1:
+ case 2:
+ modified = true;
+ skip = true;
+ if (sb == null) {
+ sb = new StringBuilder();
+ append(sb, dir, 0, i - dotcount);
+ }
+ if (dotcount == 2) {
+ sb.append("__parent__");
+ if (c != EOL) {
+ sb.append('/');
+ }
+ }
+ default:
+ // pass along
+ }
+ }
+ nondot = false;
+ dotcount = 0;
+ break;
+ case '.':
+ if (nondot) {
+ // a path element containing a dot among other things
+ // so leave it alone
+ }
+ else {
+ dotcount++;
+ skip = true;
+ }
+ break;
+ default:
+ nondot = true;
+ if (dotcount > 0) {
+ if (modified) {
+ for (int j = 0; j < dotcount; j++) {
+ sb.append('.');
+ }
+ }
+ dotcount = 0;
+ }
+ }
+ if (modified) {
+ if (sb == null) {
+ sb = new StringBuilder();
+ append(sb, dir, 0, i - dotcount);
+ }
+ if (!skip && c != EOL) {
+ sb.append(c);
+ }
+ }
}
+ if (modified) {
+ return sb.toString();
+ }
+ else {
+ return dir;
+ }
}
+
+ public static void testMakeRelative(String str, String expected, boolean samestr) {
+ String result = remotePathName(str);
+ if (!result.equals(expected)) {
+ throw new RuntimeException("input: '" + str + "', expected: '" + expected + "', result: '" + result + "'");
+ }
+ if (samestr && (str != result)) {
+ throw new RuntimeException("Expected same string for '" + str + "'");
+ }
+ System.out.println("OK '" + str + "' -> '" + result + "'");
+ }
+
+ public static void main(String[] args) {
+ testMakeRelative("onething", "onething", true);
+ testMakeRelative("two/things", "two/things", true);
+ testMakeRelative("/absolute/path", "__root__/absolute/path", false);
+ testMakeRelative("../in/the/beginning", "__parent__/in/the/beginning", false);
+ testMakeRelative("in/the/../middle", "in/the/__parent__/middle", false);
+ // nonsensical, but meh
+ testMakeRelative("/../in/the/beginning/absolute", "__root__/__parent__/in/the/beginning/absolute", false);
+ testMakeRelative("/in/the/../middle/absolute", "__root__/in/the/__parent__/middle/absolute", false);
+ testMakeRelative("../in/../many/../places", "__parent__/in/__parent__/many/__parent__/places", false);
+ testMakeRelative("/../in/../many/../places/../absolute", "__root__/__parent__/in/__parent__/many/__parent__/places/__parent__/absolute", false);
+ testMakeRelative("a/single/./dot", "a/single/dot", false);
+ testMakeRelative("double//slash", "double/slash", false);
+ testMakeRelative("./single/dot/at/start", "single/dot/at/start", false);
+ testMakeRelative("/./single/dot/at/start/absolute", "__root__/single/dot/at/start/absolute", false);
+ testMakeRelative("multiple/single/././././dots", "multiple/single/dots", false);
+ // technically this isn't valid, but that's not our problem
+ testMakeRelative("three/.../dots", "three/.../dots", true);
+ testMakeRelative("two/..valid/dots", "two/..valid/dots", true);
+ testMakeRelative("more/val..id/dots", "more/val..id/dots", true);
+ testMakeRelative("/everything/./in/../../one//single../../path", "__root__/everything/in/__parent__/__parent__/one/single../__parent__/path", false);
+ testMakeRelative("..", "__parent__", false);
+ testMakeRelative("ends/in/..", "ends/in/__parent__", false);
+ }
+ private static void append(StringBuilder sb, String str, int begin, int end) {
+ for (int i = begin; i < end; i++) {
+ sb.append(str.charAt(i));
+ }
+ }
+
+
public static class BaseName extends AbstractSingleValuedFunction {
private ArgRef<String> path;
@@ -134,13 +273,13 @@
}
private static String windowsify(String path, boolean windows) {
- if (windows) {
- return path.replace('/', '\\');
- }
- else {
- return path;
- }
- }
+ if (windows) {
+ return path.replace('/', '\\');
+ }
+ else {
+ return path;
+ }
+ }
public static class PathNames extends AbstractSingleValuedFunction {
private ArgRef<List<String>> files;
@@ -154,7 +293,7 @@
public String[] function(Stack stack) {
List<String> l = new ArrayList<String>();
for (String f : files.getValue(stack)) {
- l.add(new AbsFile(f).getPath());
+ l.add(remotePathName(new AbsFile(f).getPath()));
}
return l.toArray(new String[0]);
}
Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java 2013-07-08 08:19:11 UTC (rev 6619)
@@ -296,12 +296,7 @@
*/
public static String relativize(String name) {
name = pathOnly(name);
- if (name != null && name.length() > 0 && name.charAt(0) == '/') {
- return name.substring(1);
- }
- else {
- return name;
- }
+ return PathUtils.remotePathName(name);
}
protected boolean compatible(Type expectedType, Type actualType) {
Copied: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java (from rev 6618, trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java)
===================================================================
--- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java (rev 0)
+++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2012 University of Chicago
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.griphyn.vdl.karajan.lib.swiftscript;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+
+import k.rt.ExecutionException;
+import k.rt.Stack;
+
+import org.globus.cog.karajan.analyzer.ArgRef;
+import org.globus.cog.karajan.analyzer.Signature;
+import org.griphyn.vdl.karajan.lib.SwiftFunction;
+import org.griphyn.vdl.mapping.AbstractDataNode;
+import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.RootDataNode;
+import org.griphyn.vdl.type.Types;
+
+
+public class ExtractFloat extends SwiftFunction {
+ private ArgRef<AbstractDataNode> var;
+
+ @Override
+ protected Signature getSignature() {
+ return new Signature(params("var"));
+ }
+
+ @Override
+ public Object function(Stack stack) {
+ AbstractDataNode handle = this.var.getValue(stack);
+ try {
+ handle.waitFor(this);
+
+ String fn = argList(filename(handle), true);
+ Reader freader = new FileReader(fn);
+ BufferedReader breader = new BufferedReader(freader);
+ String str = breader.readLine();
+ freader.close();
+ DSHandle result = new RootDataNode(Types.FLOAT, Double.parseDouble(str));
+ int provid = SwiftFunction.nextProvenanceID();
+ SwiftFunction.logProvenanceResult(provid, result, "extractfloat");
+ SwiftFunction.logProvenanceParameter(provid, handle, "filename");
+ return result;
+ }
+ catch (IOException ioe) {
+ throw new ExecutionException("Reading integer content of file", ioe);
+ }
+ }
+}
Modified: branches/faster/tests/README
===================================================================
--- branches/faster/tests/README 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/tests/README 2013-07-08 08:19:11 UTC (rev 6619)
@@ -1,6 +1,6 @@
USAGENOTES contains instructions on how to operate the test suite.
It is an asciidoc document, in order to properly build it run the following command:
-$ asciidoc -a toc USAGENOTES
+$ asciidoc -a toc USAGENOTES.txt
It will generate an HTML file called USAGENOTES.
Warning:
Modified: branches/faster/tests/USAGENOTES.txt
===================================================================
--- branches/faster/tests/USAGENOTES.txt 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/tests/USAGENOTES.txt 2013-07-08 08:19:11 UTC (rev 6619)
@@ -36,7 +36,7 @@
have the conventional *_cog/modules/swift_* configuration,
and you have done an *ant dist*, you can run
*********************************************************
- suite.sh -t -o /tmp $PWD/tests/groups/group-all-local.sh
+ suite.sh -t -o /tmp $PWD/groups/group-all-local.sh
*********************************************************
or *cd* into */tmp* and run
**************************************************************
Deleted: branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b.out.expected
===================================================================
--- branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b.out.expected 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b.out.expected 2013-07-08 08:19:11 UTC (rev 6619)
@@ -1 +0,0 @@
-Hello, world!
Copied: branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b0.out.expected (from rev 6618, trunk/tests/language-behaviour/mappers/076-structured-regexp-mapper.b0.out.expected)
===================================================================
--- branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b0.out.expected (rev 0)
+++ branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b0.out.expected 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1 @@
+Hello, world!
Copied: branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b1.out.expected (from rev 6618, trunk/tests/language-behaviour/mappers/076-structured-regexp-mapper.b1.out.expected)
===================================================================
--- branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b1.out.expected (rev 0)
+++ branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.b1.out.expected 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1 @@
+Hello, world!
Modified: branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.swift
===================================================================
--- branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.swift 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/tests/language-behaviour/mappers/076-structured-regexp-mapper.swift 2013-07-08 08:19:11 UTC (rev 6619)
@@ -12,14 +12,14 @@
}
}
+messagefile outfile[] <simple_mapper; prefix="076-structured-regexp-mapper.a",suffix=".out", padding=1>;
-messagefile outfile <"076-structured-regexp-mapper.a.out">;
-messagefile outfile2 <structured_regexp_mapper;
- source=outfile,
- match="(.*)per.a(.*)",
- transform="\\1per.b\\2"
->;
+messagefile outfile2[] <structured_regexp_mapper;
+ source=outfile, match="(.*)per.a(.*)",
+ transform="\\1per.b\\2" >;
-outfile = greeting();
+foreach i in [0:1]{
+ outfile[i] = greeting();
+ outfile2[i] = greeting2(outfile[i]);
+}
-outfile2 = greeting2(outfile);
Modified: branches/faster/tests/language-behaviour/mappers/077-regexp-mapper.swift
===================================================================
--- branches/faster/tests/language-behaviour/mappers/077-regexp-mapper.swift 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/tests/language-behaviour/mappers/077-regexp-mapper.swift 2013-07-08 08:19:11 UTC (rev 6619)
@@ -6,10 +6,10 @@
}
}
-messagefile outfile <regexp_mapper;
- source="outfoo",
- match="(...)(.*)",
- transform="077-regexp-mapper.\\2.\\1">;
+messagefile infile <"077-regexpmapper-input.in">;
+messagefile outfile <regexp_mapper; source="infile",
+ match="(.*)in", transform=@strcat("077-regexp-mapper.foo.","\\1out")>;
+
outfile = greeting();
Copied: branches/faster/tests/language-behaviour/mappers/077-regexpmapper-input.in (from rev 6618, trunk/tests/language-behaviour/mappers/077-regexpmapper-input.in)
===================================================================
--- branches/faster/tests/language-behaviour/mappers/077-regexpmapper-input.in (rev 0)
+++ branches/faster/tests/language-behaviour/mappers/077-regexpmapper-input.in 2013-07-08 08:19:11 UTC (rev 6619)
@@ -0,0 +1 @@
+garuda
Modified: branches/faster/tests/sites/intrepid/sites.template.xml
===================================================================
--- branches/faster/tests/sites/intrepid/sites.template.xml 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/tests/sites/intrepid/sites.template.xml 2013-07-08 08:19:11 UTC (rev 6619)
@@ -18,14 +18,14 @@
<profile namespace="globus" key="queue">_QUEUE_</profile>
<profile namespace="globus" key="kernelprofile">zeptoos</profile>
<profile namespace="globus" key="alcfbgpnat">true</profile>
- <profile namespace="karajan" key="jobthrottle">21</profile>
+ <profile namespace="karajan" key="jobthrottle">5.11</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
<profile namespace="globus" key="jobsPerNode">1</profile>
<profile namespace="globus" key="workerLoggingLevel">DEBUG</profile>
<profile namespace="globus" key="slots">1</profile>
<profile namespace="globus" key="maxTime">900</profile> <!-- seconds -->
- <profile namespace="globus" key="nodeGranularity">64</profile>
- <profile namespace="globus" key="maxNodes">64</profile>
+ <profile namespace="globus" key="nodeGranularity">512</profile>
+ <profile namespace="globus" key="maxNodes">512</profile>
<workdirectory>_WORK_</workdirectory>
</pool>
Modified: branches/faster/tests/sites/mcs/coaster-service.conf
===================================================================
--- branches/faster/tests/sites/mcs/coaster-service.conf 2013-07-08 06:24:47 UTC (rev 6618)
+++ branches/faster/tests/sites/mcs/coaster-service.conf 2013-07-08 08:19:11 UTC (rev 6619)
@@ -33,7 +33,7 @@
# Below are various settings to give information about how to create sites.xml
export WORK=/home/${USER}/work
-export JOBS_PER_NODE=4
+export JOBSPERNODE=4
# Try to determine throttle automatically based on the number of nodes and jobs per node
export JOB_THROTTLE=$( echo "scale=5; ($JOBS_PER_NODE * $( echo $WORKER_HOSTS | wc -w ))/100 - 0.00001"|bc )
More information about the Swift-commit
mailing list