[Swift-commit] r7865 - in SwiftApps/swift-galaxy/swift: . old
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Mon May 12 14:44:03 CDT 2014
Author: ketan
Date: 2014-05-12 14:44:03 -0500 (Mon, 12 May 2014)
New Revision: 7865
Added:
SwiftApps/swift-galaxy/swift/old/2stage.xml
SwiftApps/swift-galaxy/swift/old/dogftp.sh
SwiftApps/swift-galaxy/swift/old/edrevised.mod
SwiftApps/swift-galaxy/swift/old/gftplist.swift
SwiftApps/swift-galaxy/swift/old/mysim.xml
SwiftApps/swift-galaxy/swift/old/swift-2stage
SwiftApps/swift-galaxy/swift/old/swift-3stage
SwiftApps/swift-galaxy/swift/old/swift-skeleton.sh
SwiftApps/swift-galaxy/swift/old/swift-skeleton.xml
SwiftApps/swift-galaxy/swift/old/tc
Removed:
SwiftApps/swift-galaxy/swift/2stage.xml
SwiftApps/swift-galaxy/swift/dogftp.sh
SwiftApps/swift-galaxy/swift/edrevised.mod
SwiftApps/swift-galaxy/swift/gftplist.swift
SwiftApps/swift-galaxy/swift/mysim.xml
SwiftApps/swift-galaxy/swift/swift-2stage
SwiftApps/swift-galaxy/swift/swift-3stage
SwiftApps/swift-galaxy/swift/swift-skeleton.sh
SwiftApps/swift-galaxy/swift/swift-skeleton.xml
SwiftApps/swift-galaxy/swift/tc
Log:
move old stuff in old
Deleted: SwiftApps/swift-galaxy/swift/2stage.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/2stage.xml 2014-05-12 19:42:49 UTC (rev 7864)
+++ SwiftApps/swift-galaxy/swift/2stage.xml 2014-05-12 19:44:03 UTC (rev 7865)
@@ -1,23 +0,0 @@
-<tool id="2stage" name="A Swift 2 stage workflow">
- <description>A 2 stage Swift workflow</description>
-
- <command interpreter="bash">
- swift-2stage "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file"
- </command>
-
- <inputs>
- <param name="site" type="select" label="Execution Location">
- <option value="localhost">Localhost</option>
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="stampede">Stampede</option>
- </param>
- <param name="n" type="text" value="1" label="Number of jobs to run (tested ranges 1 - 1000)"/>
- <param name="data" type="data" label="Input data file"/>
- <!--<param name="outdir" type="text" value="/tmp/outdir" label="Directory to put output data into"/>-->
- </inputs>
- <outputs>
- <data format="txt" name="log_file" type="data" label="Swift output" />
- <data format="html" name="html_file" label="2stageoutput.html" />
- </outputs>
-</tool>
Deleted: SwiftApps/swift-galaxy/swift/dogftp.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/dogftp.sh 2014-05-12 19:42:49 UTC (rev 7864)
+++ SwiftApps/swift-galaxy/swift/dogftp.sh 2014-05-12 19:44:03 UTC (rev 7865)
@@ -1,16 +0,0 @@
-#!/bin/bash -x
-
-#stagein
-#eg. globus-url-copy file:///work/01739/ketan/JSC_AdaptiveSchedule/workflow/sample.txt gsiftp://gridftp.stampede.tacc.utexas.edu:2811/scratch/01739/ketan/storage/
-
-src=$1
-dest=$2
-token=$3
-
-#echo $src > $token
-#echo $dest >> $token
-
-globus-url-copy $src $dest
-#ssh -i /home1/01739/ketan/.ssh/id_rsa maheshwari at swift.rcc.uchicago.edu globus-url-copy gsiftp://trestles-dm2.sdsc.edu:2811/home/ketan/storage/$src gsiftp://gridftp.stampede.tacc.utexas.edu:2811/work/01739/ketan/JSC_AdaptiveSchedule/workflow/dir/$dest
-
-echo $? > $token
Deleted: SwiftApps/swift-galaxy/swift/edrevised.mod
===================================================================
--- SwiftApps/swift-galaxy/swift/edrevised.mod 2014-05-12 19:42:49 UTC (rev 7864)
+++ SwiftApps/swift-galaxy/swift/edrevised.mod 2014-05-12 19:44:03 UTC (rev 7865)
@@ -1,115 +0,0 @@
-# ampl model declaration
-# V. M. Zavala - MCS/ANL 2012
-
-# scenarios
-param NS; # number of scenarios
-set SCEN := 1..NS; # set of scenarios
-
-# dispatch horizon
-param T; # horizon length
-param TH; # simulation horizon (hours)
-param dt; # length of time step [hr]
-set HORIZON := 0..T-1; # horizon minus one
-set TESTTIME:=0..T; # set of test times
-param idx_data; # auxiliary parameter
-param idx_state; # auxiliary parameter
-
-# lines
-set LIN; # set of lines
-param status{LIN}, symbolic; # line status
-param snd_bus{LIN}; # sending bus
-param rec_bus{LIN}; # receiving bus
-param X{LIN}; # reactance - Ohm
-param V{LIN}; # voltage drop - kV
-param Pmax{LIN}; # maximum line capacity - MW
-param install_cost{LIN}; # annualized installation cost $/yr
-
-# buses
-set BUS; # set of buses
-param lat_bus{BUS}; # latitude of bus
-param lon_bus{BUS}; # longitude of bus
-param ref_bus, symbolic; # reference bus
-
-# generators
-set GEN;
-param bus_gen{GEN}; # generator bus
-param np_cap{GEN}; # name place capacity - MW
-param sum_cap{GEN}; # summer capacity - MW
-param win_cap{GEN}; # winter capacity - MW
-param min_cap{GEN}; # minimum capacity - MW
-param fuel{GEN}, symbolic; # type of fuel for generator
-param min_hrate{GEN}; # heat rate at minimum capacity - BTU/kWh
-param min_power{GEN}; # power at minimum capacity - MW
-param max_ur{GEN}; # minimum up ramp - MW
-param max_dr{GEN}; # maximum down ramp - MW
-param lat_gen{GEN}; # latitude of generation bus
-param lon_gen{GEN}; # longitude of generation bus
-param gen_cost{GEN}; # generation cost $/MWhr
-param Pgen_init{GEN}; # initial conditions generators MW
-param clgen_cost{TESTTIME}; # closed-loop generation cost $/MWh
-
-# fuels
-set FUEL;
-param Description{FUEL}, symbolic; # fuel description
-param HV{FUEL}; # heating value (Mbtu/unit)
-param Unitprice{FUEL}; # unit price ($/unit)
-param Units{FUEL}, symbolic; # units
-
-# loads
-set TIME; # set of times for loads
-set LOAD; # set of loads
-param bus_load{LOAD}; # load bus
-param loads{TIME,LOAD}; # loads in space and time - MW
-param wind_totalmh{SCEN,TESTTIME}; # total wind power - MW
-
-# wind data
-set WIND;
-param wind_total{SCEN,TIME}; # total wind power - MW
-param bus_wind{WIND}; # wind bus
-param wind_share{WIND}; # wind share of total power MW/MW
-
-# variables
-var Pgen{SCEN,TESTTIME,GEN}; # generation levels - MW
-var Pload{SCEN,TESTTIME,LOAD}; # load levels - MW
-var yuc{TESTTIME,j in GEN} >= 0, <= 1; # ON/OFF Status generators
-var slackp{s in SCEN, t in TESTTIME}>=0; # slack - for modeling purposes only
-var slackm{s in SCEN, t in TESTTIME}>=0; # slack - for modeling purposes only
-var Pgen_initial{GEN}; # initial state for generators - MW
-
-# params for inference
-param fix_x;
-
-# power flow equations
-pfeq{s in SCEN, t in TESTTIME}:
- +(sum{i in GEN} Pgen[s,t,i])
- -(sum{i in LOAD} Pload[s,t,i])
- + wind_totalmh[s,t]
- = slackp[s,t]-slackm[s,t];
-
-# upper bound and lower bounds for generation flows [MW]
-Pgenequb{s in SCEN,t in TESTTIME,i in GEN}: Pgen[s,t,i] <= np_cap[i]*yuc[t,i];
-Pgeneqlb{s in SCEN,t in TESTTIME,i in GEN}: Pgen[s,t,i] >= 0;
-
-Pgenequbinit{i in GEN}: Pgen_initial[i] <= np_cap[i]*yuc[1,i];
-Pgeneqlbinit{i in GEN}: Pgen_initial[i] >= 0;
-
-# upper and lower bounds on ramps
-upramp{s in SCEN,t in HORIZON,i in GEN}: Pgen[s,t+1,i] - Pgen[s,t,i] <= +max_ur[i];
-loramp{s in SCEN,t in HORIZON,i in GEN}: Pgen[s,t,i] - Pgen[s,t+1,i] >= -max_dr[i];
-
-upramp0{s in SCEN,i in GEN}: Pgen[s,1,i] - Pgen_initial[i] <= +max_ur[i];
-loramp0{s in SCEN,i in GEN}: Pgen_initial[i] - Pgen[s,1,i] >= -max_dr[i];
-
-# time-dependent generation costs
-var tot_gen_cost{s in SCEN,t in TESTTIME} = 1e+0*sum{i in GEN} (gen_cost[i]*Pgen[s,t,i]);
-
-# time-dependent startup/shutdown costs
-var tot_start_cost{t in TESTTIME} = 1e+3*(sum{i in GEN} yuc[t,i]);
-
-# time-dependent slack costs
-var tot_slack_cost{s in SCEN,t in TESTTIME} = 1e+4*(slackp[s,t]+slackm[s,t]);
-
-# cost function
-minimize cost: sum{t in TESTTIME} tot_start_cost[t]
- + (1/(card(SCEN)))*sum{s in SCEN,t in TESTTIME} tot_gen_cost[s,t]
- + (1/(card(SCEN)))*sum{s in SCEN,t in TESTTIME} tot_slack_cost[s,t];
Deleted: SwiftApps/swift-galaxy/swift/gftplist.swift
===================================================================
--- SwiftApps/swift-galaxy/swift/gftplist.swift 2014-05-12 19:42:49 UTC (rev 7864)
+++ SwiftApps/swift-galaxy/swift/gftplist.swift 2014-05-12 19:44:03 UTC (rev 7865)
@@ -1,18 +0,0 @@
-type file;
-
-app (file _status, file _stdout, file _stderr) stage (string _src, string _dst){
- stage _src _dst @_status stdout=@_stdout stderr=@_stderr;
-}
-
-string src="gsiftp://trestles-dm2.sdsc.edu:2811/home/ketan/storage/";
-string dst="gsiftp://gridftp.stampede.tacc.utexas.edu:2811/scratch/01739/ketan/dest/";
-
-string data[] = readData("datlist.txt");
-
-foreach d, i in data{
- file status <single_file_mapper; file=@strcat("outdir/status.",i,".txt")>;
- file out <single_file_mapper; file=@strcat("outdir/std.",i,".out")>;
- file err <single_file_mapper; file=@strcat("outdir/std.",i,".err")>;
- //trace (@strcat(src, d));
- (status, out, err) = stage (@strcat(src, d), @strcat(dst, d));
-}
Deleted: SwiftApps/swift-galaxy/swift/mysim.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/mysim.xml 2014-05-12 19:42:49 UTC (rev 7864)
+++ SwiftApps/swift-galaxy/swift/mysim.xml 2014-05-12 19:44:03 UTC (rev 7865)
@@ -1,24 +0,0 @@
-<tool id="mysim" name="mysim application">
- <description>A Swift tool to run the mysim workflow</description>
-
- <command interpreter="bash">
- swift-mysim "$site" "$nsim" "$steps" "$range" "$vals" "$html_file.files_path" "$log_file" "$html_file"
- </command>
-
- <inputs>
- <param name="site" type="select" label="Execution Location">
- <option value="localhost">Localhost</option>
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="stampede">Stampede</option>
- </param>
- <param name="nsim" type="text" value="10" label="Number of simulation tasks to run "/>
- <param name="steps" type="text" value="1" label="Number of timesteps per simulation"/>
- <param name="range" type="text" value="100" label="Range of the generated random numbers"/>
- <param name="vals" type="text" value="10" label="Number of values generated per simulation"/>
- </inputs>
- <outputs>
- <data format="txt" name="log_file" type="data" label="Swift output" />
- <data format="html" name="html_file" label="mysimoutput.html" />
- </outputs>
-</tool>
Copied: SwiftApps/swift-galaxy/swift/old/2stage.xml (from rev 7862, SwiftApps/swift-galaxy/swift/2stage.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/2stage.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/2stage.xml 2014-05-12 19:44:03 UTC (rev 7865)
@@ -0,0 +1,23 @@
+<tool id="2stage" name="A Swift 2 stage workflow">
+ <description>A 2 stage Swift workflow</description>
+
+ <command interpreter="bash">
+ swift-2stage "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file"
+ </command>
+
+ <inputs>
+ <param name="site" type="select" label="Execution Location">
+ <option value="localhost">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ </param>
+ <param name="n" type="text" value="1" label="Number of jobs to run (tested ranges 1 - 1000)"/>
+ <param name="data" type="data" label="Input data file"/>
+ <!--<param name="outdir" type="text" value="/tmp/outdir" label="Directory to put output data into"/>-->
+ </inputs>
+ <outputs>
+ <data format="txt" name="log_file" type="data" label="Swift output" />
+ <data format="html" name="html_file" label="2stageoutput.html" />
+ </outputs>
+</tool>
Copied: SwiftApps/swift-galaxy/swift/old/dogftp.sh (from rev 7862, SwiftApps/swift-galaxy/swift/dogftp.sh)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/dogftp.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/dogftp.sh 2014-05-12 19:44:03 UTC (rev 7865)
@@ -0,0 +1,16 @@
+#!/bin/bash -x
+
+#stagein
+#eg. globus-url-copy file:///work/01739/ketan/JSC_AdaptiveSchedule/workflow/sample.txt gsiftp://gridftp.stampede.tacc.utexas.edu:2811/scratch/01739/ketan/storage/
+
+src=$1
+dest=$2
+token=$3
+
+#echo $src > $token
+#echo $dest >> $token
+
+globus-url-copy $src $dest
+#ssh -i /home1/01739/ketan/.ssh/id_rsa maheshwari at swift.rcc.uchicago.edu globus-url-copy gsiftp://trestles-dm2.sdsc.edu:2811/home/ketan/storage/$src gsiftp://gridftp.stampede.tacc.utexas.edu:2811/work/01739/ketan/JSC_AdaptiveSchedule/workflow/dir/$dest
+
+echo $? > $token
Copied: SwiftApps/swift-galaxy/swift/old/edrevised.mod (from rev 7862, SwiftApps/swift-galaxy/swift/edrevised.mod)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/edrevised.mod (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/edrevised.mod 2014-05-12 19:44:03 UTC (rev 7865)
@@ -0,0 +1,115 @@
+# ampl model declaration
+# V. M. Zavala - MCS/ANL 2012
+
+# scenarios
+param NS; # number of scenarios
+set SCEN := 1..NS; # set of scenarios
+
+# dispatch horizon
+param T; # horizon length
+param TH; # simulation horizon (hours)
+param dt; # length of time step [hr]
+set HORIZON := 0..T-1; # horizon minus one
+set TESTTIME:=0..T; # set of test times
+param idx_data; # auxiliary parameter
+param idx_state; # auxiliary parameter
+
+# lines
+set LIN; # set of lines
+param status{LIN}, symbolic; # line status
+param snd_bus{LIN}; # sending bus
+param rec_bus{LIN}; # receiving bus
+param X{LIN}; # reactance - Ohm
+param V{LIN}; # voltage drop - kV
+param Pmax{LIN}; # maximum line capacity - MW
+param install_cost{LIN}; # annualized installation cost $/yr
+
+# buses
+set BUS; # set of buses
+param lat_bus{BUS}; # latitude of bus
+param lon_bus{BUS}; # longitude of bus
+param ref_bus, symbolic; # reference bus
+
+# generators
+set GEN;
+param bus_gen{GEN}; # generator bus
+param np_cap{GEN}; # name place capacity - MW
+param sum_cap{GEN}; # summer capacity - MW
+param win_cap{GEN}; # winter capacity - MW
+param min_cap{GEN}; # minimum capacity - MW
+param fuel{GEN}, symbolic; # type of fuel for generator
+param min_hrate{GEN}; # heat rate at minimum capacity - BTU/kWh
+param min_power{GEN}; # power at minimum capacity - MW
+param max_ur{GEN}; # minimum up ramp - MW
+param max_dr{GEN}; # maximum down ramp - MW
+param lat_gen{GEN}; # latitude of generation bus
+param lon_gen{GEN}; # longitude of generation bus
+param gen_cost{GEN}; # generation cost $/MWhr
+param Pgen_init{GEN}; # initial conditions generators MW
+param clgen_cost{TESTTIME}; # closed-loop generation cost $/MWh
+
+# fuels
+set FUEL;
+param Description{FUEL}, symbolic; # fuel description
+param HV{FUEL}; # heating value (Mbtu/unit)
+param Unitprice{FUEL}; # unit price ($/unit)
+param Units{FUEL}, symbolic; # units
+
+# loads
+set TIME; # set of times for loads
+set LOAD; # set of loads
+param bus_load{LOAD}; # load bus
+param loads{TIME,LOAD}; # loads in space and time - MW
+param wind_totalmh{SCEN,TESTTIME}; # total wind power - MW
+
+# wind data
+set WIND;
+param wind_total{SCEN,TIME}; # total wind power - MW
+param bus_wind{WIND}; # wind bus
+param wind_share{WIND}; # wind share of total power MW/MW
+
+# variables
+var Pgen{SCEN,TESTTIME,GEN}; # generation levels - MW
+var Pload{SCEN,TESTTIME,LOAD}; # load levels - MW
+var yuc{TESTTIME,j in GEN} >= 0, <= 1; # ON/OFF Status generators
+var slackp{s in SCEN, t in TESTTIME}>=0; # slack - for modeling purposes only
+var slackm{s in SCEN, t in TESTTIME}>=0; # slack - for modeling purposes only
+var Pgen_initial{GEN}; # initial state for generators - MW
+
+# params for inference
+param fix_x;
+
+# power flow equations
+pfeq{s in SCEN, t in TESTTIME}:
+ +(sum{i in GEN} Pgen[s,t,i])
+ -(sum{i in LOAD} Pload[s,t,i])
+ + wind_totalmh[s,t]
+ = slackp[s,t]-slackm[s,t];
+
+# upper bound and lower bounds for generation flows [MW]
+Pgenequb{s in SCEN,t in TESTTIME,i in GEN}: Pgen[s,t,i] <= np_cap[i]*yuc[t,i];
+Pgeneqlb{s in SCEN,t in TESTTIME,i in GEN}: Pgen[s,t,i] >= 0;
+
+Pgenequbinit{i in GEN}: Pgen_initial[i] <= np_cap[i]*yuc[1,i];
+Pgeneqlbinit{i in GEN}: Pgen_initial[i] >= 0;
+
+# upper and lower bounds on ramps
+upramp{s in SCEN,t in HORIZON,i in GEN}: Pgen[s,t+1,i] - Pgen[s,t,i] <= +max_ur[i];
+loramp{s in SCEN,t in HORIZON,i in GEN}: Pgen[s,t,i] - Pgen[s,t+1,i] >= -max_dr[i];
+
+upramp0{s in SCEN,i in GEN}: Pgen[s,1,i] - Pgen_initial[i] <= +max_ur[i];
+loramp0{s in SCEN,i in GEN}: Pgen_initial[i] - Pgen[s,1,i] >= -max_dr[i];
+
+# time-dependent generation costs
+var tot_gen_cost{s in SCEN,t in TESTTIME} = 1e+0*sum{i in GEN} (gen_cost[i]*Pgen[s,t,i]);
+
+# time-dependent startup/shutdown costs
+var tot_start_cost{t in TESTTIME} = 1e+3*(sum{i in GEN} yuc[t,i]);
+
+# time-dependent slack costs
+var tot_slack_cost{s in SCEN,t in TESTTIME} = 1e+4*(slackp[s,t]+slackm[s,t]);
+
+# cost function
+minimize cost: sum{t in TESTTIME} tot_start_cost[t]
+ + (1/(card(SCEN)))*sum{s in SCEN,t in TESTTIME} tot_gen_cost[s,t]
+ + (1/(card(SCEN)))*sum{s in SCEN,t in TESTTIME} tot_slack_cost[s,t];
Copied: SwiftApps/swift-galaxy/swift/old/gftplist.swift (from rev 7862, SwiftApps/swift-galaxy/swift/gftplist.swift)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/gftplist.swift (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/gftplist.swift 2014-05-12 19:44:03 UTC (rev 7865)
@@ -0,0 +1,18 @@
+type file;
+
+app (file _status, file _stdout, file _stderr) stage (string _src, string _dst){
+ stage _src _dst @_status stdout=@_stdout stderr=@_stderr;
+}
+
+string src="gsiftp://trestles-dm2.sdsc.edu:2811/home/ketan/storage/";
+string dst="gsiftp://gridftp.stampede.tacc.utexas.edu:2811/scratch/01739/ketan/dest/";
+
+string data[] = readData("datlist.txt");
+
+foreach d, i in data{
+ file status <single_file_mapper; file=@strcat("outdir/status.",i,".txt")>;
+ file out <single_file_mapper; file=@strcat("outdir/std.",i,".out")>;
+ file err <single_file_mapper; file=@strcat("outdir/std.",i,".err")>;
+ //trace (@strcat(src, d));
+ (status, out, err) = stage (@strcat(src, d), @strcat(dst, d));
+}
Copied: SwiftApps/swift-galaxy/swift/old/mysim.xml (from rev 7862, SwiftApps/swift-galaxy/swift/mysim.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/mysim.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/mysim.xml 2014-05-12 19:44:03 UTC (rev 7865)
@@ -0,0 +1,24 @@
+<tool id="mysim" name="mysim application">
+ <description>A Swift tool to run the mysim workflow</description>
+
+ <command interpreter="bash">
+ swift-mysim "$site" "$nsim" "$steps" "$range" "$vals" "$html_file.files_path" "$log_file" "$html_file"
+ </command>
+
+ <inputs>
+ <param name="site" type="select" label="Execution Location">
+ <option value="localhost">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ </param>
+ <param name="nsim" type="text" value="10" label="Number of simulation tasks to run "/>
+ <param name="steps" type="text" value="1" label="Number of timesteps per simulation"/>
+ <param name="range" type="text" value="100" label="Range of the generated random numbers"/>
+ <param name="vals" type="text" value="10" label="Number of values generated per simulation"/>
+ </inputs>
+ <outputs>
+ <data format="txt" name="log_file" type="data" label="Swift output" />
+ <data format="html" name="html_file" label="mysimoutput.html" />
+ </outputs>
+</tool>
Copied: SwiftApps/swift-galaxy/swift/old/swift-2stage (from rev 7862, SwiftApps/swift-galaxy/swift/swift-2stage)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swift-2stage (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swift-2stage 2014-05-12 19:44:03 UTC (rev 7865)
@@ -0,0 +1,319 @@
+#!/bin/bash
+
+site=$1
+data=$2
+n=$3
+outdir=$4
+logfile=$5
+htmlfile=$6
+mkdir -p $outdir
+echo $@ >> /tmp/args.txt
+
+swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift
+# use "here" document for configs
+
+cat << EOF > /tmp/2stage.swift
+type file;
+type script;
+
+app (file o) s (file i) {
+ cat @i stdout=@o;
+}
+
+file out1[]<simple_mapper; location="$outdir", prefix="f1.",suffix=".txt">;
+file out2[]<simple_mapper; location="$outdir", prefix="f2.",suffix=".txt">;
+
+string d=@arg("data");
+
+foreach j in [1:@toInt(@arg("n","1"))] {
+ file data<single_file_mapper; file=d>;
+ out1[j] = s(data);
+ out2[j] = s(out1[j]);
+}
+EOF
+
+case "$site" in
+midway )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+ <pool handle="midway">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+midway echo echo
+midway cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+uc3 )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" url="none" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+uc3 echo /bin/echo
+uc3 cat /bin/cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+stampede )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="stampede">
+ <execution provider="coaster" jobmanager="gt2:gt2:slurm" url="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm"/>
+ <!--<execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>-->
+ <!--<filesystem provider="gsiftp" url="gsiftp://gridftp.stampede.tacc.utexas.edu:2811"/>-->
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+stampede echo echo
+stampede cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+localhost )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+esac
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/2stage.swift -n=$n -data=$data 2> std.err
+EXITCODE=$?
+
+swiftlog=`\ls -t *.log | head -1`
+
+#/tmp/plotswiftlogs $logfile 2>gnuplot.err
+
+cat << 'EOF' > /tmp/plotswiftlogs
+#!/bin/bash
+
+#usage: ./extract4plots <swift.logfile>
+
+SWIFTLOGFILE=$1
+#TMPDIR=`mktemp -d plotlog.XXX`
+grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
+SWIFTOUTFILE=swiftoutfile.out
+#extract start time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
+START_TIME=`date +%s -d "$TMPDATE"`
+
+#extract end time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
+END_TIME=`date +%s -d "$TMPDATE"`
+
+#duration
+DIFFTIME=$((END_TIME - START_TIME))
+#echo 0 > active.txt
+#echo 0 > cumulative.txt
+#extract active runs in a file
+(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >> active.txt)
+
+#extract successful completions in a file
+(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >> cumulative.txt)
+
+#prepare tics
+activelines=`wc -l active.txt | awk '{print $1}'`
+cumulines=`wc -l cumulative.txt | awk '{print $1}'`
+
+if [ $activelines -ne 0 ]
+then
+ activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
+fi
+seq 0 $activelinespertic $DIFFTIME > activetics.txt
+
+if [ $cumulines -ne 0 ]
+then
+ cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
+fi
+
+seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
+
+#final plot data
+paste activetics.txt active.txt > plot_active.txt
+paste cumultics.txt cumulative.txt > plot_cumulative.txt
+
+cat << EOF1 > plotit.gp
+set terminal png enhanced
+set output "activeplot.png"
+set nokey
+set xlabel "Time in sec"
+set ylabel "number of active jobs"
+set title "Active jobs"
+plot "plot_active.txt" using 1:2 with line
+
+set output "cumulativeplot.png"
+set ylabel "number of completed jobs"
+set title "Cumulative jobs"
+plot "plot_cumulative.txt" using 1:2 with lines
+EOF1
+
+gnuplot plotit.gp 2>/dev/null
+
+#cp *.png ../
+echo `pwd`>/tmp/curdir
+echo `ls`>/tmp/dirlist
+#cd ..
+EOF
+
+sh /tmp/plotswiftlogs $swiftlog
+cp *.png $outdir/
+
+cat << EOF > $htmlfile
+<!DOCTYPE HTML>
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+ <title>2stage workflow output</title>
+</head>
+<body>
+<h2>Swift run trace plots</h2>
+
+<a href="activeplot.png" alt="active plot">active plot</a>
+<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
+
+<h2>Browse output of 2stage application</h2>
+`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
+</body>
+</html>
+EOF
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat std.err >&2
+fi
+cp std.err $logfile
+rm std.err
+exit $EXITCODE
Copied: SwiftApps/swift-galaxy/swift/old/swift-3stage (from rev 7862, SwiftApps/swift-galaxy/swift/swift-3stage)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swift-3stage (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swift-3stage 2014-05-12 19:44:03 UTC (rev 7865)
@@ -0,0 +1,325 @@
+#!/bin/bash
+
+site=$1
+data=$2
+n=$3
+outdir=$4
+logfile=$5
+htmlfile=$6
+mkdir -p $outdir
+echo $@ >> /tmp/args.txt
+
+swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift
+# use "here" document for configs
+
+cat << EOF > /tmp/2stage.swift
+type file;
+type script;
+
+app (file o) s (file i) {
+ cat @i stdout=@o;
+}
+
+file out1[]<simple_mapper; location="$outdir", prefix="f1.",suffix=".txt">;
+file out2[]<simple_mapper; location="$outdir", prefix="f2.",suffix=".txt">;
+
+string d=@arg("data");
+
+foreach j in [1:@toInt(@arg("n","1"))] {
+ file data<single_file_mapper; file=d>;
+ out1[j] = s(data);
+ out2[j] = s(out1[j]);
+}
+EOF
+
+case "$site" in
+midway )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+ <pool handle="midway">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+midway echo echo
+midway cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+uc3 )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" url="none" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+uc3 echo /bin/echo
+uc3 cat /bin/cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+stampede )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="stampede">
+ <execution provider="coaster" jobmanager="gt2:gt2:slurm" url="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm"/>
+ <!--<execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>-->
+ <!--<filesystem provider="gsiftp" url="gsiftp://gridftp.stampede.tacc.utexas.edu:2811"/>-->
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+stampede echo echo
+stampede cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+localhost )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+esac
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/2stage.swift -n=$n -data=$data 2> std.err
+EXITCODE=$?
+
+swiftlog=`\ls -t *.log | head -1`
+
+#/tmp/plotswiftlogs $logfile 2>gnuplot.err
+
+cat << 'EOF' > /tmp/plotswiftlogs
+#!/bin/bash
+
+#usage: ./extract4plots <swift.logfile>
+
+SWIFTLOGFILE=$1
+#TMPDIR=`mktemp -d plotlog.XXX`
+grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
+SWIFTOUTFILE=swiftoutfile.out
+#extract start time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
+START_TIME=`date +%s -d "$TMPDATE"`
+
+#extract end time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
+END_TIME=`date +%s -d "$TMPDATE"`
+
+#duration
+DIFFTIME=$((END_TIME - START_TIME))
+#echo 0 > active.txt
+#echo 0 > cumulative.txt
+#extract active runs in a file
+(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt)
+
+#extract successful completions in a file
+(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
+
+#prepare tics
+activelines=`wc -l active.txt | awk '{print $1}'`
+cumulines=`wc -l cumulative.txt | awk '{print $1}'`
+
+if [ $activelines -ne 0 ]
+then
+ activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
+fi
+seq 0 $activelinespertic $DIFFTIME > activetics.txt
+
+if [ $cumulines -ne 0 ]
+then
+ cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
+fi
+
+seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
+
+#final plot data
+paste activetics.txt active.txt > plot_active.txt
+paste cumultics.txt cumulative.txt > plot_cumulative.txt
+
+cat << EOF1 > plotit.gp
+set terminal png enhanced
+set output "activeplot.png"
+set nokey
+set xlabel "Time in sec"
+set ylabel "number of active jobs"
+set title "Active jobs"
+plot "plot_active.txt" using 1:2 with line
+
+set output "cumulativeplot.png"
+set ylabel "number of completed jobs"
+set title "Cumulative jobs"
+plot "plot_cumulative.txt" using 1:2 with lines
+EOF1
+
+gnuplot plotit.gp 2>/dev/null
+
+#cp *.png ../
+echo `pwd`>/tmp/curdir
+echo `ls`>/tmp/dirlist
+#cd ..
+EOF
+
+sh /tmp/plotswiftlogs $swiftlog
+cp *.png $outdir/
+
+#for i in `find $outdir`
+#do
+# lastfile=$i
+#done
+#bname=`basename $lastfile`
+
+cat << EOF > $htmlfile
+<!DOCTYPE HTML>
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+ <title>2stage workflow output</title>
+</head>
+<body>
+<h2>Swift run trace plots</h2>
+
+<a href="activeplot.png" alt="active plot">active plot</a>
+<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
+
+<h2>Browse output of 2stage application</h2>
+`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
+</body>
+</html>
+EOF
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat std.err >&2
+fi
+cp std.err $logfile
+rm std.err
+exit $EXITCODE
Copied: SwiftApps/swift-galaxy/swift/old/swift-skeleton.sh (from rev 7862, SwiftApps/swift-galaxy/swift/swift-skeleton.sh)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swift-skeleton.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swift-skeleton.sh 2014-05-12 19:44:03 UTC (rev 7865)
@@ -0,0 +1,187 @@
+#!/bin/bash
+
+#set -x
+set -e
+
+site=$1
+data=$2
+n=$3
+outdir=$4
+logfile=$5
+htmlfile=$6
+mkdir -p $outdir
+
+swift=`which swift`
+# use "here" document for configs
+
+cat << EOF > /tmp/script.swift
+### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+# Place your Swift script here
+# Make sure all data references are full-path
+### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+EOF
+
+case "$site" in
+midway )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="midway">
+ <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+;;
+
+uc3 )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" url="none" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+;;
+
+stampede )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="stampede">
+ <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+;;
+
+localhost )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localhost">
+ <execution provider="local" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localhost">
+ <execution provider="local" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+;;
+esac
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+stampede echo echo
+stampede cat cat
+uc3 echo echo
+uc3 cat cat
+midway echo echo
+midway cat cat
+EOF
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/catsn.swift -n=$n -data=$data 2> $logfile
+EXITCODE=$?
+
+#for i in `find $outdir`
+#do
+# lastfile=$i
+#done
+#bname=`basename $lastfile`
+
+cat << EOF > $htmlfile
+<!DOCTYPE HTML>
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+ <title>Catsn output</title>
+</head>
+<body>
+<h2>Swift run trace plots</h2>
+
+<a href="activeplot.png" alt="active plot">active plot</a>
+<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
+
+<h2>Browse output of catsn application</h2>
+`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
+</body>
+</html>
+EOF
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat $logfile >&2
+fi
+exit $EXITCODE
Copied: SwiftApps/swift-galaxy/swift/old/swift-skeleton.xml (from rev 7862, SwiftApps/swift-galaxy/swift/swift-skeleton.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swift-skeleton.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swift-skeleton.xml 2014-05-12 19:44:03 UTC (rev 7865)
@@ -0,0 +1,22 @@
+<tool id="skeleton(replace with an id)" name="Swift Skeleton (replace with a name)">
+ <description>A description here</description>
+
+ <command interpreter="bash">
+ swift-skeleton.sh "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file"
+ </command>
+
+ <inputs>
+ <param name="site" type="select" label="Execution Location">
+ <option value="localhost">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ </param>
+ <param name="n" type="text" value="1" label="Number of tasks to run "/>
+ <param name="data" type="data" label="Input data file"/>
+ </inputs>
+ <outputs>
+ <data format="txt" name="log_file" type="data" label="Standard out" />
+ <data format="html" name="html_file" label="catsnoutput.html" />
+ </outputs>
+</tool>
Copied: SwiftApps/swift-galaxy/swift/old/tc (from rev 7862, SwiftApps/swift-galaxy/swift/tc)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/tc (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/tc 2014-05-12 19:44:03 UTC (rev 7865)
@@ -0,0 +1,7 @@
+localhost echo echo
+localhost cat cat
+localhost gensample /home/maheshwari/powergridapps/bin/gensample.py
+localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py
+localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py
+localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py
+localhost stage /scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/dogftp.sh
Deleted: SwiftApps/swift-galaxy/swift/swift-2stage
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-2stage 2014-05-12 19:42:49 UTC (rev 7864)
+++ SwiftApps/swift-galaxy/swift/swift-2stage 2014-05-12 19:44:03 UTC (rev 7865)
@@ -1,319 +0,0 @@
-#!/bin/bash
-
-site=$1
-data=$2
-n=$3
-outdir=$4
-logfile=$5
-htmlfile=$6
-mkdir -p $outdir
-echo $@ >> /tmp/args.txt
-
-swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift
-# use "here" document for configs
-
-cat << EOF > /tmp/2stage.swift
-type file;
-type script;
-
-app (file o) s (file i) {
- cat @i stdout=@o;
-}
-
-file out1[]<simple_mapper; location="$outdir", prefix="f1.",suffix=".txt">;
-file out2[]<simple_mapper; location="$outdir", prefix="f2.",suffix=".txt">;
-
-string d=@arg("data");
-
-foreach j in [1:@toInt(@arg("n","1"))] {
- file data<single_file_mapper; file=d>;
- out1[j] = s(data);
- out2[j] = s(out1[j]);
-}
-EOF
-
-case "$site" in
-midway )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="midway">
- <execution provider="coaster" jobmanager="local:slurm"/>
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">2</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.31</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-midway echo echo
-midway cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-uc3 )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" url="none" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-uc3 echo /bin/echo
-uc3 cat /bin/cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-stampede )
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="stampede">
- <execution provider="coaster" jobmanager="gt2:gt2:slurm" url="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm"/>
- <!--<execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>-->
- <!--<filesystem provider="gsiftp" url="gsiftp://gridftp.stampede.tacc.utexas.edu:2811"/>-->
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="project">TG-STA110005S</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
- <profile namespace="globus" key="slurm.mail-type">ALL</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">8000</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.15</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-stampede echo echo
-stampede cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-localhost )
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-* )
-echo "defaulting to localhost"
-
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-esac
-
-$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/2stage.swift -n=$n -data=$data 2> std.err
-EXITCODE=$?
-
-swiftlog=`\ls -t *.log | head -1`
-
-#/tmp/plotswiftlogs $logfile 2>gnuplot.err
-
-cat << 'EOF' > /tmp/plotswiftlogs
-#!/bin/bash
-
-#usage: ./extract4plots <swift.logfile>
-
-SWIFTLOGFILE=$1
-#TMPDIR=`mktemp -d plotlog.XXX`
-grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
-SWIFTOUTFILE=swiftoutfile.out
-#extract start time
-TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
-START_TIME=`date +%s -d "$TMPDATE"`
-
-#extract end time
-TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
-END_TIME=`date +%s -d "$TMPDATE"`
-
-#duration
-DIFFTIME=$((END_TIME - START_TIME))
-#echo 0 > active.txt
-#echo 0 > cumulative.txt
-#extract active runs in a file
-(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >> active.txt)
-
-#extract successful completions in a file
-(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >> cumulative.txt)
-
-#prepare tics
-activelines=`wc -l active.txt | awk '{print $1}'`
-cumulines=`wc -l cumulative.txt | awk '{print $1}'`
-
-if [ $activelines -ne 0 ]
-then
- activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
-fi
-seq 0 $activelinespertic $DIFFTIME > activetics.txt
-
-if [ $cumulines -ne 0 ]
-then
- cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
-fi
-
-seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
-
-#final plot data
-paste activetics.txt active.txt > plot_active.txt
-paste cumultics.txt cumulative.txt > plot_cumulative.txt
-
-cat << EOF1 > plotit.gp
-set terminal png enhanced
-set output "activeplot.png"
-set nokey
-set xlabel "Time in sec"
-set ylabel "number of active jobs"
-set title "Active jobs"
-plot "plot_active.txt" using 1:2 with line
-
-set output "cumulativeplot.png"
-set ylabel "number of completed jobs"
-set title "Cumulative jobs"
-plot "plot_cumulative.txt" using 1:2 with lines
-EOF1
-
-gnuplot plotit.gp 2>/dev/null
-
-#cp *.png ../
-echo `pwd`>/tmp/curdir
-echo `ls`>/tmp/dirlist
-#cd ..
-EOF
-
-sh /tmp/plotswiftlogs $swiftlog
-cp *.png $outdir/
-
-cat << EOF > $htmlfile
-<!DOCTYPE HTML>
-
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
- <title>2stage workflow output</title>
-</head>
-<body>
-<h2>Swift run trace plots</h2>
-
-<a href="activeplot.png" alt="active plot">active plot</a>
-<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
-
-<h2>Browse output of 2stage application</h2>
-`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
-</body>
-</html>
-EOF
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat std.err >&2
-fi
-cp std.err $logfile
-rm std.err
-exit $EXITCODE
Deleted: SwiftApps/swift-galaxy/swift/swift-3stage
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-3stage 2014-05-12 19:42:49 UTC (rev 7864)
+++ SwiftApps/swift-galaxy/swift/swift-3stage 2014-05-12 19:44:03 UTC (rev 7865)
@@ -1,325 +0,0 @@
-#!/bin/bash
-
-site=$1
-data=$2
-n=$3
-outdir=$4
-logfile=$5
-htmlfile=$6
-mkdir -p $outdir
-echo $@ >> /tmp/args.txt
-
-swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift
-# use "here" document for configs
-
-cat << EOF > /tmp/2stage.swift
-type file;
-type script;
-
-app (file o) s (file i) {
- cat @i stdout=@o;
-}
-
-file out1[]<simple_mapper; location="$outdir", prefix="f1.",suffix=".txt">;
-file out2[]<simple_mapper; location="$outdir", prefix="f2.",suffix=".txt">;
-
-string d=@arg("data");
-
-foreach j in [1:@toInt(@arg("n","1"))] {
- file data<single_file_mapper; file=d>;
- out1[j] = s(data);
- out2[j] = s(out1[j]);
-}
-EOF
-
-case "$site" in
-midway )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="midway">
- <execution provider="coaster" jobmanager="local:slurm"/>
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">2</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.31</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-midway echo echo
-midway cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-uc3 )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" url="none" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-uc3 echo /bin/echo
-uc3 cat /bin/cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-stampede )
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="stampede">
- <execution provider="coaster" jobmanager="gt2:gt2:slurm" url="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm"/>
- <!--<execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>-->
- <!--<filesystem provider="gsiftp" url="gsiftp://gridftp.stampede.tacc.utexas.edu:2811"/>-->
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="project">TG-STA110005S</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
- <profile namespace="globus" key="slurm.mail-type">ALL</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">8000</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.15</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-stampede echo echo
-stampede cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-localhost )
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-* )
-echo "defaulting to localhost"
-
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-esac
-
-$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/2stage.swift -n=$n -data=$data 2> std.err
-EXITCODE=$?
-
-swiftlog=`\ls -t *.log | head -1`
-
-#/tmp/plotswiftlogs $logfile 2>gnuplot.err
-
-cat << 'EOF' > /tmp/plotswiftlogs
-#!/bin/bash
-
-#usage: ./extract4plots <swift.logfile>
-
-SWIFTLOGFILE=$1
-#TMPDIR=`mktemp -d plotlog.XXX`
-grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
-SWIFTOUTFILE=swiftoutfile.out
-#extract start time
-TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
-START_TIME=`date +%s -d "$TMPDATE"`
-
-#extract end time
-TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
-END_TIME=`date +%s -d "$TMPDATE"`
-
-#duration
-DIFFTIME=$((END_TIME - START_TIME))
-#echo 0 > active.txt
-#echo 0 > cumulative.txt
-#extract active runs in a file
-(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt)
-
-#extract successful completions in a file
-(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
-
-#prepare tics
-activelines=`wc -l active.txt | awk '{print $1}'`
-cumulines=`wc -l cumulative.txt | awk '{print $1}'`
-
-if [ $activelines -ne 0 ]
-then
- activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
-fi
-seq 0 $activelinespertic $DIFFTIME > activetics.txt
-
-if [ $cumulines -ne 0 ]
-then
- cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
-fi
-
-seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
-
-#final plot data
-paste activetics.txt active.txt > plot_active.txt
-paste cumultics.txt cumulative.txt > plot_cumulative.txt
-
-cat << EOF1 > plotit.gp
-set terminal png enhanced
-set output "activeplot.png"
-set nokey
-set xlabel "Time in sec"
-set ylabel "number of active jobs"
-set title "Active jobs"
-plot "plot_active.txt" using 1:2 with line
-
-set output "cumulativeplot.png"
-set ylabel "number of completed jobs"
-set title "Cumulative jobs"
-plot "plot_cumulative.txt" using 1:2 with lines
-EOF1
-
-gnuplot plotit.gp 2>/dev/null
-
-#cp *.png ../
-echo `pwd`>/tmp/curdir
-echo `ls`>/tmp/dirlist
-#cd ..
-EOF
-
-sh /tmp/plotswiftlogs $swiftlog
-cp *.png $outdir/
-
-#for i in `find $outdir`
-#do
-# lastfile=$i
-#done
-#bname=`basename $lastfile`
-
-cat << EOF > $htmlfile
-<!DOCTYPE HTML>
-
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
- <title>2stage workflow output</title>
-</head>
-<body>
-<h2>Swift run trace plots</h2>
-
-<a href="activeplot.png" alt="active plot">active plot</a>
-<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
-
-<h2>Browse output of 2stage application</h2>
-`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
-</body>
-</html>
-EOF
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat std.err >&2
-fi
-cp std.err $logfile
-rm std.err
-exit $EXITCODE
Deleted: SwiftApps/swift-galaxy/swift/swift-skeleton.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-skeleton.sh 2014-05-12 19:42:49 UTC (rev 7864)
+++ SwiftApps/swift-galaxy/swift/swift-skeleton.sh 2014-05-12 19:44:03 UTC (rev 7865)
@@ -1,187 +0,0 @@
-#!/bin/bash
-
-#set -x
-set -e
-
-site=$1
-data=$2
-n=$3
-outdir=$4
-logfile=$5
-htmlfile=$6
-mkdir -p $outdir
-
-swift=`which swift`
-# use "here" document for configs
-
-cat << EOF > /tmp/script.swift
-### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-# Place your Swift script here
-# Make sure all data references are full-path
-### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-EOF
-
-case "$site" in
-midway )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
- <pool handle="midway">
- <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">2</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.31</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-;;
-
-uc3 )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
- <pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" url="none" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-;;
-
-stampede )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="stampede">
- <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="project">TG-STA110005S</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
- <profile namespace="globus" key="slurm.mail-type">ALL</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">8000</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.15</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
-</pool>
-</config>
-EOF
-;;
-
-localhost )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="localhost">
- <execution provider="local" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
-</pool>
-</config>
-EOF
-;;
-
-* )
-echo "defaulting to localhost"
-
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="localhost">
- <execution provider="local" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
-</pool>
-</config>
-EOF
-;;
-esac
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-stampede echo echo
-stampede cat cat
-uc3 echo echo
-uc3 cat cat
-midway echo echo
-midway cat cat
-EOF
-
-$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/catsn.swift -n=$n -data=$data 2> $logfile
-EXITCODE=$?
-
-#for i in `find $outdir`
-#do
-# lastfile=$i
-#done
-#bname=`basename $lastfile`
-
-cat << EOF > $htmlfile
-<!DOCTYPE HTML>
-
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
- <title>Catsn output</title>
-</head>
-<body>
-<h2>Swift run trace plots</h2>
-
-<a href="activeplot.png" alt="active plot">active plot</a>
-<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
-
-<h2>Browse output of catsn application</h2>
-`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
-</body>
-</html>
-EOF
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat $logfile >&2
-fi
-exit $EXITCODE
Deleted: SwiftApps/swift-galaxy/swift/swift-skeleton.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-skeleton.xml 2014-05-12 19:42:49 UTC (rev 7864)
+++ SwiftApps/swift-galaxy/swift/swift-skeleton.xml 2014-05-12 19:44:03 UTC (rev 7865)
@@ -1,22 +0,0 @@
-<tool id="skeleton(replace with an id)" name="Swift Skeleton (replace with a name)">
- <description>A description here</description>
-
- <command interpreter="bash">
- swift-skeleton.sh "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file"
- </command>
-
- <inputs>
- <param name="site" type="select" label="Execution Location">
- <option value="localhost">Localhost</option>
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="stampede">Stampede</option>
- </param>
- <param name="n" type="text" value="1" label="Number of tasks to run "/>
- <param name="data" type="data" label="Input data file"/>
- </inputs>
- <outputs>
- <data format="txt" name="log_file" type="data" label="Standard out" />
- <data format="html" name="html_file" label="catsnoutput.html" />
- </outputs>
-</tool>
Deleted: SwiftApps/swift-galaxy/swift/tc
===================================================================
--- SwiftApps/swift-galaxy/swift/tc 2014-05-12 19:42:49 UTC (rev 7864)
+++ SwiftApps/swift-galaxy/swift/tc 2014-05-12 19:44:03 UTC (rev 7865)
@@ -1,7 +0,0 @@
-localhost echo echo
-localhost cat cat
-localhost gensample /home/maheshwari/powergridapps/bin/gensample.py
-localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py
-localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py
-localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py
-localhost stage /scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/dogftp.sh
More information about the Swift-commit
mailing list