[Swift-commit] r5572 - in SwiftApps/SciColSim: . conf params
wilde at ci.uchicago.edu
wilde at ci.uchicago.edu
Thu Feb 9 16:30:17 CST 2012
Author: wilde
Date: 2012-02-09 16:30:17 -0600 (Thu, 09 Feb 2012)
New Revision: 5572
Added:
SwiftApps/SciColSim/conf/
SwiftApps/SciColSim/conf/basiclocal.xml
SwiftApps/SciColSim/conf/beagle.xml
SwiftApps/SciColSim/conf/cf
SwiftApps/SciColSim/conf/local.xml
SwiftApps/SciColSim/conf/pads.similar.beagle.xml
SwiftApps/SciColSim/conf/sites.beagle.quick.xml
SwiftApps/SciColSim/conf/sites.beagle.xml
SwiftApps/SciColSim/conf/tc
SwiftApps/SciColSim/getswift.sh
SwiftApps/SciColSim/params/
SwiftApps/SciColSim/params/ARtest01
SwiftApps/SciColSim/params/Fast01
SwiftApps/SciColSim/params/MWtest01
SwiftApps/SciColSim/swiftopt.sh
Log:
Add new end-user commands.
Added: SwiftApps/SciColSim/conf/basiclocal.xml
===================================================================
--- SwiftApps/SciColSim/conf/basiclocal.xml (rev 0)
+++ SwiftApps/SciColSim/conf/basiclocal.xml 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,10 @@
+<config>
+
+ <pool handle="localhost">
+ <execution provider="local" url="none" />
+ <filesystem provider="local"/>
+ <workdirectory>/home/wilde/swift/lab/swiftwork</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0</profile>
+ </pool>
+
+</config>
Added: SwiftApps/SciColSim/conf/beagle.xml
===================================================================
--- SwiftApps/SciColSim/conf/beagle.xml (rev 0)
+++ SwiftApps/SciColSim/conf/beagle.xml 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,60 @@
+<config>
+
+<!-- From Glen: used w/ "Swift svn swift-r4813 (swift modified locally) cog-r3175" -->
+
+ <pool handle="beagle">
+
+ <execution provider="coaster" jobmanager="local:pbs" url="none"/>
+ <profile namespace="globus" key="providerAttributes">pbs.aprun;pbs.mpp;depth=24</profile>
+ <profile key="jobsPerNode" namespace="globus">1</profile>
+
+ <profile namespace="env" key="OMP_NUM_THREADS">24</profile>
+ <profile namespace="globus" key="maxwalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">14400</profile>
+ <profile namespace="globus" key="slots">20</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+
+ <profile namespace="globus" key="project">CI-MCB000119</profile>
+ <profile namespace="globus" key="queue">route</profile>
+
+ <filesystem provider="local"/>
+ <workdirectory >/lustre/beagle/wilde/swiftwork</workdirectory>
+ </pool>
+
+<!-- From Justin's swift-devel page:
+
+<import file="sys.xml"/>
+<set name="wdir" value="/lustre/beagle/{user.name}/work"/>
+<echo message="setting workDirectory to: {wdir}"/>
+
+
+<pool handle="beagle-pbs">
+ <execution jobmanager="local:pbs" provider="coaster" url="none"/>
+ <profile namespace="globus" key="maxWallTime">1</profile>
+ <profile namespace="globus" key="maxTime">7200</profile>
+
+ <profile namespace="globus" key="providerAttributes">
+ pbs.aprun;pbs.mpp;depth=24
+ </profile>
+ <profile key="jobsPerNode" namespace="globus">24</profile>
+ <profile key="slots" namespace="globus">1</profile>
+ <profile key="nodeGranularity" namespace="globus">1</profile>
+ <profile key="maxNodes" namespace="globus">1</profile>
+ <profile key="queue" namespace="globus">batch</profile>
+ <profile key="jobThrottle" namespace="karajan">5.99</profile>
+ <profile key="initialScore" namespace="karajan">10000</profile>
+ <profile namespace="globus" key="project">_PROJECT_</profile>
+ <profile namespace="globus" key="project">_QUEUE_</profile>
+
+ <filesystem provider="local" url="none" />
+ <workdirectory>{wdir}</workdirectory>
+</pool>
+
+-->
+
+</config>
Added: SwiftApps/SciColSim/conf/cf
===================================================================
--- SwiftApps/SciColSim/conf/cf (rev 0)
+++ SwiftApps/SciColSim/conf/cf 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,7 @@
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=3
+lazy.errors=true
+status.mode=provider
+use.provider.staging=false
+provider.staging.pin.swiftfiles=false
Added: SwiftApps/SciColSim/conf/local.xml
===================================================================
--- SwiftApps/SciColSim/conf/local.xml (rev 0)
+++ SwiftApps/SciColSim/conf/local.xml 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,9 @@
+<config>
+ <pool handle="localhost" >
+ <execution provider="local" url="none" />
+ <profile namespace="karajan" key="jobThrottle">0.05</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>/gpfs/pads/swift/jonmon/Swift/work/pads</workdirectory>
+ </pool>
+</config>
Added: SwiftApps/SciColSim/conf/pads.similar.beagle.xml
===================================================================
--- SwiftApps/SciColSim/conf/pads.similar.beagle.xml (rev 0)
+++ SwiftApps/SciColSim/conf/pads.similar.beagle.xml 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,23 @@
+<config>
+ <pool handle="pads">
+ <execution provider="coaster" jobmanager="ssh:pbs" url="login.pads.ci.uchicago.edu" />
+ <filesystem provider="local"/>
+
+ <profile namespace="globus" key="project">CI-CCR000013</profile>
+ <profile namespace="env" key="SWIFT_GEN_SCRIPTS">KEEP</profile>
+
+ <profile namespace="globus" key="ppn">8</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="maxTime">3600</profile>
+ <profile namespace="globus" key="maxWallTime">00:02:00</profile>
+ <profile namespace="globus" key="slots">10</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="queue">fast</profile>
+ <profile namespace="karajan" key="jobThrottle">9.59</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/gpfs/pads/swift/jonmon/Swift/work/pads</workdirectory>
+ </pool>
+</config>
Added: SwiftApps/SciColSim/conf/sites.beagle.quick.xml
===================================================================
--- SwiftApps/SciColSim/conf/sites.beagle.quick.xml (rev 0)
+++ SwiftApps/SciColSim/conf/sites.beagle.quick.xml 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,24 @@
+<config>
+ <pool handle="beagle">
+ <execution provider="coaster" jobmanager="local:pbs"/>
+ <profile namespace="globus" key="project">CI-MCB000119</profile>
+
+ <profile namespace="env" key="SWIFT_GEN_SCRIPTS">KEEP</profile>
+
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="env" key="OMP_NUM_THREADS">24</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="providerAttributes">pbs.aprun;pbs.mpp;depth=24</profile>
+ <profile namespace="globus" key="maxTime">10000</profile>
+ <profile namespace="globus" key="maxWallTime">01:30:00</profile>
+ <profile namespace="globus" key="slots">50</profile>
+ <profile namespace="globus" key="nodeGranularity">2</profile>
+ <profile namespace="globus" key="maxNodes">2</profile>
+ <profile namespace="globus" key="queue">route</profile>
+ <profile namespace="karajan" key="jobThrottle">9.59</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory >/lustre/beagle/ketan/labs/SciColSim-Bgl/swift.workdir</workdirectory>
+ </pool>
+</config>
Added: SwiftApps/SciColSim/conf/sites.beagle.xml
===================================================================
--- SwiftApps/SciColSim/conf/sites.beagle.xml (rev 0)
+++ SwiftApps/SciColSim/conf/sites.beagle.xml 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,23 @@
+<config>
+ <pool handle="beagle">
+ <execution provider="coaster" jobmanager="local:pbs"/>
+ <profile namespace="globus" key="project">CI-MCB000119</profile>
+
+ <profile namespace="env" key="SWIFT_GEN_SCRIPTS">KEEP</profile>
+ <!--<profile namespace="globus" key="jobsPerNode">24</profile>-->
+ <profile namespace="env" key="OMP_NUM_THREADS">24</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="providerAttributes">pbs.aprun;pbs.mpp;depth=24</profile>
+ <profile namespace="globus" key="maxTime">37000</profile>
+ <profile namespace="globus" key="maxWallTime">10:00:00</profile>
+ <profile namespace="globus" key="slots">20</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="nodeGranularity">2</profile>
+ <profile namespace="globus" key="maxNodes">2</profile>
+ <profile namespace="karajan" key="jobThrottle">9.59</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory >/lustre/beagle/ketan/labs/SciColSim-Bgl/swift.workdir</workdirectory>
+ </pool>
+</config>
Added: SwiftApps/SciColSim/conf/tc
===================================================================
--- SwiftApps/SciColSim/conf/tc (rev 0)
+++ SwiftApps/SciColSim/conf/tc 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,15 @@
+localhost sh /bin/sh null null null
+localhost cat /bin/cat null null null
+
+pbs cat /bin/cat null null null
+mcs cat /bin/cat null null null
+localhost catnap /home/wilde/swift/lab/catnap.sh null null GLOBUS::maxwalltime="00:01:00"
+
+beagle optimizer /home/wilde/AndreysOptimizer/src/Optimizer null null null
+beagle optimizersh /home/wilde/AndreysOptimizer/src/optimizer.sh null null null
+
+beagle evolve /home/wilde/AndreysOptimizer/src/evolve.sh null null null
+localhost evolve /home/wilde/AndreysOptimizer/src/evolve.sh null null GLOBUS::maxwalltime="02:00:00"
+
+beagle sumloss /home/wilde/AndreysOptimizer/src/sumloss.sh null null null
+localhost sumloss /home/wilde/AndreysOptimizer/src/sumloss.sh null null GLOBUS::maxwalltime="02:00:00"
Added: SwiftApps/SciColSim/getswift.sh
===================================================================
--- SwiftApps/SciColSim/getswift.sh (rev 0)
+++ SwiftApps/SciColSim/getswift.sh 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,30 @@
+#! /bin/sh
+
+swiftrel=http://www.ci.uchicago.edu/swift/packages/swift-0.93.tar.gz
+tarfile=$(basename $swiftrel)
+release=$(basename $swiftrel .tar.gz)
+
+echo
+echo Downloading $release from $swiftrel
+
+rm -f $tarfile
+
+if [ $(uname) = Linux ]; then
+ wget -nv $swiftrel
+else
+ curl $swiftrel > $(basename $swiftrel)
+fi
+
+tar zxf $tarfile
+
+echo
+echo Swift installed at $(pwd)/$release
+echo
+
+rm -f swift # We expect this to be a symlink or non-existant
+ln -s $release swift
+
+./swift/bin/swift -version
+rm -f swift.log
+
+
Property changes on: SwiftApps/SciColSim/getswift.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/SciColSim/params/ARtest01
===================================================================
--- SwiftApps/SciColSim/params/ARtest01 (rev 0)
+++ SwiftApps/SciColSim/params/ARtest01 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,18 @@
+
+min_target_innovation = 58 # starting target innovation value to try
+max_target_innovation = 209 # stops before this target innovation value
+target_innovation_increment = 50 # increment target innovation by this amount
+
+annealing_repeats = 15 # times to repeate the entire optimization process for each target_innovation
+annealing_cycles = 100 # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns = 1000 # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers = 2 # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation = 100 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i = 0.0 # 5 evolve() parameters: must be decimal values !!!
+alpha_m = 0.0 # alpha_i and alpha_m are fixed for now
+beta = 4.0 # rest are varied by the optimization process
+gamma = 50.0
+delta = -1.0
+
Added: SwiftApps/SciColSim/params/Fast01
===================================================================
--- SwiftApps/SciColSim/params/Fast01 (rev 0)
+++ SwiftApps/SciColSim/params/Fast01 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,19 @@
+
+min_target_innovation 58 # starting target innovation value to try
+max_target_innovation 209 # stops before this target innovation value
+target_innovation_increment 50 # increment target innovation by this amount
+
+annealing_repeats 1 # times to repeate the entire optimization process for each target_innovation
+annealing_cycles 10 # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns 10 # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers 2 # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation 100 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i 0.0 # 5 evolve() parameters: must be decimal values !!!
+alpha_m 0.0 # alpha_i and alpha_m are fixed for now
+beta 4.0 # rest are varied by the optimization process
+gamma 50.0
+delta -1.0
+
+
\ No newline at end of file
Added: SwiftApps/SciColSim/params/MWtest01
===================================================================
--- SwiftApps/SciColSim/params/MWtest01 (rev 0)
+++ SwiftApps/SciColSim/params/MWtest01 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,17 @@
+
+min_target_innovation = 58
+max_target_innovation = 1009
+target_innovation_increment = 50
+
+annealing_repeats = 1
+annealing_cycles = 100
+evolve_reruns = 1000
+
+alpha_i = 0.0 # 5 parameters: must be decimal values !!!
+alpha_m = 0.0
+beta = 4.0
+gamma = 50.0
+delta = -1.0
+
+nworkers = 2
+reruns_per_opt_invocation = 100 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
Added: SwiftApps/SciColSim/swiftopt.sh
===================================================================
--- SwiftApps/SciColSim/swiftopt.sh (rev 0)
+++ SwiftApps/SciColSim/swiftopt.sh 2012-02-09 22:30:17 UTC (rev 5572)
@@ -0,0 +1,90 @@
+#! /bin/sh
+
+# Usage: swiftopt.sh [-s sitename] [-p paramfile] [-n # for dryrun ]
+
+usage="$0 [-s sitename] [-p paramfile] [-n] # -n for dryrun: just print params and time estimates"
+
+# Default settings:
+
+execsite=local
+paramfile=Fast01
+ram=2000M
+dryrun=
+
+
+# Process command line arguments
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ -s) execsite=$2; shift 2;;
+ -p) paramfile=$2; shift 2;;
+ -n) dryrun=true; shift 1;;
+ *) echo $usage 1>&2
+ exit 1;;
+ esac
+done
+
+# Create next unique run id and run directory
+
+if [ ! -f nextrun ]; then
+ echo 000 >nextrun
+else
+ cat nextrun | awk '{ printf("%03d\n", $1+1)}' >newrun
+fi
+runid=$(cat newrun)
+mv newrun nextrun
+rundir=run${runid}
+mkdir $rundir
+
+# Get optimization parameters
+
+cp params/$paramfile $rundir/paramfile
+sed -e '/^ */d' -e 's/#.*//' -e 's/ */=/' <params/$paramfile >$rundir/params.annealing
+source $rundir/params.annealing
+
+e33="-e33=$(echo -n -e '\033')"
+swift=../swift/bin/swift # relative to runNNN/ dirs
+
+echo Optimization run $runid: site=$execsite paramfile=$paramfile
+
+cp conf/{tc,cf,$execsite.xml} $rundir
+cp movie_graph.txt $rundir
+
+# Echo parameters
+
+echo Annealing parameters:
+echo
+cat $rundir/params.annealing
+echo
+
+# Echo runtime estimates
+
+echo Total jobs = $((20*$annealing_repeats*$annealing_cycles*3*$evolve_reruns)) # FIXME: Example: replace with real formulas
+
+if [ _$dryrun != _ ]; then
+ exit 0
+fi
+
+# Do the run
+
+cd $rundir
+
+SWIFT_HEAP_MAX=$ram $swift >& swift.out -tc.file tc -sites.file $execsite.xml -config cf ../annealing.swift $e33 \
+ \
+-minrange=$min_target_innovation \
+-maxrange=$max_target_innovation \
+-rangeinc=$target_innovation_increment \
+ \
+-nreps=$annealing_repeats \
+-annealingcycles=$annealing_cycles \
+-evoreruns=$evolve_reruns \
+ \
+-alphai=$alpha_i \
+-alpham=$alpha_m \
+-beta=$beta \
+-gamma=$gamma \
+-delta=$delta \
+ \
+-nworkers=$nworkers \
+-rerunsperapp=$reruns_per_opt_invocation
+
Property changes on: SwiftApps/SciColSim/swiftopt.sh
___________________________________________________________________
Added: svn:executable
+ *
More information about the Swift-commit
mailing list