[Swift-commit] r8236 - in SwiftApps: . subjobs
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Tue Sep 9 12:49:42 CDT 2014
Author: ketan
Date: 2014-09-09 12:49:41 -0500 (Tue, 09 Sep 2014)
New Revision: 8236
Added:
SwiftApps/subjobs/
SwiftApps/subjobs/apps
SwiftApps/subjobs/bg.sh
SwiftApps/subjobs/catsnsleepmpi.swift
SwiftApps/subjobs/cf
SwiftApps/subjobs/cobalt.xml
SwiftApps/subjobs/ensemble_jobs_0.pdf
SwiftApps/subjobs/flexpep.swift
SwiftApps/subjobs/in.data
SwiftApps/subjobs/mpicatnap.c
SwiftApps/subjobs/notes.txt
SwiftApps/subjobs/runbash.sh
SwiftApps/subjobs/runswift.sh
Log:
adding subjob dir
Added: SwiftApps/subjobs/apps
===================================================================
--- SwiftApps/subjobs/apps (rev 0)
+++ SwiftApps/subjobs/apps 2014-09-09 17:49:41 UTC (rev 8236)
@@ -0,0 +1,3 @@
+localhost cat /bin/cat null null null
+cluster bgsh /home/ketan/subjoblab/bg.sh null null null
+
Added: SwiftApps/subjobs/bg.sh
===================================================================
--- SwiftApps/subjobs/bg.sh (rev 0)
+++ SwiftApps/subjobs/bg.sh 2014-09-09 17:49:41 UTC (rev 8236)
@@ -0,0 +1,54 @@
+#! /bin/bash
+
+# Note: This path is for Cetus only
+#export PATH=/soft/cobalt/cetus/bgq_hardware_mapper:$PATH
+# Note: This path is for Mira or Vesta
+export PATH=/soft/cobalt/bgq_hardware_mapper:$PATH
+
+case "$SUBBLOCK_SIZE" in
+8) SHAPE="1x2x2x2x1"
+;;
+16) SHAPE="2x2x2x2x1"
+;;
+32) SHAPE="2x2x2x2x2"
+;;
+64) SHAPE="2x2x4x2x2"
+;;
+128) SHAPE="2x4x4x2x2"
+;;
+256) SHAPE="2x4x4x4x2"
+;;
+512) SHAPE="4x4x4x4x2"
+;;
+*) SHAPE="2x2x2x2x2"
+ REMAINING_ARGS="$@"
+;;
+esac
+
+export SWIFT_SUBBLOCKS=$(get-corners.py "$COBALT_PARTNAME" $SHAPE)
+export SWIFT_SUBBLOCK_ARRAY=($SWIFT_SUBBLOCKS)
+
+echo "$0": SWIFT_SUBBLOCKS="$SWIFT_SUBBLOCKS"
+
+if [ "_$SWIFT_SUBBLOCKS" = _ ]; then
+ echo ERROR: "$0": SWIFT_SUBBLOCKS is null.
+ exit 1
+fi
+
+nsb=${#SWIFT_SUBBLOCK_ARRAY[@]}
+for ((sb=0;sb<nsb;sb++))
+do
+ echo "$0: SUBBLOCK $sb: ${SWIFT_SUBBLOCK_ARRAY[$sb]}"
+done
+
+CORNER=${SWIFT_SUBBLOCK_ARRAY[$SWIFT_JOB_SLOT]}
+
+echo "$0": running BLOCK="$COBALT_PARTNAME" SLOT="$SWIFT_JOB_SLOT"
+echo "$0": running cmd: "$0" args: "$@"
+echo "$0": running runjob --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" --ranks-per-node 1 : "$REMAINING_ARGS"
+
+runjob --strace 0 --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 1 --np "$SUBBLOCK_SIZE" --timeout 100 : "$@"
+
+echo "Runjob finished."
+exit 0
+
Property changes on: SwiftApps/subjobs/bg.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/subjobs/catsnsleepmpi.swift
===================================================================
--- SwiftApps/subjobs/catsnsleepmpi.swift (rev 0)
+++ SwiftApps/subjobs/catsnsleepmpi.swift 2014-09-09 17:49:41 UTC (rev 8236)
@@ -0,0 +1,15 @@
+type file;
+
+app (file o, file sout, file serr) cat (file i)
+{
+ bgsh "/home/ketan/subjoblab/mpicatnap" @i @o arg("s","1") stdout=@sout stderr=@serr;
+}
+
+file out[]<simple_mapper; location="outdir", prefix="f.",suffix=".out">;
+
+foreach j in [1:toInt(arg("n","1"))] {
+ file data<"data.txt">;
+ file sout<single_file_mapper; file=strcat("outdir/f.",j,".stdout")>;
+ file serr<single_file_mapper; file=strcat("outdir/f.",j,".stderr")>;
+ (out[j],sout,serr) = cat(data);
+}
Added: SwiftApps/subjobs/cf
===================================================================
--- SwiftApps/subjobs/cf (rev 0)
+++ SwiftApps/subjobs/cf 2014-09-09 17:49:41 UTC (rev 8236)
@@ -0,0 +1,27 @@
+wrapperlog.always.transfer=false
+provider.staging.pin.swiftfiles=false
+sitedir.keep=false
+kickstart.always.transfer=false
+replication.limit=3
+throttle.host.submit=2
+clustering.min.time=15
+kickstart.enabled=maybe
+execution.retries=10
+pgraph=false
+use.provider.staging=false
+tracing.enabled=true
+foreach.max.threads=1024
+use.wrapper.staging=false
+replication.enabled=false
+throttle.submit=4
+throttle.score.job.factor=0.2
+lazy.errors=true
+clustering.enabled=false
+caching.algorithm=LRU
+pgraph.node.options=color="seagreen", style="filled"
+replication.min.queue.time=60
+clustering.queue.delay=4
+throttle.file.operations=8
+provenance.log=false
+throttle.transfers=4
+pgraph.graph.options=splines="compound", rankdir="TB"
Added: SwiftApps/subjobs/cobalt.xml
===================================================================
--- SwiftApps/subjobs/cobalt.xml (rev 0)
+++ SwiftApps/subjobs/cobalt.xml 2014-09-09 17:49:41 UTC (rev 8236)
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+
+<pool handle="cluster">
+<execution provider="coaster" jobmanager="local:cobalt" />
+
+<profile namespace="globus" key="slots">1</profile>
+<profile namespace="globus" key="mode">script</profile>
+
+<profile namespace="karajan" key="jobThrottle">2.99</profile>
+<profile namespace="karajan" key="initialScore">10000</profile>
+<profile namespace="globus" key="maxwalltime">00:40:00</profile>
+<profile namespace="globus" key="walltime">2050</profile>
+
+<profile namespace="globus" key="maxnodes">256</profile>
+<profile namespace="globus" key="nodegranularity">256</profile>
+<profile key="SUBBLOCK_SIZE" namespace="env">8</profile>
+<profile namespace="globus" key="jobsPerNode">32</profile>
+
+<workdirectory>/home/ketan/swiftwork</workdirectory>
+<filesystem provider="local"/>
+
+</pool>
+</config>
Added: SwiftApps/subjobs/ensemble_jobs_0.pdf
===================================================================
(Binary files differ)
Property changes on: SwiftApps/subjobs/ensemble_jobs_0.pdf
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftApps/subjobs/flexpep.swift
===================================================================
--- SwiftApps/subjobs/flexpep.swift (rev 0)
+++ SwiftApps/subjobs/flexpep.swift 2014-09-09 17:49:41 UTC (rev 8236)
@@ -0,0 +1,34 @@
+type file;
+
+app (file _scorefile, file sout, file serr) flexpep (file _pdb)
+{
+ runjob "/projects/ExM/ketan/openmp-gnu-july16-mini/bin/FlexPepDocking.staticmpi.linuxxlcdebug"
+ "-database /home/ketan/minirosetta_database"
+ "-pep_refine" "-s" @_pdb "-ex1" "-ex2aro"
+ "-use_input_sc" "-nstruct" "1" "-overwrite"
+ "-scorefile" @_scorefile
+ stdout=@sout stderr=@serr;
+}
+
+file scores[] <simple_mapper; location="outdir", prefix="f.",suffix=".sc">;
+file pdb_inputs[] <filesys_mapper; location="/home/ketan/hlac-97-D", pattern="*complex*">;
+
+foreach pdb, j in pdb_inputs {
+ file sout<single_file_mapper; file=strcat("outdir/f.",j,".stdout")>;
+ file serr<single_file_mapper; file=strcat("outdir/f.",j,".stderr")>;
+ (scores[j], sout, serr) = flexpep(pdb);
+}
+
+/*
+qsub --env BG_SHAREDMEMSIZE=32MB -t 20 -n 2 --proccount 2 \
+ --mode c1 \
+ /projects/ExM/ketan/openmp-gnu-july16-mini/bin/FlexPepDocking.staticmpi.linuxxlcdebug \
+ -database /home/ketan/minirosetta_database/ \
+ -pep_refine \
+ -s /home/ketan/hlac-97-D/hlac-97-D-AAADAAAAL_complex_0001.pdb \
+ -ex1 \
+ -ex2aro \
+ -use_input_sc \
+ -nstruct 1 -overwrite \
+ -scorefile AAADAAAAL_complex_91R.sc
+*/
Added: SwiftApps/subjobs/in.data
===================================================================
--- SwiftApps/subjobs/in.data (rev 0)
+++ SwiftApps/subjobs/in.data 2014-09-09 17:49:41 UTC (rev 8236)
@@ -0,0 +1 @@
+this is my input
Added: SwiftApps/subjobs/mpicatnap.c
===================================================================
--- SwiftApps/subjobs/mpicatnap.c (rev 0)
+++ SwiftApps/subjobs/mpicatnap.c 2014-09-09 17:49:41 UTC (rev 8236)
@@ -0,0 +1,45 @@
+#include <stdio.h>
+#include <mpi.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+int main (argc, argv)
+int argc;
+char *argv[];
+{
+ int rank, size;
+
+ MPI_Init (&argc, &argv); /* starts MPI */
+
+ char *ifile = argv[1];
+ char *ofile = argv[2];
+ int sleeptime = atoi(argv[3]);
+
+ MPI_Comm_rank (MPI_COMM_WORLD, &rank); /* get current MPI process id (rank) */
+ MPI_Comm_size (MPI_COMM_WORLD, &size); /* get number of MPI processes */
+
+ if (rank==0) {
+ printf("ifile=%s ofile=%s sleeptime=%d\n", ifile, ofile, sleeptime);
+
+ int ifd = open(ifile,O_RDONLY);
+ int ofd = open(ofile,O_WRONLY|O_CREAT,0664);
+ char buf[1024*1024];
+ for(;;) {
+ int rc = read(ifd, buf, sizeof(buf));
+ if (rc <= 0) {
+ close(ifd);
+ close(ofd);
+ break;
+ }
+ write(ofd, buf, rc);
+ }
+ }
+
+ sleep(sleeptime);
+
+ char host[512];
+ gethostname(host, 512);
+ printf( "Hello from process %d of %d on %s\n", rank, size, host);
+ MPI_Finalize();
+ return 0;
+}
Added: SwiftApps/subjobs/notes.txt
===================================================================
--- SwiftApps/subjobs/notes.txt (rev 0)
+++ SwiftApps/subjobs/notes.txt 2014-09-09 17:49:41 UTC (rev 8236)
@@ -0,0 +1,42 @@
+== Sub-block jobs ==
+
+== Components ==
+
+Outer block
+
+Inner blocks
+
+Parameters
+Blockname
+
+Blockshape
+
+Corner
+
+== Open Issues ==
+
+== Possible applications ==
+
+-- xstack mantivo and proxy by Mike Baru from los alamos
+-- Bill Gropp Mlife game
+-- XMatix Coehmm Justin MPI code
+-- Gromacs, NAMD and LAMMPS
+
+== ToDo ==
+
+robustness testing
+multiple cobalt jobs ("slots")
+multiple sites each with unique subblock sizes
+docs/tutorials
+persistent coasters
+conversion from 0.95 to trunk
+Use the "direct" mode
+cray
+integ runsubjob into swiftwrap and/or worker.pl (initially, swiftwrap)
+(low prio)
+
+all above, but for swift/t
+
+
+separate: ksub/kstat; same but through sockets? ipython tools?
+
Added: SwiftApps/subjobs/runbash.sh
===================================================================
--- SwiftApps/subjobs/runbash.sh (rev 0)
+++ SwiftApps/subjobs/runbash.sh 2014-09-09 17:49:41 UTC (rev 8236)
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+#To run:
+#qsub -n 256 -t 45 --mode script runbash.sh
+
+# Note: This path is for Cetus only
+#export PATH=/soft/cobalt/cetus/bgq_hardware_mapper:$PATH
+# Note: This path is for Mira or Vesta
+export PATH=/soft/cobalt/bgq_hardware_mapper:$PATH
+
+SHAPE="1x2x2x2x1"
+CORNERS=$(get-corners.py $COBALT_PARTNAME $SHAPE)
+
+
+# Set this low for testing.
+# Consult with systems if you need it > 128
+MAXRUNJOB=64
+
+i=0
+for j in $(seq 1 200)
+do
+ for CORNER in $CORNERS
+ do
+ echo "Run $i corner $CORNER"
+ stime=$(shuf -i 1-10 -n 1)
+ runjob --block $COBALT_PARTNAME --corner $CORNER --shape $SHAPE -p 1 --np 8 --timeout 30 : mpicatnap in.data out.data $stime >RUNJOB.$j-$i 2>&1 &
+ # Important - give some time for runjob to get initialized
+ sleep 6
+ i=$((i+1))
+ if [ $i -ge $MAXRUNJOB ]; then
+ echo "Reached MAXRUNJOB $MAXRUNJOB"
+ break;
+ fi
+ done
+ # The runjobs were backgrounded, wait for them to finish
+ wait
+ echo "Done $i"
+ i=0
+done
+
+# Cobalt job ends when this script exits
+exit 0
+
Property changes on: SwiftApps/subjobs/runbash.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/subjobs/runswift.sh
===================================================================
--- SwiftApps/subjobs/runswift.sh (rev 0)
+++ SwiftApps/subjobs/runswift.sh 2014-09-09 17:49:41 UTC (rev 8236)
@@ -0,0 +1,11 @@
+#! /bin/sh
+
+#Old method
+#qsub -A ATPESC2014 -n 512 -t 15 --mode script runswift_script.sh -config cf -tc.file apps -sites.file localcoast.xml "$@"
+
+#New method
+~/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file cobalt.xml -config cf -tc.file apps catsnsleepmpi.swift -n=32
+#~/prebuilt/swift-devel/cog/modules/swift/dist/swift-svn/bin/swift catsnsleepmpi.swift -n=10
+
+#A testblock run
+#qsub -A ATPESC2013 -n 32 -t 5 -q low --mode c16 --mode script testblock.sh # --disable_preboot
Property changes on: SwiftApps/subjobs/runswift.sh
___________________________________________________________________
Added: svn:executable
+ *
More information about the Swift-commit
mailing list