From jonmon at ci.uchicago.edu Thu Mar 1 11:04:12 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Thu, 1 Mar 2012 11:04:12 -0600 (CST) Subject: [Swift-commit] r5699 - in SwiftApps/SciColSim: . bin Message-ID: <20120301170412.081E59CD15@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-01 11:04:11 -0600 (Thu, 01 Mar 2012) New Revision: 5699 Modified: SwiftApps/SciColSim/annealing.swift SwiftApps/SciColSim/bin/showbest.sh SwiftApps/SciColSim/evolve.sh SwiftApps/SciColSim/sumloss.sh Log: o the average time of each optimizer call is now calculated and reported Modified: SwiftApps/SciColSim/annealing.swift =================================================================== --- SwiftApps/SciColSim/annealing.swift 2012-03-01 01:25:50 UTC (rev 5698) +++ SwiftApps/SciColSim/annealing.swift 2012-03-01 17:04:11 UTC (rev 5699) @@ -11,6 +11,8 @@ { float loss; float sdev; + float tloss; + float tsdev; } global boolean FIX_VARIABLES = true; @@ -174,8 +176,8 @@ if ( mlres[i][j].loss < ALOT ) { - fprintf( some_out_filename, "N, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n", - i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss, try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev ); // Note i-1 field: print that way to match with C++ output + fprintf( some_out_filename, "N, %fs, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n", + mlres[i][j].tloss, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss, try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev ); // Note i-1 field: print that way to match with C++ output // fprintf( "max_dist_swift.txt", color( Red,"multi_annealing: AF: max_dist.txt - tbd\n" ) ); // FIXME: max_dist is global set in evolve() } Modified: SwiftApps/SciColSim/bin/showbest.sh =================================================================== --- SwiftApps/SciColSim/bin/showbest.sh 2012-03-01 01:25:50 UTC (rev 5698) +++ SwiftApps/SciColSim/bin/showbest.sh 2012-03-01 17:04:11 UTC (rev 5699) @@ -1,8 +1,8 @@ #! /bin/sh awk '{ - printf( "N %2d %2d %10.5f %5.2f | %5.2f %10.5f [ %5.2f %5.2f %10.5f %10.5f %10.5f ] %10.5f\n", - $2, $3, $4, $5, $7, $8, $10, $11, $12, $13, $14, $16); + printf( "N %.2f %2d %2d %10.5f %5.2f | %5.2f %10.5f [ %5.2f %5.2f %10.5f %10.5f %10.5f ] %10.5f\n", + $2, $3, $4, $5, $7, $8, $10, $11, $12, $13, $14, $16, $17); }' Modified: SwiftApps/SciColSim/evolve.sh =================================================================== --- SwiftApps/SciColSim/evolve.sh 2012-03-01 01:25:50 UTC (rev 5698) +++ SwiftApps/SciColSim/evolve.sh 2012-03-01 17:04:11 UTC (rev 5699) @@ -3,17 +3,24 @@ touch multi_loss.data shift 1 NWORKERS=${23} +START=$(date +%s) OMP_NUM_THREADS=$NWORKERS $(dirname $0)/openmp-optimizer $* 2>&1 -if [ $? == 127 ]; +RC=$? +END=$(date +%s) +if [ $RC == 127 ]; then echo "Missing openmp-optimizer binary" >&2 exit 1 fi -if [ $? != 0 ]; +if [ $RC != 0 ]; then echo "Problem running the openmp-optimizer" >&2 exit 1 fi mv multi_loss.data $datafile +DIFF=$(($END-$START)) +echo $DIFF >> $datafile + +exit $RC Modified: SwiftApps/SciColSim/sumloss.sh =================================================================== --- SwiftApps/SciColSim/sumloss.sh 2012-03-01 01:25:50 UTC (rev 5698) +++ SwiftApps/SciColSim/sumloss.sh 2012-03-01 17:04:11 UTC (rev 5699) @@ -2,26 +2,36 @@ awk ' -BEGIN { n = 0; loss = 0; } +BEGIN { RS = ""; FS="\n"; loss = 0; loss_sq = 0; tloss = 0; tloss_sq = 0; n = 0; tn = 0; } { - loss += $1 - loss_sq += ($1*$1) - n++ + for(i=1;i Author: wilde Date: 2012-03-01 11:22:06 -0600 (Thu, 01 Mar 2012) New Revision: 5700 Modified: trunk/examples/tutorial/ParameterSweep/sweep.swift Log: revert toInt back to toint unitl we sort out camelCase issues. Should now work OK with 0.93. Modified: trunk/examples/tutorial/ParameterSweep/sweep.swift =================================================================== --- trunk/examples/tutorial/ParameterSweep/sweep.swift 2012-03-01 17:04:11 UTC (rev 5699) +++ trunk/examples/tutorial/ParameterSweep/sweep.swift 2012-03-01 17:22:06 UTC (rev 5700) @@ -16,8 +16,8 @@ # Set the size of the parameter sweep -int nMembers = @toInt(@arg("nMembers","5")); // number of members in the simulation -int nCommon = @toInt(@arg("nCommon","3")); // number of common files to each sim +int nMembers = @toint(@arg("nMembers","5")); // number of members in the simulation +int nCommon = @toint(@arg("nCommon","3")); // number of common files to each sim tracef("Running parameter sweep ensemble of %i members with %i common files\n", nMembers, nCommon); # Generate the file names to use From ketan at ci.uchicago.edu Thu Mar 1 11:37:45 2012 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 1 Mar 2012 11:37:45 -0600 (CST) Subject: [Swift-commit] r5701 - branches/release-0.93/docs/userguide Message-ID: <20120301173745.F34939CD15@svn.ci.uchicago.edu> Author: ketan Date: 2012-03-01 11:37:45 -0600 (Thu, 01 Mar 2012) New Revision: 5701 Modified: branches/release-0.93/docs/userguide/faq Log: answered faq Modified: branches/release-0.93/docs/userguide/faq =================================================================== --- branches/release-0.93/docs/userguide/faq 2012-03-01 17:22:06 UTC (rev 5700) +++ branches/release-0.93/docs/userguide/faq 2012-03-01 17:37:45 UTC (rev 5701) @@ -2,21 +2,37 @@ ================================ Q1. What is Swift? +A1. It is a parallel scripting framework for HPC and distributed computing. -Q2. Does it have nested foreach? +Q2. Does Swift have nested foreach? +A2. Yes. Q3. What is the learning curve like? +A3. If you can read/write basic C/C++ and some UNIX, the curve is on the better side of hikeable. Q4. What if my execution fails? +A4. You could get an idea of what went wrong from the Swift stdout/stderr or from the Swift log. If all else fails, write to swift team. Q5. What if the remote node on which I am running fails? +A5. Depending upon your retry settings the failed jobs from that failed node will be rescheduled on other working nodes. Q6. How do I know if my service is running? +A6. ps -u `whoami` | grep coaster Q7. What if I submit a Swift run from my laptop and shut down the laptop? +A7. The run will stop. You may resume this run from the point it stopped upon restart. A better option would be to run swift from a node expected to be up. For example a login node or a cluster node. -Q8. How do I run parallell and serial jobs together? +Q8. How do I run parallel and serial jobs together? +A8. By designating separate site pools for each of them. Q9. What kind of data movements are allowed? +A9. Any kind of data movement that could be done via local, ssh, tcp, http or gridftp protocols are allowed. Q10. Can Swift run interactive programs? +A10. No, per se. The interactive inputs could be loaded on a file and fed to swift from commandline or using a wrapper script around your application. + +Q11. How can you run two statements in sequence when there is a data dependency between them? I do not see a wait() statement in between? +A11. Write once future variables enable asynchronous execution. + +Q12. What are coasters? +A12. Coasters are pilot-job like mechanism available with Swift. Essentially a client server mechanism for many-task coordination. From jonmon at ci.uchicago.edu Thu Mar 1 14:15:13 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Thu, 1 Mar 2012 14:15:13 -0600 (CST) Subject: [Swift-commit] r5702 - trunk/etc Message-ID: <20120301201513.1CC319CCE8@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-01 14:15:12 -0600 (Thu, 01 Mar 2012) New Revision: 5702 Modified: trunk/etc/coaster-service.conf Log: o have a default setting for the kernel profile in the coaster-service.conf file Modified: trunk/etc/coaster-service.conf =================================================================== --- trunk/etc/coaster-service.conf 2012-03-01 17:37:45 UTC (rev 5701) +++ trunk/etc/coaster-service.conf 2012-03-01 20:15:12 UTC (rev 5702) @@ -41,6 +41,9 @@ # Setting for WORKER_INIT_CMD export WORKER_ENVIRONMENT= +# Kernel profile for cobalt jobs +export KERNEL=zeptoos + # Swift information for creating sites.xml export PROJECT= export WORK=$HOME/swiftwork From jonmon at ci.uchicago.edu Thu Mar 1 14:22:58 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Thu, 1 Mar 2012 14:22:58 -0600 (CST) Subject: [Swift-commit] r5703 - trunk/src/org/griphyn/vdl/engine Message-ID: <20120301202258.647469CCE8@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-01 14:22:58 -0600 (Thu, 01 Mar 2012) New Revision: 5703 Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java Log: o The deprecated function message is no longer printed, the code however marking functions as deprecated has been left in tact. Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2012-03-01 20:15:12 UTC (rev 5702) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2012-03-01 20:22:58 UTC (rev 5703) @@ -31,7 +31,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.StringTokenizer; import javax.xml.namespace.QName; @@ -670,7 +669,7 @@ ProcedureSignature proc = proceduresMap.get(procName); if (proc.isDeprecated()) { - warn(call, "Procedure " + procName + " is deprecated"); + /* warn(call, "Procedure " + procName + " is deprecated"); */ } StringTemplate callST; From jonmon at ci.uchicago.edu Thu Mar 1 18:46:25 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Thu, 1 Mar 2012 18:46:25 -0600 (CST) Subject: [Swift-commit] r5704 - SwiftApps/SciColSim/params Message-ID: <20120302004625.5EBB49CCE8@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-01 18:46:25 -0600 (Thu, 01 Mar 2012) New Revision: 5704 Modified: SwiftApps/SciColSim/params/ARtest02 Log: o changes to some of the parameters for ARtest02 Modified: SwiftApps/SciColSim/params/ARtest02 =================================================================== --- SwiftApps/SciColSim/params/ARtest02 2012-03-01 20:22:58 UTC (rev 5703) +++ SwiftApps/SciColSim/params/ARtest02 2012-03-02 00:46:25 UTC (rev 5704) @@ -1,6 +1,6 @@ min_target_innovation 58 # starting target innovation value to try -max_target_innovation 109 # stops before this target innovation value +max_target_innovation 59 # stops before this target innovation value target_innovation_increment 50 # increment target innovation by this amount annealing_repeats 1 # times to repeate the entire optimization process for each target_innovation @@ -8,7 +8,7 @@ evolve_reruns 10000 # times to perform evolve_to_target_and_save() (objective function - is batched) nworkers 24 # number of parallel threads (cores) to use per invocation of C++ optimizer -reruns_per_opt_invocation 240 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!! +reruns_per_opt_invocation 48 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!! alpha_i 0.0 # 5 evolve() parameters: must be decimal values !!! alpha_m 0.0 # alpha_i and alpha_m are fixed for now From swift at ci.uchicago.edu Fri Mar 2 11:00:03 2012 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 2 Mar 2012 11:00:03 -0600 (CST) Subject: [Swift-commit] Cog update Message-ID: <20120302170003.A273F8D00092@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3362 | jmwozniak | 2012-03-02 10:58:15 -0600 (Fri, 02 Mar 2012) | 2 lines Put WORKER_INIT_CMD after reconnect() ------------------------------------------------------------------------ Index: modules/provider-coaster/resources/worker.pl =================================================================== --- modules/provider-coaster/resources/worker.pl (revision 3361) +++ modules/provider-coaster/resources/worker.pl (working copy) @@ -406,11 +406,12 @@ if (defined $ENV{"WORKER_COPIES"}) { workerCopies($ENV{"WORKER_COPIES"}); } + + reconnect(); + if(defined $ENV{"WORKER_INIT_CMD"}) { worker_init_cmd($ENV{"WORKER_INIT_CMD"}); } - - reconnect(); } sub logsetup() { @@ -442,7 +443,7 @@ my ($cmd) = @_; wlog DEBUG, "worker_init_cmd: $cmd\n"; my $rc = system($cmd); - print "rc: $rc\n"; + print "worker_init_cmd exit code: $rc\n"; } sub trim { From jonmon at ci.uchicago.edu Wed Mar 7 10:06:18 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Wed, 7 Mar 2012 10:06:18 -0600 (CST) Subject: [Swift-commit] r5705 - SwiftApps/SciColSim Message-ID: <20120307160618.7926D9CCB6@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-07 10:06:17 -0600 (Wed, 07 Mar 2012) New Revision: 5705 Modified: SwiftApps/SciColSim/optimizer.cpp Log: o added loss limiting code in the optimizer for the SciColSim app. Needs larger scale testing. Modified: SwiftApps/SciColSim/optimizer.cpp =================================================================== --- SwiftApps/SciColSim/optimizer.cpp 2012-03-02 00:46:25 UTC (rev 5704) +++ SwiftApps/SciColSim/optimizer.cpp 2012-03-07 16:06:17 UTC (rev 5705) @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include @@ -69,6 +69,8 @@ #define FIX_VARIABLES 1 +#define MAX_LOSS 160 + using namespace boost; using namespace std; using namespace boost::numeric::ublas; @@ -114,10 +116,10 @@ GaussNum += ((double)rand()/(double)RAND_MAX - 0.5); } GaussNum = GaussNum*sqrt((double)12/(double)NumInSum); - - + + return GaussNum; - + } @@ -133,27 +135,27 @@ //================================================ //================================================================ double get_new_x(double x, double dx){ - + double new_x; // boost::variate_generator > uni(generator, uni_dist); double r = rand()/(double)(pow(2.,31)-1.); - - if (r > 0.5){ + + if (r > 0.5){ new_x = x + rand()*dx/(double)(pow(2.,31)-1.); - } else { + } else { new_x = x - rand()*dx/(double)(pow(2.,31)-1.); } - + return new_x; - + } -//=============================================== +//=============================================== string string_wrap(string ins, int mode){ - + std::ostringstream s; - + switch(mode){ case 0: s << "\033[1;29m" << ins << "\033[0m"; @@ -191,143 +193,143 @@ default: s << ins; } - + return s.str(); } //=============================================== string wrap_double(double val, int mode){ - + std::ostringstream s; s << string_wrap(strDouble(val),mode); - + return s.str(); } //=============================================== -const +const string i2string(int i){ - + std::ostringstream s; - s << "worker" + s << "worker" << lexical_cast(i); - + return s.str(); - + } //=============================================== char* i2char(int i){ - + std::ostringstream s; - s << "worker" + s << "worker" << lexical_cast(i); - + char* a=new char[s.str().size()+1]; memcpy(a,s.str().c_str(), s.str().size()); - + return a; } //================================================ class Universe { - + private: - + double alpha_i; double alpha_m; double beta; double gamma; double delta; - + double TargetNovelty; double CumulativeRelativeLoss; double CRLsquare; string id; - - + + int N_nodes; int M_edges; - + int N_epochs; int N_steps; int N_repeats; - + int current_epoch; double current_loss; int current_repeat; double current_novelty; - + int mode_identify_failed; int verbose_level; // 0 is silent, higher is more - + double k_max; - + graph_t Full_g; - + double **Prob; double **Tried; double **Dist; double **Final; double **EdgeIndex; double *Rank; - - base_generator_type generator; + + base_generator_type generator; boost::uniform_real<> uni_dist; boost::geometric_distribution geo; - + public: - - - + + + //====== Constructor ====== Universe(const std::string FileToOpen, int Epochs, int Steps, int Repeats, int identify_failed, double target, const std::string idd) { //typedef array_type2::index index2; - - + + std::ifstream inFile; //string line; - + //------------------------------- - + base_generator_type gene(42u); generator = gene; generator.seed(static_cast(std::time(0))); boost::uniform_real<> uni_d(0,1); uni_dist = uni_d; - + //-------------------------------- - + int i, k; int x, y; Edge* edge_array_mine; int num_arcs_mine, num_nodes_mine; int* weights_mine; - + TargetNovelty = target; CumulativeRelativeLoss = 0.; CRLsquare = 0.; - - + + N_epochs = Epochs; N_steps = Steps; N_repeats = Repeats; - + current_epoch = 0; current_loss = 0.; current_repeat = 0; - + id = idd; - + verbose_level = 1; - + mode_identify_failed = identify_failed; - - + + //------------------------------- // The first pass though file with the graph inFile.open(FileToOpen.c_str()); @@ -335,57 +337,57 @@ cout << "Unable to open file"; exit(1); // terminate with error }else { - + if (verbose_level > 2){ std::cout << " Opened <" << FileToOpen << ">"<> x; inFile >> y; - + if (verbose_level > 2){ std::cout << " x: " << x; std::cout << " y: " << y << std::endl; } - + if (i==0){ N_nodes=x; - M_edges=y; + M_edges=y; break; } i++; - - + + } inFile.close(); - + if (verbose_level == 2){ std::cout << N_nodes << " nodes, " << M_edges << " edges"< 2){ std::cout << " Opened <" << FileToOpen << ">"<> x && inFile >>y) { if (i > 0) { Final[x][y]=1.; Final[y][x]=1.; - - + + if (verbose_level == 2){ std::cout << "."; } } i++; - + } if (verbose_level == 2){ std::cout << std::endl; } - inFile.close(); - + inFile.close(); + k=0; for (int i=0; i p(num_edges(Full_g)); std::vector d(num_edges(Full_g)); edge_descriptor s; boost::graph_traits::vertex_descriptor u, v; - + for (int i=0; i 0.){ @@ -482,103 +484,103 @@ v = vertex(j, Full_g); remove_edge(u,v,Full_g); remove_edge(v,u,Full_g); - + } } } - - + + } - - + + //===================================================================== int sample_failed_number(double pfail){ - + //boost::geometric_distribution geo(pfail); //boost::variate_generator > geom(generator, geo); - + double r, u, g; - + r=0.; for(int i=0; i=3){ std::cout << id << " failed " << r << std::endl; } return (int) round(r); // FIXME: Andrey: please verify that round() is correct. - + } - + //============================================= double get_target(void){ return TargetNovelty; } - + //============================================= void set_target(double target){ TargetNovelty=target; } - + //============================================= int sample(){ - + //boost::variate_generator > uni(generator, uni_dist); // double r = uni(), Summa = 0.; - - - + + + double r = rand(), Summa = 0.; r /= (double)(pow(2.,31)-1.); int result = 0; int finished = 0; - + if (verbose_level==4){ std::cout << id << " sampled " << r << std::endl; } - - for(int i=0; i r){ - + Tried[i][j]+=1.; - + if (Final[i][j] > 0.){ result = 1; } - finished = 1; + finished = 1; } } } - + return result; - + } - + //=============================== void update_current_graph(void){ - + std::vector p(num_edges(Full_g)); std::vector d(num_edges(Full_g)); edge_descriptor s; boost::graph_traits::vertex_descriptor u, v; - + //property_map::type weightmap = get(edge_weight, Full_g); for (int i=0; i 0. && Tried[i][j]>0){ - //s = edge(i, j, Full_g); + //s = edge(i, j, Full_g); boost::graph_traits::edge_descriptor e1,e2; bool found1, found2; u = vertex(i, Full_g); @@ -589,64 +591,64 @@ add_edge(u,v,1,Full_g); add_edge(v,u,1,Full_g); } - + } } - + } } - + //=============================== void update_distances(void){ // put shortest paths to the *Dist[][] std::vector p(num_vertices(Full_g)); std::vector d(num_vertices(Full_g)); vertex_descriptor s; - - + + // put shortest paths to the *Dist[][] for (int j=0; j 0.){ - s = vertex(j, Full_g); + s = vertex(j, Full_g); dijkstra_shortest_paths(Full_g, s, predecessor_map(&p[0]).distance_map(&d[0])); - + //std::cout <<" Vertex "<< j << std::endl; graph_traits < graph_t >::vertex_iterator vi, vend; - + for (boost::tie(vi, vend) = vertices(Full_g); vi != vend; ++vi) { - + if (p[*vi]!=*vi){ Dist[*vi][j]=d[*vi]; Dist[j][*vi]=d[*vi]; - + if ( (int)round(Dist[*vi][j]>max_dist)) { // FIXME: Andrey: please verify that (int) cast is correct. Do we need to round()? - // also, the indent on this iff statement was way off - + // also, the indent on this iff statement was way off - // perhaps due to space v. tab? max_dist=(int)round(Dist[*vi][j]); } - - + + } else { Dist[*vi][j]=-1.; Dist[j][*vi]=-1.; } } } - + } - - + + } - + //====================================================== void update_ranks(void){ - + for(int i=0; i0. && Final[i][j] >0.){ @@ -655,30 +657,30 @@ } } } - + } - + //==================================================================== void set_world(double a_i, double a_m, double b, double g, double d){ - + alpha_i=a_i; alpha_m=a_m; gamma=g; beta=b; delta=d; - + } - + //==================================================================== void reset_world(){ - + //==================================================== std::vector p(num_edges(Full_g)); std::vector d(num_edges(Full_g)); edge_descriptor s; boost::graph_traits::vertex_descriptor u, v; - - + + for (int i=0; i 0. && Tried[i][j] > 0){ @@ -686,18 +688,18 @@ v = vertex(j, Full_g); remove_edge(u,v,Full_g); remove_edge(v,u,Full_g); - + } } } - + //================================================== - + current_loss=0; current_epoch=0; current_repeat++; current_novelty=0; - + for(int i = 0; i < N_nodes; ++i) { Rank[i]=0.; for(int j = 0; j < N_nodes; ++j) { @@ -707,28 +709,28 @@ } } } - - + + //============================================== void show_parameters(void){ - - std::cout << "Parameters: " + + std::cout << "Parameters: " << alpha_i << " " << alpha_m << " | " << beta << " " << gamma << " | " << delta << std::endl; - + } - - - + + + //=============================================== string file_name(){ - + std::ostringstream s; - s << "world_" - << lexical_cast(alpha_i) << "_" + s << "world_" + << lexical_cast(alpha_i) << "_" << lexical_cast(alpha_m) << "_" << lexical_cast(beta) << "_" << lexical_cast(gamma) << "_" @@ -736,228 +738,228 @@ << lexical_cast(N_epochs) << "_" << lexical_cast(N_steps) << "_" << lexical_cast(N_repeats) << ".txt"; - + return s.str(); - + } - - - - + + + + //================================================= void set_verbose(int verbose){ - + verbose_level = verbose; } - - + + //============================================================= void update_probabilities(void){ - - + + //========================= // Compute sampling probabilities // first pass: \xi_i,j for(int i=0; i 0.){ - + double k = Dist[i][j]; if (k >= k_max){ k = k_max-1; } - + bg = beta * log(k/k_max) + gamma * log(1. - k/k_max); - + } else { bg = delta; } - + Prob[i][j] = exp(Prob[i][j] + bg); } } - - + + // second pass: sum double Summa = 0.; - + for(int i=0; i0. && Final[i][j]>0.){ novel+=1.; } } } - + } //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ else { - + double pfail=0.; int n_failed; //, n_check = 0; - + for(int i=0; i0. && Final[i][j]>0.){ novel+=1.; } } } } - + current_novelty = novel; - - + + //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ if (verbose_level == 2){ - std::cout << (current_repeat+1) << " epoch=" << (current_epoch+1) - - << " cost=" << cost - << " novel=" << novel + std::cout << (current_repeat+1) << " epoch=" << (current_epoch+1) + + << " cost=" << cost + << " novel=" << novel << " rel_loss=" << cost/novel << std::endl; } - + current_epoch++; } - - + + //====== Destructor ====== ~Universe(){ - + delete_2Dmatrix(Final, N_nodes); delete_2Dmatrix(Dist, N_nodes); delete_2Dmatrix(Tried, N_nodes); delete_2Dmatrix(Prob, N_nodes); delete_2Dmatrix(EdgeIndex, N_nodes); - delete_1Dmatrix(Rank); + delete_1Dmatrix(Rank); } - + //================================================ // Allocate memory double** allocate_2Dmatrix(int N, int M) { double **pointer; - + if (verbose_level == 2){ std::cout<< "["< 0){ - + pointer = new double[N]; - + }else { - + pointer = NULL; } - + return pointer; - + } - + //============================================== // De-Allocate memory to prevent memory leak void delete_2Dmatrix(double **pointer, int N){ - + if (pointer != NULL){ - + for (int i = 0; i < N; ++i){ delete [] pointer[i]; } @@ -966,54 +968,98 @@ } //==================== void delete_1Dmatrix(double *pointer){ - + delete [] pointer; } - + //=========================================== double get_rel_loss(){ - + return CumulativeRelativeLoss ; } - + //=========================================== double get_rel_loss_err(){ - + return CRLsquare ; } - - - + + + //================================================================================== void evolve_to_target_and_save(int istart, int iend, double* storage, int* counters){ - - double ALOT=100000000000.; - + + double ALOT=10000.; + int check = 0; + reset_world(); - + for (int k = istart; k < iend; k++){ - - + +// Code for getting time to check + for(int i=0; i< N_epochs && current_novelty < TargetNovelty; i++){ - update_world(); + +// Check the walltime limit + + if(current_loss/current_novelty < max_loss_value((int)TargetNovelty)){ + update_world(); + } + else{ + check=1; + break; + } } - - storage[k]=current_loss/current_novelty; + + if( check ){ + storage[k]=ALOT; + } + else{ + storage[k]=current_loss/current_novelty; + } counters[k]=1; - - + reset_world(); } - + } + //============================================== + double max_loss_value(int tn){ + /* Poor check */ + /* Do not check for equality, check for within some error */ + if( tn == 58. ) return 128.286; + else if( tn == 108. ) return 131.866; + else if( tn == 158. ) return 135.551; + else if( tn == 208. ) return 139.694; + else if( tn == 258. ) return 144.163; + else if( tn == 308. ) return 148.967; + else if( tn == 358. ) return 154.201; + else if( tn == 408. ) return 159.962; + else if( tn == 458. ) return 166.441; + else if( tn == 508. ) return 173.655; + else if( tn == 558. ) return 181.921; + else if( tn == 608. ) return 191.246; + else if( tn == 658. ) return 202.150; + else if( tn == 708. ) return 215.197; + else if( tn == 758. ) return 202.150; // Verify with Andrey. Same as TargetNovelty of 658 + else if( tn == 808. ) return 251.698; + else if( tn == 858. ) return 279.201; + else if( tn == 908. ) return 320.112; + else if( tn == 958. ) return 394.774; + else if( tn == 1008. ) return 1052.38; // Huge number here. Why? + else return 10000.; /* Pray that this does not occur. Ask Andrey what to do */ + } + + + //============================================== int get_reruns(void){ return N_repeats; } - + //============================================== double get_parameter(int i){ - + switch(i){ case 0: return alpha_i; @@ -1026,72 +1072,72 @@ case 4: return delta; default: - + std::cout << "Erroneous parameter id!!!!\n\n\n"; return 0.; } } - - + + //============================================== void evolve_to_target(){ - + reset_world(); if (beta < -1. || gamma < -1.){ CumulativeRelativeLoss = 100000000000.; CRLsquare = 0.; return; } - - + + for (int k=0; k< N_repeats; k++){ - - + + for(int i=0; i 4) {return 0;} - + else { - + switch(position){ case 0: alpha_i=value; @@ -1109,17 +1155,17 @@ delta=value; return 1; } - + } - + return 0; } - -#ifdef notdef + +#ifdef notdef //================================================================= - void try_annealing(double starting_jump, int iterations, + void try_annealing(double starting_jump, int iterations, double temp_start, double temp_end, double target_rejection){ - + double dx[5]={0.,0.,0.,0.,0}; double x[5]={0.,0.,0.,0.,0}; double rejection[5]={0., 0., 0., 0., 0.}; @@ -1128,41 +1174,41 @@ double ratio, r; int cycle=10; boost::variate_generator > uni(generator, uni_dist); - + // set up parameter for annealing - + x[0]=alpha_i; x[1]=alpha_m; x[2]=beta; x[3]=gamma; x[4]=delta; - + for(int i=0;i<5;i++){ dx[i] = starting_jump; } - + // establish the current value - + //.......................................... - evolve_to_target(); + evolve_to_target(); std::cout << CumulativeRelativeLoss << " +- " << CRLsquare << std::endl; - + curr_x = CumulativeRelativeLoss; curr_err = CRLsquare; CumulativeRelativeLoss = 0; CRLsquare = 0; //........................................... - + // optimization cycle for(int i=0; i ratio){ - - std::cout << string_wrap(id, 4) <<" "<< (i+1) << ","<< (j) - <<" "<< (i+1) << " Did not accept " + + std::cout << string_wrap(id, 4) <<" "<< (i+1) << ","<< (j) + <<" "<< (i+1) << " Did not accept " << x_tmp << "(" << j << ")" << std::endl; std::cout << alpha_i << " "<< alpha_m << " " - << beta << " " << gamma << " " + << beta << " " << gamma << " " << delta << " " << std::endl; set_parameter(x[j], j); CumulativeRelativeLoss = 0; CRLsquare = 0; - + rejection[j]+=1.; } - + else { - + curr_x = CumulativeRelativeLoss; curr_err = CRLsquare; x[j] = x_tmp; CumulativeRelativeLoss = 0; CRLsquare = 0; - std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8) - << wrap_double(rejection[0],2) + std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8) + << wrap_double(rejection[0],2) << " "<< wrap_double(rejection[1], 7) << " " - << wrap_double(rejection[2],5) << " " << wrap_double(rejection[2],9) << " " - << wrap_double(rejection[4],6) << " " + << wrap_double(rejection[2],5) << " " << wrap_double(rejection[2],9) << " " + << wrap_double(rejection[4],6) << " " << std::endl << std::endl; - - std::cout << string_wrap(id, 4) <<" "<< (i+1) <<","<< (j) + + std::cout << string_wrap(id, 4) <<" "<< (i+1) <<","<< (j) <<" " - << string_wrap((string) "***** Did accept! ", 3) - << wrap_double(alpha_i,2) + << string_wrap((string) "***** Did accept! ", 3) + << wrap_double(alpha_i,2) << " "<< wrap_double(alpha_m, 7) << " " - << wrap_double(beta,5) << " " - << wrap_double(gamma,9) << " " - << wrap_double(delta,6) << " " + << wrap_double(beta,5) << " " + << wrap_double(gamma,9) << " " + << wrap_double(delta,6) << " " << std::endl << std::endl; - + } - //........................................................ - + //........................................................ + } - + } - + } - + #endif - + }; //============================================================ #ifdef P_DISPATCH -std::pair multi_loss(dispatch_group_t group, - Universe* un[], +std::pair multi_loss(dispatch_group_t group, + Universe* un[], dispatch_queue_t* CustomQueues, double* Results, int* Counters, double* params){ #else -std::pair multi_loss(Universe* un[], +std::pair multi_loss(Universe* un[], double* Results, int* Counters, double* params){ #endif - + int N = un[0]->get_reruns(); int step = (int)floor((double)N/(double)(Nworkers)); // FIXME: Andrey: please check change in cast grouping and use of floor int istart=0; int iend = istart+step; - + double Loss=0., LossSquare=0.; - + timeval startTime, endTime; double elapsedTime; gettimeofday(&startTime, NULL); @@ -1282,20 +1328,20 @@ un[i]->set_parameter(params[j],j); } } - -#ifdef P_DISPATCH + +#ifdef P_DISPATCH for(int i=0; ievolve_to_target_and_save(istart, iend, Results, Counters); un[i]->evolve_to_target_and_save(i*step, min((i+1)*step,N), Results, Counters); }); - + std::cout << "queued: i=" << i << " N=" << N << " istart=" << istart << " iend=" << iend << "\n"; // istart += step; // iend = min(istart+step,N); - + } dispatch_group_wait(group, DISPATCH_TIME_FOREVER); //dispatch_release(group); @@ -1316,23 +1362,23 @@ un[i]->evolve_to_target_and_save(i*step, min((i+1)*step,N), Results, Counters); } #endif - + for (int i=0; i Res; Res.first=Loss; Res.second=two_std; - + gettimeofday(&endTime, NULL); elapsedTime = (endTime.tv_sec - startTime.tv_sec) * 1000.0; // sec to ms elapsedTime += (endTime.tv_usec - startTime.tv_usec) / 1000.0; // us to ms @@ -1340,30 +1386,30 @@ cout << "multi_loss(N=" << N << ", target=" << un[0]->get_target() << ") elapsed time: " << elapsedTime << " seconds " << elapsedTime/60. << " minutes\n\n"; return Res; - - + + } //============================================================ //============================================================ #ifdef P_DISPATCH -void multi_annealing( dispatch_group_t group, - Universe* un[], - dispatch_queue_t* CustomQueues, - double T_start, double T_end, - double Target_rejection, - int Annealing_repeats, +void multi_annealing( dispatch_group_t group, + Universe* un[], + dispatch_queue_t* CustomQueues, + double T_start, double T_end, + double Target_rejection, + int Annealing_repeats, double starting_jump, double* Results, int* Counters, double* params0, double annealing_cycles){ #else -void multi_annealing(Universe* un[], - double T_start, double T_end, - double Target_rejection, - int Annealing_repeats, +void multi_annealing(Universe* un[], + double T_start, double T_end, + double Target_rejection, + int Annealing_repeats, double starting_jump, double* Results, int* Counters, @@ -1372,7 +1418,7 @@ #endif //................................. // re-implement annealing - + double dx[5]={0.,0.,0.,0.,0}; double x[5]={0.,0.,0.,0.,0}; double rejection[5]={0., 0., 0., 0., 0.}; @@ -1381,30 +1427,30 @@ double ratio, r; int cycle=10; //boost::variate_generator > uni(generator, uni_dist); - + // set up parameter for annealing - + x[0]=params0[0]; x[1]=params0[1]; x[2]=params0[2]; x[3]=params0[3]; x[4]=params0[4]; - + for(int i=0;i<5;i++){ dx[i] = starting_jump; } - + // establish the current value std::pairRes; -#ifdef P_DISPATCH +#ifdef P_DISPATCH Res = multi_loss(group, un, CustomQueues, Results, Counters, x); #else Res = multi_loss( un, Results, Counters, x); #endif std::cout << "Returned from initial multi_loss:" << std::endl; std::cout << Res.first << " +- " << Res.second << std::endl; - + if ( operation == 'm' ) { FILE *f; int N = un[0]->get_reruns(); @@ -1419,19 +1465,19 @@ curr_x = Res.first; curr_err = Res.second; - + // optimization cycle - + for(int i=0; iset_parameter(x_tmp, j); } - -#ifdef P_DISPATCH + +#ifdef P_DISPATCH Res = multi_loss(group, un, CustomQueues, Results, Counters, x); #else Res = multi_loss( un, Results, Counters, x); #endif std::cout << Res.first << " +- " << Res.second << std::endl; - + ratio = min(1.,exp(-(Res.first-curr_x)/temperature)); r = rand()/(double)(pow(2.,31)-1.); std::cout << r << " vs " << ratio << std::endl; - + double ALOT=100000000000.; - + if (Res.first < ALOT) { ofstream filestr; - + filestr.open ("best_opt_some.txt", ofstream::app); - + // >> i/o operations here << filestr << "N, " << i << ", " << j << ", " << dx[j] << ", " << rejection[j] << ", |, " // FIXME: MW-DEBUGGING - - << un[0]->get_target() << ", " - << Res.first - << ", " << un[0]->get_parameter(0) - << ", " << un[0]->get_parameter(1) - << ", " << un[0]->get_parameter(2) - << ", " << un[0]->get_parameter(3) + + << un[0]->get_target() << ", " + << Res.first + << ", " << un[0]->get_parameter(0) + << ", " << un[0]->get_parameter(1) + << ", " << un[0]->get_parameter(2) + << ", " << un[0]->get_parameter(3) << ", " << un[0]->get_parameter(4) << ", " << Res.second << ",\n"; - + filestr.close(); - - + + filestr.open ("max_dist.txt", ofstream::app); - + // >> i/o operations here << filestr << max_dist << ",\n"; - + filestr.close(); FILE *bf; bf = fopen("bestdb.txt","a"); fprintf(bf, "N %2d %2d %10.5f %5.2f | %5.2f %10.5f [ %5.2f %5.2f %10.5f %10.5f %10.5f ] %10.5f\n", i, j, dx[j], rejection[j], un[0]->get_target(), - Res.first, - un[0]->get_parameter(0), - un[0]->get_parameter(1), - un[0]->get_parameter(2), + Res.first, + un[0]->get_parameter(0), + un[0]->get_parameter(1), + un[0]->get_parameter(2), un[0]->get_parameter(3), un[0]->get_parameter(4), Res.second); fclose(bf); } - - + + if (r > ratio){ - - std::cout << " "<< (i+1) << ","<< (j) - <<" "<< (i+1) << " Did not accept " + + std::cout << " "<< (i+1) << ","<< (j) + <<" "<< (i+1) << " Did not accept " << x_tmp << "(" << j << ")" << std::endl; - std::cout << un[0]->get_parameter(0) - << " " << un[0]->get_parameter(1) - << " " << un[0]->get_parameter(2) - << " " << un[0]->get_parameter(3) + std::cout << un[0]->get_parameter(0) + << " " << un[0]->get_parameter(1) + << " " << un[0]->get_parameter(2) + << " " << un[0]->get_parameter(3) << " " << un[0]->get_parameter(4) << " " << std::endl; - + x[j]=x_hold; for(int w=0; wset_parameter(x[j], j); } - - - //set_parameter(x[j], j); + + + //set_parameter(x[j], j); rejection[j]+=1.; } - + else { - + curr_x = Res.first; curr_err = Res.second; x[j] = x_tmp; - + for(int w=0; wset_parameter(x[j], j); } - - std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8) - << wrap_double(rejection[0],2) << " " + + std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8) + << wrap_double(rejection[0],2) << " " << wrap_double(rejection[1],7) << " " - << wrap_double(rejection[2],5) << " " - << wrap_double(rejection[3],9) << " " - << wrap_double(rejection[4],6) << " " + << wrap_double(rejection[2],5) << " " + << wrap_double(rejection[3],9) << " " + << wrap_double(rejection[4],6) << " " << std::endl << std::endl; - - std::cout << " "<< (i+1) <<","<< (j) + + std::cout << " "<< (i+1) <<","<< (j) <<" " - << string_wrap((string) "***** Did accept! ", 3) + << string_wrap((string) "***** Did accept! ", 3) << wrap_double(un[0]->get_parameter(0),2) << " " << wrap_double(un[0]->get_parameter(1),7) << " " - << wrap_double(un[0]->get_parameter(2),5) << " " - << wrap_double(un[0]->get_parameter(3),9) << " " - << wrap_double(un[0]->get_parameter(4),6) << " " + << wrap_double(un[0]->get_parameter(2),5) << " " + << wrap_double(un[0]->get_parameter(3),9) << " " + << wrap_double(un[0]->get_parameter(4),6) << " " << std::endl << std::endl; - - - + + + } - //........................................................ - + //........................................................ + } } - + } - + } @@ -1588,43 +1634,43 @@ int main(int argc, char* argv[]) { - + double params0[6] = {0., 0., 0., 0., 0., 0.2}, target=50., range; string par_names0[6] = {"alpha_i", "alpha_m", "beta", "gamma", "delta", "target"}; string par_names1[4] = {"n_epochs", "n_steps", "n_reruns", "range"}; string par_names2[5] = {"T_start", "T_end", "Annealing_steps","Target_rejection","Starting_jump"}; - string par_names3[5] = {"FREEZE_alpha_i", "FREEZE_alpha_m", "FREEZE_beta", "FREEZE_gamma", "FREEZE_delta"}; + string par_names3[5] = {"FREEZE_alpha_i", "FREEZE_alpha_m", "FREEZE_beta", "FREEZE_gamma", "FREEZE_delta"}; string par_names4[3] = {"Operation", "Nworkers", "initSeed"}; int params1[4] = {300, 50, 1000, 10}; int params3[5] = { 0, 0, 0, 0, 0}; - + // temperature_start, temperature_end, annealing_steps target_rejection Starting_jump double params2[5] = {1, 0.001, 100, 0.3, 1.5}; - + int verbose_level = 2; const std::string one="one", two="two"; static Universe* un[MAXNworkers]; #ifdef P_DISPATCH static dispatch_queue_t CustomQueues[MAXNworkers]; -#endif +#endif static double* Results; static int* Counters; - + timeval t1, t2; double elapsedTime; // start timer gettimeofday(&t1, NULL); - - + + if (argc < 8) { - std::cout << "Usage: super_optimizer alpha_i alpha_m beta gamma delta target_innov [n_epochs n_steps n_reruns] [range] [verbose_level]\n"; - std::cout << " [T_start T_end Annealing_steps Target_rejection Starting_jump]\n"; - std::cout << " [FREEZE_alpha_i FREEZE_alpha_m FREEZE_beta FREEZE_gamma FREEZE_delta]\n"; - + std::cout << "Usage: super_optimizer alpha_i alpha_m beta gamma delta target_innov [n_epochs n_steps n_reruns] [range] [verbose_level]\n"; + std::cout << " [T_start T_end Annealing_steps Target_rejection Starting_jump]\n"; + std::cout << " [FREEZE_alpha_i FREEZE_alpha_m FREEZE_beta FREEZE_gamma FREEZE_delta]\n"; + system("pwd"); - - + + return(1); } else { @@ -1663,25 +1709,25 @@ initSeed = atoi(argv[nArg]); std::cout << par_names4[2] << ": " << initSeed << std::endl; } - - + + } - + } - + for (int j=0; j<5; j++){ - + cout << j << " | " << var_fixed[j] << " (fixed) \n"; } - + target=params0[5]; range = (double)params1[3]; int identify_failed = 0; char* filename= (char *)"movie_graph.txt"; int n_ep=params1[0], n_st=params1[1], n_rep=params1[2]; - + //............................... - + for(int i=0; i 0){ - + Results = new double[n_rep]; Counters = new int[n_rep]; - + }else { - + Results = NULL; Counters = NULL; std::cout << " Number of reruns should be positive! " << std::endl; return 0; - + } //............................... //srand(time(0)); //srandomdev(); - + if ( initSeed != 0.0 ) { srand(initSeed); } @@ -1721,84 +1767,84 @@ { double r=0; for (int j=0; j<100; j++){ - - - + + + r = rand()/(double)(pow(2.,31)-1.); std::cout << r << " "; } std::cout << "\n"; } //random initiation of starting parameters - + if (range > 0.){ - + for (int i=0; i < 5; i++){ - + if (params0[i]==-100.){ - + double r1 = (rand()/(double)(pow(2.,31)-1.)); double r2 = (rand()/(double)(pow(2.,31)-1.)); double sign = 1.; - + if(r1 > 0.5){ sign=-1.; } - + params0[i] = sign*r2*range; - + std::cout << par_names0[i] << ": " << params0[i] << std::endl; } } - + } - - + + double T_start=params2[0], T_end=params2[1], Target_rejection=params2[3], starting_jump=params2[4]; int Annealing_repeats = (int) params2[2]; - -#ifdef P_DISPATCH + +#ifdef P_DISPATCH dispatch_group_t group = dispatch_group_create(); - + //............................. - multi_annealing(group, un, CustomQueues, T_start, T_end, Target_rejection, Annealing_repeats, + multi_annealing(group, un, CustomQueues, T_start, T_end, Target_rejection, Annealing_repeats, starting_jump, Results, Counters, params0, Annealing_repeats); - + //dispatch_group_wait(group, DISPATCH_TIME_FOREVER); dispatch_release(group); //............................. #else - multi_annealing( un, T_start, T_end, Target_rejection, Annealing_repeats, + multi_annealing( un, T_start, T_end, Target_rejection, Annealing_repeats, starting_jump, Results, Counters, params0, Annealing_repeats); #endif - - + + // stop timer gettimeofday(&t2, NULL); - + // compute and print the elapsed time in millisec elapsedTime = (t2.tv_sec - t1.tv_sec) * 1000.0; // sec to ms elapsedTime += (t2.tv_usec - t1.tv_usec) / 1000.0; // us to ms elapsedTime /= 1000.; cout << "\n*** optimizer completed, elapsed time=" << elapsedTime << " seconds " << elapsedTime/60. << " minutes)\n\n"; - + //..................... - + for(int i=0; i 0){ - + delete [] Results; delete [] Counters; - + } - + return 0; - - - + + + } From swift at ci.uchicago.edu Wed Mar 7 17:45:03 2012 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 7 Mar 2012 17:45:03 -0600 (CST) Subject: [Swift-commit] Cog update Message-ID: <20120307234504.2AF9A8D00092@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3363 | davidkelly999 | 2012-03-07 17:40:35 -0600 (Wed, 07 Mar 2012) | 2 lines Fix to start the correct number of workers ------------------------------------------------------------------------ Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java (revision 3362) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java (working copy) @@ -315,7 +315,7 @@ */ protected void writeMultiJobPreamble(Writer wr, String exitcodefile) throws IOException { - wr.write("NODES=`cat $PE_HOSTFILE | awk '{ for(i=0;i<$2;i++){print $1} }'`\n"); + wr.write("NODES=`cat $PE_HOSTFILE | awk '{print $1}'`\n"); wr.write("ECF=" + exitcodefile + "\n"); wr.write("INDEX=0\n"); wr.write("for NODE in $NODES; do\n"); From jonmon at ci.uchicago.edu Sun Mar 11 19:33:39 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Sun, 11 Mar 2012 19:33:39 -0500 (CDT) Subject: [Swift-commit] r5707 - SwiftApps/SciColSim/params Message-ID: <20120312003339.E2E799CCA4@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-11 19:33:39 -0500 (Sun, 11 Mar 2012) New Revision: 5707 Added: SwiftApps/SciColSim/params/ARFull Log: added full scale parameter file for the SciColSim application Added: SwiftApps/SciColSim/params/ARFull =================================================================== --- SwiftApps/SciColSim/params/ARFull (rev 0) +++ SwiftApps/SciColSim/params/ARFull 2012-03-12 00:33:39 UTC (rev 5707) @@ -0,0 +1,18 @@ + +min_target_innovation 58 # starting target innovation value to try +max_target_innovation 1009 # stops before this target innovation value +target_innovation_increment 50 # increment target innovation by this amount + +annealing_repeats 1 # times to repeate the entire optimization process for each target_innovation +annealing_cycles 100 # times to perform the simulated annealing loop for all non-fixed parameters + +evolve_reruns 10000 # times to perform evolve_to_target_and_save() (objective function - is batched) +nworkers 24 # number of parallel threads (cores) to use per invocation of C++ optimizer +reruns_per_opt_invocation 240 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!! + +alpha_i 0.0 # 5 evolve() parameters: must be decimal values !!! +alpha_m 0.0 +beta 4.0 +gamma 50.0 +delta -1.0 + From jonmon at ci.uchicago.edu Mon Mar 12 14:18:13 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Mon, 12 Mar 2012 14:18:13 -0500 (CDT) Subject: [Swift-commit] r5708 - SwiftApps/SciColSim/params Message-ID: <20120312191813.0FB849CCB5@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-12 14:18:12 -0500 (Mon, 12 Mar 2012) New Revision: 5708 Modified: SwiftApps/SciColSim/params/Fast01 Log: checking in small test case configured for beagle. Modified: SwiftApps/SciColSim/params/Fast01 =================================================================== --- SwiftApps/SciColSim/params/Fast01 2012-03-12 00:33:39 UTC (rev 5707) +++ SwiftApps/SciColSim/params/Fast01 2012-03-12 19:18:12 UTC (rev 5708) @@ -1,14 +1,14 @@ min_target_innovation 58 # starting target innovation value to try -max_target_innovation 109 # stops before this target innovation value +max_target_innovation 59 # stops before this target innovation value target_innovation_increment 50 # increment target innovation by this amount annealing_repeats 1 # times to repeate the entire optimization process for each target_innovation annealing_cycles 10 # times to perform the simulated annealing loop for all non-fixed parameters -evolve_reruns 8 # times to perform evolve_to_target_and_save() (objective function - is batched) -nworkers 1 # number of parallel threads (cores) to use per invocation of C++ optimizer -reruns_per_opt_invocation 2 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!! +evolve_reruns 100 # times to perform evolve_to_target_and_save() (objective function - is batched) +nworkers 24 # number of parallel threads (cores) to use per invocation of C++ optimizer +reruns_per_opt_invocation 48 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!! alpha_i 0.0 # 5 evolve() parameters: must be decimal values !!! alpha_m 0.0 # alpha_i and alpha_m are fixed for now From wozniak at ci.uchicago.edu Mon Mar 12 15:46:29 2012 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Mon, 12 Mar 2012 15:46:29 -0500 (CDT) Subject: [Swift-commit] r5709 - SwiftApps/SciColSim Message-ID: <20120312204629.A2E249CCA4@svn.ci.uchicago.edu> Author: wozniak Date: 2012-03-12 15:46:29 -0500 (Mon, 12 Mar 2012) New Revision: 5709 Modified: SwiftApps/SciColSim/evolve.sh Log: Currently, sh is not bash on Ubuntu Modified: SwiftApps/SciColSim/evolve.sh =================================================================== --- SwiftApps/SciColSim/evolve.sh 2012-03-12 19:18:12 UTC (rev 5708) +++ SwiftApps/SciColSim/evolve.sh 2012-03-12 20:46:29 UTC (rev 5709) @@ -1,4 +1,4 @@ -#! /bin/sh +#! /bin/bash datafile=$1 touch multi_loss.data shift 1 From lgadelha at ci.uchicago.edu Mon Mar 12 22:23:19 2012 From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu) Date: Mon, 12 Mar 2012 22:23:19 -0500 (CDT) Subject: [Swift-commit] r5710 - provenancedb Message-ID: <20120313032319.6C4AA9CCA4@svn.ci.uchicago.edu> Author: lgadelha Date: 2012-03-12 22:23:19 -0500 (Mon, 12 Mar 2012) New Revision: 5710 Modified: provenancedb/prov-init.sql provenancedb/prov-to-sql.sh Log: Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2012-03-12 20:46:29 UTC (rev 5709) +++ provenancedb/prov-init.sql 2012-03-13 03:23:19 UTC (rev 5710) @@ -73,7 +73,21 @@ start_time numeric, duration numeric, final_state varchar(32), - site varchar(256) + site varchar(256), + maxrss numeric, + walltime numeric, + systime numeric, + usertime numeric, + cpu numeric, + fsin numeric, + fsout numeric, + timesswapped numeric, + socketrecv numeric, + socketsent numeric, + majorpagefaults numeric, + minorpagefaults numeric, + contextswitchesinv numeric, + contextswitchesvol numeric ); -- app execution runtime info extracted from the /proc filesystem (assumes the app executed @@ -283,7 +297,7 @@ drop view runtime_info; create view runtime_info as - select app_exec_id as application_execution_id, tstamp as timestamp, cpu_usage, max_phys_mem, max_virt_mem, io_read, io_write + select app_exec_id as application_execution_id, to_timestamp(tstamp) as timestamp, cpu_usage, max_phys_mem, max_virt_mem, io_read, io_write from rt_info; drop view produces; Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2012-03-12 20:46:29 UTC (rev 5709) +++ provenancedb/prov-to-sql.sh 2012-03-13 03:23:19 UTC (rev 5710) @@ -137,13 +137,28 @@ if [ -f runtime.txt ]; then while read execute2_id runtime; do - timestamp=$(echo $runtime | awk -F "," '{print $1}' | awk -F ":" '{print $2}') - cpu_usage=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - max_phys_mem=$(echo $runtime | awk -F "," '{print $3}' | awk -F ":" '{print $2}') - max_virtual_mem=$(echo $runtime | awk -F "," '{print $4}' | awk -F ":" '{print $2}') - io_read_bytes=$(echo $runtime | awk -F "," '{print $5}' | awk -F ":" '{print $2}') - io_write_bytes=$(echo $runtime | awk -F "," '{print $6}' | awk -F ":" '{print $2}') - echo "INSERT INTO runtime_info (app_execution_id, tstamp, cpu_usage, max_phys_mem, max_virtual_mem, io_read_bytes, io_write_bytes) VALUES ('$execute2_id', $timestamp, $cpu_usage, $max_phys_mem, $max_virtual_mem, $io_read_bytes, $io_write_bytes);" >> /tmp/$RUNID-6.sql + #timestamp=$(echo $runtime | awk -F "," '{print $1}' | awk -F ":" '{print $2}') + #cpu_usage=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') + #max_phys_mem=$(echo $runtime | awk -F "," '{print $3}' | awk -F ":" '{print $2}') + #max_virtual_mem=$(echo $runtime | awk -F "," '{print $4}' | awk -F ":" '{print $2}') + #io_read_bytes=$(echo $runtime | awk -F "," '{print $5}' | awk -F ":" '{print $2}') + #io_write_bytes=$(echo $runtime | awk -F "," '{print $6}' | awk -F ":" '{print $2}') + #echo "INSERT INTO rt_info (app_exec_id, tstamp, cpu_usage, max_phys_mem, max_virt_mem, io_read, io_write) VALUES ('$execute2_id', $timestamp, $cpu_usage, $max_phys_mem, $max_virtual_mem, $io_read_bytes, $io_write_bytes);" >> /tmp/$RUNID-6.sql + maxrss=$(echo $runtime | awk -F "," '{print $1}' | awk -F ":" '{print $2}') + walltime=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') + systime=$(echo $runtime | awk -F "," '{print $3}' | awk -F ":" '{print $2}') + usertime=$(echo $runtime | awk -F "," '{print $4}' | awk -F ":" '{print $2}') + cpu=$(echo $runtime | awk -F "," '{print $5}' | awk -F ":" '{print $2}') + fsin=$(echo $runtime | awk -F "," '{print $6}' | awk -F ":" '{print $2}') + fsout=$(echo $runtime | awk -F "," '{print $7}' | awk -F ":" '{print $2}') + timesswapped=$(echo $runtime | awk -F "," '{print $8}' | awk -F ":" '{print $2}') + socketrecv=$(echo $runtime | awk -F "," '{print $9}' | awk -F ":" '{print $2}') + socketsent=$(echo $runtime | awk -F "," '{print $10}' | awk -F ":" '{print $2}') + majorpagefaults=$(echo $runtime | awk -F "," '{print $11}' | awk -F ":" '{print $2}') + minorpagefaults=$(echo $runtime | awk -F "," '{print $12}' | awk -F ":" '{print $2}') + contextswitchesinv=$(echo $runtime | awk -F "," '{print $13}' | awk -F ":" '{print $2}') + contextswitchesvol=$(echo $runtime | awk -F "," '{print $14}' | awk -F ":" '{print $2}') + echo "UPDATE app_exec SET maxrss=$maxrss, walltime=$walltime, systime=$systime, usertime=$usertime,cpu=$cpu,fsin=$fsin,fsout=$fsout,timesswapped=$timesswapped,socketrecv=$socketrecv,socketsent=$socketsent,majorpagefaults=$majorpagefaults,minorpagefaults=$minorpagefaults,contextswitchesinv=$contextswitchesinv,contextswitchesvol=$contextswitchesvol where app_exec_id=$execute2_id;" >> /tmp/$RUNID-6.sql done < runtime.txt fi From lgadelha at ci.uchicago.edu Mon Mar 12 22:49:46 2012 From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu) Date: Mon, 12 Mar 2012 22:49:46 -0500 (CDT) Subject: [Swift-commit] r5711 - provenancedb Message-ID: <20120313034946.7F1A89CCA4@svn.ci.uchicago.edu> Author: lgadelha Date: 2012-03-12 22:49:46 -0500 (Mon, 12 Mar 2012) New Revision: 5711 Modified: provenancedb/info-to-runtime provenancedb/prov-to-sql.sh Log: Modified: provenancedb/info-to-runtime =================================================================== --- provenancedb/info-to-runtime 2012-03-13 03:23:19 UTC (rev 5710) +++ provenancedb/info-to-runtime 2012-03-13 03:49:46 UTC (rev 5711) @@ -10,7 +10,7 @@ if [ "X$record" != "X" ] && [ -f $record ] ; then - grep '^RUNTIME_INFO=' $record | sed "s/^RUNTIME_INFO=\(.*\)$/$globalid \1/" + grep '^RUNTIME_AGGR=' $record | sed "s/^RUNTIME_AGGR=\(.*\)$/$globalid \1/" else echo no wrapper log for $id >&2 Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2012-03-13 03:23:19 UTC (rev 5710) +++ provenancedb/prov-to-sql.sh 2012-03-13 03:49:46 UTC (rev 5711) @@ -148,7 +148,7 @@ walltime=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') systime=$(echo $runtime | awk -F "," '{print $3}' | awk -F ":" '{print $2}') usertime=$(echo $runtime | awk -F "," '{print $4}' | awk -F ":" '{print $2}') - cpu=$(echo $runtime | awk -F "," '{print $5}' | awk -F ":" '{print $2}') + cpu=$(echo $runtime | awk -F "," '{print $5}' | awk -F ":" '{print $2}' | awk -F "%" '{print $1}') fsin=$(echo $runtime | awk -F "," '{print $6}' | awk -F ":" '{print $2}') fsout=$(echo $runtime | awk -F "," '{print $7}' | awk -F ":" '{print $2}') timesswapped=$(echo $runtime | awk -F "," '{print $8}' | awk -F ":" '{print $2}') From lgadelha at ci.uchicago.edu Mon Mar 12 23:43:23 2012 From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu) Date: Mon, 12 Mar 2012 23:43:23 -0500 (CDT) Subject: [Swift-commit] r5712 - provenancedb Message-ID: <20120313044323.1B4E29CCB5@svn.ci.uchicago.edu> Author: lgadelha Date: 2012-03-12 23:43:22 -0500 (Mon, 12 Mar 2012) New Revision: 5712 Modified: provenancedb/prov-to-sql.sh Log: Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2012-03-13 03:49:46 UTC (rev 5711) +++ provenancedb/prov-to-sql.sh 2012-03-13 04:43:22 UTC (rev 5712) @@ -158,7 +158,7 @@ minorpagefaults=$(echo $runtime | awk -F "," '{print $12}' | awk -F ":" '{print $2}') contextswitchesinv=$(echo $runtime | awk -F "," '{print $13}' | awk -F ":" '{print $2}') contextswitchesvol=$(echo $runtime | awk -F "," '{print $14}' | awk -F ":" '{print $2}') - echo "UPDATE app_exec SET maxrss=$maxrss, walltime=$walltime, systime=$systime, usertime=$usertime,cpu=$cpu,fsin=$fsin,fsout=$fsout,timesswapped=$timesswapped,socketrecv=$socketrecv,socketsent=$socketsent,majorpagefaults=$majorpagefaults,minorpagefaults=$minorpagefaults,contextswitchesinv=$contextswitchesinv,contextswitchesvol=$contextswitchesvol where app_exec_id=$execute2_id;" >> /tmp/$RUNID-6.sql + echo "UPDATE app_exec SET maxrss=$maxrss, walltime=$walltime, systime=$systime, usertime=$usertime,cpu=$cpu,fsin=$fsin,fsout=$fsout,timesswapped=$timesswapped,socketrecv=$socketrecv,socketsent=$socketsent,majorpagefaults=$majorpagefaults,minorpagefaults=$minorpagefaults,contextswitchesinv=$contextswitchesinv,contextswitchesvol=$contextswitchesvol where app_exec_id='$execute2_id';" >> /tmp/$RUNID-6.sql done < runtime.txt fi From lgadelha at ci.uchicago.edu Tue Mar 13 07:53:18 2012 From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu) Date: Tue, 13 Mar 2012 07:53:18 -0500 (CDT) Subject: [Swift-commit] r5713 - provenancedb Message-ID: <20120313125318.761269CCA4@svn.ci.uchicago.edu> Author: lgadelha Date: 2012-03-13 07:53:17 -0500 (Tue, 13 Mar 2012) New Revision: 5713 Modified: provenancedb/prov-to-sql.sh Log: Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2012-03-13 04:43:22 UTC (rev 5712) +++ provenancedb/prov-to-sql.sh 2012-03-13 12:53:17 UTC (rev 5713) @@ -158,17 +158,17 @@ minorpagefaults=$(echo $runtime | awk -F "," '{print $12}' | awk -F ":" '{print $2}') contextswitchesinv=$(echo $runtime | awk -F "," '{print $13}' | awk -F ":" '{print $2}') contextswitchesvol=$(echo $runtime | awk -F "," '{print $14}' | awk -F ":" '{print $2}') - echo "UPDATE app_exec SET maxrss=$maxrss, walltime=$walltime, systime=$systime, usertime=$usertime,cpu=$cpu,fsin=$fsin,fsout=$fsout,timesswapped=$timesswapped,socketrecv=$socketrecv,socketsent=$socketsent,majorpagefaults=$majorpagefaults,minorpagefaults=$minorpagefaults,contextswitchesinv=$contextswitchesinv,contextswitchesvol=$contextswitchesvol where app_exec_id='$execute2_id';" >> /tmp/$RUNID-6.sql + echo "UPDATE app_exec SET maxrss=$maxrss, walltime=$walltime, systime=$systime, usertime=$usertime, cpu=$cpu, fsin=$fsin, fsout=$fsout, timesswapped=$timesswapped, socketrecv=$socketrecv, socketsent=$socketsent, majorpagefaults=$majorpagefaults, minorpagefaults=$minorpagefaults, contextswitchesinv=$contextswitchesinv, contextswitchesvol=$contextswitchesvol where id='$execute2_id';" >> /tmp/$RUNID-6.sql done < runtime.txt fi for i in `seq 1 6` do cat /tmp/$RUNID-$i.sql | sort | uniq >> /tmp/$RUNID.sql - #rm /tmp/$RUNID-$i.sql + rm /tmp/$RUNID-$i.sql done echo "COMMIT;" >> /tmp/$RUNID.sql $SQLCMD -f /tmp/$RUNID.sql -#rm /tmp/$RUNID.sql +rm /tmp/$RUNID.sql echo Finished sending SQL to DB \ No newline at end of file From jonmon at ci.uchicago.edu Wed Mar 14 11:49:49 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Wed, 14 Mar 2012 11:49:49 -0500 (CDT) Subject: [Swift-commit] r5714 - trunk/libexec Message-ID: <20120314164949.7B06A9CCBF@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-14 11:49:49 -0500 (Wed, 14 Mar 2012) New Revision: 5714 Modified: trunk/libexec/vdl-int.k Log: fixed an error in vdl-int.k where there were too many aruments being passed to checkErrorFile Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2012-03-13 12:53:17 UTC (rev 5713) +++ trunk/libexec/vdl-int.k 2012-03-14 16:49:49 UTC (rev 5714) @@ -54,7 +54,7 @@ ) ) ) - throw(checkErrorFile(rhost, wfdir, jobid, tr, jobdir)) + throw(checkErrorFile(rhost, wfdir, jobid, jobdir)) ) ) From lgadelha at ci.uchicago.edu Wed Mar 14 14:44:33 2012 From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu) Date: Wed, 14 Mar 2012 14:44:33 -0500 (CDT) Subject: [Swift-commit] r5715 - in provenancedb: . apps apps/SciColSim Message-ID: <20120314194433.998AF9CCBF@svn.ci.uchicago.edu> Author: lgadelha Date: 2012-03-14 14:44:33 -0500 (Wed, 14 Mar 2012) New Revision: 5715 Added: provenancedb/apps/SciColSim/ provenancedb/apps/SciColSim/SciColSim_extractor.sh Modified: provenancedb/prov-to-sql.sh provenancedb/swift-prov-import-all-logs Log: SciColSim annotation extractor, minor updates. Added: provenancedb/apps/SciColSim/SciColSim_extractor.sh =================================================================== --- provenancedb/apps/SciColSim/SciColSim_extractor.sh (rev 0) +++ provenancedb/apps/SciColSim/SciColSim_extractor.sh 2012-03-14 19:44:33 UTC (rev 5715) @@ -0,0 +1,67 @@ +#!/bin/bash + +# Annotation extractor for the SciColSim application +# + +PROVDB_HOME=~/provenancedb +SCICOLSIM_HOME=/home/lgadelha/SciColSim +IMPORT_HOME=/tmp + +#cp $PROVDB_HOME/etc/provenance.config.ci $PROVDB_HOME/etc/provenance.config +source $PROVDB_HOME/etc/provenance.config + +cd $SCICOLSIM_HOME +for k in $(find . -name "annealing-*-*-*.log"); +do + run_dir=$(echo "$k" | awk -F "annealing-" '{print $1}') + cd $SCICOLSIM_HOME/$run_dir + full_dir=$(pwd) + log_suffix=$(echo "$k" | awk -F "^." '{print $2}') + count=$($SQLCMD --tuples-only -c "select count(*) from script_run where log_filename like '%$log_suffix';" | awk '{print $1}') + if [ "$count" -eq "0" ]; then + echo "export LOGREPO=$full_dir" > $PROVDB_HOME/etc/provenance.config + echo "export SQLCMD=\"$SQLCMD\"" >> $PROVDB_HOME/etc/provenance.config + cd /tmp + $PROVDB_HOME/swift-prov-import-all-logs + SCICOLSIM_RUNID=$(echo "$run_dir" | awk -F "/" '{print $2}') + SCRIPT_RUN_ID=$($SQLCMD --tuples-only -c "SELECT id FROM script_run WHERE log_filename like '%$log_suffix';" | awk '{print $1}') + echo "Annotating script run $SCRIPT_RUN_ID with key-value(text) pair ('scicolsim_run_id', '$SCICOLSIM_RUNID')" + $SQLCMD -c "INSERT INTO a_run_t VALUES ('$SCRIPT_RUN_ID', 'scicolsim_run_id', '$SCICOLSIM_RUNID')" 1> /dev/null + cd $SCICOLSIM_HOME/$run_dir + while read key value rest + do + if [ -n "$key" ]; then + echo "Annotating script run $SCRIPT_RUN_ID with key-value(numeric) pair ('$key', $value)." + $SQLCMD -c "INSERT INTO a_run_n VALUES ('$SCRIPT_RUN_ID', '$key', $value)" 1> /dev/null + fi + done < paramfile + + cd /tmp + while read t d id rest; do + cd $SCICOLSIM_HOME + record=$(find "$run_dir" -name ${id}-info | grep -v swiftwork) + globalid=$EXECUTE2PREFIX$id + if [ -n "$record" -a -f "$record" ] ; then + outf=$(grep '^OUTF=' $record | awk -F "=" '{print $2}' | awk -F "|" '{print $1}') + while read keyc value rest + do + key=$(echo $keyc | awk -F ":" '{print $1'}) + if [ "$key" != "$keyc" -a "$key" != "multi_loss" ]; then + PROC_ID=$($SQLCMD --tuples-only -c "SELECT app_inv_id FROM app_exec WHERE id='execute2:$SCRIPT_RUN_ID:$globalid';" | awk '{print $1}') + if [ "$key" == "Operation" ]; then + echo "Annotating function call $PROC_ID with key-value(text) pair ('$key', '$value')." + $SQLCMD -c "INSERT INTO a_proc_t VALUES ('$PROC_ID', '$key', '$value')" 1> /dev/null + else + echo "Annotating function call $PROC_ID with key-value(numeric) pair ('$key', $value)." + $SQLCMD -c "INSERT INTO a_proc_n VALUES ('$PROC_ID', '$key', $value)" 1> /dev/null + fi + fi + done < $SCICOLSIM_HOME/$run_dir/$outf + else + echo no wrapper log for $id >&2 + fi + done < execute2.event + + fi + cd $SCICOLSIM_HOME +done Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2012-03-14 16:49:49 UTC (rev 5714) +++ provenancedb/prov-to-sql.sh 2012-03-14 19:44:33 UTC (rev 5715) @@ -7,18 +7,20 @@ # TODO is there already a URI form for identifying workflows? export WF="${RUNID}" -echo Generating SQL for $RUNID +echo "Generating SQL for $RUNID:" # this gives a distinction between the root process for a workflow and the # workflow itself. perhaps better to model the workflow as a process -#echo "BEGIN TRANSACTION;" > /tmp/$RUNID.sql +echo " - Root thread." echo "INSERT INTO proc (id, type, name, run_id) VALUES ('${WFID}0', 'rootthread', '$RUNID', '$WF');" >> /tmp/$RUNID.sql +echo " - Function calls." while read time duration thread localthread endstate tr_name scratch; do echo "INSERT INTO proc (id, type, run_id) VALUES ('$thread', 'execute', '$WF');" >> /tmp/$RUNID-1.sql echo "INSERT INTO app_inv (id, proc_name, start_time, duration, final_state, scratch) VALUES ('$thread', '$tr_name', $time, $duration, '$endstate', '$scratch');" >> /tmp/$RUNID-2.sql done < execute.global.event +echo " - Application executions." while read start_time duration globalid id endstate thread site scratch; do # cut off the last component of the thread, so that we end up at the # parent thread id which should correspond with the execute-level ID @@ -26,17 +28,19 @@ echo "INSERT INTO app_exec (id, app_inv_id, start_time, duration, final_state, site) VALUES ('$globalid', '$inv_id', $start_time, $duration, '$endstate', '$site');" >> /tmp/$RUNID-3.sql done < execute2.global.event - +echo " - Mapped variables." while read dataset filename; do echo "INSERT INTO ds (id) VALUES ('$dataset');" >> /tmp/$RUNID-4.sql echo "INSERT INTO file (id, name) VALUES ('$dataset', '$filename');" >> /tmp/$RUNID-5.sql done < dataset-filenames.txt +echo " - Primitive variables." while read dataset idtype equal value rest; do echo "INSERT INTO ds (id) VALUES ('$dataset');" >> /tmp/$RUNID-4.sql echo "INSERT INTO in_mem (id, value) VALUES ('$dataset', '$value');" >> /tmp/$RUNID-5.sql done < dataset-values.txt +echo " - Arrays and structures." while read outer inner; do echo "INSERT INTO ds (id) VALUES ('$outer');" >> /tmp/$RUNID-4.sql echo "INSERT INTO ds (id) VALUES ('$inner');" >> /tmp/$RUNID-4.sql @@ -46,7 +50,7 @@ echo "INSERT INTO ds_out (proc_id, ds_id, param) VALUES ('${WFID}constructor:$outer', '$outer', 'collection');" >> /tmp/$RUNID-5.sql done < tie-containers.txt - +echo " - Operators." while read col1 col2 col3 col4 col5 thread name lhs rhs result; do thread=$(echo $thread | awk 'BEGIN { FS = "=" }; {print $2}') name=$(echo $name | awk 'BEGIN { FS = "=" }; {print $2}') @@ -65,6 +69,7 @@ echo "INSERT INTO ds_out (proc_id, ds_id, param) VALUES ('$operatorid', '$result', 'result');" >> /tmp/$RUNID-5.sql done < operators.txt +echo " - Built-in functions." while read id name output; do echo "INSERT INTO ds (id) VALUES ('$output');" >> /tmp/$RUNID-4.sql echo "INSERT INTO proc (id, type, name, run_id) VALUES ('$id', 'function', '$name', '$WF');" >> /tmp/$RUNID-1.sql @@ -76,11 +81,12 @@ echo "INSERT INTO ds_in (proc_id, ds_id, param) VALUES ('$id', '$value', 'undefined');" >> /tmp/$RUNID-5.sql done < function-inputs.txt - +echo " - Function call names." while read thread appname; do echo "UPDATE proc SET name='$appname' WHERE id='$thread';" >> /tmp/$RUNID-3.sql done < invocation-procedure-names.txt +echo " - Script run events." while read start duration wfid rest; do echo "UPDATE run SET start_time=$start WHERE id='$WF';" >> /tmp/$RUNID-1.sql echo "UPDATE run SET duration=$duration WHERE id='$WF';" >> /tmp/$RUNID-1.sql @@ -88,6 +94,7 @@ # TODO this could merge with other naming tables +echo " - Compound functions." while read start duration thread final_state procname ; do if [ "$duration" != "last-event-line" ]; then compoundid=$WFID$thread @@ -107,6 +114,7 @@ echo "INSERT INTO proc (id, type, name, run_id) VALUES ('$thread', 'scope', 'scope', '$WF');" >> /tmp/$RUNID-1.sql done < scopes.txt +echo " - Variable consumption and production." while read thread direction dataset variable rest; do if [ "$direction" == "input" ] ; then table=ds_in @@ -118,6 +126,7 @@ echo "INSERT INTO $table (proc_id, ds_id, param) VALUES ('$thread', '$dataset', '$variable');" >> /tmp/$RUNID-5.sql done < tie-data-invocs.txt +echo " - Wrapper log extra info." if [ -f extrainfo.txt ]; then while read execute2_id extrainfo; do echo $extrainfo | awk -F ";" '{ for (i = 1; i <= NF; i++) @@ -135,6 +144,7 @@ done < extrainfo.txt fi +echo " - Wrapper log resource consumption info." if [ -f runtime.txt ]; then while read execute2_id runtime; do #timestamp=$(echo $runtime | awk -F "," '{print $1}' | awk -F ":" '{print $2}') @@ -168,7 +178,10 @@ rm /tmp/$RUNID-$i.sql done -echo "COMMIT;" >> /tmp/$RUNID.sql -$SQLCMD -f /tmp/$RUNID.sql +#echo "COMMIT;" >> /tmp/$RUNID.sql + +echo "Finished SQL generation." +echo "Exporting provenance to database..." +$SQLCMD -f /tmp/$RUNID.sql 1> /tmp/$RUNID-provenancedb-import.log rm /tmp/$RUNID.sql -echo Finished sending SQL to DB \ No newline at end of file +echo "Finished exporting provenance to database." \ No newline at end of file Modified: provenancedb/swift-prov-import-all-logs =================================================================== --- provenancedb/swift-prov-import-all-logs 2012-03-14 16:49:49 UTC (rev 5714) +++ provenancedb/swift-prov-import-all-logs 2012-03-14 19:44:33 UTC (rev 5715) @@ -57,7 +57,7 @@ export WF="${RUNID}" - echo "BEGIN TRANSACTION;" > /tmp/$WF.sql + #echo "BEGIN TRANSACTION;" > /tmp/$WF.sql echo "INSERT INTO run (id, log_filename, swift_version, cog_version, final_state) VALUES ('$WF','$filename','$version', '$COG_VERSION', '$wfstatus');" >> /tmp/$WF.sql echo version $version in log file $filename From lgadelha at ci.uchicago.edu Thu Mar 15 19:12:05 2012 From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu) Date: Thu, 15 Mar 2012 19:12:05 -0500 (CDT) Subject: [Swift-commit] r5716 - provenancedb/apps/SciColSim Message-ID: <20120316001205.8E4B19CCA4@svn.ci.uchicago.edu> Author: lgadelha Date: 2012-03-15 19:12:05 -0500 (Thu, 15 Mar 2012) New Revision: 5716 Modified: provenancedb/apps/SciColSim/SciColSim_extractor.sh Log: Minor fix. Modified: provenancedb/apps/SciColSim/SciColSim_extractor.sh =================================================================== --- provenancedb/apps/SciColSim/SciColSim_extractor.sh 2012-03-14 19:44:33 UTC (rev 5715) +++ provenancedb/apps/SciColSim/SciColSim_extractor.sh 2012-03-16 00:12:05 UTC (rev 5716) @@ -4,7 +4,7 @@ # PROVDB_HOME=~/provenancedb -SCICOLSIM_HOME=/home/lgadelha/SciColSim +SCICOLSIM_HOME=~/scicolsimruns IMPORT_HOME=/tmp #cp $PROVDB_HOME/etc/provenance.config.ci $PROVDB_HOME/etc/provenance.config @@ -16,7 +16,7 @@ run_dir=$(echo "$k" | awk -F "annealing-" '{print $1}') cd $SCICOLSIM_HOME/$run_dir full_dir=$(pwd) - log_suffix=$(echo "$k" | awk -F "^." '{print $2}') + log_suffix=$(echo "$k" | awk -F "/" '{print $3}') count=$($SQLCMD --tuples-only -c "select count(*) from script_run where log_filename like '%$log_suffix';" | awk '{print $1}') if [ "$count" -eq "0" ]; then echo "export LOGREPO=$full_dir" > $PROVDB_HOME/etc/provenance.config From lgadelha at ci.uchicago.edu Fri Mar 16 04:54:51 2012 From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu) Date: Fri, 16 Mar 2012 04:54:51 -0500 (CDT) Subject: [Swift-commit] r5717 - provenancedb/apps/SciColSim Message-ID: <20120316095451.BC9329CCA4@svn.ci.uchicago.edu> Author: lgadelha Date: 2012-03-16 04:54:50 -0500 (Fri, 16 Mar 2012) New Revision: 5717 Modified: provenancedb/apps/SciColSim/SciColSim_extractor.sh Log: Gather multi_loss. Modified: provenancedb/apps/SciColSim/SciColSim_extractor.sh =================================================================== --- provenancedb/apps/SciColSim/SciColSim_extractor.sh 2012-03-16 00:12:05 UTC (rev 5716) +++ provenancedb/apps/SciColSim/SciColSim_extractor.sh 2012-03-16 09:54:50 UTC (rev 5717) @@ -55,6 +55,12 @@ echo "Annotating function call $PROC_ID with key-value(numeric) pair ('$key', $value)." $SQLCMD -c "INSERT INTO a_proc_n VALUES ('$PROC_ID', '$key', $value)" 1> /dev/null fi + if [ "$value" == "+-" ]; then + echo "Annotating function call $PROC_ID with key-value(numeric) pair ('multi_loss', $key)." + $SQLCMD -c "INSERT INTO a_proc_n VALUES ('$PROC_ID', 'multi_loss', $key)" 1> /dev/null + echo "Annotating function call $PROC_ID with key-value(numeric) pair ('multi_loss_std_dev', $rest)." + $SQLCMD -c "INSERT INTO a_proc_n VALUES ('$PROC_ID', 'multi_loss_std_dev', $rest)" 1> /dev/null + fi fi done < $SCICOLSIM_HOME/$run_dir/$outf else From swift at ci.uchicago.edu Sat Mar 17 03:01:54 2012 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Sat, 17 Mar 2012 03:01:54 -0500 (CDT) Subject: [Swift-commit] Cog update Message-ID: <20120317080154.A906B8D0009A@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3363 | davidkelly999 | 2012-03-07 17:40:35 -0600 (Wed, 07 Mar 2012) | 2 lines Fix to start the correct number of workers ------------------------------------------------------------------------ Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java (revision 3362) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java (working copy) @@ -315,7 +315,7 @@ */ protected void writeMultiJobPreamble(Writer wr, String exitcodefile) throws IOException { - wr.write("NODES=`cat $PE_HOSTFILE | awk '{ for(i=0;i<$2;i++){print $1} }'`\n"); + wr.write("NODES=`cat $PE_HOSTFILE | awk '{print $1}'`\n"); wr.write("ECF=" + exitcodefile + "\n"); wr.write("INDEX=0\n"); wr.write("for NODE in $NODES; do\n"); From ketan at ci.uchicago.edu Sun Mar 18 21:31:40 2012 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Sun, 18 Mar 2012 21:31:40 -0500 (CDT) Subject: [Swift-commit] r5718 - SwiftApps/SciColSim/docs Message-ID: <20120319023140.2D3379CF91@svn.ci.uchicago.edu> Author: ketan Date: 2012-03-18 21:31:40 -0500 (Sun, 18 Mar 2012) New Revision: 5718 Modified: SwiftApps/SciColSim/docs/plotit Log: plotting comments Modified: SwiftApps/SciColSim/docs/plotit =================================================================== --- SwiftApps/SciColSim/docs/plotit 2012-03-16 09:54:50 UTC (rev 5717) +++ SwiftApps/SciColSim/docs/plotit 2012-03-19 02:31:40 UTC (rev 5718) @@ -1,3 +1,5 @@ +#To be run as a gnuplot script: +# $ gnuplot plotit set terminal png enhanced #set term postscript eps enhanced #set terminal svg enhanced size 1000 1000 From ketan at ci.uchicago.edu Mon Mar 19 10:01:27 2012 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 19 Mar 2012 10:01:27 -0500 (CDT) Subject: [Swift-commit] r5719 - SwiftApps/SciColSim/bin Message-ID: <20120319150127.C59199CCFC@svn.ci.uchicago.edu> Author: ketan Date: 2012-03-19 10:01:27 -0500 (Mon, 19 Mar 2012) New Revision: 5719 Modified: SwiftApps/SciColSim/bin/extract4plots Log: minor update Modified: SwiftApps/SciColSim/bin/extract4plots =================================================================== --- SwiftApps/SciColSim/bin/extract4plots 2012-03-19 02:31:40 UTC (rev 5718) +++ SwiftApps/SciColSim/bin/extract4plots 2012-03-19 15:01:27 UTC (rev 5719) @@ -1,6 +1,6 @@ #!/bin/bash -#usage: ./swiftoutput2plot +#usage: ./extract4plots SWIFTOUTFILE=$1 From jonmon at ci.uchicago.edu Tue Mar 20 14:11:05 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Tue, 20 Mar 2012 14:11:05 -0500 (CDT) Subject: [Swift-commit] r5720 - SwiftApps/SciColSim Message-ID: <20120320191105.1CB949CC9C@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-20 14:11:04 -0500 (Tue, 20 Mar 2012) New Revision: 5720 Modified: SwiftApps/SciColSim/annealing.swift SwiftApps/SciColSim/optimizer.cpp SwiftApps/SciColSim/sumloss.sh Log: o committed changes for calculation the timing average o optimizer.cpp now has working loss limiting code Modified: SwiftApps/SciColSim/annealing.swift =================================================================== --- SwiftApps/SciColSim/annealing.swift 2012-03-19 15:01:27 UTC (rev 5719) +++ SwiftApps/SciColSim/annealing.swift 2012-03-20 19:11:04 UTC (rev 5720) @@ -11,12 +11,12 @@ { float loss; float sdev; - float tloss; + float tavg; float tsdev; } global boolean FIX_VARIABLES = true; -global int var_fixed[] = [1,1,0,0,0]; +global int var_fixed[] = [0,0,0,0,0]; global int Nworkers = @toint(@arg("nworkers","4")); global int rerunsPerApp= @toint(@arg("rerunsperapp", "100")); @@ -159,16 +159,8 @@ } tracef( @strcat( "multi_annealing: AR: ", color( 10,"%f" ), " ", color( 9,"%i" ),"\n" ), try_x[j], j ); - if( (try_x[2] < -1.2) || (try_x[2] > 12.0) ) # FIXME: Temp debugging hack: REMOVE THIS!!! - { - mlres[i][j].loss = 140.0; - mlres[i][j].sdev = 0.8; - } - else - { - // Up to here, x[] and dx[] are only set for previous i - mlres[i][j] = multi_loss( T_start, T_end, annealing_cycles, Target_rejection, starting_jump, i, j, try_x, target_innov, evolve_reruns ); // do the N evolve()'s, N=evolve_reruns - } + mlres[i][j] = multi_loss( T_start, T_end, annealing_cycles, Target_rejection, starting_jump, i, j, try_x, target_innov, evolve_reruns ); // do the N evolve()'s, N=evolve_rerusn + tracef( "multi_annealing: AR: %f +- %f\n", mlres[i][j].loss, mlres[i][j].sdev ); // Beyond this point, x[] and dx[] are being set for this i,j @@ -177,14 +169,14 @@ if ( mlres[i][j].loss < ALOT ) { fprintf( some_out_filename, "N, %fs, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n", - mlres[i][j].tloss, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss, try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev ); // Note i-1 field: print that way to match with C++ output + mlres[i][j].tavg, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss, try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev ); // Note i-1 field: print that way to match with C++ output // fprintf( "max_dist_swift.txt", color( Red,"multi_annealing: AF: max_dist.txt - tbd\n" ) ); // FIXME: max_dist is global set in evolve() } else // does this ever occur? if so did we want to still do the ratio computation above??? { - fprintf( some_out_filename, "A %i %i %f %f | %i, %f, %f, %f, %f, %f, %f, %f\n", - i, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss, try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev ); + fprintf( some_out_filename, "A, %fs, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n", + mlres[i][j].tavg, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss, try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev ); // Note i-1 field: print that way to match with C++ output //tracef( "multi_annealing: Loss %f > ALOT at [i][j] = [%d][%d]\n", mlres[i][j].loss, i ,j ); } @@ -290,7 +282,7 @@ @strcat(t_start), @strcat(t_end), @strcat(annealing_steps), @strcat(t_rejection), @strcat(starting_jump), // FREEZE: alpha_i alpha_m beta gamma delta - "1", "1", "0", "0", "0", + "0", "0", "0", "0", "0", // operation-code:(m,a) Nworkers seed "m", @strcat(Nworkers), @arg("seed", "0" ) ]; Modified: SwiftApps/SciColSim/optimizer.cpp =================================================================== --- SwiftApps/SciColSim/optimizer.cpp 2012-03-19 15:01:27 UTC (rev 5719) +++ SwiftApps/SciColSim/optimizer.cpp 2012-03-20 19:11:04 UTC (rev 5720) @@ -990,25 +990,23 @@ void evolve_to_target_and_save(int istart, int iend, double* storage, int* counters){ double ALOT=10000.; + double ratio = 0.0; int check = 0; reset_world(); for (int k = istart; k < iend; k++){ -// Code for getting time to check - for(int i=0; i< N_epochs && current_novelty < TargetNovelty; i++){ -// Check the walltime limit - - if(current_loss/current_novelty < max_loss_value((int)TargetNovelty)){ + if(ratio < max_loss_value((int)TargetNovelty)*2){ update_world(); } else{ - check=1; + check = 1; break; } + ratio = current_loss/current_novelty; } if( check ){ @@ -1026,28 +1024,27 @@ //============================================== double max_loss_value(int tn){ - /* Poor check */ - /* Do not check for equality, check for within some error */ - if( tn == 58. ) return 128.286; - else if( tn == 108. ) return 131.866; - else if( tn == 158. ) return 135.551; - else if( tn == 208. ) return 139.694; - else if( tn == 258. ) return 144.163; - else if( tn == 308. ) return 148.967; - else if( tn == 358. ) return 154.201; - else if( tn == 408. ) return 159.962; - else if( tn == 458. ) return 166.441; - else if( tn == 508. ) return 173.655; - else if( tn == 558. ) return 181.921; - else if( tn == 608. ) return 191.246; - else if( tn == 658. ) return 202.150; - else if( tn == 708. ) return 215.197; - else if( tn == 758. ) return 202.150; // Verify with Andrey. Same as TargetNovelty of 658 - else if( tn == 808. ) return 251.698; - else if( tn == 858. ) return 279.201; - else if( tn == 908. ) return 320.112; - else if( tn == 958. ) return 394.774; - else if( tn == 1008. ) return 1052.38; // Huge number here. Why? + + if( tn == 58 ) return 128.286; + else if( tn == 108 ) return 131.866; + else if( tn == 158 ) return 135.551; + else if( tn == 208 ) return 139.694; + else if( tn == 258 ) return 144.163; + else if( tn == 308 ) return 148.967; + else if( tn == 358 ) return 154.201; + else if( tn == 408 ) return 159.962; + else if( tn == 458 ) return 166.441; + else if( tn == 508 ) return 173.655; + else if( tn == 558 ) return 181.921; + else if( tn == 608 ) return 191.246; + else if( tn == 658 ) return 202.150; + else if( tn == 708 ) return 215.197; + else if( tn == 758 ) return 202.150; // Verify with Andrey. Same as TargetNovelty of 658 + else if( tn == 808 ) return 251.698; + else if( tn == 858 ) return 279.201; + else if( tn == 908 ) return 320.112; + else if( tn == 958 ) return 394.774; + else if( tn == 1000 ) return 1052.38; // Huge number here. Why? else return 10000.; /* Pray that this does not occur. Ask Andrey what to do */ } Modified: SwiftApps/SciColSim/sumloss.sh =================================================================== --- SwiftApps/SciColSim/sumloss.sh 2012-03-19 15:01:27 UTC (rev 5719) +++ SwiftApps/SciColSim/sumloss.sh 2012-03-20 19:11:04 UTC (rev 5720) @@ -1,8 +1,17 @@ #! /bin/sh +# Calculates the average loss, loss standard deviation, average running time, and average running time standard deviation +# of a set of loss files all produced by evolve.sh + +# loss = average loss +# sdev = standard deviation of loss +# tavg = average timing data(name should be changed) +# tsdev = standard deviation of timing data + +# process each file individually and treat each number as a field awk ' -BEGIN { RS = ""; FS="\n"; loss = 0; loss_sq = 0; tloss = 0; tloss_sq = 0; n = 0; tn = 0; } +BEGIN { RS = ""; FS="\n"; loss = 0; loss_sq = 0; tavg = 0; tavg_sq = 0; n = 0; tn = 0; } { for(i=1;i Res; -# Res.first=Loss; -# Res.second=two_std; - - -# Older logic, literal transcription from C++ (didnt handle multiline files correctly): - -# awk ' - -# BEGIN { n = 0; loss = 0; } - -# { -# loss += $1 -# loss_sq += ($1*$1) -# n++ -# } - -# END { - -# loss /= n; -# loss_sq /= n; - -# x = (loss_sq - (loss*loss))/n -# sdev = 2.0 * sqrt(x) -# printf "loss sdev\n" -# printf "%f %f\n", loss, sdev -# } ' $* From wozniak at ci.uchicago.edu Wed Mar 21 16:19:15 2012 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Wed, 21 Mar 2012 16:19:15 -0500 (CDT) Subject: [Swift-commit] r5721 - trunk/docs/userguide Message-ID: <20120321211915.E411F9CCC8@svn.ci.uchicago.edu> Author: wozniak Date: 2012-03-21 16:19:15 -0500 (Wed, 21 Mar 2012) New Revision: 5721 Modified: trunk/docs/userguide/cdm Log: Add CDM debugging notes Modified: trunk/docs/userguide/cdm =================================================================== --- trunk/docs/userguide/cdm 2012-03-20 19:11:04 UTC (rev 5720) +++ trunk/docs/userguide/cdm 2012-03-21 21:19:15 UTC (rev 5721) @@ -133,6 +133,15 @@ unlink these links; if it does, the application will not be able to find or create Swift-compatible data. +=== Debugging + +To troubleshoot CDM, check the Swift log and the wrapper logs +(+*-info+ files). These will indicate the CDM policy that Swift finds +for each file and resulting action, such as skipping stage in for +DIRECT and the creation of links. Common errors include specifying +the wrong directory which will result in an error that the file was +not found, or that a link could not be created. + // .Gather // ----- // property GATHER_LIMIT 500000000 # 500 MB From jonmon at ci.uchicago.edu Fri Mar 23 16:22:39 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Fri, 23 Mar 2012 16:22:39 -0500 (CDT) Subject: [Swift-commit] r5722 - in SwiftApps/SciColSim: . openmp_test openmp_test/900074.sdb Message-ID: <20120323212239.13FC49CCF7@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-23 16:22:38 -0500 (Fri, 23 Mar 2012) New Revision: 5722 Added: SwiftApps/SciColSim/openmp_test/ SwiftApps/SciColSim/openmp_test/900074.sdb/ SwiftApps/SciColSim/openmp_test/900074.sdb/movie_graph.txt SwiftApps/SciColSim/openmp_test/900074.sdb/openmp-optimizer SwiftApps/SciColSim/openmp_test/900074.sdb/run.sh SwiftApps/SciColSim/openmp_test/900074.sdb/top.data SwiftApps/SciColSim/openmp_test/movie_graph.txt SwiftApps/SciColSim/openmp_test/openmp-optimizer SwiftApps/SciColSim/openmp_test/openmp-verify.pbs SwiftApps/SciColSim/openmp_test/run.sh Log: added test showing producing evidence that we are indeed utilizing the 24 cores in openmp Added: SwiftApps/SciColSim/openmp_test/900074.sdb/movie_graph.txt =================================================================== --- SwiftApps/SciColSim/openmp_test/900074.sdb/movie_graph.txt (rev 0) +++ SwiftApps/SciColSim/openmp_test/900074.sdb/movie_graph.txt 2012-03-23 21:22:38 UTC (rev 5722) @@ -0,0 +1,1010 @@ +500 1008 +0 56 +0 436 +0 446 +1 256 +1 476 +1 260 +1 262 +1 9 +1 266 +1 13 +1 142 +1 273 +1 388 +1 30 +1 36 +1 37 +1 423 +1 168 +1 425 +1 255 +1 175 +1 307 +1 58 +1 316 +1 319 +1 323 +1 68 +1 455 +1 216 +1 82 +1 88 +1 346 +1 348 +1 225 +1 228 +1 236 +1 365 +1 111 +1 424 +1 370 +1 246 +1 376 +1 122 +1 191 +1 252 +1 382 +1 127 +2 3 +3 262 +3 263 +3 392 +3 172 +3 138 +3 395 +3 13 +3 18 +3 150 +3 408 +3 154 +3 156 +3 30 +3 291 +3 37 +3 168 +3 300 +3 52 +3 158 +3 310 +3 265 +3 187 +3 61 +3 319 +3 65 +3 322 +3 54 +3 130 +3 71 +3 332 +3 461 +3 397 +3 419 +3 473 +3 346 +3 95 +3 352 +3 98 +3 103 +3 107 +3 414 +3 368 +3 498 +3 499 +4 224 +4 449 +4 419 +4 356 +4 72 +4 330 +4 173 +4 80 +4 242 +4 180 +4 24 +4 138 +5 33 +5 459 +5 484 +5 106 +5 299 +5 430 +5 16 +5 336 +5 50 +5 147 +5 406 +5 87 +5 91 +5 190 +6 7 +6 264 +6 137 +6 276 +6 149 +6 279 +6 153 +6 29 +6 415 +6 293 +6 166 +6 296 +6 42 +6 44 +6 48 +6 49 +6 438 +6 55 +6 314 +6 69 +6 454 +6 463 +6 84 +6 85 +6 470 +6 349 +6 94 +6 229 +6 233 +6 364 +6 117 +6 248 +6 380 +6 426 +7 129 +7 131 +7 470 +7 136 +7 471 +7 271 +7 402 +7 19 +7 404 +7 282 +7 286 +7 491 +7 35 +7 421 +7 166 +7 295 +7 427 +7 428 +7 301 +7 349 +7 436 +7 439 +7 56 +7 445 +7 446 +7 320 +7 123 +7 451 +7 69 +7 454 +7 78 +7 463 +7 212 +7 213 +7 342 +7 343 +7 474 +7 93 +7 483 +7 357 +7 102 +7 235 +7 237 +7 239 +7 371 +7 379 +8 112 +8 164 +8 10 +8 12 +8 398 +8 176 +8 120 +8 178 +8 373 +8 24 +8 124 +9 262 +10 11 +10 12 +10 47 +10 335 +10 124 +11 96 +11 289 +11 66 +11 283 +11 97 +11 361 +11 47 +11 432 +11 335 +11 215 +11 27 +11 124 +11 189 +12 112 +12 434 +12 47 +13 32 +13 130 +13 260 +13 37 +13 39 +13 202 +13 305 +13 461 +13 145 +13 275 +13 448 +13 54 +13 183 +13 344 +13 36 +14 387 +14 267 +14 155 +14 36 +14 165 +14 51 +14 308 +14 440 +14 441 +14 59 +14 444 +14 199 +14 328 +14 462 +14 81 +14 210 +14 214 +14 91 +14 99 +14 60 +14 106 +14 109 +14 110 +14 244 +15 193 +15 73 +15 400 +15 341 +15 27 +15 31 +16 33 +16 258 +16 353 +16 329 +16 204 +16 207 +16 336 +16 108 +16 50 +16 22 +16 119 +16 169 +16 381 +16 223 +17 321 +17 482 +17 197 +17 276 +17 149 +17 315 +17 94 +18 32 +18 461 +18 305 +18 498 +18 86 +18 25 +19 320 +19 421 +19 334 +19 314 +19 349 +19 446 +20 48 +20 28 +20 83 +20 44 +20 45 +21 288 +21 176 +21 347 +21 325 +21 392 +21 490 +21 366 +21 272 +21 209 +21 115 +21 182 +21 247 +21 184 +21 25 +21 217 +21 27 +21 86 +21 478 +22 353 +23 257 +23 100 +23 25 +24 134 +24 146 +24 302 +24 151 +24 290 +24 291 +24 43 +24 173 +24 46 +24 176 +24 180 +24 245 +24 453 +24 457 +24 208 +24 338 +24 363 +24 116 +24 373 +24 126 +25 32 +25 128 +25 257 +25 86 +26 161 +26 357 +26 193 +26 78 +26 341 +26 69 +27 361 +27 490 +27 333 +27 272 +27 312 +27 468 +27 41 +27 184 +27 429 +27 124 +27 341 +28 64 +28 354 +28 67 +28 135 +28 200 +28 42 +28 75 +28 45 +28 206 +28 48 +28 359 +28 53 +28 238 +29 229 +29 199 +29 296 +29 233 +29 42 +29 274 +29 117 +29 152 +29 155 +29 62 +30 386 +30 225 +30 76 +30 322 +30 367 +30 318 +31 200 +31 73 +31 333 +31 494 +31 45 +31 312 +32 257 +32 305 +32 101 +32 486 +32 39 +32 231 +32 79 +32 145 +32 275 +33 459 +33 336 +33 372 +33 119 +33 409 +34 283 +35 491 +35 243 +35 237 +35 326 +36 260 +36 266 +36 285 +36 292 +36 39 +36 424 +36 171 +36 181 +36 183 +36 440 +36 58 +36 323 +36 328 +36 81 +36 82 +36 87 +36 99 +36 228 +36 101 +36 111 +37 168 +37 202 +38 92 +39 448 +39 101 +39 486 +39 231 +39 145 +39 344 +40 345 +40 355 +40 179 +40 59 +40 125 +41 160 +41 226 +41 230 +41 104 +41 361 +41 492 +41 341 +41 215 +41 121 +41 189 +42 64 +42 67 +42 199 +42 359 +42 109 +42 206 +42 143 +42 48 +42 274 +42 117 +44 193 +44 69 +44 161 +44 73 +44 48 +44 83 +44 279 +45 200 +45 73 +45 83 +46 242 +46 485 +46 126 +46 173 +48 64 +48 206 +50 259 +50 261 +50 329 +50 299 +50 204 +50 430 +50 207 +50 405 +50 59 +50 159 +51 67 +51 101 +51 135 +51 210 +51 244 +51 214 +51 441 +51 249 +52 368 +52 217 +52 347 +52 86 +53 101 +53 327 +53 75 +53 238 +53 79 +53 249 +54 130 +55 84 +57 162 +57 331 +58 266 +58 82 +59 384 +59 259 +59 132 +59 267 +59 496 +59 355 +59 152 +59 91 +59 221 +59 159 +60 91 +60 165 +62 152 +62 155 +62 221 +63 456 +63 334 +63 303 +63 472 +63 314 +63 380 +65 154 +65 499 +65 374 +65 103 +66 96 +66 160 +66 215 +67 359 +67 135 +67 109 +67 143 +67 214 +69 161 +69 166 +69 239 +69 404 +69 85 +69 279 +69 153 +70 410 +70 467 +71 130 +72 138 +73 193 +74 241 +74 82 +74 188 +74 133 +74 469 +75 140 +75 79 +75 90 +75 350 +76 386 +77 120 +77 306 +77 452 +77 254 +77 311 +78 239 +78 357 +79 257 +79 100 +79 101 +80 138 +81 99 +81 171 +82 256 +82 133 +82 348 +82 205 +82 372 +82 181 +82 119 +82 409 +82 255 +82 316 +82 122 +82 469 +84 277 +84 264 +84 149 +84 380 +84 415 +85 153 +85 166 +86 247 +86 217 +87 181 +87 147 +87 285 +89 192 +89 200 +89 333 +89 468 +89 443 +89 222 +90 272 +90 100 +90 350 +91 165 +91 106 +91 430 +91 406 +91 159 +92 258 +93 136 +93 464 +93 493 +93 326 +94 482 +94 233 +94 114 +94 276 +95 107 +95 265 +95 186 +95 332 +96 289 +96 324 +96 102 +96 401 +97 283 +97 220 +97 434 +97 164 +99 101 +99 171 +99 244 +99 251 +100 272 +101 231 +101 466 +101 244 +101 249 +101 251 +102 141 +102 237 +102 369 +102 243 +102 212 +102 121 +102 340 +102 411 +104 121 +104 230 +105 396 +105 309 +106 147 +106 308 +106 406 +106 444 +106 190 +109 199 +109 143 +109 387 +109 214 +110 267 +112 164 +113 195 +114 233 +114 482 +114 315 +114 412 +116 465 +117 296 +118 163 +119 188 +119 409 +120 297 +120 363 +120 398 +120 254 +121 160 +121 230 +121 492 +121 369 +121 212 +122 256 +122 316 +124 176 +125 394 +125 497 +125 179 +125 351 +125 479 +126 465 +126 290 +126 339 +129 491 +130 397 +131 464 +131 136 +132 383 +133 273 +133 232 +133 205 +133 241 +133 316 +134 203 +135 249 +135 354 +136 464 +136 474 +137 276 +138 419 +138 414 +138 158 +139 169 +140 200 +140 222 +140 350 +144 334 +146 363 +147 308 +147 285 +148 175 +149 456 +149 407 +151 422 +152 488 +152 233 +152 221 +152 351 +154 187 +154 358 +154 158 +155 199 +155 267 +155 221 +155 437 +156 332 +156 318 +156 310 +157 436 +157 301 +158 396 +158 187 +158 414 +159 430 +160 215 +161 193 +163 220 +164 297 +164 398 +164 220 +166 427 +167 201 +167 442 +167 170 +167 383 +168 319 +169 280 +169 270 +169 198 +170 353 +170 403 +170 269 +170 207 +170 211 +173 242 +174 194 +174 195 +174 362 +174 177 +174 250 +174 219 +174 412 +175 460 +175 246 +175 447 +176 291 +176 294 +176 366 +176 178 +176 182 +177 219 +179 345 +179 394 +181 372 +182 184 +182 429 +182 294 +184 429 +185 358 +188 280 +188 469 +189 215 +192 272 +192 443 +192 468 +192 287 +193 400 +195 488 +195 233 +196 389 +196 317 +197 321 +198 280 +199 387 +200 333 +200 222 +201 284 +203 311 +207 353 +207 329 +207 269 +207 240 +212 230 +212 341 +212 439 +213 349 +214 462 +215 226 +216 388 +216 236 +217 347 +218 465 +219 450 +219 394 +219 250 +219 298 +219 479 +220 297 +220 283 +221 267 +221 437 +222 443 +222 350 +225 236 +227 472 +227 321 +227 315 +227 412 +227 390 +228 424 +228 260 +230 341 +232 280 +232 241 +233 480 +233 412 +234 243 +234 340 +238 249 +238 354 +240 353 +241 280 +242 356 +243 385 +243 326 +244 251 +246 382 +249 354 +249 327 +249 466 +250 488 +250 479 +250 351 +252 376 +253 384 +253 269 +254 363 +254 302 +254 311 +258 353 +258 317 +258 477 +259 384 +259 405 +259 269 +260 424 +262 352 +262 346 +263 392 +263 300 +263 325 +264 438 +267 437 +268 413 +269 384 +269 383 +270 381 +270 431 +272 443 +272 350 +272 287 +273 316 +276 377 +278 390 +280 417 +281 376 +282 391 +283 289 +283 432 +285 308 +285 292 +288 325 +288 478 +289 401 +289 413 +291 325 +291 478 +292 328 +292 308 +297 398 +297 495 +302 453 +302 311 +304 493 +307 388 +308 328 +309 396 +311 453 +312 333 +312 494 +313 375 +314 487 +314 364 +314 380 +315 482 +315 412 +319 346 +321 456 +326 491 +326 435 +327 466 +328 440 +331 472 +332 408 +332 489 +333 468 +334 446 +337 380 +338 465 +341 400 +343 436 +345 394 +345 399 +347 392 +347 368 +349 364 +349 470 +351 496 +351 479 +353 393 +358 360 +358 396 +365 375 +366 478 +368 392 +372 409 +374 467 +378 411 +379 436 +380 456 +380 415 +382 433 +383 384 +394 481 +394 479 +396 449 +416 420 +416 422 +417 447 +418 446 +450 481 +456 475 +458 471 +463 470 +472 475 +496 497 + Added: SwiftApps/SciColSim/openmp_test/900074.sdb/openmp-optimizer =================================================================== (Binary files differ) Property changes on: SwiftApps/SciColSim/openmp_test/900074.sdb/openmp-optimizer ___________________________________________________________________ Added: svn:executable + * Added: svn:mime-type + application/octet-stream Added: SwiftApps/SciColSim/openmp_test/900074.sdb/run.sh =================================================================== --- SwiftApps/SciColSim/openmp_test/900074.sdb/run.sh (rev 0) +++ SwiftApps/SciColSim/openmp_test/900074.sdb/run.sh 2012-03-23 21:22:38 UTC (rev 5722) @@ -0,0 +1,8 @@ +#!/bin/bash + +OMP_NUM_THREADS=24 openmp-optimizer 1.3361867203970093 1.0553507574550953 0.29712814586892483 -0.49955400395753874 4.186059352113561 58 40000 20 2400 2 1 2 0.01 100 0.3 2.3 0 0 0 0 0 m 24 0 & +pid=`ps -u jonmon | grep openmp-optimize | awk '{print $1}'` + +top -b -n 10 -p $pid > top.data + + Property changes on: SwiftApps/SciColSim/openmp_test/900074.sdb/run.sh ___________________________________________________________________ Added: svn:executable + * Added: SwiftApps/SciColSim/openmp_test/900074.sdb/top.data =================================================================== --- SwiftApps/SciColSim/openmp_test/900074.sdb/top.data (rev 0) +++ SwiftApps/SciColSim/openmp_test/900074.sdb/top.data 2012-03-23 21:22:38 UTC (rev 5722) @@ -0,0 +1,329 @@ +top - 16:19:59 up 15 days, 23:25, 0 users, load average: 0.17, 1.93, 2.33 +Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie +Cpu0 : 68.1%us, 0.4%sy, 0.0%ni, 28.5%id, 0.0%wa, 3.0%hi, 0.0%si, 0.0%st +Cpu1 : 68.7%us, 0.9%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu2 : 68.9%us, 0.8%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu3 : 68.8%us, 0.8%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu4 : 68.5%us, 1.1%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu5 : 68.7%us, 0.9%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu6 : 68.7%us, 1.0%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu7 : 69.2%us, 0.4%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu8 : 68.9%us, 0.7%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu9 : 68.7%us, 0.9%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu10 : 68.6%us, 1.0%sy, 0.0%ni, 30.4%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu11 : 68.6%us, 1.0%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu12 : 68.7%us, 1.0%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu13 : 68.8%us, 0.9%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu14 : 68.8%us, 0.8%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu15 : 68.8%us, 0.8%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu16 : 68.7%us, 1.0%sy, 0.0%ni, 30.4%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu17 : 68.8%us, 0.8%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu18 : 68.7%us, 0.9%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu19 : 68.8%us, 0.9%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu20 : 68.7%us, 0.9%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu21 : 68.6%us, 1.1%sy, 0.0%ni, 30.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu22 : 53.3%us, 1.0%sy, 0.0%ni, 45.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu23 : 53.5%us, 0.8%sy, 0.0%ni, 45.5%id, 0.2%wa, 0.0%hi, 0.0%si, 0.0%st +Mem: 33086604k total, 690676k used, 32395928k free, 0k buffers +Swap: 0k total, 0k used, 0k free, 26676k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +25593 jonmon 20 0 219m 203m 1324 R 92 0.6 0:00.48 openmp-optimize + + +top - 16:20:02 up 15 days, 23:25, 0 users, load average: 0.17, 1.93, 2.33 +Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie +Cpu0 : 97.7%us, 0.0%sy, 0.0%ni, 2.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu1 : 99.7%us, 0.3%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu2 : 97.7%us, 0.0%sy, 0.0%ni, 2.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu3 : 97.7%us, 0.3%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu4 : 98.0%us, 0.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu5 : 97.7%us, 0.3%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu6 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu7 : 97.7%us, 0.0%sy, 0.0%ni, 2.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu8 : 97.7%us, 0.0%sy, 0.0%ni, 2.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu9 : 98.0%us, 0.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu10 : 98.0%us, 0.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu11 : 97.7%us, 0.0%sy, 0.0%ni, 2.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu12 : 98.0%us, 0.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu13 : 97.7%us, 0.0%sy, 0.0%ni, 2.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu14 : 98.0%us, 0.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu15 : 97.7%us, 0.0%sy, 0.0%ni, 2.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu16 : 97.7%us, 0.0%sy, 0.0%ni, 2.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu17 : 98.0%us, 0.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu18 : 98.0%us, 0.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu19 : 98.0%us, 0.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu20 : 98.0%us, 0.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu21 : 98.0%us, 0.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu22 : 97.7%us, 0.0%sy, 0.0%ni, 2.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu23 : 98.0%us, 0.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Mem: 33086604k total, 711668k used, 32374936k free, 0k buffers +Swap: 0k total, 0k used, 0k free, 26828k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +25593 jonmon 20 0 550m 233m 1380 R 2241 0.7 1:08.08 openmp-optimize + + +top - 16:20:05 up 15 days, 23:26, 0 users, load average: 2.08, 2.30, 2.45 +Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie +Cpu0 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu1 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu2 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu3 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu4 : 99.7%us, 0.3%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu5 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu6 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu7 : 99.7%us, 0.3%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu8 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu9 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu10 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu11 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu12 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu13 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu14 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu15 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu16 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu17 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu18 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu19 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu20 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu21 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu22 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu23 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Mem: 33086604k total, 711548k used, 32375056k free, 0k buffers +Swap: 0k total, 0k used, 0k free, 26832k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +25593 jonmon 20 0 550m 233m 1380 R 2302 0.7 2:17.34 openmp-optimize + + +top - 16:20:08 up 15 days, 23:26, 0 users, load average: 3.83, 2.66, 2.56 +Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie +Cpu0 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu1 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu2 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu3 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu4 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu5 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu6 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu7 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu8 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu9 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu10 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu11 : 99.7%us, 0.3%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu12 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu13 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu14 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu15 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu16 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu17 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu18 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu19 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu20 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu21 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu22 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu23 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Mem: 33086604k total, 711548k used, 32375056k free, 0k buffers +Swap: 0k total, 0k used, 0k free, 26832k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +25593 jonmon 20 0 550m 233m 1380 R 2298 0.7 3:26.37 openmp-optimize + + +top - 16:20:11 up 15 days, 23:26, 0 users, load average: 3.83, 2.66, 2.56 +Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie +Cpu0 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu1 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu2 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu3 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu4 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu5 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu6 : 0.3%us, 0.0%sy, 0.0%ni, 99.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu7 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu8 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu9 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu10 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu11 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu12 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu13 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu14 : 99.7%us, 0.3%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu15 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu16 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu17 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu18 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu19 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu20 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu21 : 99.7%us, 0.3%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu22 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu23 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Mem: 33086604k total, 711424k used, 32375180k free, 0k buffers +Swap: 0k total, 0k used, 0k free, 26832k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +25593 jonmon 20 0 550m 233m 1380 R 2300 0.7 4:35.55 openmp-optimize + + +top - 16:20:14 up 15 days, 23:26, 0 users, load average: 5.45, 3.01, 2.68 +Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie +Cpu0 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu1 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu2 : 95.3%us, 0.0%sy, 0.0%ni, 4.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu3 : 96.4%us, 0.0%sy, 0.0%ni, 3.6%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu4 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu5 : 94.4%us, 0.0%sy, 0.0%ni, 5.6%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu6 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu7 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu8 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu9 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu10 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu11 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu12 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu13 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu14 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu15 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu16 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu17 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu18 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu19 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu20 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu21 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu22 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu23 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Mem: 33086604k total, 711424k used, 32375180k free, 0k buffers +Swap: 0k total, 0k used, 0k free, 26832k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +25593 jonmon 20 0 550m 233m 1380 R 2288 0.7 5:44.38 openmp-optimize + + +top - 16:20:17 up 15 days, 23:26, 0 users, load average: 5.45, 3.01, 2.68 +Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie +Cpu0 : 8.4%us, 0.0%sy, 0.0%ni, 91.6%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu1 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu2 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu3 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu4 : 4.3%us, 0.0%sy, 0.0%ni, 95.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu5 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu6 : 0.3%us, 0.0%sy, 0.0%ni, 99.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu7 : 36.3%us, 0.0%sy, 0.0%ni, 63.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu8 : 35.0%us, 0.0%sy, 0.0%ni, 65.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu9 : 37.0%us, 0.0%sy, 0.0%ni, 63.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu10 : 41.7%us, 0.0%sy, 0.0%ni, 58.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu11 : 44.3%us, 0.0%sy, 0.0%ni, 55.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu12 : 50.7%us, 0.0%sy, 0.0%ni, 49.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu13 : 52.5%us, 0.0%sy, 0.0%ni, 47.5%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu14 : 46.8%us, 0.0%sy, 0.0%ni, 53.2%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu15 : 64.7%us, 0.0%sy, 0.0%ni, 35.3%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu16 : 37.3%us, 0.0%sy, 0.0%ni, 62.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu17 : 34.3%us, 0.0%sy, 0.0%ni, 65.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu18 : 45.0%us, 0.0%sy, 0.0%ni, 55.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu19 : 50.0%us, 0.0%sy, 0.0%ni, 50.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu20 : 50.0%us, 0.0%sy, 0.0%ni, 50.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu21 : 42.3%us, 0.0%sy, 0.0%ni, 57.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu22 : 50.8%us, 0.0%sy, 0.0%ni, 49.2%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu23 : 57.3%us, 0.0%sy, 0.0%ni, 42.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Mem: 33086604k total, 711424k used, 32375180k free, 0k buffers +Swap: 0k total, 0k used, 0k free, 26836k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +25593 jonmon 20 0 550m 233m 1380 R 893 0.7 6:11.28 openmp-optimize + + +top - 16:20:20 up 15 days, 23:26, 0 users, load average: 5.17, 2.99, 2.67 +Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie +Cpu0 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu1 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu2 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu3 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu4 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu5 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu6 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu7 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu8 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu9 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu10 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu11 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu12 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu13 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu14 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu15 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu16 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu17 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu18 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu19 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu20 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu21 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu22 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu23 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Mem: 33086604k total, 711408k used, 32375196k free, 0k buffers +Swap: 0k total, 0k used, 0k free, 26840k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +25593 jonmon 20 0 550m 233m 1380 R 100 0.7 6:14.29 openmp-optimize + + +top - 16:20:23 up 15 days, 23:26, 0 users, load average: 4.92, 2.98, 2.67 +Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie +Cpu0 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu1 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu2 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu3 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu4 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu5 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu6 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu7 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu8 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu9 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu10 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu11 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu12 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu13 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu14 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu15 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu16 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu17 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu18 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu19 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu20 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu21 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu22 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu23 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Mem: 33086604k total, 711408k used, 32375196k free, 0k buffers +Swap: 0k total, 0k used, 0k free, 26840k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +25593 jonmon 20 0 550m 233m 1380 R 100 0.7 6:17.28 openmp-optimize + + +top - 16:20:26 up 15 days, 23:26, 0 users, load average: 4.92, 2.98, 2.67 +Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie +Cpu0 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu1 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu2 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu3 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu4 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu5 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu6 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu7 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu8 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu9 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu10 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu11 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu12 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu13 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu14 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu15 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu16 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu17 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu18 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu19 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu20 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu21 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu22 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Cpu23 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st +Mem: 33086604k total, 711288k used, 32375316k free, 0k buffers +Swap: 0k total, 0k used, 0k free, 26844k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +25593 jonmon 20 0 550m 233m 1380 R 100 0.7 6:20.29 openmp-optimize + Added: SwiftApps/SciColSim/openmp_test/movie_graph.txt =================================================================== --- SwiftApps/SciColSim/openmp_test/movie_graph.txt (rev 0) +++ SwiftApps/SciColSim/openmp_test/movie_graph.txt 2012-03-23 21:22:38 UTC (rev 5722) @@ -0,0 +1,1010 @@ +500 1008 +0 56 +0 436 +0 446 +1 256 +1 476 +1 260 +1 262 +1 9 +1 266 +1 13 +1 142 +1 273 +1 388 +1 30 +1 36 +1 37 +1 423 +1 168 +1 425 +1 255 +1 175 +1 307 +1 58 +1 316 +1 319 +1 323 +1 68 +1 455 +1 216 +1 82 +1 88 +1 346 +1 348 +1 225 +1 228 +1 236 +1 365 +1 111 +1 424 +1 370 +1 246 +1 376 +1 122 +1 191 +1 252 +1 382 +1 127 +2 3 +3 262 +3 263 +3 392 +3 172 +3 138 +3 395 +3 13 +3 18 +3 150 +3 408 +3 154 +3 156 +3 30 +3 291 +3 37 +3 168 +3 300 +3 52 +3 158 +3 310 +3 265 +3 187 +3 61 +3 319 +3 65 +3 322 +3 54 +3 130 +3 71 +3 332 +3 461 +3 397 +3 419 +3 473 +3 346 +3 95 +3 352 +3 98 +3 103 +3 107 +3 414 +3 368 +3 498 +3 499 +4 224 +4 449 +4 419 +4 356 +4 72 +4 330 +4 173 +4 80 +4 242 +4 180 +4 24 +4 138 +5 33 +5 459 +5 484 +5 106 +5 299 +5 430 +5 16 +5 336 +5 50 +5 147 +5 406 +5 87 +5 91 +5 190 +6 7 +6 264 +6 137 +6 276 +6 149 +6 279 +6 153 +6 29 +6 415 +6 293 +6 166 +6 296 +6 42 +6 44 +6 48 +6 49 +6 438 +6 55 +6 314 +6 69 +6 454 +6 463 +6 84 +6 85 +6 470 +6 349 +6 94 +6 229 +6 233 +6 364 +6 117 +6 248 +6 380 +6 426 +7 129 +7 131 +7 470 +7 136 +7 471 +7 271 +7 402 +7 19 +7 404 +7 282 +7 286 +7 491 +7 35 +7 421 +7 166 +7 295 +7 427 +7 428 +7 301 +7 349 +7 436 +7 439 +7 56 +7 445 +7 446 +7 320 +7 123 +7 451 +7 69 +7 454 +7 78 +7 463 +7 212 +7 213 +7 342 +7 343 +7 474 +7 93 +7 483 +7 357 +7 102 +7 235 +7 237 +7 239 +7 371 +7 379 +8 112 +8 164 +8 10 +8 12 +8 398 +8 176 +8 120 +8 178 +8 373 +8 24 +8 124 +9 262 +10 11 +10 12 +10 47 +10 335 +10 124 +11 96 +11 289 +11 66 +11 283 +11 97 +11 361 +11 47 +11 432 +11 335 +11 215 +11 27 +11 124 +11 189 +12 112 +12 434 +12 47 +13 32 +13 130 +13 260 +13 37 +13 39 +13 202 +13 305 +13 461 +13 145 +13 275 +13 448 +13 54 +13 183 +13 344 +13 36 +14 387 +14 267 +14 155 +14 36 +14 165 +14 51 +14 308 +14 440 +14 441 +14 59 +14 444 +14 199 +14 328 +14 462 +14 81 +14 210 +14 214 +14 91 +14 99 +14 60 +14 106 +14 109 +14 110 +14 244 +15 193 +15 73 +15 400 +15 341 +15 27 +15 31 +16 33 +16 258 +16 353 +16 329 +16 204 +16 207 +16 336 +16 108 +16 50 +16 22 +16 119 +16 169 +16 381 +16 223 +17 321 +17 482 +17 197 +17 276 +17 149 +17 315 +17 94 +18 32 +18 461 +18 305 +18 498 +18 86 +18 25 +19 320 +19 421 +19 334 +19 314 +19 349 +19 446 +20 48 +20 28 +20 83 +20 44 +20 45 +21 288 +21 176 +21 347 +21 325 +21 392 +21 490 +21 366 +21 272 +21 209 +21 115 +21 182 +21 247 +21 184 +21 25 +21 217 +21 27 +21 86 +21 478 +22 353 +23 257 +23 100 +23 25 +24 134 +24 146 +24 302 +24 151 +24 290 +24 291 +24 43 +24 173 +24 46 +24 176 +24 180 +24 245 +24 453 +24 457 +24 208 +24 338 +24 363 +24 116 +24 373 +24 126 +25 32 +25 128 +25 257 +25 86 +26 161 +26 357 +26 193 +26 78 +26 341 +26 69 +27 361 +27 490 +27 333 +27 272 +27 312 +27 468 +27 41 +27 184 +27 429 +27 124 +27 341 +28 64 +28 354 +28 67 +28 135 +28 200 +28 42 +28 75 +28 45 +28 206 +28 48 +28 359 +28 53 +28 238 +29 229 +29 199 +29 296 +29 233 +29 42 +29 274 +29 117 +29 152 +29 155 +29 62 +30 386 +30 225 +30 76 +30 322 +30 367 +30 318 +31 200 +31 73 +31 333 +31 494 +31 45 +31 312 +32 257 +32 305 +32 101 +32 486 +32 39 +32 231 +32 79 +32 145 +32 275 +33 459 +33 336 +33 372 +33 119 +33 409 +34 283 +35 491 +35 243 +35 237 +35 326 +36 260 +36 266 +36 285 +36 292 +36 39 +36 424 +36 171 +36 181 +36 183 +36 440 +36 58 +36 323 +36 328 +36 81 +36 82 +36 87 +36 99 +36 228 +36 101 +36 111 +37 168 +37 202 +38 92 +39 448 +39 101 +39 486 +39 231 +39 145 +39 344 +40 345 +40 355 +40 179 +40 59 +40 125 +41 160 +41 226 +41 230 +41 104 +41 361 +41 492 +41 341 +41 215 +41 121 +41 189 +42 64 +42 67 +42 199 +42 359 +42 109 +42 206 +42 143 +42 48 +42 274 +42 117 +44 193 +44 69 +44 161 +44 73 +44 48 +44 83 +44 279 +45 200 +45 73 +45 83 +46 242 +46 485 +46 126 +46 173 +48 64 +48 206 +50 259 +50 261 +50 329 +50 299 +50 204 +50 430 +50 207 +50 405 +50 59 +50 159 +51 67 +51 101 +51 135 +51 210 +51 244 +51 214 +51 441 +51 249 +52 368 +52 217 +52 347 +52 86 +53 101 +53 327 +53 75 +53 238 +53 79 +53 249 +54 130 +55 84 +57 162 +57 331 +58 266 +58 82 +59 384 +59 259 +59 132 +59 267 +59 496 +59 355 +59 152 +59 91 +59 221 +59 159 +60 91 +60 165 +62 152 +62 155 +62 221 +63 456 +63 334 +63 303 +63 472 +63 314 +63 380 +65 154 +65 499 +65 374 +65 103 +66 96 +66 160 +66 215 +67 359 +67 135 +67 109 +67 143 +67 214 +69 161 +69 166 +69 239 +69 404 +69 85 +69 279 +69 153 +70 410 +70 467 +71 130 +72 138 +73 193 +74 241 +74 82 +74 188 +74 133 +74 469 +75 140 +75 79 +75 90 +75 350 +76 386 +77 120 +77 306 +77 452 +77 254 +77 311 +78 239 +78 357 +79 257 +79 100 +79 101 +80 138 +81 99 +81 171 +82 256 +82 133 +82 348 +82 205 +82 372 +82 181 +82 119 +82 409 +82 255 +82 316 +82 122 +82 469 +84 277 +84 264 +84 149 +84 380 +84 415 +85 153 +85 166 +86 247 +86 217 +87 181 +87 147 +87 285 +89 192 +89 200 +89 333 +89 468 +89 443 +89 222 +90 272 +90 100 +90 350 +91 165 +91 106 +91 430 +91 406 +91 159 +92 258 +93 136 +93 464 +93 493 +93 326 +94 482 +94 233 +94 114 +94 276 +95 107 +95 265 +95 186 +95 332 +96 289 +96 324 +96 102 +96 401 +97 283 +97 220 +97 434 +97 164 +99 101 +99 171 +99 244 +99 251 +100 272 +101 231 +101 466 +101 244 +101 249 +101 251 +102 141 +102 237 +102 369 +102 243 +102 212 +102 121 +102 340 +102 411 +104 121 +104 230 +105 396 +105 309 +106 147 +106 308 +106 406 +106 444 +106 190 +109 199 +109 143 +109 387 +109 214 +110 267 +112 164 +113 195 +114 233 +114 482 +114 315 +114 412 +116 465 +117 296 +118 163 +119 188 +119 409 +120 297 +120 363 +120 398 +120 254 +121 160 +121 230 +121 492 +121 369 +121 212 +122 256 +122 316 +124 176 +125 394 +125 497 +125 179 +125 351 +125 479 +126 465 +126 290 +126 339 +129 491 +130 397 +131 464 +131 136 +132 383 +133 273 +133 232 +133 205 +133 241 +133 316 +134 203 +135 249 +135 354 +136 464 +136 474 +137 276 +138 419 +138 414 +138 158 +139 169 +140 200 +140 222 +140 350 +144 334 +146 363 +147 308 +147 285 +148 175 +149 456 +149 407 +151 422 +152 488 +152 233 +152 221 +152 351 +154 187 +154 358 +154 158 +155 199 +155 267 +155 221 +155 437 +156 332 +156 318 +156 310 +157 436 +157 301 +158 396 +158 187 +158 414 +159 430 +160 215 +161 193 +163 220 +164 297 +164 398 +164 220 +166 427 +167 201 +167 442 +167 170 +167 383 +168 319 +169 280 +169 270 +169 198 +170 353 +170 403 +170 269 +170 207 +170 211 +173 242 +174 194 +174 195 +174 362 +174 177 +174 250 +174 219 +174 412 +175 460 +175 246 +175 447 +176 291 +176 294 +176 366 +176 178 +176 182 +177 219 +179 345 +179 394 +181 372 +182 184 +182 429 +182 294 +184 429 +185 358 +188 280 +188 469 +189 215 +192 272 +192 443 +192 468 +192 287 +193 400 +195 488 +195 233 +196 389 +196 317 +197 321 +198 280 +199 387 +200 333 +200 222 +201 284 +203 311 +207 353 +207 329 +207 269 +207 240 +212 230 +212 341 +212 439 +213 349 +214 462 +215 226 +216 388 +216 236 +217 347 +218 465 +219 450 +219 394 +219 250 +219 298 +219 479 +220 297 +220 283 +221 267 +221 437 +222 443 +222 350 +225 236 +227 472 +227 321 +227 315 +227 412 +227 390 +228 424 +228 260 +230 341 +232 280 +232 241 +233 480 +233 412 +234 243 +234 340 +238 249 +238 354 +240 353 +241 280 +242 356 +243 385 +243 326 +244 251 +246 382 +249 354 +249 327 +249 466 +250 488 +250 479 +250 351 +252 376 +253 384 +253 269 +254 363 +254 302 +254 311 +258 353 +258 317 +258 477 +259 384 +259 405 +259 269 +260 424 +262 352 +262 346 +263 392 +263 300 +263 325 +264 438 +267 437 +268 413 +269 384 +269 383 +270 381 +270 431 +272 443 +272 350 +272 287 +273 316 +276 377 +278 390 +280 417 +281 376 +282 391 +283 289 +283 432 +285 308 +285 292 +288 325 +288 478 +289 401 +289 413 +291 325 +291 478 +292 328 +292 308 +297 398 +297 495 +302 453 +302 311 +304 493 +307 388 +308 328 +309 396 +311 453 +312 333 +312 494 +313 375 +314 487 +314 364 +314 380 +315 482 +315 412 +319 346 +321 456 +326 491 +326 435 +327 466 +328 440 +331 472 +332 408 +332 489 +333 468 +334 446 +337 380 +338 465 +341 400 +343 436 +345 394 +345 399 +347 392 +347 368 +349 364 +349 470 +351 496 +351 479 +353 393 +358 360 +358 396 +365 375 +366 478 +368 392 +372 409 +374 467 +378 411 +379 436 +380 456 +380 415 +382 433 +383 384 +394 481 +394 479 +396 449 +416 420 +416 422 +417 447 +418 446 +450 481 +456 475 +458 471 +463 470 +472 475 +496 497 + Added: SwiftApps/SciColSim/openmp_test/openmp-optimizer =================================================================== (Binary files differ) Property changes on: SwiftApps/SciColSim/openmp_test/openmp-optimizer ___________________________________________________________________ Added: svn:executable + * Added: svn:mime-type + application/octet-stream Added: SwiftApps/SciColSim/openmp_test/openmp-verify.pbs =================================================================== --- SwiftApps/SciColSim/openmp_test/openmp-verify.pbs (rev 0) +++ SwiftApps/SciColSim/openmp_test/openmp-verify.pbs 2012-03-23 21:22:38 UTC (rev 5722) @@ -0,0 +1,22 @@ +#!/bin/bash + +#PBS -N openmp-verify +#PBS -l walltime=0:01:00 +#PBS -l mppwidth=1,mppnppn=1,mppdepth=24 +#PBS -q development +#PBS -j oe + +cd $PBS_O_WORKDIR + +LUSTREDIR=/lustre/beagle/$USER/Swift/SciColSim/openmp_test/${PBS_JOBID} +echo $LUSTREDIR +mkdir -p $LUSTREDIR + +cd $LUSTREDIR + +cp ../movie_graph.txt . +cp ../openmp-optimizer . +cp ../run.sh . + + +OMP_NUM_THREADS=24 aprun -n 1 -d 24 ./run.sh Added: SwiftApps/SciColSim/openmp_test/run.sh =================================================================== --- SwiftApps/SciColSim/openmp_test/run.sh (rev 0) +++ SwiftApps/SciColSim/openmp_test/run.sh 2012-03-23 21:22:38 UTC (rev 5722) @@ -0,0 +1,8 @@ +#!/bin/bash + +OMP_NUM_THREADS=24 openmp-optimizer 1.3361867203970093 1.0553507574550953 0.29712814586892483 -0.49955400395753874 4.186059352113561 58 40000 20 2400 2 1 2 0.01 100 0.3 2.3 0 0 0 0 0 m 24 0 & +pid=`ps -u jonmon | grep openmp-optimize | awk '{print $1}'` + +top -b -n 10 -p $pid > top.data + + Property changes on: SwiftApps/SciColSim/openmp_test/run.sh ___________________________________________________________________ Added: svn:executable + * From jonmon at ci.uchicago.edu Fri Mar 23 16:24:34 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Fri, 23 Mar 2012 16:24:34 -0500 (CDT) Subject: [Swift-commit] r5723 - SwiftApps/SciColSim/openmp_test Message-ID: <20120323212434.936C89CCF7@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-23 16:24:34 -0500 (Fri, 23 Mar 2012) New Revision: 5723 Modified: SwiftApps/SciColSim/openmp_test/run.sh Log: use environment variable $USER Modified: SwiftApps/SciColSim/openmp_test/run.sh =================================================================== --- SwiftApps/SciColSim/openmp_test/run.sh 2012-03-23 21:22:38 UTC (rev 5722) +++ SwiftApps/SciColSim/openmp_test/run.sh 2012-03-23 21:24:34 UTC (rev 5723) @@ -1,7 +1,7 @@ #!/bin/bash OMP_NUM_THREADS=24 openmp-optimizer 1.3361867203970093 1.0553507574550953 0.29712814586892483 -0.49955400395753874 4.186059352113561 58 40000 20 2400 2 1 2 0.01 100 0.3 2.3 0 0 0 0 0 m 24 0 & -pid=`ps -u jonmon | grep openmp-optimize | awk '{print $1}'` +pid=`ps -u $USER | grep openmp-optimize | awk '{print $1}'` top -b -n 10 -p $pid > top.data From swift at ci.uchicago.edu Sat Mar 24 04:00:06 2012 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Sat, 24 Mar 2012 04:00:06 -0500 (CDT) Subject: [Swift-commit] Cog update Message-ID: <20120324090006.A061F8D0009B@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3365 | davidkelly999 | 2012-03-24 03:58:40 -0500 (Sat, 24 Mar 2012) | 5 lines Fixes to get Swift loading into Eclipse without errors Many modules had a classpath setup pointing to non-existent versions of log4j Others modules were relying on Eclipse variables for the location of log4j I think this is better - give each module a real jar it can use (but use relative symlinks to conserve space) ------------------------------------------------------------------------ Index: modules/provider-mpichg2/.classpath =================================================================== --- modules/provider-mpichg2/.classpath (revision 3364) +++ modules/provider-mpichg2/.classpath (working copy) @@ -14,6 +14,6 @@ - + Index: modules/provider-mpichg2/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-mpichg2/lib/log4j-1.2.16.jar (revision 0) +++ modules/provider-mpichg2/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/provider-mpichg2/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/provider-localscheduler/.classpath =================================================================== --- modules/provider-localscheduler/.classpath (revision 3364) +++ modules/provider-localscheduler/.classpath (working copy) @@ -11,6 +11,6 @@ - + Index: modules/provider-localscheduler/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-localscheduler/lib/log4j-1.2.16.jar (revision 0) +++ modules/provider-localscheduler/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/provider-localscheduler/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/provider-local/.classpath =================================================================== --- modules/provider-local/.classpath (revision 3364) +++ modules/provider-local/.classpath (working copy) @@ -10,6 +10,6 @@ - + Index: modules/provider-local/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-local/lib/log4j-1.2.16.jar (revision 0) +++ modules/provider-local/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/provider-local/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/provider-gt2/.classpath =================================================================== --- modules/provider-gt2/.classpath (revision 3364) +++ modules/provider-gt2/.classpath (working copy) @@ -11,6 +11,6 @@ - + Index: modules/provider-gt2/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-gt2/lib/log4j-1.2.16.jar (revision 0) +++ modules/provider-gt2/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/provider-gt2/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/provider-gt4_0_0/.classpath =================================================================== --- modules/provider-gt4_0_0/.classpath (revision 3364) +++ modules/provider-gt4_0_0/.classpath (working copy) @@ -46,6 +46,6 @@ - + Index: modules/provider-gt4_0_0/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-gt4_0_0/lib/log4j-1.2.16.jar (revision 0) +++ modules/provider-gt4_0_0/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/provider-gt4_0_0/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/provider-ssh/.classpath =================================================================== --- modules/provider-ssh/.classpath (revision 3364) +++ modules/provider-ssh/.classpath (working copy) @@ -12,6 +12,7 @@ - + + Index: modules/provider-ssh/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-ssh/lib/log4j-1.2.16.jar (revision 0) +++ modules/provider-ssh/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/provider-ssh/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/provider-coaster/.classpath =================================================================== --- modules/provider-coaster/.classpath (revision 3364) +++ modules/provider-coaster/.classpath (working copy) @@ -13,6 +13,6 @@ - + Index: modules/provider-coaster/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-coaster/lib/log4j-1.2.16.jar (revision 0) +++ modules/provider-coaster/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/provider-coaster/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/abstraction-common/.classpath =================================================================== --- modules/abstraction-common/.classpath (revision 3364) +++ modules/abstraction-common/.classpath (working copy) @@ -4,8 +4,8 @@ + - Index: modules/abstraction-common/lib/log4j-1.2.16.jar =================================================================== --- modules/abstraction-common/lib/log4j-1.2.16.jar (revision 0) +++ modules/abstraction-common/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/abstraction-common/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/provider-dcache/.classpath =================================================================== --- modules/provider-dcache/.classpath (revision 3364) +++ modules/provider-dcache/.classpath (working copy) @@ -4,6 +4,6 @@ - + Index: modules/provider-dcache/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-dcache/lib/log4j-1.2.16.jar (revision 0) +++ modules/provider-dcache/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/provider-dcache/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/setup/.classpath =================================================================== --- modules/setup/.classpath (revision 3364) +++ modules/setup/.classpath (working copy) @@ -7,6 +7,6 @@ - + Index: modules/setup/lib/log4j-1.2.16.jar =================================================================== --- modules/setup/lib/log4j-1.2.16.jar (revision 0) +++ modules/setup/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/setup/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/provider-webdav/.classpath =================================================================== --- modules/provider-webdav/.classpath (revision 3364) +++ modules/provider-webdav/.classpath (working copy) @@ -11,6 +11,6 @@ - + Index: modules/provider-webdav/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-webdav/lib/log4j-1.2.16.jar (revision 0) +++ modules/provider-webdav/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/provider-webdav/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/provider-slocal/.classpath =================================================================== --- modules/provider-slocal/.classpath (revision 3364) +++ modules/provider-slocal/.classpath (working copy) @@ -12,6 +12,6 @@ - + Index: modules/provider-slocal/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-slocal/lib/log4j-1.2.16.jar (revision 0) +++ modules/provider-slocal/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/provider-slocal/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/certrequest/.classpath =================================================================== --- modules/certrequest/.classpath (revision 3364) +++ modules/certrequest/.classpath (working copy) @@ -8,6 +8,6 @@ - + Index: modules/certrequest/lib/log4j-1.2.16.jar =================================================================== --- modules/certrequest/lib/log4j-1.2.16.jar (revision 0) +++ modules/certrequest/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/certrequest/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/karajan/.classpath =================================================================== --- modules/karajan/.classpath (revision 3364) +++ modules/karajan/.classpath (working copy) @@ -11,6 +11,6 @@ - + Index: modules/karajan/lib/log4j-1.2.16.jar =================================================================== --- modules/karajan/lib/log4j-1.2.16.jar (revision 0) +++ modules/karajan/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/karajan/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/grapheditor/.classpath =================================================================== --- modules/grapheditor/.classpath (revision 3364) +++ modules/grapheditor/.classpath (working copy) @@ -6,9 +6,8 @@ - + - Index: modules/grapheditor/lib/log4j-1.2.16.jar =================================================================== --- modules/grapheditor/lib/log4j-1.2.16.jar (revision 0) +++ modules/grapheditor/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/grapheditor/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/all/.classpath =================================================================== --- modules/all/.classpath (revision 3364) +++ modules/all/.classpath (working copy) @@ -2,9 +2,10 @@ + - + Index: modules/all/lib/log4j-1.2.16.jar =================================================================== --- modules/all/lib/log4j-1.2.16.jar (revision 0) +++ modules/all/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/all/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/provider-condor/.classpath =================================================================== --- modules/provider-condor/.classpath (revision 3364) +++ modules/provider-condor/.classpath (working copy) @@ -9,6 +9,6 @@ - + Index: modules/provider-condor/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-condor/lib/log4j-1.2.16.jar (revision 0) +++ modules/provider-condor/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/provider-condor/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/util/.classpath =================================================================== --- modules/util/.classpath (revision 3364) +++ modules/util/.classpath (working copy) @@ -4,6 +4,6 @@ - + Index: modules/util/lib/log4j-1.2.16.jar =================================================================== --- modules/util/lib/log4j-1.2.16.jar (revision 0) +++ modules/util/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/util/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Index: modules/examples/.classpath =================================================================== --- modules/examples/.classpath (revision 3364) +++ modules/examples/.classpath (working copy) @@ -8,7 +8,6 @@ - - + Index: modules/examples/lib/log4j-1.2.16.jar =================================================================== --- modules/examples/lib/log4j-1.2.16.jar (revision 0) +++ modules/examples/lib/log4j-1.2.16.jar (revision 3365) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: modules/examples/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * From davidk at ci.uchicago.edu Sat Mar 24 04:05:59 2012 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Sat, 24 Mar 2012 04:05:59 -0500 (CDT) Subject: [Swift-commit] r5724 - in trunk: . bin lib src/org/griphyn/vdl/karajan src/org/griphyn/vdl/mapping src/org/griphyn/vdl/mapping/file Message-ID: <20120324090559.A861B9CCC5@svn.ci.uchicago.edu> Author: davidk Date: 2012-03-24 04:05:58 -0500 (Sat, 24 Mar 2012) New Revision: 5724 Added: trunk/lib/log4j-1.2.16.jar Modified: trunk/.classpath trunk/bin/swift trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java trunk/src/org/griphyn/vdl/mapping/AbsFile.java trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java trunk/src/org/griphyn/vdl/mapping/PathElementComparator.java trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java Log: Some fixes to get Swift loading in Eclipse with no errors Mostly just involves some issues with @override Updates to classpath bin/swift gives warnings on modern Linux systems where /bin/sh is not bash Explicitly use bash Increase the default max heap size from 256M to 1024M Modified: trunk/.classpath =================================================================== --- trunk/.classpath 2012-03-23 21:24:34 UTC (rev 5723) +++ trunk/.classpath 2012-03-24 09:05:58 UTC (rev 5724) @@ -19,7 +19,7 @@ - + Modified: trunk/bin/swift =================================================================== --- trunk/bin/swift 2012-03-23 21:24:34 UTC (rev 5723) +++ trunk/bin/swift 2012-03-24 09:05:58 UTC (rev 5724) @@ -10,7 +10,7 @@ CYGWIN= CPDELIM=":" -HEAPMAX=256M +HEAPMAX=1024M if echo `uname` | grep -i "cygwin"; then CYGWIN="yes" Added: trunk/lib/log4j-1.2.16.jar =================================================================== --- trunk/lib/log4j-1.2.16.jar (rev 0) +++ trunk/lib/log4j-1.2.16.jar 2012-03-24 09:05:58 UTC (rev 5724) @@ -0,0 +1 @@ +link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Property changes on: trunk/lib/log4j-1.2.16.jar ___________________________________________________________________ Added: svn:special + * Modified: trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2012-03-23 21:24:34 UTC (rev 5723) +++ trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2012-03-24 09:05:58 UTC (rev 5724) @@ -470,8 +470,7 @@ } } - @Override - public void resourceUpdated(Service service, String name, String value) { + public void resourceUpdated(Service service, String name, String value) { if (logger.isInfoEnabled()) { logger.info(service + " resource updated: " + name + " -> " + value); } Modified: trunk/src/org/griphyn/vdl/mapping/AbsFile.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/AbsFile.java 2012-03-23 21:24:34 UTC (rev 5723) +++ trunk/src/org/griphyn/vdl/mapping/AbsFile.java 2012-03-24 09:05:58 UTC (rev 5724) @@ -182,7 +182,6 @@ return getURIAsString(); } - @Override public void clean() { try { getFileResource().deleteFile(path); Modified: trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java 2012-03-23 21:24:34 UTC (rev 5723) +++ trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java 2012-03-24 09:05:58 UTC (rev 5724) @@ -77,12 +77,10 @@ throw new UnsupportedOperationException(); } - @Override public void clean(Path path) { // no cleaning by default } - @Override public boolean isPersistent(Path path) { // persistent unless explicitly overridden return true; Modified: trunk/src/org/griphyn/vdl/mapping/PathElementComparator.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/PathElementComparator.java 2012-03-23 21:24:34 UTC (rev 5723) +++ trunk/src/org/griphyn/vdl/mapping/PathElementComparator.java 2012-03-24 09:05:58 UTC (rev 5724) @@ -22,7 +22,6 @@ public class PathElementComparator implements Comparator> { @SuppressWarnings("unchecked") - @Override public int compare(Comparable p1, Comparable p2) { Comparable e1 = (Comparable) p1; Comparable e2 = (Comparable) p2; Modified: trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java 2012-03-23 21:24:34 UTC (rev 5723) +++ trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java 2012-03-24 09:05:58 UTC (rev 5724) @@ -70,7 +70,6 @@ return !PARAM_TEMP.getBooleanValue(this); } - @Override public PhysicalFormat map(Path path) { if (remap == null) { if (map == null) { @@ -83,12 +82,10 @@ } } - @Override public Collection existing() { return Collections.singletonList(Path.EMPTY_PATH); } - @Override public boolean isStatic() { return PARAM_STATIC.getBooleanValue(this); } From wilde at ci.uchicago.edu Mon Mar 26 12:03:12 2012 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Mon, 26 Mar 2012 12:03:12 -0500 (CDT) Subject: [Swift-commit] r5725 - SwiftApps/SciColSim Message-ID: <20120326170312.14E2C9CCF7@svn.ci.uchicago.edu> Author: wilde Date: 2012-03-26 12:03:11 -0500 (Mon, 26 Mar 2012) New Revision: 5725 Modified: SwiftApps/SciColSim/optimizer.cpp Log: Changed max_loss_valuue() to use a switch. Corrected bad check for t.i. 1008. Modified: SwiftApps/SciColSim/optimizer.cpp =================================================================== --- SwiftApps/SciColSim/optimizer.cpp 2012-03-24 09:05:58 UTC (rev 5724) +++ SwiftApps/SciColSim/optimizer.cpp 2012-03-26 17:03:11 UTC (rev 5725) @@ -1023,7 +1023,7 @@ } //============================================== - double max_loss_value(int tn){ + double Omax_loss_value(int tn){ if( tn == 58 ) return 128.286; else if( tn == 108 ) return 131.866; @@ -1044,11 +1044,37 @@ else if( tn == 858 ) return 279.201; else if( tn == 908 ) return 320.112; else if( tn == 958 ) return 394.774; - else if( tn == 1000 ) return 1052.38; // Huge number here. Why? + else if( tn == 1008 ) return 1052.38; // Huge number here. Why? else return 10000.; /* Pray that this does not occur. Ask Andrey what to do */ } + double max_loss_value(int tn){ + switch(tn) { + case 58: return 128.286; + case 108: return 131.866; + case 158: return 135.551; + case 208: return 139.694; + case 258: return 144.163; + case 308: return 148.967; + case 358: return 154.201; + case 408: return 159.962; + case 458: return 166.441; + case 508: return 173.655; + case 558: return 181.921; + case 608: return 191.246; + case 658: return 202.150; + case 708: return 215.197; + case 758: return 202.150; // Verify with Andrey. Same as TargetNovelty of 658 + case 808: return 251.698; + case 858: return 279.201; + case 908: return 320.112; + case 958: return 394.774; + case 1008: return 1052.38; // Huge number here. Why? + default: return 10000.; /* Pray that this does not occur. Ask Andrey what to do */ + } + } + //============================================== int get_reruns(void){ return N_repeats; From wilde at ci.uchicago.edu Mon Mar 26 12:06:06 2012 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Mon, 26 Mar 2012 12:06:06 -0500 (CDT) Subject: [Swift-commit] r5726 - SwiftApps/SciColSim Message-ID: <20120326170606.AFC589CCF7@svn.ci.uchicago.edu> Author: wilde Date: 2012-03-26 12:06:06 -0500 (Mon, 26 Mar 2012) New Revision: 5726 Modified: SwiftApps/SciColSim/testopt.py Log: Adjusted testopt.py, mw2 param set as well as command invocation; changed to 0 0 0 0 0 for param vaues and 0 0 0 0 0 for fixed params. Modified: SwiftApps/SciColSim/testopt.py =================================================================== --- SwiftApps/SciColSim/testopt.py 2012-03-26 17:03:11 UTC (rev 5725) +++ SwiftApps/SciColSim/testopt.py 2012-03-26 17:06:06 UTC (rev 5726) @@ -10,9 +10,9 @@ app = "./openmp-optimizer"; # For Mac or Linux: Using OpenMP (Default) #paramset="default" -#paramset="mw2" +paramset="mw2" #paramset="VERYFASTTEST" -paramset="VERYFASTTEST" +#paramset="VERYFASTTEST" print("Running with parameter set " + paramset + "\n\n"); @@ -116,9 +116,9 @@ elif paramset == "mw2": # Timing TEST PARAMETERS by Mike - startTarget = 900 - endTarget = 1059 - incrTarget = 1 + startTarget = 58 + endTarget = 1009 + incrTarget = 50 optimizerRepeats = 1 evolveReruns = 4 annealingSteps = 1 @@ -141,14 +141,15 @@ for target in range(startTarget,endTarget,incrTarget): for i in range(optimizerRepeats): args = "rm -f bestdb.txt; " + \ - openmp + " " + app + " 0 0 4 50 -1 " + str(target) + " 40000 20 " + str(evolveReruns) + \ - " 2 1 2. 0.01 " + str(annealingSteps) + " 0.3 2.3 1 1 0 0 0 " + operation + " " + NWorkers + " " + seed + \ - " >& out.T"+str(target)+".i"+str(i) + "; mv bestdb.txt best.T" + str(target) + ".i" + str(i) + openmp + " " + app + " 0 0 0 0 0 " + str(target) + " 40000 20 " + str(evolveReruns) + \ + " 2 1 2. 0.01 " + str(annealingSteps) + " 0.3 2.3 0 0 0 0 0 " + operation + " " + NWorkers + " " + seed + \ + " >& out.T"+str(target)+".i"+str(i) + "; #mv bestdb.txt best.T" + str(target) + ".i" + str(i) print("\n**** Calling optimizer: "+args+"\n") os.system(args); print sys.argv[0] + " Done!" +# openmp + " " + app + " 0 0 4 50 -1 " + str(target) + " 40000 20 " + str(evolveReruns) + \ ### Argument worksheet: From jonmon at ci.uchicago.edu Mon Mar 26 12:07:08 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Mon, 26 Mar 2012 12:07:08 -0500 (CDT) Subject: [Swift-commit] r5727 - SwiftApps/SciColSim Message-ID: <20120326170708.881AB9CCF7@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-26 12:07:08 -0500 (Mon, 26 Mar 2012) New Revision: 5727 Modified: SwiftApps/SciColSim/optimizer.cpp Log: Changed ALOT in evolve_to_target_and_save to 100000000000. Modified: SwiftApps/SciColSim/optimizer.cpp =================================================================== --- SwiftApps/SciColSim/optimizer.cpp 2012-03-26 17:06:06 UTC (rev 5726) +++ SwiftApps/SciColSim/optimizer.cpp 2012-03-26 17:07:08 UTC (rev 5727) @@ -989,7 +989,7 @@ //================================================================================== void evolve_to_target_and_save(int istart, int iend, double* storage, int* counters){ - double ALOT=10000.; + double ALOT=100000000000.; double ratio = 0.0; int check = 0; @@ -1050,7 +1050,7 @@ double max_loss_value(int tn){ switch(tn) { - case 58: return 128.286; + case 58: return 128.286; case 108: return 131.866; case 158: return 135.551; case 208: return 139.694; From wilde at ci.uchicago.edu Mon Mar 26 18:10:25 2012 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Mon, 26 Mar 2012 18:10:25 -0500 (CDT) Subject: [Swift-commit] r5728 - SwiftApps/SciColSim Message-ID: <20120326231025.DA0ED9CCF7@svn.ci.uchicago.edu> Author: wilde Date: 2012-03-26 18:10:25 -0500 (Mon, 26 Mar 2012) New Revision: 5728 Modified: SwiftApps/SciColSim/Makefile SwiftApps/SciColSim/optimizer.cpp SwiftApps/SciColSim/testopt.py Log: Changed optimizer.cpp to run all reruns across all workers. Added -O and static to makefile. Adjusted testopt.py to test optimizer.cpp change. Modified: SwiftApps/SciColSim/Makefile =================================================================== --- SwiftApps/SciColSim/Makefile 2012-03-26 17:07:08 UTC (rev 5727) +++ SwiftApps/SciColSim/Makefile 2012-03-26 23:10:25 UTC (rev 5728) @@ -5,7 +5,7 @@ all: openmp-optimizer openmp-optimizer: optimizer.cpp - g++ -DP_OPENMP -fopenmp -I boost_1_47_0 -o openmp-optimizer optimizer.cpp + g++ -DP_OPENMP -static -O -fopenmp -I boost_1_47_0 -o openmp-optimizer optimizer.cpp clean: @rm -rvf openmp-optimizer Modified: SwiftApps/SciColSim/optimizer.cpp =================================================================== --- SwiftApps/SciColSim/optimizer.cpp 2012-03-26 17:07:08 UTC (rev 5727) +++ SwiftApps/SciColSim/optimizer.cpp 2012-03-26 23:10:25 UTC (rev 5728) @@ -11,6 +11,8 @@ #define MAXNworkers 24 int Nworkers=MAXNworkers; +#define NUniverse 240 + // Add operation code to enable existing code to be used at lower level from Swift scripts: char operation = 'n'; // n: normal; m: do one multi_loss (with n_reruns). @@ -1008,7 +1010,6 @@ } ratio = current_loss/current_novelty; } - if( check ){ storage[k]=ALOT; } @@ -1346,7 +1347,12 @@ double elapsedTime; gettimeofday(&startTime, NULL); - for(int i=0; i NUniverse ) { + std::cerr << "Error: Number of reruns=" << N << " is greater than max NUniverse=" << NUniverse << "\n"; + exit(1); + } + + for(int i=0; iset_parameter(params[j],j); } @@ -1371,30 +1377,19 @@ #else int i; - // Print debug in serial code to get coherent trace output + // Execute rerun loop in parallel - for(i=0; ievolve_to_target_and_save(i*step, min((i+1)*step,N), Results, Counters); + for(i=0; ievolve_to_target_and_save(i, i+1, Results, Counters); } #endif for (int i=0; i Author: jonmon Date: 2012-03-27 11:27:09 -0500 (Tue, 27 Mar 2012) New Revision: 5729 Modified: SwiftApps/SciColSim/swiftopt.sh Log: changed to a more accurate calculation of the number of app calls necessary. still now correct though Modified: SwiftApps/SciColSim/swiftopt.sh =================================================================== --- SwiftApps/SciColSim/swiftopt.sh 2012-03-26 23:10:25 UTC (rev 5728) +++ SwiftApps/SciColSim/swiftopt.sh 2012-03-27 16:27:09 UTC (rev 5729) @@ -35,10 +35,10 @@ # Default settings execsite=local paramfile=Fast01 -ram=2000M +ram=2048M dryrun= escapecode=$(printf '\033') -fixed_params=2 # Currently CONSTANT, will want to have this vary +variable_params=5 # Currently CONSTANT, will want to have this vary # Process command line arguments while [ $# -gt 0 ]; do @@ -103,7 +103,7 @@ echo # Echo runtime estimates -total_jobs=`python -c "from math import ceil; print int(ceil(($max_target_innovation.00 - $min_target_innovation.00)/$target_innovation_increment.00) * $annealing_repeats * $fixed_params * $annealing_cycles * ($evolve_reruns/$reruns_per_opt_invocation) * $nworkers)"` +total_jobs=`python -c "from math import ceil; print int(ceil(($max_target_innovation.00 - $min_target_innovation.00)/$target_innovation_increment.00) * $annealing_repeats * $variable_params * $annealing_cycles * $evolve_reruns/$reruns_per_opt_invocation"` echo Total jobs = $total_jobs cd $rundir @@ -128,7 +128,7 @@ echo "Run dir=$rundir" >> ABOUT echo "Work dir=$WORK" >> ABOUT echo "Total jobs=$total_jobs" >> ABOUT -echo "Run Command: SWIFT_HEAP_MAX=$ram SWIFT_LIB=.. $swift >> swift.out 2>&1 -tc.file tc.data -sites.file $1 -config cf annealing.swift -e33="$escapecode" -minrange=$min_target_innovation -maxrange=$max_target_innovation-rangeinc=$target_innovation_increment -nreps=$annealing_repeats -annealingcycles=$annealing_cycles -evoreruns=$evolve_reruns -alphai=$alpha_i -alpham=$alpha_m -beta=$beta -gamma=$gamma -delta=$delta -nworkers=$nworkers -rerunsperapp=$reruns_per_opt_invocation " >> ABOUT +echo "Run Command: SWIFT_HEAP_MAX=$ram SWIFT_LIB=.. $swift >> swift.out 2>&1 -tc.file tc.data -sites.file $execsite.xml -config cf annealing.swift -minrange=$min_target_innovation -maxrange=$max_target_innovation-rangeinc=$target_innovation_increment -nreps=$annealing_repeats -annealingcycles=$annealing_cycles -evoreruns=$evolve_reruns -alphai=$alpha_i -alpham=$alpha_m -beta=$beta -gamma=$gamma -delta=$delta -nworkers=$nworkers -rerunsperapp=$reruns_per_opt_invocation -e33="$escapecode"" >> ABOUT if [ _$dryrun != _ ]; then exit 0 @@ -141,6 +141,7 @@ runswift "$execsite.xml" fi +# Run the convertbest.sh script on the generated txt files from within the rundir ../bin/convertbest.sh *.txt exit From jonmon at ci.uchicago.edu Tue Mar 27 15:22:54 2012 From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu) Date: Tue, 27 Mar 2012 15:22:54 -0500 (CDT) Subject: [Swift-commit] r5730 - SwiftApps/SciColSim/docs Message-ID: <20120327202254.758E69CCA5@svn.ci.uchicago.edu> Author: jonmon Date: 2012-03-27 15:22:54 -0500 (Tue, 27 Mar 2012) New Revision: 5730 Added: SwiftApps/SciColSim/docs/single_rerun_all_0.tdata Log: added timing data gather by doing a single rerun with all parameters starting at 0 Added: SwiftApps/SciColSim/docs/single_rerun_all_0.tdata =================================================================== --- SwiftApps/SciColSim/docs/single_rerun_all_0.tdata (rev 0) +++ SwiftApps/SciColSim/docs/single_rerun_all_0.tdata 2012-03-27 20:22:54 UTC (rev 5730) @@ -0,0 +1,20 @@ +out.T58.i0:multi_loss(N=1, target=58) elapsed time: 0.091136 seconds 0.00151893 minutes +out.T108.i0:multi_loss(N=1, target=108) elapsed time: 0.134665 seconds 0.00224442 minutes +out.T158.i0:multi_loss(N=1, target=158) elapsed time: 0.201696 seconds 0.0033616 minutes +out.T208.i0:multi_loss(N=1, target=208) elapsed time: 0.279621 seconds 0.00466035 minutes +out.T258.i0:multi_loss(N=1, target=258) elapsed time: 0.381066 seconds 0.0063511 minutes +out.T308.i0:multi_loss(N=1, target=308) elapsed time: 0.475485 seconds 0.00792475 minutes +out.T358.i0:multi_loss(N=1, target=358) elapsed time: 0.616931 seconds 0.0102822 minutes +out.T408.i0:multi_loss(N=1, target=408) elapsed time: 0.869909 seconds 0.0144985 minutes +out.T458.i0:multi_loss(N=1, target=458) elapsed time: 1.08077 seconds 0.0180128 minutes +out.T508.i0:multi_loss(N=1, target=508) elapsed time: 1.32565 seconds 0.0220941 minutes +out.T558.i0:multi_loss(N=1, target=558) elapsed time: 1.7256 seconds 0.02876 minutes +out.T608.i0:multi_loss(N=1, target=608) elapsed time: 2.14264 seconds 0.0357107 minutes +out.T658.i0:multi_loss(N=1, target=658) elapsed time: 2.5721 seconds 0.0428683 minutes +out.T708.i0:multi_loss(N=1, target=708) elapsed time: 3.23249 seconds 0.0538748 minutes +out.T758.i0:multi_loss(N=1, target=758) elapsed time: 4.31466 seconds 0.0719109 minutes +out.T808.i0:multi_loss(N=1, target=808) elapsed time: 5.22954 seconds 0.0871589 minutes +out.T858.i0:multi_loss(N=1, target=858) elapsed time: 6.60829 seconds 0.110138 minutes +out.T908.i0:multi_loss(N=1, target=908) elapsed time: 8.0576 seconds 0.134293 minutes +out.T958.i0:multi_loss(N=1, target=958) elapsed time: 13.3817 seconds 0.223029 minutes +out.T1008.i0:multi_loss(N=1, target=1008) elapsed time: 36.0038 seconds 0.600064 minutes From ketan at ci.uchicago.edu Thu Mar 29 22:21:10 2012 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 29 Mar 2012 22:21:10 -0500 (CDT) Subject: [Swift-commit] r5731 - SwiftApps/SciColSim/docs Message-ID: <20120330032110.B46E89CC99@svn.ci.uchicago.edu> Author: ketan Date: 2012-03-29 22:21:10 -0500 (Thu, 29 Mar 2012) New Revision: 5731 Added: SwiftApps/SciColSim/docs/plotit.gp Removed: SwiftApps/SciColSim/docs/plotit Log: small change in plotting script Deleted: SwiftApps/SciColSim/docs/plotit =================================================================== --- SwiftApps/SciColSim/docs/plotit 2012-03-27 20:22:54 UTC (rev 5730) +++ SwiftApps/SciColSim/docs/plotit 2012-03-30 03:21:10 UTC (rev 5731) @@ -1,44 +0,0 @@ -#To be run as a gnuplot script: -# $ gnuplot plotit -set terminal png enhanced -#set term postscript eps enhanced -#set terminal svg enhanced size 1000 1000 -#set style line 1 linecolor rgb "blue" -set output "activeplot.png" -set nokey -set xlabel "Time in sec" -set ylabel "number of active jobs" -set title "Active jobs" -plot "plot_active.txt" using 1:2 with line - -set output "cumulativeplot-openmp.png" -set xlabel "Time in seconds" -set ylabel "number of completed jobs" -set title "Cumulative SciColSim-openMP jobs" -plot "plot_cumulative.txt" using 1:($2*24) with lines - -set output "cumulativeplot.png" -set xlabel "Time in seconds" -set ylabel "number of completed jobs" -set title "Cumulative jobs" -plot "plot_cumulative.txt" using 1:2 with lines - -set output "scs.png" -set xlabel "Evolution" -set ylabel "Value of T" -set title "SciColSim evolution Results" -plot "T.data" using 1 with lines - -set output "scs_loss.png" -set title "SciColSim evolution loss Results" -set xlabel "Evolution" -set ylabel "Value of loss(AR)" -plot "anneal.data" using 1 with lines - -set output "multiloss.png" -set title "SciColSim evolution loss Results" -set key auto -set yrange [0:200] -set xlabel "Evolution" -set ylabel "loss" -plot "multiloss.txt" using 3 with lines title "multiloss mean val", "multiloss.txt" using ($3+$4) with lines title "+stddev", "multiloss.txt" using ($3-$4) with lines title "-stddev" Copied: SwiftApps/SciColSim/docs/plotit.gp (from rev 5730, SwiftApps/SciColSim/docs/plotit) =================================================================== --- SwiftApps/SciColSim/docs/plotit.gp (rev 0) +++ SwiftApps/SciColSim/docs/plotit.gp 2012-03-30 03:21:10 UTC (rev 5731) @@ -0,0 +1,48 @@ +#To be run as a gnuplot script as follows: + +# $ gnuplot plotit + +set terminal png enhanced +#set term postscript eps enhanced +#set terminal svg enhanced size 1000 1000 +#set style line 1 linecolor rgb "blue" +set output "activeplot.png" +set nokey +set xlabel "Time in sec" +set ylabel "number of active jobs" +set title "Active jobs" +plot "plot_active.txt" using 1:2 with line + +#multiplies the active jobs by 24 to take into account the number of openmp +#procs running on Beagle +set output "cumulativeplot-openmp.png" +set xlabel "Time in seconds" +set ylabel "number of completed jobs" +set title "Cumulative SciColSim-openMP jobs" +plot "plot_cumulative.txt" using 1:($2*24) with lines + +set output "cumulativeplot.png" +set xlabel "Time in seconds" +set ylabel "number of completed jobs" +set title "Cumulative jobs" +plot "plot_cumulative.txt" using 1:2 with lines + +set output "scs.png" +set xlabel "Evolution" +set ylabel "Value of T" +set title "SciColSim evolution Results" +plot "T.data" using 1 with lines + +set output "scs_loss.png" +set title "SciColSim evolution loss Results" +set xlabel "Evolution" +set ylabel "Value of loss(AR)" +plot "anneal.data" using 1 with lines + +set output "multiloss.png" +set title "SciColSim evolution loss Results" +set key auto +set yrange [0:200] +set xlabel "Evolution" +set ylabel "loss" +plot "multiloss.txt" using 3 with lines title "multiloss mean val", "multiloss.txt" using ($3+$4) with lines title "+stddev", "multiloss.txt" using ($3-$4) with lines title "-stddev" From wilde at ci.uchicago.edu Fri Mar 30 11:26:08 2012 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Fri, 30 Mar 2012 11:26:08 -0500 (CDT) Subject: [Swift-commit] r5732 - in SwiftApps/ParVis: . HiRAMTools Message-ID: <20120330162608.900959CFE3@svn.ci.uchicago.edu> Author: wilde Date: 2012-03-30 11:26:08 -0500 (Fri, 30 Mar 2012) New Revision: 5732 Added: SwiftApps/ParVis/HiRAMTools/ SwiftApps/ParVis/HiRAMTools/checkgroups.sh SwiftApps/ParVis/HiRAMTools/combine.chk.sh SwiftApps/ParVis/HiRAMTools/combine.orig.sh SwiftApps/ParVis/HiRAMTools/combine.sh SwiftApps/ParVis/HiRAMTools/combine.swift SwiftApps/ParVis/HiRAMTools/combine2.swift SwiftApps/ParVis/HiRAMTools/combine_realization.sh SwiftApps/ParVis/HiRAMTools/find_anomalies.sh SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh SwiftApps/ParVis/HiRAMTools/get_real_copy.sh SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh SwiftApps/ParVis/HiRAMTools/makeyearly.sh SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh SwiftApps/ParVis/HiRAMTools/runall.sh SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh SwiftApps/ParVis/HiRAMTools/runcombine.sh SwiftApps/ParVis/HiRAMTools/y1.sh Log: Initial version. Added: SwiftApps/ParVis/HiRAMTools/checkgroups.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/checkgroups.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/checkgroups.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,21 @@ +#! /bin/bash + +# /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history/19500101/atmos_4xdaily.tile1.nc + +sed -e 's/^.*history.//' groups | sort >todo +find /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016 -type f -print | fgrep .nc | sed -e 's/^.*run016.//' | sort >done + +echo +echo diff: "done" +echo +diff todo done + +echo +echo last 20 files produced were: +echo +ls -ltd /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016/*/* | head -20 + +echo +echo File lengths: +echo +ls -ltd /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016/*/* | awk '{print $5}' | sort | uniq -c Property changes on: SwiftApps/ParVis/HiRAMTools/checkgroups.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/combine.chk.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/combine.chk.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/combine.chk.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,33 @@ +#! /bin/bash + +set -o errexit + +mkdir -p /scratch/wilde + +tmp=$(mktemp -d /scratch/wilde/combine_XXXXXX) + +echo mktemp error code: $? + +cd $tmp + +outdir=$1 +inpattern=$2 +date=$(basename $(dirname $inpattern)) +basepattern=$(basename $inpattern) + +infiles="$2*" +for f in $infiles; do + dd if=$f of=$(basename $f) bs=4M +done + +/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}* + +mkdir -p $outdir/$date +dd if=out.nc of=$outdir/$date/$basepattern.trans bs=4M +mv $outdir/$date/$basepattern.trans $outdir/$date/$basepattern + +cd .. +rm -rf $tmp + + + Property changes on: SwiftApps/ParVis/HiRAMTools/combine.chk.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/combine.orig.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/combine.orig.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/combine.orig.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,28 @@ +#! /bin/bash + +set -x + +mkdir -p /scratch/wilde +tmp=$(mktemp -d /scratch/wilde/combine_XXXX) +cd $tmp + +outdir=$1 +inpattern=$2 +date=$(basename $(dirname $inpattern)) +basepattern=$(basename $inpattern) + +infiles="$2*" +for f in $infiles; do + dd if=$f of=$(basename $f) bs=4M +done + +/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}* + +mkdir -p $outdir/$date +dd if=out.nc of=$outdir/$date/$basepattern bs=4M + +cd .. +rm -rf $tmp + + + Property changes on: SwiftApps/ParVis/HiRAMTools/combine.orig.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/combine.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/combine.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/combine.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,29 @@ +#! /bin/bash + +set -x + +mkdir -p /scratch/wilde +tmp=$(mktemp -d /scratch/wilde/combine_XXXX) +cd $tmp + +outdir=$1 +inpattern=$2 +date=$(basename $(dirname $inpattern)) +basepattern=$(basename $inpattern) + +infiles="$2*" +for f in $infiles; do + dd if=$f of=$(basename $f) bs=4M +done + +/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}* + +mkdir -p $outdir/$date +dd if=out.nc of=$outdir/$date/$basepattern.trans bs=4M +mv $outdir/$date/$basepattern.trans $outdir/$date/$basepattern + +cd .. +rm -rf $tmp + + + Property changes on: SwiftApps/ParVis/HiRAMTools/combine.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/combine.swift =================================================================== --- SwiftApps/ParVis/HiRAMTools/combine.swift (rev 0) +++ SwiftApps/ParVis/HiRAMTools/combine.swift 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,16 @@ + +type file; + +string rundir=@arg("rundir","NO-OUTDIR-SPECIFIED"); +string groupfile=@arg("groupfile","groups"); + +app combine (string dir, string pattern) +{ + combine dir pattern; +} + +string group[] = readData(groupfile); + +foreach g in group { + combine(rundir,g); +} Added: SwiftApps/ParVis/HiRAMTools/combine2.swift =================================================================== --- SwiftApps/ParVis/HiRAMTools/combine2.swift (rev 0) +++ SwiftApps/ParVis/HiRAMTools/combine2.swift 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,18 @@ + +type file; + +string rundir=@arg("rundir","NO-OUTDIR-SPECIFIED"); +string groupfile=@arg("groupfile","groups"); +string realizatoinfile=@arg("ralizationsfile","realizations"); + +app combine (string dir, string pattern) +{ + combine dir pattern; +} + +string group[] = readData(groupfile); +string reals[] = readData(realizationFile); + +foreach g in group { + combine(rundir,g); +} Added: SwiftApps/ParVis/HiRAMTools/combine_realization.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/combine_realization.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,82 @@ +#! /bin/bash + +hdir=$1 +outdir=$2 + +/bin/ls -1 -d ${hdir}* >dirs + +echo '=== ' $(basename $0) doing dirs: +cat dirs + +for d in $(cat dirs); do + find $d | grep .nc.[0-9][0-9][0-9][0-9] | sed -e 's/.....$//' | sort | uniq +done >groups + +# Create new runNNN directory + +rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' ) +mkdir $rundir + +cat >$rundir/cf <$rundir/sites.xml < + + + + .01 + 10000 + + $PWD/swiftwork + + + + + + + MTCScienceApps + default + 2.56 + 10000 + 8 + + + 32 + 28000 + 00:20:00 + 100 + 100 + 1 + 1 + $PWD/$rundir/swiftwork + + + + +END + +cat >$rundir/tc <& swift.out + + Property changes on: SwiftApps/ParVis/HiRAMTools/combine_realization.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/find_anomalies.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/find_anomalies.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/find_anomalies.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,18 @@ +#! /bin/sh + +reals=/intrepid-fs0/users/wilde/persistent/LZ/combined +cd $reals + +for real in *; do + echo Realization: $real + ( cd $real + for y in *; do + echo " Year: " $y + ( cd $y + for f in *.nc; do + echo -n $f; ncdump -h $f | grep 'time =' || echo + done | awk '{print " ",$1,$7}' | grep 'atmos.*\.tile[0-9].* (1' | grep -v scalar + ) + done + ) +done Property changes on: SwiftApps/ParVis/HiRAMTools/find_anomalies.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,17 @@ +#! /bin/bash + +src=/intrepid-fs0/users/lzamboni/persistent +dest=/intrepid-fs0/users/wilde/persistent/LZ/combined + +f=atmos_scalar.nc + +reals=$(cat /home/wilde/LZ/realizations) + +for r in $reals; do + cd $src/$r/history + years=$( /bin/ls -1d [0-9]??????? ) + for y in $years; do + cp $y/$f $dest/$r/$y + done +done + Property changes on: SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/get_real_copy.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/get_real_copy.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/get_real_copy.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,26 @@ +#! /bin/bash + +# src=/intrepid-fs0/users/lzamboni/persistent + +src=/intrepid-fs0/users/wilde/persistent/LZ/combined +dest=$PWD + +real=$1 + +if [ _$1 = _ ]; then + echo realization to copy was not specified - exiting; exit 1 +fi + +if [ -e _$1 ]; then + echo $1 already exits - exiting; exit 1 +fi + +cd $src/$real +dirs=$(find * -type d -print) + +for d in $dirs; do + mkdir -p $dest/$real/$d + for f in $d/*.nc; do + ln -s $PWD/$f $dest/$real/$f + done +done Property changes on: SwiftApps/ParVis/HiRAMTools/get_real_copy.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,26 @@ +#! /bin/bash + +# src=/intrepid-fs0/users/lzamboni/persistent + +src=/intrepid-fs0/users/wilde/persistent/LZ/combined +dest=$PWD + +real=$1 + +if [ _$1 = _ ]; then + echo realization to copy was not specified - exiting; exit 1 +fi + +if [ -e _$1 ]; then + echo $1 already exits - exiting; exit 1 +fi + +cd $src/$real +dirs=$(find * -type d -print) + +for d in $dirs; do + mkdir -p $dest/$real/history/$d + for f in $d/*.nc; do + ln -s $PWD/$f $dest/$real/history/$f + done +done Property changes on: SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/makeyearly.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/makeyearly.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/makeyearly.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,231 @@ +#! /bin/bash + +set -x + +infile=$1 # must be full path +outdir=$2 # must be full path + +realdir=$(dirname $(dirname $1)) # realization dir, e.g., en* +realname=$(basename $realdir) # realization name +f=$(basename $1) +d1=$(dirname $1) +yfirst=$(echo $(basename $d1) | sed -e 's/....$//') +ymid=$((yfirst+2)) +#ydir1=${yfirst}0101 +d2=$realdir/${ymid}0701 + +period=$(basename $f .nc | sed -e 's/^.*_//' -e 's/\..*$//' -e 's/_.*//') + +leap() +{ + case $1 in + 1952|1956|1960|1964|1968|1972|1976|1980|1984|1988|1992|1996|2000) echo 1 ;; + *) echo 0 ;; + esac +} + +units() +{ + p=$1 # period + y=$2 # year + case $period in + 4xdaily) echo $((4*(365+$(leap $y)))) ;; + 8xdaily) echo $((8*(365+$(leap $y)))) ;; + daily) echo $((1*(365+$(leap $y)))) ;; + month) echo 12 ;; + * ) + esac +} + +mkdir -p /scratch/wilde +tmp=$(mktemp -d /scratch/wilde/makeyearly_XXXXXX) +startdir=$PWD +cd $tmp + +export PATH=$PATH:/soft/apps/nco-3.9.9/bin +# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/apps/nco-3.9.9/lib + +o=$outdir/$realname + +yd1=$o/$((yfirst+0))0101 +yd2=$o/$((yfirst+1))0101 +yd3=$o/$((yfirst+2))0101 +yd4=$o/$((yfirst+3))0101 +yd5=$o/$((yfirst+4))0101 + +y1=$((yfirst+0)) +y2=$((yfirst+1)) +y3=$((yfirst+2)) +y4=$((yfirst+3)) +y5=$((yfirst+4)) + +set +x +e=echo + +$e mkdir -p $yd1 $yd2 $yd3 $yd4 $yd5 + +$e dd if=$d1/$f of=$f bs=4M # copy in first 2.5 year fle + +su=1 # start unit +u1=$(units $period $y1) +eu=$u1 # end unit + +$e ncks -F -d time,$su,$eu $f out.nc # extract year 1 +$e dd if=out.nc of=$yd1/$f bs=4M +$e rm out.nc + +su=$((eu+1)) +u2=$(units $period $y2) +eu=$((eu+$u2)) + +$e ncks -F -d time,$su,$eu $f out.nc # extract year 2 +$e dd if=out.nc of=$yd2/$f bs=4M +$e rm out.nc + +su=$((eu+1)) +u3=$(units $period $y3) +heu=$((u3/2)) +eu=$((eu+$heu)) + +$e ncks -F -d time,$su,$eu $f tmp.a.nc # extract year 3 part a +$e dd if=$d2/$f of=$f bs=4M # copy in second 2.y year file + +su=1 +eu=$heu + +$e ncks -F -d time,$su,$eu $f tmp.b.nc # extract year 3 part b + +$e ncrcat tmp.a.nc tmp.b.nc out.nc # merge year3 parts a and b +$e dd if=out.nc of=$yd3/$f bs=4M +$e rm out.nc tmp.a.nc tmp.b.nc + +su=$((eu+1)) +u4=$(units $period $y4) +eu=$((eu+$u4)) + +$e ncks -F -d time,$su,$eu $f out.nc # extract year 4 +$e dd if=out.nc of=$yd4/$f bs=4M +$e rm out.nc + +su=$((eu+1)) +u5=$(units $period $y5) +eu=$((eu+$u4)) + +$e ncks -F -d time,$su,$eu $f out.nc # extract year 5 +$e dd if=out.nc of=$yd5/$f bs=4M +$e rm out.nc $f + +$e cd $startdir +rm -rf $tmp + +exit + +###################### +# + +the origin of the problem is that in the first 6 month of the year (Jan-Jun) +there are less days than in the second half of the year (Jul-Dec), and that +2.5 years refers to the # of months. + +The difference is 3 days in a normal year, 2 in a leap year. + +So the variations are not simply due by the presence of a leap year. These are +the categories. Let me define these before: + +# normal= no leap in the 2.5 directory. +# 1st half=directoryName is yyyy0101 +# 2nd half=directoryName is yyyy0701 +# +# N1 normal, 1st half -->3644 steps (911 days*4) +# N2 normal, 2nd half -->3656 steps (914 days*4) +# L1 leap, 1st half -->3648 steps (912 days*4) +# L2 leap, 2nd half -->3660 steps. (915 days*4) + +# leap: 1952 1956 1960 1964 1968 1972 1976 1980 1984 1988 1992 1996 2000 + +# Jan, Feb, Mar, Apr, May, June = (31+28+31+30+31+30) = 181 + +# N-year: 181+184 x4= 724 + 736 = 1460 +# L-year: 182+184 x4= 728 + 736 = 1474 + + +19500101 1 L1: 3648 +1951 1 +1952 / L + +19520701 / N2: 3656 +1953 1 +1954 1 +---- +19550101 1 L1: 3648 +1956 1 L +1957 / + +19570701 + N2: 3656 +1958 1 +1959 1 +---- +19600101 1 L L1: 3648 +1961 1 +1962 / + +19620701 + L2: 3660 +1963 1 +1964 1 L +---- +19650101 1 N1: 3648 +1966 1 +1967 / + +19670701 + L2: 3660 +1968 1 L +1969 1 +---- +19700101 1 L1: 3648 +1971 1 +1972 / L + +19720701 + N2: 3656 +1973 1 +1974 1 +---- +19750101 1 L1: 3648 +1976 1 L +1977 / + +19770701 + N2: 3656 +1978 1 +1979 1 +---- +19800101 1 L L1: 3648 +1981 1 +1982 / + +19820701 + L2: 3660 +1983 1 +1984 1 L +---- +19850101 1 N1: 3648 +1986 1 +1987 / + +19870701 + L2: 3660 +1988 1 L +1989 1 +---- +19900101 1 L1: 3648 +1991 1 +1992 / L + +19920701 + N2: 3656 +1993 1 +1994 1 +---- +19950101 1 L1: 3648 +1995 1 +1996 / L + +19970701 + N2: 3656 +1998 1 +1999 1 +---- Property changes on: SwiftApps/ParVis/HiRAMTools/makeyearly.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,83 @@ +#! /bin/bash + +cdir=$1 # combined dir +outdir=$2 # outdir for equivalent yearly files + + +for y in $(seq 1950 5 1997); do + startyear=${y}0101; + nextyear=$((y+2))0701 + + find $cdir/$startyear -type f | fgrep .nc +done >ncfiles + +exit + +# Create new runNNN directory + +rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' ) +mkdir $rundir + +cat >$rundir/cf <$rundir/sites.xml < + + + + .01 + 10000 + + $PWD/swiftwork + + + + + + + MTCScienceApps + default + 2.56 + 10000 + 8 + + + 32 + 28000 + 00:20:00 + 100 + 100 + 1 + 1 + $PWD/$rundir/swiftwork + + + + +END + +cat >$rundir/tc <& swift.out + + Property changes on: SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,5 @@ +#! /bin/sh + +./combine_realization.sh \ + /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history/19 \ + /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/ Property changes on: SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,48 @@ +#! /bin/sh + +# ls -1d en*/history/[0-9][0-9][0-9][0-9]???? eur$ ls -1d en*/history/[0-9][0-9][0-9][0-9]???? + +# The following can be made into parameters to this script. Set here for now. + +realdir=/intrepid-fs0/users/lzamboni/persistent # Base dir of realizations +realtodo=real.todo # File of names of realization subdirs to process +realdone=real.done # File of names of realization subdirs completed +outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined # Base dir for combined output + +# Helpful patterns for filename matching + +yyyy="[0-9][0-9][0-9][0-9]" +yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]" + +mkdir -p $outdir + +while true; do + + real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do + if [ _$real = _ ]; then + break + fi + + echo Combining $real + + find $realdir/$real/history/$yyyymmdd | + grep .nc.$yyyy | + sed -e 's/.....$//' | sort | uniq > $real.groups + + echo Created $real.groups + + echo ./combine_realization.sh \ + "$realdir/$real/history/$yyyymmdd" \ + $outdir/$real + + ./combine_realization.sh \ + "$realdir/$real/history/$yyyymmdd" \ + $outdir/$real + + echo Completed $real + echo $real >>$realdone + +done + +echo All combines are done. + Property changes on: SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/runall.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/runall.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/runall.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,48 @@ +#! /bin/sh + +# ls -1d en*/history/[0-9][0-9][0-9][0-9]???? eur$ ls -1d en*/history/[0-9][0-9][0-9][0-9]???? + +# The following can be made into parameters to this script. Set here for now. + +realdir=/intrepid-fs0/users/wilde/persistent/LZ/combined # Base dir of realizations +realtodo=yearly.todo # File of names of realization subdirs to process +realdone=yearly.done # File of names of realization subdirs completed +outdir=/intrepid-fs0/users/wilde/persistent/LZ/yearly # Base dir for combined output + +# Helpful patterns for filename matching + +yyyy="[0-9][0-9][0-9][0-9]" +yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]" + +mkdir -p $outdir + +while true; do + + real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do + if [ _$real = _ ]; then + break + fi + + echo RunAll: Processing $real + + find $realdir/$real/history/$yyyymmdd | + grep .nc.$yyyy | + sed -e 's/.....$//' | sort | uniq > $real.groups + + echo Created $real.groups + + echo ./makeyearly_realization.sh \ + "$realdir/$real/$yyyymmdd" \ + $outdir/$real + + ./makeyearly_realization.sh \ + "$realdir/$real/$yyyymmdd" \ + $outdir/$real + + echo RunAll: Completed $real + echo $real >>$realdone + +done + +echo RunAll: all realizations are done. + Property changes on: SwiftApps/ParVis/HiRAMTools/runall.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,48 @@ +#! /bin/sh + +# ls -1d en*/history/[0-9][0-9][0-9][0-9]???? eur$ ls -1d en*/history/[0-9][0-9][0-9][0-9]???? + +# The following can be made into parameters to this script. Set here for now. + +realdir=/intrepid-fs0/users/lzamboni/persistent # Base dir of realizations +realtodo=real4.todo # File of names of realization subdirs to process +realdone=real4.done # File of names of realization subdirs completed +outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined4 # Base dir for combined output + +# Helpful patterns for filename matching + +yyyy="[0-9][0-9][0-9][0-9]" +yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]" + +mkdir -p $outdir + +while true; do + + real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do + if [ _$real = _ ]; then + break + fi + + echo Combining $real + + find $realdir/$real/history/$yyyymmdd | + grep .nc.$yyyy | + sed -e 's/.....$//' | sort | uniq > $real.groups + + echo Created $real.groups + + echo ./combine_realization.sh \ + "$realdir/$real/history/$yyyymmdd" \ + $outdir/$real + + ./combine_realization.sh \ + "$realdir/$real/history/$yyyymmdd" \ + $outdir/$real + + echo Completed $real + echo $real >>$realdone + +done + +echo All combines are done. + Property changes on: SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/runcombine.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/runcombine.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/runcombine.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,5 @@ +#! /bin/sh + +$HOME/LZ/combine.csh \ + -a /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history \ + -s 19500101 -n 0 -m 30 # -q Property changes on: SwiftApps/ParVis/HiRAMTools/runcombine.sh ___________________________________________________________________ Added: svn:executable + Added: SwiftApps/ParVis/HiRAMTools/y1.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/y1.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/y1.sh 2012-03-30 16:26:08 UTC (rev 5732) @@ -0,0 +1,29 @@ +#! /bin/sh + +export PATH=$PATH:/soft/apps/nco-3.9.9/bin +# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/apps/nco-3.9.9/lib + +d1=19500101 +d2=19520701 +f=ice_month.nc + +o=testyears +y1=$o/19500101 +y2=$o/19510101 +y3=$o/19520101 +y4=$o/19530101 +y5=$o/19540101 + +mkdir -p $y1 $y2 $y3 $y4 $y5 + +ncks -F -d time,1,12 $d1/$f $y1/$f + +ncks -F -d time,13,24 $d1/$f $y2/$f + +ncks -F -d time,25,30 $d1/$f $y3/$f.a +ncks -F -d time,1,6 $d2/$f $y3/$f.b +ncrcat $y3/$f.a $y3/$f.b $y3/$f + +ncks -F -d time,7,18 $d2/$f $y4/$f + +ncks -F -d time,19,30 $d2/$f $y5/$f Property changes on: SwiftApps/ParVis/HiRAMTools/y1.sh ___________________________________________________________________ Added: svn:executable + From wilde at ci.uchicago.edu Fri Mar 30 11:32:09 2012 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Fri, 30 Mar 2012 11:32:09 -0500 (CDT) Subject: [Swift-commit] r5733 - SwiftApps/ParVis/HiRAMTools Message-ID: <20120330163209.8A68F9CFE3@svn.ci.uchicago.edu> Author: wilde Date: 2012-03-30 11:32:09 -0500 (Fri, 30 Mar 2012) New Revision: 5733 Added: SwiftApps/ParVis/HiRAMTools/README Log: Add Readme Added: SwiftApps/ParVis/HiRAMTools/README =================================================================== --- SwiftApps/ParVis/HiRAMTools/README (rev 0) +++ SwiftApps/ParVis/HiRAMTools/README 2012-03-30 16:32:09 UTC (rev 5733) @@ -0,0 +1,6 @@ + +1. Add Swift to your path. For now we will use a Swift version maintained by + Swift team member Jon Monette: + +PATH=ls ~jonmon/Library/Swift/swift-0.93:$PATH + From wilde at ci.uchicago.edu Fri Mar 30 11:34:00 2012 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Fri, 30 Mar 2012 11:34:00 -0500 (CDT) Subject: [Swift-commit] r5734 - SwiftApps/ParVis/HiRAMTools Message-ID: <20120330163400.EE9E99CFE3@svn.ci.uchicago.edu> Author: wilde Date: 2012-03-30 11:34:00 -0500 (Fri, 30 Mar 2012) New Revision: 5734 Modified: SwiftApps/ParVis/HiRAMTools/README Log: Fix README Modified: SwiftApps/ParVis/HiRAMTools/README =================================================================== --- SwiftApps/ParVis/HiRAMTools/README 2012-03-30 16:32:09 UTC (rev 5733) +++ SwiftApps/ParVis/HiRAMTools/README 2012-03-30 16:34:00 UTC (rev 5734) @@ -2,5 +2,5 @@ 1. Add Swift to your path. For now we will use a Swift version maintained by Swift team member Jon Monette: -PATH=ls ~jonmon/Library/Swift/swift-0.93:$PATH +PATH=/home/jonmon/Library/Swift/swift-0.93:$PATH From wilde at ci.uchicago.edu Fri Mar 30 11:37:57 2012 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Fri, 30 Mar 2012 11:37:57 -0500 (CDT) Subject: [Swift-commit] r5735 - SwiftApps/ParVis/HiRAMTools Message-ID: <20120330163757.D499F9CFE3@svn.ci.uchicago.edu> Author: wilde Date: 2012-03-30 11:37:57 -0500 (Fri, 30 Mar 2012) New Revision: 5735 Modified: SwiftApps/ParVis/HiRAMTools/README Log: Fix README Modified: SwiftApps/ParVis/HiRAMTools/README =================================================================== --- SwiftApps/ParVis/HiRAMTools/README 2012-03-30 16:34:00 UTC (rev 5734) +++ SwiftApps/ParVis/HiRAMTools/README 2012-03-30 16:37:57 UTC (rev 5735) @@ -2,5 +2,5 @@ 1. Add Swift to your path. For now we will use a Swift version maintained by Swift team member Jon Monette: -PATH=/home/jonmon/Library/Swift/swift-0.93:$PATH +PATH=/home/jonmon/Library/Swift/swift-0.93/bin:/home/jonmon/Library/jdk1.7.0_03/bin:$PATH From wilde at ci.uchicago.edu Fri Mar 30 11:47:11 2012 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Fri, 30 Mar 2012 11:47:11 -0500 (CDT) Subject: [Swift-commit] r5736 - SwiftApps/ParVis/HiRAMTools Message-ID: <20120330164711.A1F109CFE3@svn.ci.uchicago.edu> Author: wilde Date: 2012-03-30 11:47:11 -0500 (Fri, 30 Mar 2012) New Revision: 5736 Added: SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.config Modified: SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh Log: Added config file. Added: SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.config =================================================================== --- SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.config (rev 0) +++ SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.config 2012-03-30 16:47:11 UTC (rev 5736) @@ -0,0 +1,4 @@ +realdir=/intrepid-fs0/users/lzamboni/persistent # Base dir of realizations +realtodo=real.todo # File of names of realization subdirs to process +realdone=real.done # File of names of realization subdirs completed +outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined4 # Base dir for combined output Modified: SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh 2012-03-30 16:37:57 UTC (rev 5735) +++ SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh 2012-03-30 16:47:11 UTC (rev 5736) @@ -9,6 +9,8 @@ realdone=real.done # File of names of realization subdirs completed outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined # Base dir for combined output +source ./runall.combine_realizations.config # can overwrite above config params + # Helpful patterns for filename matching yyyy="[0-9][0-9][0-9][0-9]" From wilde at ci.uchicago.edu Fri Mar 30 12:02:30 2012 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Fri, 30 Mar 2012 12:02:30 -0500 (CDT) Subject: [Swift-commit] r5737 - SwiftApps/ParVis/HiRAMTools Message-ID: <20120330170230.99ACF9CFE3@svn.ci.uchicago.edu> Author: wilde Date: 2012-03-30 12:02:30 -0500 (Fri, 30 Mar 2012) New Revision: 5737 Modified: SwiftApps/ParVis/HiRAMTools/README SwiftApps/ParVis/HiRAMTools/combine.chk.sh Log: Removed wilde deps. Modified: SwiftApps/ParVis/HiRAMTools/README =================================================================== --- SwiftApps/ParVis/HiRAMTools/README 2012-03-30 16:47:11 UTC (rev 5736) +++ SwiftApps/ParVis/HiRAMTools/README 2012-03-30 17:02:30 UTC (rev 5737) @@ -4,3 +4,8 @@ PATH=/home/jonmon/Library/Swift/swift-0.93/bin:/home/jonmon/Library/jdk1.7.0_03/bin:$PATH +2. Edit runall.combine_realizations.config + +3. run: + + ./runall.combine_realizations.sh >& runall.out Modified: SwiftApps/ParVis/HiRAMTools/combine.chk.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/combine.chk.sh 2012-03-30 16:47:11 UTC (rev 5736) +++ SwiftApps/ParVis/HiRAMTools/combine.chk.sh 2012-03-30 17:02:30 UTC (rev 5737) @@ -2,9 +2,9 @@ set -o errexit -mkdir -p /scratch/wilde +mkdir -p /scratch/$USER -tmp=$(mktemp -d /scratch/wilde/combine_XXXXXX) +tmp=$(mktemp -d /scratch/$USER/combine_XXXXXX) echo mktemp error code: $? From wilde at ci.uchicago.edu Fri Mar 30 12:04:56 2012 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Fri, 30 Mar 2012 12:04:56 -0500 (CDT) Subject: [Swift-commit] r5738 - SwiftApps/ParVis/HiRAMTools Message-ID: <20120330170456.ED8A89CFE3@svn.ci.uchicago.edu> Author: wilde Date: 2012-03-30 12:04:56 -0500 (Fri, 30 Mar 2012) New Revision: 5738 Modified: SwiftApps/ParVis/HiRAMTools/combine_realization.sh Log: Use .chk version of combine.sh Modified: SwiftApps/ParVis/HiRAMTools/combine_realization.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2012-03-30 17:02:30 UTC (rev 5737) +++ SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2012-03-30 17:04:56 UTC (rev 5738) @@ -66,7 +66,7 @@ cat >$rundir/tc < Author: wilde Date: 2012-03-30 12:14:31 -0500 (Fri, 30 Mar 2012) New Revision: 5739 Modified: SwiftApps/ParVis/HiRAMTools/combine_realization.sh Log: Removed project from sites.xml Modified: SwiftApps/ParVis/HiRAMTools/combine_realization.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2012-03-30 17:04:56 UTC (rev 5738) +++ SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2012-03-30 17:14:31 UTC (rev 5739) @@ -42,7 +42,7 @@ - MTCScienceApps + default 2.56 10000