From wilde at ci.uchicago.edu Tue Jan 1 16:42:53 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 1 Jan 2013 16:42:53 -0600 (CST) Subject: [Swift-commit] r6114 - in SwiftApps/SciColSim: . params Message-ID: <20130101224253.578F09CCC0@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-01 16:42:52 -0600 (Tue, 01 Jan 2013) New Revision: 6114 Modified: SwiftApps/SciColSim/Makefile SwiftApps/SciColSim/TODO SwiftApps/SciColSim/annealing.swift SwiftApps/SciColSim/optimizer.cpp SwiftApps/SciColSim/params/ARtest03 SwiftApps/SciColSim/testopt.py Log: annealing.swift: do indivudual target_innovation annealing loops iteratively instead of in parallel. Will make parallel an option in a future rev (code is still there). Removed unused unmapped file which was causing a spurious warning '(input) no files'; Makefile: addded targets for non-OpenMP optimizer. optimizer.cpp: return 3*max_loss when ration exceeds max_loss; added some debug traces. Modified: SwiftApps/SciColSim/Makefile =================================================================== --- SwiftApps/SciColSim/Makefile 2012-12-30 19:55:18 UTC (rev 6113) +++ SwiftApps/SciColSim/Makefile 2013-01-01 22:42:52 UTC (rev 6114) @@ -2,14 +2,13 @@ ifeq ($(UNAME), Linux) -all: openmp-optimizer graphsim openmptest +all: optimizer openmp-optimizer graphsim openmptest openmp-optimizer: optimizer.cpp - g++ -DP_OPENMP -static -O -fopenmp -I boost_1_47_0 -o openmp-optimizer optimizer.cpp -# g++ -static -O -I boost_1_47_0 -o openmp-optimizer optimizer.cpp + g++ -DP_OPENMP -static -O2 -fopenmp -I boost_1_47_0 -o openmp-optimizer optimizer.cpp -#t: t.cpp -# g++ -DP_OPENMP -static -O -fopenmp -I boost_1_47_0 -o t t.cpp +optimizer: optimizer.cpp + g++ -static -O2 -I boost_1_47_0 -o optimizer optimizer.cpp graphsim: graphsim.cpp g++ -DP_OPENMP -static -pg -O2 -fopenmp -I boost_1_47_0 -o graphsim graphsim.cpp @@ -24,11 +23,14 @@ ifeq ($(UNAME), Darwin) -all: openmp-optimizer dispatch-optimizer orig-optimizer graphsim openmptest +all: optimizer openmp-optimizer dispatch-optimizer orig-optimizer graphsim openmptest openmp-optimizer: optimizer.cpp g++ -DP_OPENMP -fopenmp -I boost_1_47_0 -o openmp-optimizer optimizer.cpp +optimizer: optimizer.cpp + g++ -I boost_1_47_0 -o optimizer optimizer.cpp + graphsim: graphsim.cpp g++ -DP_OPENMP -fopenmp -I boost_1_47_0 -g -o graphsim graphsim.cpp Modified: SwiftApps/SciColSim/TODO =================================================================== --- SwiftApps/SciColSim/TODO 2012-12-30 19:55:18 UTC (rev 6113) +++ SwiftApps/SciColSim/TODO 2013-01-01 22:42:52 UTC (rev 6114) @@ -1,5 +1,49 @@ +--- 2012.1231 +Check running times under actual OPENMP & Node settings. Running 24X? +Reduce mem to 24X + +Why large neg expos? Eg: + +--- +multi_annealing: AR: 3487.405033 +- 23.971373 +multi_loss: returning: ci=1 cj=0 r.loss=3487.405033 r.sdev=23.971373 +Progress: time: Mon, 31 Dec 2012 06:14:55 +0000 Checking status:1 Finished successfully:841 +multi_loss: returning: ci=0 cj=0 r.loss=3427.572341 r.sdev=24.738546 +multi_annealing: AR: initial: 3427.572341 +- 24.738546 +multi_annealing: AR: 0.9889519936665272 vs 1.0174100270514812E-13 +math/min: result=1.0174100270514812E-13 +multi_annealing: AR: 1,0 1 Did not accept: 1.8783001222853395 (0) +multi_annealing: AR: 1.8783001222853395 0.0 0.0 0.0 0.0 +multi_loss appCalls=420 +multi_annealing: AR: -0.15939764554651975 1 +[0.0,-0.15939764554651975,0.0,0.0,0.0] +--- + +Why 2X jobs in first wave???? + +Run 1000X x 5 TIs first? (parallel run) + +Add select: ser or par + +Need 100 Ann cycles? + +Check calcs in sumloss.sh + +get more resource telemetry (MEM, etc) + +Debug 0.94 + +Try Swift/T + +Improve/reduce staging? + +--- + + + + --- Questions and Open Issues --- When sampling the graph, we compress the node ids. Is this OK? Modified: SwiftApps/SciColSim/annealing.swift =================================================================== --- SwiftApps/SciColSim/annealing.swift 2012-12-30 19:55:18 UTC (rev 6113) +++ SwiftApps/SciColSim/annealing.swift 2013-01-01 22:42:52 UTC (rev 6114) @@ -68,7 +68,7 @@ int annealing_cycles) { int cycle = 10; // const - int NEVOPARAMS = 5; // const - 5 params, alpha 1,m through delta, does not include target_innovation + int NEVOPARAMS = 5; // const - 5 params: alpha_i, alpha_m, beta, gamma, delta float rejection[][]; // [i][j] where i is cycle and j is evolve-parameter (alpha_i, alpha_m, beta, gamma, delta) float trejection[][]; @@ -229,7 +229,7 @@ { tracef("%q\n", x); file rfile[]; - file ofile[]; // FIXME: to obtain timings and otehr stats + //file ofile[]; // FIXME: to obtain timings and otehr stats tracef( "multi_loss: entered: ci=%i cj=%i target_innov=%f evolve_reruns=%i x=%q\n",ci, cj, target_innov,evolve_reruns,x ); @@ -268,6 +268,47 @@ // s = readStat(statsfile); FIXME: to obtain timings and other stats } +optimizer_serial_sweep() // Implements logic of python driver script +{ + + int minrange = @toint(@arg("minrange", "58")); + int maxrange = @toint(@arg("maxrange", "59")); + int rangeinc = @toint(@arg("rangeinc", "50")); + + // FIXME: add provision for random priming and random param values when x[i] == -100 (see optimizer.cpp main()) + + int nreps = @toint(@arg("nreps", "1")); + +//file bestfile ; +//file maxfile ; + +# foreach target_innov in [minrange:maxrange:rangeinc] { + iterate i { + int target_innov = minrange + (i*rangeinc); + foreach rep in [1:nreps] { + + string some_out_filename = @strcat( "best.T", @strcat(target_innov), ".R", @strcat(rep), ".txt" ); + file outfile; // ; + file lossfile; // ; + + (outfile,lossfile) = multi_annealing(some_out_filename, + @tofloat(@arg("tstart", "2.0")), + @tofloat(@arg("tend", "0.01")), + @tofloat(@arg("trejection", "0.3")), + @toint(@arg("evoreruns", "100")), + @tofloat(@arg("startingjump", "2.3")), + [ @tofloat(@arg("alphai", "0.0")), + @tofloat(@arg("alpham", "0.0")), + @tofloat(@arg("beta", "4.0")), + @tofloat(@arg("gamma", "50.0")), + @tofloat(@arg("delta", "-1.0")) + ], + @tofloat(target_innov), + @toint(@arg("annealingcycles", "50")) ); + } + } until(target_innov >= (maxrange-rangeinc)); +} + optimizer_sweep() // Implements logic of python driver script { @@ -309,7 +350,7 @@ main() { - optimizer_sweep(); + optimizer_serial_sweep(); } main(); Modified: SwiftApps/SciColSim/optimizer.cpp =================================================================== --- SwiftApps/SciColSim/optimizer.cpp 2012-12-30 19:55:18 UTC (rev 6113) +++ SwiftApps/SciColSim/optimizer.cpp 2013-01-01 22:42:52 UTC (rev 6114) @@ -694,8 +694,8 @@ std::vector d(num_edges(Full_g)); edge_descriptor s; boost::graph_traits::vertex_descriptor u, v; + cout << "reset_world: start id=" << id << endl; - for (int i=0; i 0. && Tried[i][j] > 0){ @@ -723,6 +723,7 @@ Tried[i][j]=0.; } } + cout << "reset_world: end id=" << id << endl; } @@ -1004,7 +1005,8 @@ //================================================================================== void evolve_to_target_and_save(int istart, int iend, double* storage, int* counters){ - double ALOT=100000000000.; + //double ALOT=100000000000.; + double ALOT= 3.0 * max_loss; // Per Andrey, 2012.12.29 double ratio = 0.0; int check = 0; @@ -1012,6 +1014,7 @@ reset_world(); for (int k = istart; k < iend; k++){ + cout << "evolve_ttas: k=" << k << endl; for(int i=0; i< N_epochs && current_novelty < TargetNovelty; i++){ Modified: SwiftApps/SciColSim/params/ARtest03 =================================================================== --- SwiftApps/SciColSim/params/ARtest03 2012-12-30 19:55:18 UTC (rev 6113) +++ SwiftApps/SciColSim/params/ARtest03 2013-01-01 22:42:52 UTC (rev 6114) @@ -4,11 +4,11 @@ target_innovation_increment 50 # increment target innovation by this amount annealing_repeats 1 # times to repeate the entire optimization process for each target_innovation -annealing_cycles 10 # times to perform the simulated annealing loop for all non-fixed parameters +annealing_cycles 30 # times to perform the simulated annealing loop for all non-fixed parameters -evolve_reruns 32 # times to perform evolve_to_target_and_save() (objective function - is batched) -nworkers 16 # number of parallel threads (cores) to use per invocation of C++ optimizer -reruns_per_opt_invocation 16 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!! +evolve_reruns 48 # times to perform evolve_to_target_and_save() (objective function - is batched) +nworkers 24 # number of parallel threads (cores) to use per invocation of C++ optimizer +reruns_per_opt_invocation 24 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!! alpha_i 0.0 # 5 evolve() parameters: must be decimal values !!! alpha_m 0.0 # alpha_i and alpha_m are fixed for now Modified: SwiftApps/SciColSim/testopt.py =================================================================== --- SwiftApps/SciColSim/testopt.py 2012-12-30 19:55:18 UTC (rev 6113) +++ SwiftApps/SciColSim/testopt.py 2013-01-01 22:42:52 UTC (rev 6114) @@ -10,7 +10,7 @@ app = "./openmp-optimizer"; # For Mac or Linux: Using OpenMP (Default) #paramset="default" -paramset="mw2" +paramset="mw3" verboseLevel = "1" print("Running with parameter set " + paramset + "\n\n"); @@ -131,6 +131,26 @@ app = "./openmp-optimizer" verboseLevel = "1" +elif paramset == "mw3": # Timing TEST PARAMETERS by Mike + + startTarget = 135 + endTarget = 136 + incrTarget = 135 + optimizerRepeats = 1 + evolveReruns = 1 + evolveReruns = 2 + annealingSteps = 1 + NWorkers = "1" + openmp = "OMP_NUM_THREADS=" + NWorkers + openmp = "" + + operation = "m" # n=normal, m=manual (runs 1 multi_loss call) + seed = "123456" + app = "./graphsim" + app = "./openmp-optimizer" + app = "./optimizer" + verboseLevel = "99" + # Ensure we dont pass new parameters to original optimizer versions # (but they would be ignored) @@ -144,9 +164,9 @@ for target in range(startTarget,endTarget,incrTarget): for i in range(optimizerRepeats): args = "rm -f bestdb.txt; " + \ - openmp + " " + app + " 0 0 0 0 0 " + str(target) + " 40000 20 " + str(evolveReruns) + \ - " 2 " + verboseLevel + " 2. 0.01 " + str(annealingSteps) + " 0.3 2.3 0 0 0 0 0 " + operation + " " + NWorkers + " " + seed + \ - " >& out.T"+str(target)+".i"+str(i) + "; #mv bestdb.txt best.T" + str(target) + ".i" + str(i) + openmp + " " + "/usr/bin/time -v " + app + " 0 -0.673572334121749 -2.0320313590203978 0 0 " + str(target) + " 40000 20 " + str(evolveReruns) + \ + " 2 " + verboseLevel + " 2. 0.01 " + str(annealingSteps) + " 0.3 2.3 0 0 0 0 0 " + operation + " " + NWorkers + " " + seed \ + # + " >& out.T"+str(target)+".i"+str(i) + "; #mv bestdb.txt best.T" + str(target) + ".i" + str(i) print("\n**** Calling optimizer: "+args+"\n") os.system(args); From wilde at ci.uchicago.edu Tue Jan 1 17:37:49 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 1 Jan 2013 17:37:49 -0600 (CST) Subject: [Swift-commit] r6115 - in SwiftApps/SciColSim: . conf params Message-ID: <20130101233749.184F09CCFC@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-01 17:37:48 -0600 (Tue, 01 Jan 2013) New Revision: 6115 Modified: SwiftApps/SciColSim/annealing.swift SwiftApps/SciColSim/conf/beagle03.cf SwiftApps/SciColSim/conf/beagle03.xml SwiftApps/SciColSim/movie_graph.txt SwiftApps/SciColSim/params/ARtest03 Log: Adjust iterate loop exit tests to account for Swift changes from 0.93 to 0.94. Iterate loops now exit as soon as the until() clause, tested at the end of the loop body, is true. Modified: SwiftApps/SciColSim/annealing.swift =================================================================== --- SwiftApps/SciColSim/annealing.swift 2013-01-01 22:42:52 UTC (rev 6114) +++ SwiftApps/SciColSim/annealing.swift 2013-01-01 23:37:48 UTC (rev 6115) @@ -220,8 +220,8 @@ curr_sdev[curr] = curr_sdev[prev]; // dx[i][j] not set for fixed vars } - } until( j == (NEVOPARAMS-1) ); - } until( iter_i == (annealing_cycles-1) ); + } until( j == NEVOPARAMS ); + } until( iter_i == annealing_cycles ); } (Res r) multi_loss(float t_start, float t_end, int annealing_steps, float t_rejection, float starting_jump, Modified: SwiftApps/SciColSim/conf/beagle03.cf =================================================================== --- SwiftApps/SciColSim/conf/beagle03.cf 2013-01-01 22:42:52 UTC (rev 6114) +++ SwiftApps/SciColSim/conf/beagle03.cf 2013-01-01 23:37:48 UTC (rev 6115) @@ -4,6 +4,7 @@ lazy.errors=true status.mode=provider use.provider.staging=false +use.wrapper.staging=false provider.staging.pin.swiftfiles=false #app local sumloss=$PWD/../sumloss.sh Modified: SwiftApps/SciColSim/conf/beagle03.xml =================================================================== --- SwiftApps/SciColSim/conf/beagle03.xml 2013-01-01 22:42:52 UTC (rev 6114) +++ SwiftApps/SciColSim/conf/beagle03.xml 2013-01-01 23:37:48 UTC (rev 6115) @@ -21,9 +21,9 @@ 100 pbs.aprun;pbs.mpp;depth=24 1800 - 2 - 1 - 5 + 1 + 2 + 2 scalability 12.00 Modified: SwiftApps/SciColSim/movie_graph.txt =================================================================== --- SwiftApps/SciColSim/movie_graph.txt 2013-01-01 22:42:52 UTC (rev 6114) +++ SwiftApps/SciColSim/movie_graph.txt 2013-01-01 23:37:48 UTC (rev 6115) @@ -1,1010 +1,1356 @@ -500 1008 -0 56 -0 436 -0 446 -1 256 -1 476 -1 260 -1 262 -1 9 -1 266 -1 13 -1 142 -1 273 -1 388 -1 30 -1 36 -1 37 -1 423 -1 168 -1 425 -1 255 -1 175 -1 307 -1 58 -1 316 -1 319 -1 323 -1 68 -1 455 -1 216 -1 82 -1 88 -1 346 -1 348 -1 225 -1 228 -1 236 -1 365 -1 111 -1 424 -1 370 -1 246 -1 376 -1 122 -1 191 -1 252 -1 382 -1 127 -2 3 -3 262 -3 263 -3 392 -3 172 -3 138 -3 395 -3 13 -3 18 -3 150 -3 408 -3 154 -3 156 -3 30 -3 291 -3 37 -3 168 -3 300 -3 52 -3 158 -3 310 -3 265 -3 187 -3 61 -3 319 -3 65 -3 322 -3 54 -3 130 -3 71 -3 332 -3 461 -3 397 -3 419 -3 473 -3 346 -3 95 -3 352 -3 98 -3 103 -3 107 -3 414 -3 368 -3 498 -3 499 -4 224 -4 449 -4 419 -4 356 -4 72 -4 330 -4 173 -4 80 -4 242 -4 180 -4 24 -4 138 -5 33 -5 459 -5 484 -5 106 -5 299 -5 430 -5 16 -5 336 -5 50 -5 147 -5 406 -5 87 -5 91 -5 190 -6 7 -6 264 -6 137 -6 276 -6 149 -6 279 -6 153 -6 29 -6 415 -6 293 -6 166 -6 296 -6 42 -6 44 -6 48 -6 49 -6 438 -6 55 -6 314 -6 69 -6 454 -6 463 -6 84 -6 85 -6 470 -6 349 -6 94 -6 229 -6 233 -6 364 -6 117 -6 248 -6 380 -6 426 -7 129 -7 131 -7 470 -7 136 -7 471 -7 271 -7 402 -7 19 -7 404 -7 282 -7 286 -7 491 -7 35 -7 421 -7 166 -7 295 -7 427 -7 428 -7 301 -7 349 -7 436 -7 439 -7 56 -7 445 -7 446 -7 320 -7 123 -7 451 -7 69 -7 454 -7 78 -7 463 -7 212 -7 213 -7 342 -7 343 -7 474 -7 93 -7 483 -7 357 -7 102 -7 235 -7 237 -7 239 -7 371 -7 379 -8 112 -8 164 -8 10 -8 12 -8 398 -8 176 -8 120 -8 178 -8 373 -8 24 -8 124 -9 262 -10 11 -10 12 -10 47 -10 335 -10 124 -11 96 -11 289 -11 66 -11 283 -11 97 -11 361 -11 47 -11 432 -11 335 -11 215 -11 27 -11 124 -11 189 -12 112 -12 434 -12 47 -13 32 -13 130 -13 260 -13 37 -13 39 -13 202 -13 305 -13 461 -13 145 -13 275 -13 448 -13 54 -13 183 -13 344 -13 36 -14 387 -14 267 -14 155 -14 36 -14 165 -14 51 -14 308 -14 440 -14 441 -14 59 -14 444 -14 199 -14 328 -14 462 -14 81 -14 210 -14 214 -14 91 -14 99 -14 60 -14 106 -14 109 -14 110 -14 244 -15 193 -15 73 -15 400 -15 341 -15 27 -15 31 -16 33 -16 258 -16 353 -16 329 -16 204 -16 207 -16 336 -16 108 -16 50 -16 22 -16 119 -16 169 -16 381 -16 223 -17 321 -17 482 -17 197 -17 276 -17 149 -17 315 -17 94 -18 32 -18 461 -18 305 -18 498 -18 86 -18 25 -19 320 -19 421 -19 334 -19 314 -19 349 -19 446 -20 48 -20 28 -20 83 -20 44 -20 45 -21 288 -21 176 -21 347 -21 325 -21 392 -21 490 -21 366 -21 272 -21 209 -21 115 -21 182 -21 247 -21 184 -21 25 -21 217 -21 27 -21 86 -21 478 -22 353 -23 257 -23 100 -23 25 -24 134 -24 146 -24 302 -24 151 -24 290 -24 291 -24 43 -24 173 -24 46 -24 176 -24 180 -24 245 -24 453 -24 457 -24 208 -24 338 -24 363 -24 116 -24 373 -24 126 -25 32 -25 128 -25 257 -25 86 -26 161 -26 357 -26 193 -26 78 -26 341 -26 69 -27 361 -27 490 -27 333 -27 272 -27 312 -27 468 -27 41 -27 184 -27 429 -27 124 -27 341 -28 64 -28 354 -28 67 -28 135 -28 200 -28 42 -28 75 -28 45 -28 206 -28 48 -28 359 -28 53 -28 238 -29 229 -29 199 -29 296 -29 233 -29 42 -29 274 -29 117 -29 152 -29 155 -29 62 -30 386 -30 225 -30 76 -30 322 -30 367 -30 318 -31 200 -31 73 -31 333 -31 494 -31 45 -31 312 -32 257 -32 305 -32 101 -32 486 -32 39 -32 231 -32 79 -32 145 -32 275 -33 459 -33 336 -33 372 -33 119 -33 409 -34 283 -35 491 -35 243 -35 237 -35 326 -36 260 -36 266 -36 285 -36 292 -36 39 -36 424 -36 171 -36 181 -36 183 -36 440 -36 58 -36 323 -36 328 -36 81 -36 82 -36 87 -36 99 -36 228 -36 101 -36 111 -37 168 -37 202 -38 92 -39 448 -39 101 -39 486 -39 231 -39 145 -39 344 -40 345 -40 355 -40 179 -40 59 -40 125 -41 160 -41 226 -41 230 -41 104 -41 361 -41 492 -41 341 -41 215 -41 121 -41 189 -42 64 -42 67 -42 199 -42 359 -42 109 -42 206 -42 143 -42 48 -42 274 -42 117 -44 193 -44 69 -44 161 -44 73 -44 48 -44 83 -44 279 -45 200 -45 73 -45 83 -46 242 -46 485 -46 126 -46 173 -48 64 -48 206 -50 259 -50 261 -50 329 -50 299 -50 204 -50 430 -50 207 -50 405 -50 59 -50 159 -51 67 -51 101 -51 135 -51 210 -51 244 -51 214 -51 441 -51 249 -52 368 -52 217 -52 347 -52 86 -53 101 -53 327 -53 75 -53 238 -53 79 -53 249 -54 130 -55 84 -57 162 -57 331 -58 266 -58 82 -59 384 -59 259 -59 132 -59 267 -59 496 -59 355 -59 152 -59 91 -59 221 -59 159 -60 91 -60 165 -62 152 -62 155 -62 221 -63 456 -63 334 -63 303 -63 472 -63 314 -63 380 -65 154 -65 499 -65 374 -65 103 -66 96 -66 160 -66 215 -67 359 -67 135 -67 109 -67 143 -67 214 -69 161 -69 166 -69 239 -69 404 -69 85 -69 279 -69 153 -70 410 -70 467 -71 130 -72 138 -73 193 -74 241 -74 82 -74 188 -74 133 -74 469 -75 140 -75 79 -75 90 -75 350 -76 386 -77 120 -77 306 -77 452 -77 254 -77 311 -78 239 -78 357 -79 257 -79 100 -79 101 -80 138 -81 99 -81 171 -82 256 -82 133 -82 348 -82 205 -82 372 -82 181 -82 119 -82 409 -82 255 -82 316 -82 122 -82 469 -84 277 -84 264 -84 149 -84 380 -84 415 -85 153 -85 166 -86 247 -86 217 -87 181 -87 147 -87 285 -89 192 -89 200 -89 333 -89 468 -89 443 -89 222 -90 272 -90 100 -90 350 -91 165 -91 106 -91 430 -91 406 -91 159 -92 258 -93 136 -93 464 -93 493 -93 326 -94 482 -94 233 -94 114 -94 276 -95 107 -95 265 -95 186 -95 332 -96 289 -96 324 -96 102 -96 401 -97 283 -97 220 -97 434 -97 164 -99 101 -99 171 -99 244 -99 251 -100 272 -101 231 -101 466 -101 244 -101 249 -101 251 -102 141 -102 237 -102 369 -102 243 -102 212 -102 121 -102 340 -102 411 -104 121 -104 230 -105 396 -105 309 -106 147 -106 308 -106 406 -106 444 -106 190 -109 199 -109 143 -109 387 -109 214 -110 267 -112 164 -113 195 -114 233 -114 482 -114 315 -114 412 -116 465 -117 296 -118 163 -119 188 -119 409 -120 297 -120 363 -120 398 -120 254 -121 160 -121 230 -121 492 -121 369 -121 212 -122 256 -122 316 -124 176 -125 394 -125 497 -125 179 -125 351 -125 479 -126 465 -126 290 -126 339 -129 491 -130 397 -131 464 -131 136 -132 383 -133 273 -133 232 -133 205 -133 241 -133 316 -134 203 -135 249 -135 354 -136 464 -136 474 -137 276 -138 419 -138 414 -138 158 -139 169 -140 200 -140 222 -140 350 -144 334 -146 363 -147 308 -147 285 -148 175 -149 456 -149 407 -151 422 -152 488 -152 233 -152 221 -152 351 -154 187 -154 358 -154 158 -155 199 -155 267 -155 221 -155 437 -156 332 -156 318 -156 310 -157 436 -157 301 -158 396 -158 187 -158 414 -159 430 -160 215 -161 193 -163 220 -164 297 -164 398 -164 220 -166 427 -167 201 -167 442 -167 170 -167 383 -168 319 -169 280 -169 270 -169 198 -170 353 -170 403 -170 269 -170 207 -170 211 -173 242 -174 194 -174 195 -174 362 -174 177 -174 250 -174 219 -174 412 -175 460 -175 246 -175 447 -176 291 -176 294 -176 366 -176 178 -176 182 -177 219 -179 345 -179 394 -181 372 -182 184 -182 429 -182 294 -184 429 -185 358 -188 280 -188 469 -189 215 -192 272 -192 443 -192 468 -192 287 -193 400 -195 488 -195 233 -196 389 -196 317 -197 321 -198 280 -199 387 -200 333 -200 222 -201 284 -203 311 -207 353 -207 329 -207 269 -207 240 -212 230 -212 341 -212 439 -213 349 -214 462 -215 226 -216 388 -216 236 -217 347 -218 465 -219 450 -219 394 -219 250 -219 298 -219 479 -220 297 -220 283 -221 267 -221 437 -222 443 -222 350 -225 236 -227 472 -227 321 -227 315 -227 412 -227 390 -228 424 -228 260 -230 341 -232 280 -232 241 -233 480 -233 412 -234 243 -234 340 -238 249 -238 354 -240 353 -241 280 -242 356 -243 385 -243 326 -244 251 -246 382 -249 354 -249 327 -249 466 -250 488 -250 479 -250 351 -252 376 -253 384 -253 269 -254 363 -254 302 -254 311 -258 353 -258 317 -258 477 -259 384 -259 405 -259 269 -260 424 -262 352 -262 346 -263 392 -263 300 -263 325 -264 438 -267 437 -268 413 -269 384 -269 383 -270 381 -270 431 -272 443 -272 350 -272 287 -273 316 -276 377 -278 390 -280 417 -281 376 -282 391 -283 289 -283 432 -285 308 -285 292 -288 325 -288 478 -289 401 -289 413 -291 325 -291 478 -292 328 -292 308 -297 398 -297 495 -302 453 -302 311 -304 493 -307 388 -308 328 -309 396 -311 453 -312 333 -312 494 -313 375 -314 487 -314 364 -314 380 -315 482 -315 412 -319 346 -321 456 -326 491 -326 435 -327 466 -328 440 -331 472 -332 408 -332 489 -333 468 -334 446 -337 380 -338 465 -341 400 -343 436 -345 394 -345 399 -347 392 -347 368 -349 364 -349 470 -351 496 -351 479 -353 393 -358 360 -358 396 -365 375 -366 478 -368 392 -372 409 -374 467 -378 411 -379 436 -380 456 -380 415 -382 433 -383 384 -394 481 -394 479 -396 449 -416 420 -416 422 -417 447 -418 446 -450 481 -456 475 -458 471 -463 470 -472 475 -496 497 - +1886 1356 +0 1 +2 3 +4 5 +4 6 +4 7 +4 8 +9 10 +9 11 +9 12 +9 13 +9 14 +15 16 +15 17 +15 18 +15 19 +15 20 +15 21 +15 22 +15 23 +15 24 +25 26 +25 27 +25 28 +25 29 +25 30 +25 31 +25 32 +25 33 +34 35 +34 36 +34 37 +34 38 +34 39 +34 40 +41 42 +41 43 +41 44 +45 46 +47 48 +47 49 +47 50 +51 52 +51 53 +51 54 +51 55 +51 56 +51 57 +51 58 +51 59 +51 60 +51 61 +62 63 +62 64 +62 65 +62 66 +62 67 +62 68 +62 69 +62 70 +62 71 +62 72 +62 73 +62 74 +75 76 +77 78 +79 80 +79 81 +82 83 +82 84 +82 85 +82 86 +82 87 +82 88 +82 89 +82 90 +91 92 +91 93 +91 94 +91 95 +91 96 +91 97 +98 99 +98 100 +98 101 +98 102 +98 103 +104 105 +104 106 +107 108 +109 110 +109 111 +109 112 +113 114 +115 116 +117 118 +117 119 +117 120 +117 121 +122 123 +124 125 +124 126 +124 127 +124 128 +129 130 +129 131 +132 133 +134 135 +136 137 +136 138 +139 140 +139 141 +142 143 +142 144 +142 145 +142 146 +142 147 +142 148 +142 149 +142 150 +151 152 +151 153 +154 155 +156 157 +156 158 +159 160 +159 161 +159 162 +163 164 +165 166 +167 168 +167 169 +167 170 +167 171 +172 173 +172 174 +175 176 +177 178 +179 180 +181 182 +181 183 +181 184 +185 186 +185 187 +155 188 +155 189 +155 190 +155 191 +155 192 +193 194 +195 196 +197 198 +197 199 +200 201 +202 203 +204 205 +204 206 +204 207 +204 208 +209 210 +209 211 +209 184 +212 213 +214 215 +216 217 +218 219 +218 220 +221 222 +221 223 +224 225 +226 227 +228 229 +228 230 +231 232 +231 233 +231 234 +231 235 +231 236 +231 237 +231 238 +213 239 +213 240 +203 241 +242 243 +242 244 +245 246 +247 248 +249 144 +249 250 +249 251 +249 252 +249 253 +249 254 +255 256 +255 257 +255 258 +255 259 +255 260 +261 262 +263 264 +263 265 +78 266 +78 267 +78 268 +78 269 +78 270 +271 272 +273 274 +273 275 +276 277 +278 279 +280 281 +282 283 +284 285 +284 286 +287 288 +289 290 +291 292 +293 294 +295 296 +295 297 +295 298 +299 300 +299 301 +299 302 +299 303 +299 304 +232 305 +232 306 +232 307 +232 308 +309 310 +309 311 +309 312 +313 314 +315 316 +315 317 +318 319 +318 320 +318 321 +318 322 +323 324 +325 326 +327 328 +327 329 +327 330 +327 331 +332 333 +334 335 +336 337 +336 338 +336 339 +336 340 +341 342 +341 343 +344 345 +346 347 +346 348 +346 349 +350 351 +352 353 +352 190 +352 354 +352 355 +356 357 +356 358 +359 360 +361 362 +363 364 +365 366 +367 250 +368 369 +368 370 +371 372 +373 374 +373 375 +373 376 +377 378 +377 379 +377 380 +381 382 +381 383 +384 385 +386 387 +386 388 +386 389 +386 390 +386 391 +386 392 +386 393 +386 394 +196 395 +196 396 +196 397 +196 398 +399 353 +399 400 +399 401 +399 402 +399 403 +404 405 +404 406 +404 407 +152 408 +152 409 +410 411 +410 36 +412 413 +412 201 +414 415 +414 416 +414 417 +418 419 +418 420 +421 422 +423 424 +425 426 +425 427 +425 428 +425 105 +425 429 +425 430 +431 432 +431 433 +431 434 +431 435 +431 436 +437 438 +437 312 +437 439 +440 49 +440 441 +440 442 +440 443 +440 444 +445 446 +445 447 +448 449 +448 450 +451 452 +451 453 +451 454 +451 455 +456 457 +458 459 +458 460 +458 461 +458 462 +458 463 +458 464 +385 465 +385 466 +467 468 +467 469 +467 470 +467 471 +472 473 +472 474 +472 475 +472 476 +472 477 +478 479 +478 480 +481 482 +481 483 +481 484 +485 486 +485 487 +485 488 +485 489 +490 491 +492 493 +492 494 +495 496 +497 498 +497 499 +168 500 +501 466 +502 503 +502 504 +505 506 +507 508 +509 510 +509 511 +512 513 +514 515 +514 516 +517 518 +519 389 +520 521 +522 439 +523 524 +378 525 +378 526 +378 527 +378 528 +529 530 +531 532 +531 533 +531 534 +531 535 +531 536 +537 538 +539 540 +539 541 +539 542 +335 543 +335 544 +545 546 +545 547 +545 298 +545 548 +188 549 +188 550 +188 312 +188 551 +188 552 +188 553 +554 555 +205 556 +557 558 +557 559 +557 560 +561 562 +563 564 +565 566 +567 568 +569 570 +569 571 +569 572 +569 573 +569 574 +569 575 +568 576 +568 577 +568 578 +568 579 +580 581 +582 583 +582 558 +584 125 +584 585 +584 586 +584 587 +584 588 +584 589 +590 591 +592 593 +592 594 +239 595 +596 597 +596 598 +596 599 +596 600 +596 601 +596 602 +596 603 +596 604 +605 606 +605 607 +605 608 +609 610 +611 612 +613 614 +615 616 +617 618 +617 619 +617 620 +617 621 +617 622 +623 624 +625 626 +627 628 +629 630 +629 631 +629 632 +629 633 +634 395 +634 635 +634 636 +634 637 +634 638 +634 639 +634 640 +634 641 +634 642 +634 643 +644 645 +646 647 +646 648 +646 649 +646 650 +646 651 +652 653 +652 654 +652 655 +652 656 +652 657 +652 658 +652 659 +449 133 +660 661 +662 663 +662 664 +665 447 +666 667 +666 668 +630 669 +630 310 +670 671 +670 672 +673 674 +673 675 +673 676 +677 678 +679 680 +679 681 +679 511 +679 682 +679 683 +684 685 +532 686 +532 687 +532 688 +689 690 +691 692 +693 694 +695 696 +695 697 +695 698 +699 700 +593 701 +702 703 +704 705 +704 292 +706 707 +504 708 +504 709 +710 711 +710 712 +710 713 +710 626 +353 714 +715 716 +717 718 +719 720 +719 407 +721 722 +721 723 +721 724 +725 726 +727 728 +727 729 +727 730 +727 731 +732 733 +732 734 +735 736 +737 738 +739 740 +741 742 +741 743 +741 744 +741 745 +746 747 +746 748 +746 749 +746 750 +751 752 +753 754 +755 756 +757 758 +759 760 +759 761 +762 763 +764 765 +764 766 +160 767 +160 768 +160 769 +770 771 +770 772 +773 774 +775 776 +775 777 +778 779 +778 780 +778 394 +778 781 +778 782 +783 784 +457 785 +786 787 +319 788 +319 789 +319 790 +144 791 +144 792 +793 794 +274 795 +382 796 +797 798 +799 800 +799 758 +799 801 +802 803 +576 804 +805 806 +805 807 +808 809 +810 811 +810 812 +813 814 +815 330 +815 816 +811 817 +818 819 +820 790 +821 822 +821 823 +824 825 +824 586 +296 826 +296 827 +296 828 +296 829 +296 830 +831 832 +833 314 +833 834 +806 835 +806 836 +837 838 +839 493 +839 840 +839 841 +842 843 +844 482 +845 518 +846 722 +846 847 +846 848 +849 391 +849 850 +851 852 +853 854 +855 856 +855 857 +229 858 +859 860 +861 862 +863 864 +865 866 +791 867 +791 868 +791 869 +870 871 +870 872 +873 874 +875 876 +875 877 +878 879 +878 880 +881 882 +881 883 +884 885 +884 886 +887 661 +887 888 +887 364 +887 889 +887 890 +887 891 +887 892 +887 893 +894 895 +894 896 +897 898 +897 899 +900 137 +900 901 +902 903 +902 904 +612 905 +612 906 +612 907 +908 909 +910 911 +910 912 +913 914 +913 915 +916 917 +918 919 +920 921 +360 922 +923 312 +923 924 +925 926 +927 928 +927 929 +930 931 +932 933 +934 935 +934 936 +937 938 +937 939 +707 940 +707 941 +707 942 +943 944 +943 945 +946 947 +946 948 +949 491 +949 26 +950 951 +950 952 +555 953 +954 955 +669 956 +957 958 +959 960 +961 962 +963 964 +965 966 +965 967 +968 969 +970 971 +970 972 +970 973 +243 974 +243 975 +976 912 +977 978 +222 979 +222 980 +947 981 +947 982 +983 984 +983 985 +986 987 +491 988 +491 989 +491 990 +991 992 +991 993 +994 995 +994 996 +997 998 +999 577 +999 1000 +992 1001 +726 1002 +1003 1004 +1005 1006 +1005 1007 +1008 1009 +1008 422 +1010 1011 +1010 1012 +1013 1014 +1015 1016 +1015 521 +1017 1018 +1019 1020 +1021 1022 +1023 1024 +1025 1026 +1025 1027 +1028 1029 +558 1030 +558 1031 +558 1032 +272 1033 +1034 1035 +1036 311 +1037 1038 +1039 1040 +251 1041 +1042 1043 +1044 1045 +1044 1046 +42 1047 +42 1048 +1049 1050 +1049 1051 +1052 290 +1053 1009 +1053 1054 +1055 951 +1056 1057 +1058 1059 +722 1060 +1061 1062 +1063 1064 +1063 1065 +1063 1066 +1063 1067 +1063 1068 +1069 1070 +1069 541 +1069 1071 +1072 1073 +1072 1074 +1075 1076 +281 5 +281 1077 +1078 1079 +1080 1081 +1080 1082 +1083 1084 +1085 1086 +1087 1088 +1087 1089 +1090 1091 +1091 1092 +895 1093 +1038 1094 +1095 1096 +1097 1030 +1097 108 +856 1098 +219 1099 +219 1100 +1101 1102 +1103 1104 +1103 1105 +1106 1107 +1106 1108 +10 1109 +606 1110 +401 1111 +401 1112 +401 1113 +401 298 +401 1114 +1115 1116 +1116 1117 +1118 1119 +1120 1121 +1122 1123 +549 1124 +1125 1126 +1009 279 +1127 1128 +1129 1130 +1131 736 +1132 1133 +479 1134 +879 1135 +765 426 +1136 1137 +1136 1138 +1139 1140 +1141 1142 +1143 760 +1144 626 +1145 1146 +305 1147 +1045 631 +533 736 +1002 1148 +1002 1149 +1002 1150 +1151 1152 +1153 1154 +1155 1156 +912 1157 +1158 1159 +1158 1160 +1161 1162 +1161 1163 +1161 1164 +847 1165 +1140 1166 +1140 1167 +1168 1169 +1168 1170 +1171 1172 +1173 1174 +1175 1176 +1177 1178 +1177 1179 +1180 1181 +530 1182 +1183 1184 +1185 1186 +1185 1187 +1185 1188 +1185 1189 +1190 1191 +1192 161 +1193 1194 +1195 1196 +1195 1051 +1197 564 +1198 267 +1199 1200 +740 1201 +1202 1203 +1204 1205 +1206 1207 +1206 1208 +1209 1210 +1070 1211 +1070 1212 +1213 1214 +1215 1216 +1030 1217 +1218 1219 +1220 1221 +728 290 +161 1222 +1027 752 +1223 838 +1223 1224 +1225 1226 +290 1227 +333 1228 +333 1229 +333 1230 +547 1231 +1232 1233 +598 1234 +1235 1236 +1235 1237 +1238 1239 +835 1240 +1241 1242 +1243 1244 +1245 1246 +210 1247 +210 1248 +1249 1250 +1249 106 +1251 1252 +1251 206 +1253 1254 +1255 1256 +1247 1257 +1247 1258 +1247 1259 +1260 1261 +1262 1263 +1262 1264 +1265 1266 +1267 67 +1268 560 +1269 1270 +1271 1272 +814 1273 +814 1274 +1275 1276 +1277 1278 +1277 1279 +1280 1281 +1282 1283 +1284 1285 +798 1286 +1287 1288 +1287 1289 +1290 1176 +1291 1292 +1293 1294 +1064 1295 +1296 867 +1296 1297 +800 1298 +581 1299 +1184 1300 +1184 1301 +118 252 +426 1302 +426 1303 +426 1304 +426 1305 +426 1306 +1307 1308 +697 1309 +697 1310 +1311 37 +1312 1313 +1314 1315 +1316 1317 +1316 1318 +1319 1320 +1321 466 +1322 1323 +1324 1325 +1326 1327 +1328 1329 +1330 1331 +1332 1333 +698 1334 +712 1335 +712 1336 +712 1337 +1338 1339 +1340 1341 +1342 1343 +1344 1345 +1346 1347 +1348 1349 +1350 1351 +1352 1353 +1086 1354 +1355 1356 +1355 1357 +1355 1358 +1359 1360 +1176 1361 +1176 1362 +1176 1363 +1176 1364 +1365 1366 +1367 1368 +1369 1370 +1371 1372 +1371 1373 +1374 1375 +1374 1376 +1377 1378 +1379 656 +1380 1381 +1382 1383 +922 1384 +1385 1386 +1387 1388 +1387 1389 +577 1390 +577 1391 +1392 37 +1393 1394 +1375 1395 +1396 1397 +1398 1399 +1400 1401 +1400 1402 +1403 1404 +1405 1406 +1407 1378 +146 1408 +1409 1410 +1411 1412 +1413 1414 +474 1415 +1416 1417 +1368 1418 +1419 1420 +1421 1422 +767 1423 +1424 1425 +1426 1427 +1428 1429 +1428 1430 +1431 1432 +1433 1434 +1435 1159 +1436 1437 +1438 1439 +1440 1441 +1442 1443 +1442 1444 +1445 1446 +1445 1447 +199 1448 +199 1449 +199 1450 +1159 1451 +1159 1452 +1453 59 +1454 1455 +1456 1457 +1458 1459 +1458 1460 +1458 1461 +1462 1463 +1464 482 +1465 1466 +1467 1468 +1469 1470 +1471 1472 +1378 1473 +1474 1475 +1476 1477 +1478 1420 +754 1479 +754 1480 +758 1481 +1482 331 +1254 1483 +1040 1484 +1040 1485 +1486 1487 +1486 1488 +1489 1490 +1491 1492 +1493 1494 +1495 1496 +1429 1497 +1498 1499 +1500 1501 +1502 1503 +1504 1505 +1506 1507 +1508 1509 +1510 1511 +1512 1513 +1512 1514 +1515 1516 +1517 1518 +1519 1520 +1521 1001 +1522 1523 +1299 1524 +1525 1526 +1092 1527 +1092 1528 +1529 1530 +1531 1532 +1531 1533 +1518 238 +1534 1535 +1536 1537 +1415 1538 +1539 1540 +1541 1542 +1543 1544 +1361 1545 +1361 1546 +1547 1548 +1549 766 +28 1550 +1551 1417 +1552 1001 +1439 1553 +1439 1554 +1555 1556 +1555 1557 +114 1558 +1559 1560 +1457 1561 +1562 1563 +1562 1564 +1565 1566 +1565 1331 +1567 1568 +1569 1570 +1571 1134 +1572 1573 +1523 1574 +743 3 +402 1575 +1576 1577 +1578 1579 +1580 1581 +928 1582 +862 1583 +1584 1585 +1586 637 +1587 1588 +1589 112 +1590 1591 +1592 1554 +1593 1594 +1595 1596 +1597 1598 +1597 1599 +1600 1351 +1351 1601 +1602 1603 +1604 1605 +1050 1606 +1050 1607 +1050 1608 +1050 1609 +1610 566 +1611 1612 +1613 1614 +892 1615 +1616 1617 +1618 184 +1113 1619 +1620 1621 +1622 1623 +1624 1625 +1626 1627 +1628 1629 +1073 1630 +1631 1632 +1633 1634 +655 1635 +1636 1637 +1258 1638 +1639 1317 +1640 573 +1641 1642 +1641 321 +1643 1644 +1645 1646 +1647 1648 +1649 1650 +1651 1652 +1653 246 +1654 1655 +671 1656 +1657 1658 +607 1659 +1660 995 +1660 1661 +1662 1663 +1664 1665 +1666 1667 +1668 237 +1669 1417 +1670 876 +1671 1672 +1673 1674 +1675 1676 +1677 1304 +1678 1679 +1680 1681 +1682 1683 +1001 1684 +1001 1685 +1686 761 +1687 901 +1688 1689 +1690 1691 +1692 1693 +1694 1695 +1696 1697 +1698 1699 +1700 1701 +1702 1703 +1399 1704 +227 1705 +1706 1707 +1708 1709 +1710 1711 +1710 1712 +1713 1714 +1715 1716 +1717 1718 +1719 1720 +936 1721 +1722 1723 +1724 1725 +1726 1727 +1728 659 +375 1729 +1730 1731 +1732 1733 +1734 1735 +1736 1737 +1738 1716 +1739 1740 +1741 1742 +1743 1744 +1745 1550 +1746 1731 +1746 1747 +1748 1749 +1750 1751 +1752 1200 +1753 1754 +1755 1756 +1757 1758 +1735 1759 +1760 1761 +1760 1762 +1763 1764 +1765 1766 +1767 1768 +1376 1769 +626 1770 +1771 668 +1772 1773 +1774 929 +1775 1776 +901 1777 +1778 1779 +1780 1781 +877 1782 +951 1783 +951 1784 +1785 1786 +1787 1788 +1789 480 +1789 150 +1790 1791 +1792 1793 +1051 659 +1607 1794 +1795 1796 +1656 1797 +1798 416 +1731 1799 +1800 812 +1801 1802 +1803 1804 +1805 1806 +112 750 +259 1807 +812 1783 +380 1808 +1809 1810 +1809 1811 +1812 1813 +1812 304 +1814 1815 +1766 1816 +1817 1818 +1608 1799 +1819 1820 +1821 1822 +1823 1824 +1825 1826 +1827 1828 +1829 1830 +1831 1832 +1833 1834 +1835 1836 +1837 1838 +1783 1839 +1716 1840 +1472 1841 +1842 106 +1843 38 +603 1844 +1845 1846 +1847 1848 +566 1849 +1850 1851 +1852 1853 +1585 1854 +1855 1856 +1077 1857 +1858 1859 +1860 1861 +1862 1863 +1423 1864 +1865 1866 +1867 1868 +1869 1870 +166 1871 +1872 1873 +1874 1875 +1634 1876 +1634 1875 +1877 1878 +1879 1880 +1881 1882 +1883 1811 +1884 1840 +1885 1777 Modified: SwiftApps/SciColSim/params/ARtest03 =================================================================== --- SwiftApps/SciColSim/params/ARtest03 2013-01-01 22:42:52 UTC (rev 6114) +++ SwiftApps/SciColSim/params/ARtest03 2013-01-01 23:37:48 UTC (rev 6115) @@ -1,10 +1,10 @@ -min_target_innovation 58 # starting target innovation value to try -max_target_innovation 59 # stops before this target innovation value -target_innovation_increment 50 # increment target innovation by this amount +min_target_innovation 135 # starting target innovation value to try +max_target_innovation 136 # stops before this target innovation value +target_innovation_increment 135 # increment target innovation by this amount annealing_repeats 1 # times to repeate the entire optimization process for each target_innovation -annealing_cycles 30 # times to perform the simulated annealing loop for all non-fixed parameters +annealing_cycles 5 # times to perform the simulated annealing loop for all non-fixed parameters evolve_reruns 48 # times to perform evolve_to_target_and_save() (objective function - is batched) nworkers 24 # number of parallel threads (cores) to use per invocation of C++ optimizer From wilde at ci.uchicago.edu Tue Jan 1 17:41:01 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 1 Jan 2013 17:41:01 -0600 (CST) Subject: [Swift-commit] r6116 - SwiftApps/SciColSim Message-ID: <20130101234101.231CF9CCFC@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-01 17:41:00 -0600 (Tue, 01 Jan 2013) New Revision: 6116 Added: SwiftApps/SciColSim/movie_graph.txt.orig Log: Preserve original test graph. Added: SwiftApps/SciColSim/movie_graph.txt.orig =================================================================== --- SwiftApps/SciColSim/movie_graph.txt.orig (rev 0) +++ SwiftApps/SciColSim/movie_graph.txt.orig 2013-01-01 23:41:00 UTC (rev 6116) @@ -0,0 +1,1010 @@ +500 1008 +0 56 +0 436 +0 446 +1 256 +1 476 +1 260 +1 262 +1 9 +1 266 +1 13 +1 142 +1 273 +1 388 +1 30 +1 36 +1 37 +1 423 +1 168 +1 425 +1 255 +1 175 +1 307 +1 58 +1 316 +1 319 +1 323 +1 68 +1 455 +1 216 +1 82 +1 88 +1 346 +1 348 +1 225 +1 228 +1 236 +1 365 +1 111 +1 424 +1 370 +1 246 +1 376 +1 122 +1 191 +1 252 +1 382 +1 127 +2 3 +3 262 +3 263 +3 392 +3 172 +3 138 +3 395 +3 13 +3 18 +3 150 +3 408 +3 154 +3 156 +3 30 +3 291 +3 37 +3 168 +3 300 +3 52 +3 158 +3 310 +3 265 +3 187 +3 61 +3 319 +3 65 +3 322 +3 54 +3 130 +3 71 +3 332 +3 461 +3 397 +3 419 +3 473 +3 346 +3 95 +3 352 +3 98 +3 103 +3 107 +3 414 +3 368 +3 498 +3 499 +4 224 +4 449 +4 419 +4 356 +4 72 +4 330 +4 173 +4 80 +4 242 +4 180 +4 24 +4 138 +5 33 +5 459 +5 484 +5 106 +5 299 +5 430 +5 16 +5 336 +5 50 +5 147 +5 406 +5 87 +5 91 +5 190 +6 7 +6 264 +6 137 +6 276 +6 149 +6 279 +6 153 +6 29 +6 415 +6 293 +6 166 +6 296 +6 42 +6 44 +6 48 +6 49 +6 438 +6 55 +6 314 +6 69 +6 454 +6 463 +6 84 +6 85 +6 470 +6 349 +6 94 +6 229 +6 233 +6 364 +6 117 +6 248 +6 380 +6 426 +7 129 +7 131 +7 470 +7 136 +7 471 +7 271 +7 402 +7 19 +7 404 +7 282 +7 286 +7 491 +7 35 +7 421 +7 166 +7 295 +7 427 +7 428 +7 301 +7 349 +7 436 +7 439 +7 56 +7 445 +7 446 +7 320 +7 123 +7 451 +7 69 +7 454 +7 78 +7 463 +7 212 +7 213 +7 342 +7 343 +7 474 +7 93 +7 483 +7 357 +7 102 +7 235 +7 237 +7 239 +7 371 +7 379 +8 112 +8 164 +8 10 +8 12 +8 398 +8 176 +8 120 +8 178 +8 373 +8 24 +8 124 +9 262 +10 11 +10 12 +10 47 +10 335 +10 124 +11 96 +11 289 +11 66 +11 283 +11 97 +11 361 +11 47 +11 432 +11 335 +11 215 +11 27 +11 124 +11 189 +12 112 +12 434 +12 47 +13 32 +13 130 +13 260 +13 37 +13 39 +13 202 +13 305 +13 461 +13 145 +13 275 +13 448 +13 54 +13 183 +13 344 +13 36 +14 387 +14 267 +14 155 +14 36 +14 165 +14 51 +14 308 +14 440 +14 441 +14 59 +14 444 +14 199 +14 328 +14 462 +14 81 +14 210 +14 214 +14 91 +14 99 +14 60 +14 106 +14 109 +14 110 +14 244 +15 193 +15 73 +15 400 +15 341 +15 27 +15 31 +16 33 +16 258 +16 353 +16 329 +16 204 +16 207 +16 336 +16 108 +16 50 +16 22 +16 119 +16 169 +16 381 +16 223 +17 321 +17 482 +17 197 +17 276 +17 149 +17 315 +17 94 +18 32 +18 461 +18 305 +18 498 +18 86 +18 25 +19 320 +19 421 +19 334 +19 314 +19 349 +19 446 +20 48 +20 28 +20 83 +20 44 +20 45 +21 288 +21 176 +21 347 +21 325 +21 392 +21 490 +21 366 +21 272 +21 209 +21 115 +21 182 +21 247 +21 184 +21 25 +21 217 +21 27 +21 86 +21 478 +22 353 +23 257 +23 100 +23 25 +24 134 +24 146 +24 302 +24 151 +24 290 +24 291 +24 43 +24 173 +24 46 +24 176 +24 180 +24 245 +24 453 +24 457 +24 208 +24 338 +24 363 +24 116 +24 373 +24 126 +25 32 +25 128 +25 257 +25 86 +26 161 +26 357 +26 193 +26 78 +26 341 +26 69 +27 361 +27 490 +27 333 +27 272 +27 312 +27 468 +27 41 +27 184 +27 429 +27 124 +27 341 +28 64 +28 354 +28 67 +28 135 +28 200 +28 42 +28 75 +28 45 +28 206 +28 48 +28 359 +28 53 +28 238 +29 229 +29 199 +29 296 +29 233 +29 42 +29 274 +29 117 +29 152 +29 155 +29 62 +30 386 +30 225 +30 76 +30 322 +30 367 +30 318 +31 200 +31 73 +31 333 +31 494 +31 45 +31 312 +32 257 +32 305 +32 101 +32 486 +32 39 +32 231 +32 79 +32 145 +32 275 +33 459 +33 336 +33 372 +33 119 +33 409 +34 283 +35 491 +35 243 +35 237 +35 326 +36 260 +36 266 +36 285 +36 292 +36 39 +36 424 +36 171 +36 181 +36 183 +36 440 +36 58 +36 323 +36 328 +36 81 +36 82 +36 87 +36 99 +36 228 +36 101 +36 111 +37 168 +37 202 +38 92 +39 448 +39 101 +39 486 +39 231 +39 145 +39 344 +40 345 +40 355 +40 179 +40 59 +40 125 +41 160 +41 226 +41 230 +41 104 +41 361 +41 492 +41 341 +41 215 +41 121 +41 189 +42 64 +42 67 +42 199 +42 359 +42 109 +42 206 +42 143 +42 48 +42 274 +42 117 +44 193 +44 69 +44 161 +44 73 +44 48 +44 83 +44 279 +45 200 +45 73 +45 83 +46 242 +46 485 +46 126 +46 173 +48 64 +48 206 +50 259 +50 261 +50 329 +50 299 +50 204 +50 430 +50 207 +50 405 +50 59 +50 159 +51 67 +51 101 +51 135 +51 210 +51 244 +51 214 +51 441 +51 249 +52 368 +52 217 +52 347 +52 86 +53 101 +53 327 +53 75 +53 238 +53 79 +53 249 +54 130 +55 84 +57 162 +57 331 +58 266 +58 82 +59 384 +59 259 +59 132 +59 267 +59 496 +59 355 +59 152 +59 91 +59 221 +59 159 +60 91 +60 165 +62 152 +62 155 +62 221 +63 456 +63 334 +63 303 +63 472 +63 314 +63 380 +65 154 +65 499 +65 374 +65 103 +66 96 +66 160 +66 215 +67 359 +67 135 +67 109 +67 143 +67 214 +69 161 +69 166 +69 239 +69 404 +69 85 +69 279 +69 153 +70 410 +70 467 +71 130 +72 138 +73 193 +74 241 +74 82 +74 188 +74 133 +74 469 +75 140 +75 79 +75 90 +75 350 +76 386 +77 120 +77 306 +77 452 +77 254 +77 311 +78 239 +78 357 +79 257 +79 100 +79 101 +80 138 +81 99 +81 171 +82 256 +82 133 +82 348 +82 205 +82 372 +82 181 +82 119 +82 409 +82 255 +82 316 +82 122 +82 469 +84 277 +84 264 +84 149 +84 380 +84 415 +85 153 +85 166 +86 247 +86 217 +87 181 +87 147 +87 285 +89 192 +89 200 +89 333 +89 468 +89 443 +89 222 +90 272 +90 100 +90 350 +91 165 +91 106 +91 430 +91 406 +91 159 +92 258 +93 136 +93 464 +93 493 +93 326 +94 482 +94 233 +94 114 +94 276 +95 107 +95 265 +95 186 +95 332 +96 289 +96 324 +96 102 +96 401 +97 283 +97 220 +97 434 +97 164 +99 101 +99 171 +99 244 +99 251 +100 272 +101 231 +101 466 +101 244 +101 249 +101 251 +102 141 +102 237 +102 369 +102 243 +102 212 +102 121 +102 340 +102 411 +104 121 +104 230 +105 396 +105 309 +106 147 +106 308 +106 406 +106 444 +106 190 +109 199 +109 143 +109 387 +109 214 +110 267 +112 164 +113 195 +114 233 +114 482 +114 315 +114 412 +116 465 +117 296 +118 163 +119 188 +119 409 +120 297 +120 363 +120 398 +120 254 +121 160 +121 230 +121 492 +121 369 +121 212 +122 256 +122 316 +124 176 +125 394 +125 497 +125 179 +125 351 +125 479 +126 465 +126 290 +126 339 +129 491 +130 397 +131 464 +131 136 +132 383 +133 273 +133 232 +133 205 +133 241 +133 316 +134 203 +135 249 +135 354 +136 464 +136 474 +137 276 +138 419 +138 414 +138 158 +139 169 +140 200 +140 222 +140 350 +144 334 +146 363 +147 308 +147 285 +148 175 +149 456 +149 407 +151 422 +152 488 +152 233 +152 221 +152 351 +154 187 +154 358 +154 158 +155 199 +155 267 +155 221 +155 437 +156 332 +156 318 +156 310 +157 436 +157 301 +158 396 +158 187 +158 414 +159 430 +160 215 +161 193 +163 220 +164 297 +164 398 +164 220 +166 427 +167 201 +167 442 +167 170 +167 383 +168 319 +169 280 +169 270 +169 198 +170 353 +170 403 +170 269 +170 207 +170 211 +173 242 +174 194 +174 195 +174 362 +174 177 +174 250 +174 219 +174 412 +175 460 +175 246 +175 447 +176 291 +176 294 +176 366 +176 178 +176 182 +177 219 +179 345 +179 394 +181 372 +182 184 +182 429 +182 294 +184 429 +185 358 +188 280 +188 469 +189 215 +192 272 +192 443 +192 468 +192 287 +193 400 +195 488 +195 233 +196 389 +196 317 +197 321 +198 280 +199 387 +200 333 +200 222 +201 284 +203 311 +207 353 +207 329 +207 269 +207 240 +212 230 +212 341 +212 439 +213 349 +214 462 +215 226 +216 388 +216 236 +217 347 +218 465 +219 450 +219 394 +219 250 +219 298 +219 479 +220 297 +220 283 +221 267 +221 437 +222 443 +222 350 +225 236 +227 472 +227 321 +227 315 +227 412 +227 390 +228 424 +228 260 +230 341 +232 280 +232 241 +233 480 +233 412 +234 243 +234 340 +238 249 +238 354 +240 353 +241 280 +242 356 +243 385 +243 326 +244 251 +246 382 +249 354 +249 327 +249 466 +250 488 +250 479 +250 351 +252 376 +253 384 +253 269 +254 363 +254 302 +254 311 +258 353 +258 317 +258 477 +259 384 +259 405 +259 269 +260 424 +262 352 +262 346 +263 392 +263 300 +263 325 +264 438 +267 437 +268 413 +269 384 +269 383 +270 381 +270 431 +272 443 +272 350 +272 287 +273 316 +276 377 +278 390 +280 417 +281 376 +282 391 +283 289 +283 432 +285 308 +285 292 +288 325 +288 478 +289 401 +289 413 +291 325 +291 478 +292 328 +292 308 +297 398 +297 495 +302 453 +302 311 +304 493 +307 388 +308 328 +309 396 +311 453 +312 333 +312 494 +313 375 +314 487 +314 364 +314 380 +315 482 +315 412 +319 346 +321 456 +326 491 +326 435 +327 466 +328 440 +331 472 +332 408 +332 489 +333 468 +334 446 +337 380 +338 465 +341 400 +343 436 +345 394 +345 399 +347 392 +347 368 +349 364 +349 470 +351 496 +351 479 +353 393 +358 360 +358 396 +365 375 +366 478 +368 392 +372 409 +374 467 +378 411 +379 436 +380 456 +380 415 +382 433 +383 384 +394 481 +394 479 +396 449 +416 420 +416 422 +417 447 +418 446 +450 481 +456 475 +458 471 +463 470 +472 475 +496 497 + From swift at ci.uchicago.edu Tue Jan 1 22:45:06 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Tue, 1 Jan 2013 22:45:06 -0600 (CST) Subject: [Swift-commit] cog r3535 Message-ID: <20130102044506.F366A8D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3535 | davidkelly999 | 2013-01-01 22:43:29 -0600 (Tue, 01 Jan 2013) | 2 lines A real slurm provider that uses sbatch/srun, scancel, and squeue directly ------------------------------------------------------------------------ Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java (revision 3534) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java (working copy) @@ -4,9 +4,7 @@ import java.io.Writer; import java.text.DecimalFormat; import java.text.NumberFormat; -import java.util.Collection; import java.util.Date; -import java.util.Iterator; import org.apache.log4j.Logger; import org.globus.cog.abstraction.impl.common.execution.WallTime; @@ -22,336 +20,189 @@ public class SlurmExecutor extends AbstractExecutor { public static final Logger logger = Logger.getLogger(SlurmExecutor.class); - /** - Number of program invocations - */ - int count = 1; - - /** - PBS processes-per-node - */ - int ppn = 1; - - /** - PBS mppdepth: number of available threads per node - */ - int depth = 1; - - /** - Unique number for automatic task names - */ - private static int unique = 0; + // Number of program invocations + private int count = 1; + private static NumberFormat IDF = new DecimalFormat("000000"); + // Used for task name generation + private static int unique = 0; + public SlurmExecutor(Task task, ProcessListener listener) { super(task, listener); } - private static NumberFormat IDF = new DecimalFormat("000000"); - - /** - The job name is limited to 15 characters: - http://doesciencegrid.org/public/pbs/qsub.html + /** + * Write attribute if non-null + * @throws IOException */ - protected void validate(Task task) { - String name = task.getName(); - if (name == null) { - int i = 0; - synchronized(SlurmExecutor.class) { - i = unique++; - } - name = "cog-" + IDF.format(i); - if (logger.isDebugEnabled()) { - logger.debug("Slurm name: for: " + task.getIdentity() + - " is: " + name); - } - } - else if (name.length() > 15) { - task.setName(name.substring(0, 15)); - } - } - - /** - Write attribute if non-null - @throws IOException - */ protected void writeAttr(String attrName, String arg, Writer wr) - throws IOException { + throws IOException { Object value = getSpec().getAttribute(attrName); if (value != null) { - wr.write("#PBS " + arg + String.valueOf(value) + '\n'); + wr.write("#SBATCH " + arg + "=" + String.valueOf(value) + '\n'); } } - /** - Write attribute if non-null and non-empty - @throws IOException + /** + * Write attribute if non-null and non-empty + * @throws IOException */ - protected void writeNonEmptyAttr(String attrName, String arg, - Writer wr) - throws IOException { + protected void writeNonEmptyAttr(String attrName, String arg, Writer wr) + throws IOException { Object value = getSpec().getAttribute(attrName); if (value != null) { String v = String.valueOf(value); - if (v.length() > 0 ) - wr.write("#PBS " + arg + v + '\n'); + if (v.length() > 0) + wr.write("#SBATCH " + arg + "=" + v + '\n'); } } - + + /** + * Write walltime in hh:mm:ss + * @param wr + * @throws IOException + */ protected void writeWallTime(Writer wr) throws IOException { Object walltime = getSpec().getAttribute("maxwalltime"); if (walltime != null) { - wr.write("#PBS -l walltime=" + wr.write("#SBATCH --time=" + WallTime.normalize(walltime.toString(), "pbs-native") + '\n'); } } - private int parseAndValidateInt(Object obj, String name) { - try { - assert(obj != null); - return Integer.parseInt(obj.toString()); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("Illegal value for " + name + ". Must be an integer."); - } + /** + * Ensure tasks have a valid name + */ + protected void validate(Task task) { + String name = task.getName(); + if (name == null) { + int i = 0; + synchronized (SlurmExecutor.class) { + i = unique++; + } + name = "cog-" + IDF.format(i); + task.setName(name); + } else if (name.length() > 15) { + task.setName(name.substring(0, 15)); + } + if (logger.isDebugEnabled()) { + logger.debug("Slurm name: for: " + task.getIdentity() + " is: " + name); + } } - + /** - Obtains profile settings regarding job size from - JobSpecification and writes them into the PBS file. - Looks for profiles count, ppn, ppts, and pbs.mpp - count: mandatory, default 1 (number of processes) - depth: default 1 (number of threads per node) - ppn: optional, default 1 (processes per node) - pbs.mpp: output mppwidth/mppnppn instead of nodes/ppn - pbs.properties: extra PBS properties - pbs.resource_list: extra PBS -l line - - Note that the semantics are different for the pbs.mpp setting: - mppwidth is the total number of cores while nodes is the number - of nodes. - - http://www.clusterresources.com/torquedocs/2.1jobsubmission.shtml - @return true if this is a multi-core job + * Verify that an object contains a valid int + * @param obj + * @param name + * @return */ - protected boolean writeCountAndPPN(JobSpecification spec, - Writer wr) - throws IOException { - boolean result = false; - - Object o; - - // Number of program invocations - o = getSpec().getAttribute("count"); - if (o != null) - count = parseAndValidateInt(o, "count"); - if (count != 1) - result = true; - - o = spec.getAttribute("ppn"); - if (o != null) - ppn = parseAndValidateInt(o, "ppn"); - - o = spec.getAttribute("depth"); - if (o != null) - depth = parseAndValidateInt(o, "depth"); - - String slurmProperties = - (String) getSpec().getAttribute("slurm.properties"); - - boolean mpp = false; - if (spec.getAttribute("pbs.mpp") != null) - mpp = true; - - StringBuilder sb = new StringBuilder(512); - sb.append("#PBS -l "); - if (mpp) { - sb.append("mppwidth=").append(count); - sb.append(","); - sb.append("mppnppn=").append(ppn); - sb.append(","); - sb.append("mppdepth=").append(depth); - } - else { - sb.append("nodes="); - sb.append(count); - sb.append(":"); - sb.append("ppn="); - sb.append(ppn); - } - - if (slurmProperties != null && - slurmProperties.length() > 0 ) { - sb.append(":"); - sb.append(slurmProperties); - } - - sb.append('\n'); - - wr.write(sb.toString()); - - return result; + private int parseAndValidateInt(Object obj, String name) { + try { + assert (obj != null); + return Integer.parseInt(obj.toString()); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("Illegal value for " + name + ". Must be an integer."); + } } - /* - private boolean parseAndValidateBool(Object obj, String name) - { - try { - return Boolean.parseBoolean(obj.toString()); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException - ("Illegal value for " + name + ". Must be true/false."); - } - } - */ - @Override - protected void writeScript(Writer wr, String exitcodefile, String stdout, - String stderr) - throws IOException { + protected void writeScript(Writer wr, String exitcodefile, String stdout, String stderr) + throws IOException { Task task = getTask(); JobSpecification spec = getSpec(); Properties properties = Properties.getProperties(); + validate(task); + writeHeader(wr); - getSpec().unpackProviderAttributes(); - - validate(task); - writeHeader(wr); - - wr.write("#PBS -S /bin/bash\n"); - wr.write("#PBS -N " + task.getName() + '\n'); - wr.write("#PBS -m n\n"); - writeNonEmptyAttr("project", "-A ", wr); - boolean multiple = writeCountAndPPN(spec, wr); + Object countValue = getSpec().getAttribute("count"); + if (countValue != null) + count = parseAndValidateInt(countValue, "count"); + wr.write("#SBATCH --job-name=" + task.getName() + '\n'); + wr.write("#SBATCH --output=" + quote(stdout) + '\n'); + wr.write("#SBATCH --error=" + quote(stderr) + '\n'); + wr.write("#SBATCH --nodes=" + count + '\n'); + wr.write("#SBATCH --exclusive\n"); + wr.write("#SBATCH --ntasks-per-node=1\n"); + writeNonEmptyAttr("ppn", "--cpus-per-task", wr); + writeNonEmptyAttr("project", "--account", wr); + writeNonEmptyAttr("queue", "--partition", wr); writeWallTime(wr); - writeNonEmptyAttr("queue", "-q ", wr); - wr.write("#PBS -o " + quote(stdout) + '\n'); - wr.write("#PBS -e " + quote(stderr) + '\n'); + wr.write("\n"); for (String name : spec.getEnvironmentVariableNames()) { - // "export" is necessary on the Cray XT5 Crow - wr.write("export "); - wr.write(name); - wr.write('='); - wr.write(quote(spec.getEnvironmentVariable(name))); - wr.write('\n'); + wr.write("export " + name + '=' + quote(spec.getEnvironmentVariable(name)) + '\n'); } - if (spec.getEnvironmentVariableNames().size() > 0) { - wr.write("#PBS -v " + makeList(spec.getEnvironmentVariableNames()) + '\n'); + // Handle all slurm attributes specified by the user + for (String a : spec.getAttributeNames()) { + if (a != null && a.startsWith("slurm.")) { + String attributeName[] = a.split("slurm."); + wr.write("#SBATCH --" + attributeName[1] + " = " + spec.getAttribute(a) + '\n'); + } } - String resources = - (String) spec.getAttribute("slurm.resource_list"); - if (resources != null && resources.length() > 0) { - if (logger.isDebugEnabled()) - logger.debug("slurm.resource_list: " + resources); - wr.write("#PBS -l " + resources + '\n'); - } - - // aprun option specifically for Cray Beagle, Franklin - boolean aprun = false; - if (spec.getAttribute("pbs.aprun") != null) - aprun = true; - String type = (String) spec.getAttribute("jobType"); if (logger.isDebugEnabled()) logger.debug("Job type: " + type); - if ("multiple".equals(type)) - multiple = true; - else if("single".equals(type)) - multiple = false; - if (aprun) - multiple = false; - if (multiple) - writeMultiJobPreamble(wr, exitcodefile); if (type != null) { - String wrapper = - properties.getProperty("wrapper." + type); + String wrapper = properties.getProperty("wrapper." + type); if (logger.isDebugEnabled()) { logger.debug("Wrapper: " + wrapper); } + if (wrapper != null) { wrapper = replaceVars(wrapper); wr.write(wrapper); wr.write(' '); } - if (logger.isDebugEnabled()) { + + if (logger.isDebugEnabled()) { logger.debug("Wrapper after variable substitution: " + wrapper); } } + + wr.write("srun /bin/bash -c \'"); if (spec.getDirectory() != null) { wr.write("cd " + quote(spec.getDirectory()) + " && "); } - if (aprun) - wr.write("aprun -n " + count + " -N 1 -cc none -d " + - depth + " -F exclusive /bin/sh -c '"); - wr.write(quote(spec.getExecutable())); writeQuotedList(wr, spec.getArgumentsAsList()); - if (aprun) - wr.write("'"); - if (spec.getStdInput() != null) { - wr.write(" < " + quote(spec.getStdInput())); - } - if (multiple) { - writeMultiJobPostamble(wr); + wr.write(" < " + quote(spec.getStdInput())); } - else { - wr.write('\n'); - wr.write("/bin/echo $? >" + exitcodefile + '\n'); + + if("multiple".equals(type)) { + wr.write("; /bin/echo $? >" + exitcodefile + ".$SLURM_PROCID\'\n"); + } else { + wr.write("; /bin/echo $? >" + exitcodefile + "\'\n"); } wr.close(); } - void writeHeader(Writer writer) - throws IOException { + void writeHeader(Writer writer) throws IOException { + writer.write("#!/bin/bash\n\n"); writer.write("#CoG This script generated by CoG\n"); writer.write("#CoG by class: " + SlurmExecutor.class + '\n'); writer.write("#CoG on date: " + new Date() + "\n\n"); } - - - private String makeList(Collection names) { - StringBuilder sb = new StringBuilder(); - Iterator i = names.iterator(); - while (i.hasNext()) { - sb.append(i.next()); - if (i.hasNext()) { - sb.append(", "); - } - } - return sb.toString(); - } - protected void writeMultiJobPreamble(Writer wr, String exitcodefile) - throws IOException { - wr.write("NODES=`cat $PBS_NODEFILE`\n"); - wr.write("ECF=" + exitcodefile + "\n"); - wr.write("INDEX=0\n"); - wr.write("for NODE in $NODES; do\n"); - wr.write(" echo \"N\" >$ECF.$INDEX\n"); - wr.write(" ssh $NODE /bin/bash -c \\\" \""); - } - - @Override -protected String getName() { - return "PBS"; + protected String getName() { + return "Slurm"; } @Override -protected AbstractProperties getProperties() { + protected AbstractProperties getProperties() { return Properties.getProperties(); } @Override -protected Job createJob(String jobid, String stdout, + protected Job createJob(String jobid, String stdout, FileLocation stdOutputLocation, String stderr, FileLocation stdErrorLocation, String exitcode, AbstractExecutor executor) { @@ -362,8 +213,8 @@ private static QueuePoller poller; @Override -protected AbstractQueuePoller getQueuePoller() { - synchronized(SlurmExecutor.class) { + protected AbstractQueuePoller getQueuePoller() { + synchronized (SlurmExecutor.class) { if (poller == null) { poller = new QueuePoller(getProperties()); poller.start(); @@ -371,4 +222,14 @@ return poller; } } -} + + protected String parseSubmitCommandOutput(String out) throws IOException { + if ("".equals(out)) { + throw new IOException(getProperties().getSubmitCommandName() + + " returned an empty job ID"); + } + String outArray[] = out.split(" "); + return outArray[outArray.length-1].trim(); + } + +} \ No newline at end of file Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/QueuePoller.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/QueuePoller.java (revision 3534) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/QueuePoller.java (working copy) @@ -8,6 +8,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import org.apache.log4j.Logger; @@ -19,142 +20,96 @@ public static final Logger logger = Logger.getLogger(QueuePoller.class); public static final int FULL_LIST_THRESHOLD = 16; - private Set processed; + private Set processed; public QueuePoller(AbstractProperties properties) { - super("PBS provider queue poller", properties); - processed = new HashSet(); + super("Slurm provider queue poller", properties); + processed = new HashSet(); } private static String[] CMDARRAY; protected synchronized String[] getCMDArray() { - if (getJobs().size() <= FULL_LIST_THRESHOLD) { - String[] cmda = new String[2 + getJobs().size()]; - cmda[0] = getProperties().getPollCommand(); - cmda[1] = "-f"; - int i = 2; - for (Job j : getJobs().values()) { - cmda[i++] = j.getJobID(); - } - return cmda; - } - else { - if (CMDARRAY == null) { - CMDARRAY = new String[] { getProperties().getPollCommand(), "-f" }; - } - } + if(CMDARRAY == null) { + if (getJobs().size() <= FULL_LIST_THRESHOLD) { + CMDARRAY = new String[4]; + CMDARRAY[0] = getProperties().getPollCommand(); + CMDARRAY[1] = "--noheader"; + CMDARRAY[2] = "--jobs"; + boolean first=true; + for (Job j : getJobs().values()) { + if(first) { + CMDARRAY[3] = j.getJobID(); + first=false; + } else { + CMDARRAY[3] += "," + j.getJobID(); + } + } + } else { + CMDARRAY = new String[] { getProperties().getPollCommand(), "--noheader" }; + } + } + return CMDARRAY; } @Override - protected int getError(int ec, String stderr) { - if (ec != 0) { - BufferedReader sr = new BufferedReader(new StringReader(stderr)); - try { - String line = sr.readLine(); - while (line != null) { - if (!line.contains("Unknown Job Id")) { - return ec; - } - line = sr.readLine(); - } - } - catch (IOException e) { - // should not occur while reading from a string reader - e.printStackTrace(); - } - return 0; - } - else { - return ec; - } - } + protected int getError(int ec, String stderr) { + if (ec != 0) { + BufferedReader sr = new BufferedReader(new StringReader(stderr)); + try { + String line = sr.readLine(); + while (line != null) { + if (!line.contains("Unknown Job Id")) { + return ec; + } + line = sr.readLine(); + } + } catch (IOException e) { + // should not occur while reading from a string reader + e.printStackTrace(); + } + return 0; + } else { + return ec; + } + } - protected void processStdout(InputStream is) throws IOException { + protected void processStdout(InputStream is) throws IOException { + BufferedReader br = new BufferedReader(new InputStreamReader(is)); - processed.clear(); String line; - String currentJobID = null; - Job currentJob = null; + processed.clear(); + do { line = br.readLine(); - if (line != null) { - try { - line = line.trim(); - if (line.startsWith("Job Id: ")) { - currentJobID = line.substring("Job Id: ".length()); - processed.add(currentJobID); - currentJob = getJob(currentJobID); - continue; - } - if (currentJob != null) { - if (line.startsWith("job_state = ")) { - if (logger.isDebugEnabled()) { - logger.debug("Status line: " + line); - } - switch (line.substring("job_state = ".length()) - .charAt(0)) { - case 'Q': { - if (logger.isDebugEnabled()) { - logger.debug("Status for " - + currentJobID + " is Q"); - } - currentJob.setState(Job.STATE_QUEUED); - break; - } - case 'R': { - if (logger.isDebugEnabled()) { - logger.debug("Status for " - + currentJobID + " is R"); - } - currentJob.setState(Job.STATE_RUNNING); - break; - } - case 'C': { - // for sites where keep_completed is there, - // don't wait - // for the job to be removed from the queue - if (logger.isDebugEnabled()) { - logger.debug("Status for " - + currentJobID + " is C"); - } - addDoneJob(currentJob.getJobID()); - break; - } - } - } - else if (line.startsWith("exit_status = ")) { - try { - int ec = Integer.parseInt(line.substring( - "exit_status = ".length()).trim()); - currentJob.setExitcode(ec); - } - catch (Exception e) { - if (logger.isDebugEnabled()) { - logger.debug("Could not parse exit_status", - e); - } - } - } - } + if(line != null) { + String words[] = line.split("\\s+"); + String jobid = words[0].trim(); + String state = words[4].trim(); + if (jobid == null || jobid.equals("") || state == null || state.equals("")) { + throw new IOException("Failed to parse squeue line: " + line); } - catch (Exception e) { - logger.warn("Exception caught while handling " - + getProperties().getPollCommandName() - + " output: " + line, e); + + Job job = getJob(jobid); + if (job == null){ continue; } + processed.add(jobid); + + if (state.equals("PD")) { + job.setState(Job.STATE_QUEUED); } + else if(state.equals("R")) { + job.setState(Job.STATE_RUNNING); + } } } while (line != null); - Iterator i = getJobs().entrySet().iterator(); + + Iterator> i = getJobs().entrySet().iterator(); while (i.hasNext()) { - Map.Entry e = (Map.Entry) i.next(); - String id = (String) e.getKey(); + Map.Entry e = i.next(); + String id = e.getKey(); if (!processed.contains(id)) { - Job job = (Job) e.getValue(); - if (logger.isDebugEnabled()) { - logger.debug("Status for " + id + " is Done"); - } + Job job = e.getValue(); job.setState(Job.STATE_DONE); if (job.getState() == Job.STATE_DONE) { addDoneJob(id); @@ -162,7 +117,7 @@ } } } - + protected void processStderr(InputStream is) throws IOException { } } Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/Properties.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/Properties.java (revision 3534) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/Properties.java (working copy) @@ -8,10 +8,9 @@ public static final String PROPERTIES = "provider-slurm.properties"; public static final String POLL_INTERVAL = "poll.interval"; - public static final String QSUB = "qsub"; - public static final String QSTAT = "qstat"; - public static final String QDEL = "qdel"; - public static final String USE_MPPWIDTH = "use.mppwidth"; + public static final String SBATCH = "sbatch"; + public static final String SQUEUE = "squeue"; + public static final String SCANCEL = "scancel"; private static Properties properties; @@ -25,23 +24,23 @@ protected void setDefaults() { setPollInterval(5); - setSubmitCommand("qsub"); - setPollCommand("qstat"); - setRemoveCommand("qdel"); + setSubmitCommand("sbatch"); + setPollCommand("squeue"); + setRemoveCommand("scancel"); } public String getPollCommandName() { - return QSTAT; + return SQUEUE; } public String getRemoveCommandName() { - return QDEL; + return SCANCEL; } public String getSubmitCommandName() { - return QSUB; + return SBATCH; } } From swift at ci.uchicago.edu Tue Jan 1 22:50:06 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Tue, 1 Jan 2013 22:50:06 -0600 (CST) Subject: [Swift-commit] cog r3536 Message-ID: <20130102045007.039938D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3536 | davidkelly999 | 2013-01-01 22:45:28 -0600 (Tue, 01 Jan 2013) | 2 lines Updated slurm properties ------------------------------------------------------------------------ Index: modules/provider-localscheduler/etc/provider-slurm.properties =================================================================== --- modules/provider-localscheduler/etc/provider-slurm.properties (revision 3535) +++ modules/provider-localscheduler/etc/provider-slurm.properties (working copy) @@ -6,28 +6,17 @@ poll.interval=5 # -# The path to qsub. The default assumes that qsub is in PATH +# The path to sbatch. The default assumes that sbatch is in PATH # -qsub=slurm-qsub +sbatch=sbatch # -# The path to qstat. The default assumes that qstat is in PATH +# The path to squeue. The default assumes that squeue is in PATH # -qstat=slurm-qstat +squeue=squeue # -# The path to qdel. The default assumes that qdel is in PATH +# The path to scancel. The default assumes that scancel is in PATH # -qdel=slurm-qdel +scancel=scancel -# If true, use "#PBS -l mppwidth=" instead of "#PBS -l nodes=" -# in PBS script -use.mppwidth=false - -# If the jobType attribute is specified, then the PBS provider -# will look for a property named "wrapper." and prepend -# that to the executable line in the PBS script. It will also -# substitute value of attributes in the job specification, using -# the "$attrName" notation. -# -wrapper.mpi=mpirun -np $count From swift at ci.uchicago.edu Wed Jan 2 00:20:04 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 2 Jan 2013 00:20:04 -0600 (CST) Subject: [Swift-commit] cog r3537 Message-ID: <20130102062005.0E65B8D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3537 | davidkelly999 | 2013-01-02 00:15:17 -0600 (Wed, 02 Jan 2013) | 2 lines Fix for user defined slurm commands ------------------------------------------------------------------------ Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java (revision 3536) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java (working copy) @@ -129,20 +129,22 @@ writeNonEmptyAttr("project", "--account", wr); writeNonEmptyAttr("queue", "--partition", wr); writeWallTime(wr); - wr.write("\n"); - - for (String name : spec.getEnvironmentVariableNames()) { - wr.write("export " + name + '=' + quote(spec.getEnvironmentVariable(name)) + '\n'); - } - + // Handle all slurm attributes specified by the user for (String a : spec.getAttributeNames()) { if (a != null && a.startsWith("slurm.")) { String attributeName[] = a.split("slurm."); - wr.write("#SBATCH --" + attributeName[1] + " = " + spec.getAttribute(a) + '\n'); + wr.write("#SBATCH --" + attributeName[1] + "=" + spec.getAttribute(a) + '\n'); } } + + wr.write("\n"); + for (String name : spec.getEnvironmentVariableNames()) { + wr.write("export " + name + '=' + quote(spec.getEnvironmentVariable(name)) + '\n'); + } + + String type = (String) spec.getAttribute("jobType"); if (logger.isDebugEnabled()) logger.debug("Job type: " + type); From swift at ci.uchicago.edu Wed Jan 2 11:50:05 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 2 Jan 2013 11:50:05 -0600 (CST) Subject: [Swift-commit] cog r3538 Message-ID: <20130102175006.0E7598D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3538 | davidkelly999 | 2013-01-02 11:48:23 -0600 (Wed, 02 Jan 2013) | 2 lines LSF provider ------------------------------------------------------------------------ Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/QueuePoller.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/QueuePoller.java (revision 0) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/QueuePoller.java (revision 3538) @@ -0,0 +1,148 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +package org.globus.cog.abstraction.impl.scheduler.lsf; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; + +import org.apache.log4j.Logger; +import org.globus.cog.abstraction.impl.scheduler.common.AbstractProperties; +import org.globus.cog.abstraction.impl.scheduler.common.AbstractQueuePoller; +import org.globus.cog.abstraction.impl.scheduler.common.Job; + +public class QueuePoller extends AbstractQueuePoller { + public static final Logger logger = Logger.getLogger(QueuePoller.class); + public static final int FULL_LIST_THRESHOLD = 16; + + private Set processed; + + public QueuePoller(AbstractProperties properties) { + super("LSF provider queue poller", properties); + processed = new HashSet(); + } + + private static String[] CMDARRAY; + + protected synchronized String[] getCMDArray() { + if (CMDARRAY == null) { + CMDARRAY = new String[] { getProperties().getPollCommand(), "-a" }; + } + return CMDARRAY; + } + + @Override + protected int getError(int ec, String stderr) { + if (ec != 0) { + BufferedReader sr = new BufferedReader(new StringReader(stderr)); + try { + String line = sr.readLine(); + while (line != null) { + if (!line.contains("is not found")) { + return ec; + } + line = sr.readLine(); + } + } + catch (IOException e) { + // should not occur while reading from a string reader + e.printStackTrace(); + } + return 0; + } + else { + return ec; + } + } + + protected void processStdout(InputStream is) throws IOException { + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + String line; + String header = br.readLine(); + if (header == null) { + logger.warn("Failed to read bjobs header"); + return; + } + int jobIDIndex = header.indexOf("JOBID"); + int stateIndex = header.indexOf("STAT"); + int locationIndex = header.indexOf("QUEUE"); + + if (jobIDIndex == -1 || stateIndex == -1 || locationIndex == -1) { + throw new IOException("Invalid bjobs header: " + header); + } + + processed.clear(); + + do { + line = br.readLine(); + if (line != null) { + String jobid = parseToWhitespace(line, jobIDIndex); + String state = parseToWhitespace(line, stateIndex); + if (jobid == null || jobid.equals("") || state == null + || state.equals("")) { + throw new IOException("Failed to parse bjobs line: " + + line); + } + + Job job = getJob(jobid); + if (job == null){ continue; } + processed.add(jobid); + if (state.equals("PEND")) { + if (logger.isDebugEnabled()) { + logger.debug("Status for " + jobid + " is PEND"); + } + job.setState(Job.STATE_QUEUED); + } + else if (state.equals("RUN")) { + if (logger.isDebugEnabled()) { + logger.debug("Status for " + jobid + " is RUN"); + } + job.setState(Job.STATE_RUNNING); + } + else if (state.equals("DONE")) { + if (logger.isDebugEnabled()) { + logger.debug("Status for " + jobid + " is DONE"); + } + addDoneJob(job.getJobID()); + } + else if (state.equals("EXIT")) { + if(logger.isDebugEnabled()) { + logger.debug("Status for " + jobid + " is EXIT"); + } + addDoneJob(job.getJobID()); + } + } + } while (line != null); + + + Iterator> i = getJobs().entrySet().iterator(); + while (i.hasNext()) { + Map.Entry e = i.next(); + String id = (String) e.getKey(); + if (!processed.contains(id)) { + Job job = (Job) e.getValue(); + if (logger.isDebugEnabled()) { + logger.debug("Status for " + id + " is Done"); + } + job.setState(Job.STATE_DONE); + if (job.getState() == Job.STATE_DONE) { + addDoneJob(id); + } + } + } + } + + protected void processStderr(InputStream is) throws IOException { + } +} \ No newline at end of file Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/execution/TaskHandlerImpl.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/execution/TaskHandlerImpl.java (revision 0) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/execution/TaskHandlerImpl.java (revision 3538) @@ -0,0 +1,32 @@ +// ---------------------------------------------------------------------- +// This code is developed as part of the Java CoG Kit project +// The terms of the license can be found at http://www.cogkit.org/license +// This message may not be removed or altered. +// ---------------------------------------------------------------------- + +package org.globus.cog.abstraction.impl.scheduler.lsf.execution; + +import org.globus.cog.abstraction.interfaces.DelegatedTaskHandler; + +/** + *Provides a local LSF TaskHandler + *for job submission to the local resource without + *any security context. + * + */ +public class TaskHandlerImpl extends + org.globus.cog.abstraction.impl.common.execution.TaskHandlerImpl { + + protected DelegatedTaskHandler newDelegatedTaskHandler() { + return new JobSubmissionTaskHandler(); + } + + protected String getName() { + return "LSF"; + } + + public String toString() + { + return "TaskHandlerImpl(execution LSF)"; + } +} Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/execution/JobSubmissionTaskHandler.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/execution/JobSubmissionTaskHandler.java (revision 0) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/execution/JobSubmissionTaskHandler.java (revision 3538) @@ -0,0 +1,12 @@ +package org.globus.cog.abstraction.impl.scheduler.lsf.execution; +import org.globus.cog.abstraction.impl.scheduler.lsf.LSFExecutor; +import org.globus.cog.abstraction.impl.scheduler.common.AbstractExecutor; +import org.globus.cog.abstraction.impl.scheduler.common.AbstractJobSubmissionTaskHandler; +import org.globus.cog.abstraction.interfaces.Task; + +public class JobSubmissionTaskHandler extends AbstractJobSubmissionTaskHandler { + protected AbstractExecutor newExecutor(Task task, + AbstractJobSubmissionTaskHandler th) { + return new LSFExecutor(task, th); + } +} \ No newline at end of file Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/LSFExecutor.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/LSFExecutor.java (revision 0) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/LSFExecutor.java (revision 3538) @@ -0,0 +1,326 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +package org.globus.cog.abstraction.impl.scheduler.lsf; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.util.Date; +import java.util.List; + +import org.apache.log4j.Logger; +import org.globus.cog.abstraction.impl.scheduler.lsf.Properties; +import org.globus.cog.abstraction.impl.scheduler.common.AbstractExecutor; +import org.globus.cog.abstraction.impl.scheduler.common.AbstractProperties; +import org.globus.cog.abstraction.impl.scheduler.common.AbstractQueuePoller; +import org.globus.cog.abstraction.impl.scheduler.common.Job; +import org.globus.cog.abstraction.impl.scheduler.common.ProcessListener; +import org.globus.cog.abstraction.interfaces.FileLocation; +import org.globus.cog.abstraction.interfaces.JobSpecification; +import org.globus.cog.abstraction.interfaces.Task; + +public class LSFExecutor extends AbstractExecutor { + public static final Logger logger = Logger.getLogger(LSFExecutor.class); + private int count = 1; + + public LSFExecutor(Task task, ProcessListener listener) { + super(task, listener); + } + + @Override + protected Job createJob(String jobid, String stdout, + FileLocation stdOutputLocation, String stderr, + FileLocation stdErrorLocation, String exitcode, + AbstractExecutor executor) { + return new Job(jobid, stdout, stdOutputLocation, stderr, + stdErrorLocation, exitcode, executor); + } + + @Override + protected String getName() { + return "LSF"; + } + + @Override + protected AbstractProperties getProperties() { + return Properties.getProperties(); + } + + void writeHeader(Writer writer) + throws IOException { + writer.write("#CoG This script generated by CoG\n"); + writer.write("#CoG by class: " + LSFExecutor.class + '\n'); + writer.write("#CoG on date: " + new Date() + "\n\n"); + } + + /** + Write attribute if non-null and non-empty + @throws IOException + */ + protected void writeNonEmptyAttr(String attrName, String arg, + Writer wr) + throws IOException { + Object value = getSpec().getAttribute(attrName); + if (value != null) { + String v = String.valueOf(value); + if (v.length() > 0 ) + wr.write("#BSUB " + arg + " " + v + '\n'); + } + } + + private int parseAndValidateInt(Object obj, String name) { + try { + assert(obj != null); + return Integer.parseInt(obj.toString()); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("Illegal value for " + name + ". Must be an integer."); + } + } + + /** + @return true if this is a multi-core job + **/ + protected boolean writeCountAndPPN(JobSpecification spec, + Writer wr) + throws IOException { + boolean result = false; + + Object o; + + // Number of program invocations + o = getSpec().getAttribute("count"); + if (o != null) + count = parseAndValidateInt(o, "count"); + if (count != 1) + result = true; + + wr.write("#BSUB -n " + count + "\n"); + return result; + } + + protected void writeMultiJobPreamble(Writer wr, String exitcodefile) + throws IOException { + wr.write("ECF=" + exitcodefile + "\n"); + wr.write("INDEX=0\n"); + wr.write("for NODE in $LSB_HOSTS; do\n"); + wr.write(" echo \"N\" >$ECF.$INDEX\n"); + wr.write(" ssh $NODE /bin/bash -c \\\" \""); + } + + @Override + protected void writeScript(Writer wr, String exitcodefile, String stdout, + String stderr) + throws IOException { + Task task = getTask(); + JobSpecification spec = getSpec(); + Properties properties = Properties.getProperties(); + validate(task); + writeHeader(wr); + + wr.write("#BSUB -L /bin/bash\n"); + wr.write("#BSUB -J " + task.getName() + '\n'); + wr.write("#BSUB -o " + quote(stdout) + '\n'); + wr.write("#BSUB -e " + quote(stderr) + '\n'); + + writeNonEmptyAttr("project", "-P", wr); + writeNonEmptyAttr("queue", "-q", wr); + boolean multiple = writeCountAndPPN(spec, wr); + + // Convert maxwalltime to HH:MM format + int hours=0, minutes=0; + String walltime = getSpec().getAttribute("maxwalltime").toString(); + if(walltime != null) { + String walltimeSplit[] = walltime.split(":"); + if(walltimeSplit.length == 1) { + minutes = Integer.valueOf(walltimeSplit[0]); + } else if(walltimeSplit.length > 1) { + hours = Integer.valueOf(walltimeSplit[0]); + minutes = Integer.valueOf(walltimeSplit[1]); + // LSF ignores seconds + } + if(minutes >= 60) { + hours = minutes / 60; + minutes = minutes % 60; + } + wr.write("#BSUB -W " + hours + ":" + minutes + '\n'); + } + + if (getSpec().getDirectory() != null) + wr.write("#BSUB -cwd " + getSpec().getDirectory() + '\n'); + + for (String name : spec.getEnvironmentVariableNames()) { + wr.write("export "); + wr.write(name); + wr.write('='); + wr.write(quote(spec.getEnvironmentVariable(name))); + wr.write('\n'); + } + + String type = (String) spec.getAttribute("jobType"); + if (logger.isDebugEnabled()) + logger.debug("Job type: " + type); + if ("multiple".equals(type)) + multiple = true; + else if("single".equals(type)) + multiple = false; + if (multiple) + writeMultiJobPreamble(wr, exitcodefile); + + if (type != null) { + String wrapper = + properties.getProperty("wrapper." + type); + if (logger.isDebugEnabled()) { + logger.debug("Wrapper: " + wrapper); + } + if (wrapper != null) { + wrapper = replaceVars(wrapper); + wr.write(wrapper); + wr.write(' '); + } + if (logger.isDebugEnabled()) { + logger.debug("Wrapper after variable substitution: " + wrapper); + } + } + if (spec.getDirectory() != null) { + wr.write("cd " + quote(spec.getDirectory()) + " && "); + } + + wr.write(quote(spec.getExecutable())); + writeQuotedList(wr, spec.getArgumentsAsList()); + + // Handle all LSF attributes specified by the user + for(String a : spec.getAttributeNames()) { + if(a != null && a.startsWith("lsf.")) { + String attributeName[] = a.split("lsf."); + wr.write(attributeName[1] + " = " + spec.getAttribute(a) + '\n'); + } + } + + if (spec.getStdInput() != null) { + wr.write(" < " + quote(spec.getStdInput())); + } + + if (multiple) { + writeMultiJobPostamble(wr); + } else { + wr.write('\n'); + wr.write("/bin/echo $? >" + exitcodefile + '\n'); + } + wr.close(); + } + + protected void addAttr(String attrName, String option, List l) { + addAttr(attrName, option, l, null); + } + + protected void addAttr(String attrName, String option, List l, boolean round) { + addAttr(attrName, option, l, null, round); + } + + protected void addAttr(String attrName, String option, List l, String defval) { + addAttr(attrName, option, l, defval, false); + } + + protected void addAttr(String attrName, String option, List l, + String defval, boolean round) { + Object value = getSpec().getAttribute(attrName); + if (value != null) { + if (round) { + value = round(value); + } + l.add(option); + l.add(String.valueOf(value)); + } + else if (defval != null) { + l.add(option); + l.add(defval); + } + } + + protected Object round(Object value) { + if (value instanceof Number) { + return new Integer(((Number) value).intValue()); + } + else { + return value; + } + } + + protected String parseSubmitCommandOutput(String out) throws IOException { + if ("".equals(out)) { + throw new IOException(getProperties().getSubmitCommandName() + + " returned an empty job ID"); + } + String outArray[] = out.split(" "); + String jobString = outArray[1]; + jobString = jobString.replaceAll("<", ""); + jobString = jobString.replaceAll(">", ""); + return jobString; + } + + protected String[] buildCommandLine(File jobdir, File script, + String exitcode, String stdout, String stderr) + throws IOException { + + writeScript(new BufferedWriter(new FileWriter(script)), exitcode, + stdout, stderr); + if (logger.isDebugEnabled()) { + logger.debug("Wrote " + getName() + " script to " + script); + } + + String[] cmdline = { "/bin/bash", "-c", (getProperties().getSubmitCommand() + " < " + script.getAbsolutePath()) }; + return cmdline; + } + + + @Override + protected String quote(String s) { + boolean quotes = false; + if (s.indexOf(' ') != -1) { + quotes = true; + } + StringBuffer sb = new StringBuffer(); + if (quotes) { + sb.append('"'); + } + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '"' || c == '\\') { + sb.append('\\'); + break; + } + sb.append(c); + } + if (quotes) { + sb.append('"'); + } + return sb.toString(); + } + + @Override + protected void cleanup() { + super.cleanup(); + new File(getStdout()).delete(); + new File(getStderr()).delete(); + } + + private static AbstractQueuePoller poller; + + @Override + protected AbstractQueuePoller getQueuePoller() { + synchronized(LSFExecutor.class) { + if (poller == null) { + poller = new QueuePoller(getProperties()); + poller.start(); + } + return poller; + } + } +} Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/Properties.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/Properties.java (revision 0) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/lsf/Properties.java (revision 3538) @@ -0,0 +1,49 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +package org.globus.cog.abstraction.impl.scheduler.lsf; +import org.globus.cog.abstraction.impl.scheduler.common.AbstractProperties; + +public class Properties extends AbstractProperties { + private static final long serialVersionUID = 1L; + public static final String PROPERTIES = "provider-lsf.properties"; + + + public static final String BSUB = "bsub"; + public static final String BJOBS = "bjobs"; + public static final String BKILL = "bkill"; + + private static Properties properties; + + public static synchronized Properties getProperties() { + if (properties == null) { + properties = new Properties(); + properties.load(PROPERTIES); + } + return properties; + } + + protected void setDefaults() { + setPollInterval(5); + setSubmitCommand("bsub"); + setPollCommand("bjobs"); + setRemoveCommand("bkill"); + } + + public String getPollCommandName() { + return BJOBS; + } + + public String getRemoveCommandName() { + return BKILL; + } + + public String getSubmitCommandName() { + return BSUB; + } + + +} Index: modules/provider-localscheduler/etc/provider-lsf.properties =================================================================== --- modules/provider-localscheduler/etc/provider-lsf.properties (revision 0) +++ modules/provider-localscheduler/etc/provider-lsf.properties (revision 3538) @@ -0,0 +1,21 @@ +# +# The interval, in seconds, at which the provider will poll the Cobalt +# queue for status updates. There is at most one poll thread per JVM, +# which is shared by all the jobs submitted through the Cobalt provider. +# +poll.interval=5 + +# +# The path to bsub. The default assumes that bsub is in PATH +# +bsub=bsub + +# +# The path to bjobs. The default assumes that bjobs is in PATH +# +bjobs=bjobs + +# +# The path to bkill. The default assumes that bkill is in PATH +# +bkill=bkill From swift at ci.uchicago.edu Wed Jan 2 12:00:09 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 2 Jan 2013 12:00:09 -0600 (CST) Subject: [Swift-commit] cog r3539 Message-ID: <20130102180009.A2CAA8D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3539 | davidkelly999 | 2013-01-02 11:59:19 -0600 (Wed, 02 Jan 2013) | 2 lines Add lsf to list of providers ------------------------------------------------------------------------ Index: modules/provider-localscheduler/resources/cog-provider.properties =================================================================== --- modules/provider-localscheduler/resources/cog-provider.properties (revision 3538) +++ modules/provider-localscheduler/resources/cog-provider.properties (working copy) @@ -29,3 +29,7 @@ executionTaskHandler=org.globus.cog.abstraction.impl.scheduler.slurm.execution.TaskHandlerImpl securityContext=org.globus.cog.abstraction.impl.common.task.SecurityContextImpl +provider=lsf +sandbox=false +executionTaskHandler=org.globus.cog.abstraction.impl.scheduler.lsf.execution.TaskHandlerImpl +securityContext=org.globus.cog.abstraction.impl.common.task.SecurityContextImpl From davidk at ci.uchicago.edu Wed Jan 2 15:00:33 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 2 Jan 2013 15:00:33 -0600 (CST) Subject: [Swift-commit] r6118 - trunk/docs/siteguide Message-ID: <20130102210033.7B69F9CCC8@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-02 15:00:33 -0600 (Wed, 02 Jan 2013) New Revision: 6118 Removed: trunk/docs/siteguide/pads Modified: trunk/docs/siteguide/beagle trunk/docs/siteguide/fusion trunk/docs/siteguide/futuregrid trunk/docs/siteguide/intrepid trunk/docs/siteguide/mcs trunk/docs/siteguide/uc3 Log: Rewrite uc3 site guide to use automatic coasters with condor Some reformatting Modified: trunk/docs/siteguide/beagle =================================================================== --- trunk/docs/siteguide/beagle 2013-01-02 00:14:03 UTC (rev 6117) +++ trunk/docs/siteguide/beagle 2013-01-02 21:00:33 UTC (rev 6118) @@ -1,4 +1,4 @@ -Cray XE6: Beagle +Beagle (Cray XE6) ---------------- Beagle is a Cray XE6 supercomputer at UChicago. It employs a batch-oriented Modified: trunk/docs/siteguide/fusion =================================================================== --- trunk/docs/siteguide/fusion 2013-01-02 00:14:03 UTC (rev 6117) +++ trunk/docs/siteguide/fusion 2013-01-02 21:00:33 UTC (rev 6118) @@ -1,4 +1,4 @@ -x86 Cluster: Fusion +Fusion (x86 cluster) ------------------- Fusion is a 320-node computing cluster for the Argonne Modified: trunk/docs/siteguide/futuregrid =================================================================== --- trunk/docs/siteguide/futuregrid 2013-01-02 00:14:03 UTC (rev 6117) +++ trunk/docs/siteguide/futuregrid 2013-01-02 21:00:33 UTC (rev 6118) @@ -1,5 +1,5 @@ -x86 Cloud: Futuregrid Quickstart Guide --------------------------------------- +Futuregrid (x86 cloud) +---------------------- FutureGrid is a distributed, high-performance test-bed that allows scientists to collaboratively develop and test innovative approaches Modified: trunk/docs/siteguide/intrepid =================================================================== --- trunk/docs/siteguide/intrepid 2013-01-02 00:14:03 UTC (rev 6117) +++ trunk/docs/siteguide/intrepid 2013-01-02 21:00:33 UTC (rev 6118) @@ -1,4 +1,4 @@ -Blue Gene/P: Intrepid +Intrepid (Blue Gene/P) --------------------- Intrepid is an IBM Blue Gene/P supercomputer located at the Argonne Modified: trunk/docs/siteguide/mcs =================================================================== --- trunk/docs/siteguide/mcs 2013-01-02 00:14:03 UTC (rev 6117) +++ trunk/docs/siteguide/mcs 2013-01-02 21:00:33 UTC (rev 6118) @@ -1,4 +1,4 @@ -x86 Workstations: MCS Compute Servers +MCS Compute Servers (x86 workstations) ------------------------------------- This sections describes how to use the general use compute servers for Deleted: trunk/docs/siteguide/pads =================================================================== --- trunk/docs/siteguide/pads 2013-01-02 00:14:03 UTC (rev 6117) +++ trunk/docs/siteguide/pads 2013-01-02 21:00:33 UTC (rev 6118) @@ -1,183 +0,0 @@ -x86 Cluster: PADS ------------------ - -PADS is a petabyte-scale, data intense computing resource located -at the joint Argonne National Laboratory/University of Chicago -Computation Institute. More information about PADS can be found -at http://pads.ci.uchicago.edu. - -Requesting Access -~~~~~~~~~~~~~~~~~ -If you do not already have a Computation Institute account, you can request -access at https://www.ci.uchicago.edu/accounts. This page will give you a list -of resources you can request access to. Be sure that PADS is selected. If -you already have an existing CI account, but do not have access to PADS, -send an email to support at ci.uchicago.edu to request access. - -SSH Keys -~~~~~~~~ -Before accessing PADS, be sure to have your SSH keys configured correctly. -There is some basic information about SSH and how to generate your key at -http://www.ci.uchicago.edu/wiki/bin/view/Resources/SshKeys. Once you have -followed those instructions, you can add your key at -https://www.ci.uchicago.edu/support/sshkeys/. - -Connecting to a login node -~~~~~~~~~~~~~~~~~~~~~~~~~~ -Once your keys are configured, you should be able to access a PADS login -node with the following command: - ------ -ssh yourusername at login.pads.ci.uchicago.edu ------ - -Adding Software Packages -~~~~~~~~~~~~~~~~~~~~~~~~ -Softenv is a system used for managing applications. In order to run Swift, -the softenv environment will have to be modified slightly. Softenv is -configured by a file in your home directory called .soft. Edit this file -to look like this: ------ -+java-sun -+maui -+torque - at default ------ - -Log out of PADS, and log back in for these changes to take effect. - -Which project(s) are you a member of? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -PADS requires that you are a member of a project. You can determine this by -running the following command: - ------ -$ projects --available - -The following projects are available for your use - -Project PI Title -CI-CCR000013 Michael Wilde The Swift Parallel Scripting System ------ - -If you are not a member of a project, you must first request access -to a project at http://www.ci.uchicago.edu/hpc/projects. - -You should make sure that you have a project set as default. Run -the projects command with no arguments to determine if you have a default. - ------- -$ projects -You have no default project set. ------ - -To set your default project, use projects --set ------- -$ projects --set CI-CCR000013 --all -Your default project for all CI clusters has been set to CI-CCR000013. ------ - -Creating sites.xml -^^^^^^^^^^^^^^^^^^ -Swift relies on various configuration files to determine how to -run. This section will provide a working configuration file which -you can copy and paste to get running quickly. The sites.xml file -tells Swift how to submit jobs, where working directories are -located, and various other configuration information. More -information on sites.xml can be found in the Swift User's Guide. - -The first step is to paste the text below into a file named sites.xml. - ------ -include::../../tests/providers/pads/coasters/sites.template.xml[] ------ - -This file will require just a few customizations. First, create a -directory called swiftwork. Modify \_WORK_ in sites.xml -to point to this new directory. For example ------ -/home/myhome/swiftwork ------ - -Creating tc.data -^^^^^^^^^^^^^^^^ -The tc.data configuration file gives information about the applications -that will be called by Swift. More information about the format -of tc.data can be found in the Swift User's guide. - -Paste the following example into a file named tc.data - ------ -include::../../tests/providers/pads/coasters/tc.template.data[] ------ - -Copy a Swift Script -^^^^^^^^^^^^^^^^^^^ -Within the Swift directory is an examples directory which contains -several introductory Swift scripts. The example we will use in this -section is called catsn.swift. Copy this script to the same directory -that your sites.xml and tc.data files are located. - ------ -$ cp ~/swift-0.93/examples/misc/catsn.swift . -$ cp ~/swift-0.93/examples/misc/data.txt . ------ - -TIP: The location of your swift directory may vary depending on how -you installed it. Change this to the examples/misc directory of your -installation as needed. - - -Run Swift -^^^^^^^^^ - -Finally, run the script: - ------ - swift -sites.file sites.xml -tc.file tc.data catsn.swift ------ - -You should see several new files being created, called catsn.0001.out, catsn.0002.out, etc. Each of these -files should contain the contents of what you placed into data.txt. If this happens, your job has run -successfully on PADS! - -TIP: Make sure your default project is defined. Read on for more -information. - -Read on for more detailed information about running Swift on PADS. - - -Queues -^^^^^^ - -As you run more application in the future, you will likely need -to change queues. - -PADS has several different queues you can submit jobs to depending on -the type of work you will be doing. The command "qstat -q" will print -the most up to date list of this information. - -.PADS Queues -[options="header"] -|========================================================= -|Queue |Memory|CPU Time|Walltime|Node|Run|Que|Lm |State -|route |-- |-- |-- |-- | 0| 0|-- | E R -|short |-- |-- |04:00:00|-- | 64| 0|-- | E R -|extended|-- |-- |-- |-- | 0| 0|-- | E R -|fast |-- |-- |01:00:00|1 | 0|152|-- | E R -|long |-- |-- |24:00:00|-- |232|130|-- | E R -|========================================================= - -When you determine your computing requirements, modify this line in your -sites.xml: - ------ -fast ------ - -More Help -~~~~~~~~~ -The best place for additional help is the Swift user mailing list. You can subscribe to this list at -https://lists.ci.uchicago.edu/cgi-bin/mailman/listinfo/swift-user. When submitting information, please send your sites.xml file, your tc.data, and any Swift log files that were created during your attempt. - - Modified: trunk/docs/siteguide/uc3 =================================================================== --- trunk/docs/siteguide/uc3 2013-01-02 00:14:03 UTC (rev 6117) +++ trunk/docs/siteguide/uc3 2013-01-02 21:00:33 UTC (rev 6118) @@ -1,53 +1,135 @@ -x86 Cluster: UC3 +UC3 (x86 cluster) ---------------- -Create a coaster-service.conf -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To begin, copy the text below and paste it into your Swift distribution's etc -directory. Name the file coaster-service.conf. +Requesting Access +~~~~~~~~~~~~~~~~~ +To request access to UC3, you must have a University of Chicago CNetID +and be a meimber of the UC3 group. More information about UC3 can be +found at https://wiki.uchicago.edu/display/uc3/UC3+Home or +uc3-support at lists.uchicago.edu. +Connecting to a login node +~~~~~~~~~~~~~~~~~~~~~~~~~~ +To access the UC3 login node, you will use your CNetID and password. + ----- -include::../../tests/providers/uc3/coaster-service.conf[] +ssh -l uc3-sub.uchicago.edu ----- -Starting the Coaster Service -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Installing Swift +~~~~~~~~~~~~~~~~ +Swift should be available by default on the UC3 login nodes. You can verify +this by running the following command: +----- +swift -version +----- -Change directories to the location you would like to run a -Swift script and start the coaster service with this -command: +If for some reason Swift is not available, you can following the instructions at +http://www.ci.uchicago.edu/swift/guides/release-0.93/quickstart/quickstart.html. +Swift 0.94 or later is required to work with the condor provider on UC3. +Creating sites.xml +~~~~~~~~~~~~~~~~~~ +This section will provide a working configuration file which you can copy and paste +to get running quickly. The sites.xml file tells Swift how to submit jobs, where +working directories are located, and various other configuration information. +More information on sites.xml can be found in the Swift User?s Guide. + +The first step is to paste the text below into a file named sites.xml: ----- -start-coaster-service + + + + 999.99 + 10000 + 1 + 3600 + 1 + 100 + 100 + 1000 + 1 + "group_friends.{env.USER}" + nonshared + + . + + ----- -This will create a configuration file that Swift needs -called sites.xml. +Creating tc.data +~~~~~~~~~~~~~~~~ +The tc.data configuration file gives information about the applications that will be called by Swift. +More information about the format of tc.data can be found in the Swift User?s guide. -WARNING: Any existing sites.xml files in this directory -will be overwritten. Be sure to make a copy of any -custom configuration files you may have. +Paste the following example into a file named tc.data: +----- +uc3 echo /bin/echo null null null +----- -Run Swift -~~~~~~~~~ +Create a configuration file +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +A swift configuration file enables and disables some settings in Swift. More information on what +these settings do can be found in the Swift User's guide. -Next, run Swift. If you do not have a particular script -in mind, you can test Swift by using a Swift script in -the examples/ directory. +Paste the following lines into a file called cf: +----- +wrapperlog.always.transfer=false +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false +use.wrapper.staging=false +----- -Run the following command to run the script: +Creating echo.swift +~~~~~~~~~~~~~~~~~~~ +Now we need to create a swift script to test with. Let's use a simple application that calls /bin/echo. + ----- -swift -sites.file sites.xml -tc.file tc.data yourscript.swift +type file; + +app (file o) echo (string s) { + echo s stdout=@o; +} + +foreach i in [1:5] { + file output_file ; + output_file = echo( @strcat("This is test number ", i) ); +} ----- -Stopping the Coaster Service -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The coaster service will run indefinitely. The stop-coaster-service -script will terminate the coaster service. +Running Swift +~~~~~~~~~~~~~ +Putting everything together now, run your Swift script with the following command: ----- -$ stop-coaster-service +swift -sites.file sites.xml -tc.file tc.data -config cf echo.swift ----- -This will kill the coaster service and kill the worker scripts on remote systems. +If everything runs successfully, you will see 5 files get created in the output directory. + +Controlling where jobs run +~~~~~~~~~~~~~~~~~~~~~~~~~~ +Swift will automatically generate condor scripts for you with the basic information about +how to run. However, condor has hundreds of commands that let you customize how things work. +If you need one of these advanced commands, you can add it to your sites.xml. The basic +template for this is: + +----- +value +----- + +For example, let's assume that you want to control where your jobs run by adding a +requirement. The condor command that will control the run is: +----- +Requirements = UidDomain == "osg-gk.mwt2.org" +----- + +To have this generated by Swift, you will add a line to your sites.xml in the key/value style +shown above. +UidDomain == "osg-gk.mwt2.org" + From davidk at ci.uchicago.edu Wed Jan 2 15:47:09 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 2 Jan 2013 15:47:09 -0600 (CST) Subject: [Swift-commit] r6119 - trunk/docs/siteguide Message-ID: <20130102214709.A040B9CCE5@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-02 15:47:09 -0600 (Wed, 02 Jan 2013) New Revision: 6119 Modified: trunk/docs/siteguide/siteguide.txt trunk/docs/siteguide/uc3 Log: Fix a few formatting issues Modified: trunk/docs/siteguide/siteguide.txt =================================================================== --- trunk/docs/siteguide/siteguide.txt 2013-01-02 21:00:33 UTC (rev 6118) +++ trunk/docs/siteguide/siteguide.txt 2013-01-02 21:47:09 UTC (rev 6119) @@ -10,18 +10,16 @@ include::prereqs[] -include::pads[] +include::beagle[] include::fusion[] -include::beagle[] +include::futuregrid[] +include::grid[] + include::intrepid[] -include::futuregrid[] - include::mcs[] -include::grid[] - include::uc3[] Modified: trunk/docs/siteguide/uc3 =================================================================== --- trunk/docs/siteguide/uc3 2013-01-02 21:00:33 UTC (rev 6118) +++ trunk/docs/siteguide/uc3 2013-01-02 21:47:09 UTC (rev 6119) @@ -131,5 +131,6 @@ To have this generated by Swift, you will add a line to your sites.xml in the key/value style shown above. +----- UidDomain == "osg-gk.mwt2.org" - +----- From wilde at ci.uchicago.edu Thu Jan 3 18:20:09 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 3 Jan 2013 18:20:09 -0600 (CST) Subject: [Swift-commit] r6120 - SwiftApps/SciColSim Message-ID: <20130104002009.493769CCE5@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-03 18:20:09 -0600 (Thu, 03 Jan 2013) New Revision: 6120 Modified: SwiftApps/SciColSim/annealing.swift SwiftApps/SciColSim/evolve.sh Log: Added compatibility for swift 0.93-0.94 iterate until() changes; stubbed out extra file log file handling for faster execution (need to add it back later as an option). Modified: SwiftApps/SciColSim/annealing.swift =================================================================== --- SwiftApps/SciColSim/annealing.swift 2013-01-02 21:47:09 UTC (rev 6119) +++ SwiftApps/SciColSim/annealing.swift 2013-01-04 00:20:09 UTC (rev 6120) @@ -15,6 +15,8 @@ float tsdev; } +global int iterate_adjust = @toint(@arg("iterate_adjust","1")); // FIXME!!! : 1 for 0.93, 0 for 0.94 + global boolean FIX_VARIABLES = true; global int var_fixed[] = [0,0,0,0,0]; global int Nworkers = @toint(@arg("nworkers","4")); @@ -35,9 +37,9 @@ // tracef("newx(%f,%f)=%f\n",x,dx,nx); } -app (file outfile, file loss ) evolve ( string args[], file graph ) +app (/*file outfile,*/ file loss ) evolve ( string args[], file graph ) { - evolve @loss args stdout=@outfile; // graph is passed implicitly + evolve @loss args /*stdout=@outfile*/; // FIXME: graph is passed implicitly } app ( file x ) sumloss( file loss[] ) @@ -220,8 +222,8 @@ curr_sdev[curr] = curr_sdev[prev]; // dx[i][j] not set for fixed vars } - } until( j == NEVOPARAMS ); - } until( iter_i == annealing_cycles ); + } until( j == (NEVOPARAMS-iterate_adjust) ); + } until( iter_i == (annealing_cycles-iterate_adjust) ); } (Res r) multi_loss(float t_start, float t_end, int annealing_steps, float t_rejection, float starting_jump, @@ -238,7 +240,7 @@ tracef("multi_loss appCalls=%i\n", appCalls); foreach i in [1:appCalls] { // repeats of the evolove() - same as n_reruns - file outfile; // FIXME: map and save in future + //file outfile; // FIXME: map and save in future string args[] = [ // FIXME: move this to a setargs() function // alpha_i alpha_m beta gamma delta target_innov @strcat(x[0]), @strcat(x[1]), @strcat(x[2]), @strcat(x[3]), @strcat(x[4]), @strcat(target_innov), @@ -256,7 +258,7 @@ file graph <"movie_graph.txt">; - (outfile, rfile[i]) = evolve(args, graph); + (/*outfile,*/ rfile[i]) = evolve(args, graph); tracef("multi_loss: i=%i calling evolve, args=%q\n", i, args); // tracef("multi_loss: after evolve: i=%i %k %k\n", i, outfile, rfile[i]); } @@ -350,7 +352,7 @@ main() { - optimizer_serial_sweep(); + optimizer_sweep(); } main(); Modified: SwiftApps/SciColSim/evolve.sh =================================================================== --- SwiftApps/SciColSim/evolve.sh 2013-01-02 21:47:09 UTC (rev 6119) +++ SwiftApps/SciColSim/evolve.sh 2013-01-04 00:20:09 UTC (rev 6120) @@ -4,7 +4,7 @@ shift 1 NWORKERS=${23} START=$(date +%s) -OMP_NUM_THREADS=$NWORKERS $(dirname $0)/openmp-optimizer $* 2>&1 +OMP_NUM_THREADS=$NWORKERS $(dirname $0)/openmp-optimizer $* >/dev/null 2>&1 RC=$? END=$(date +%s) if [ $RC == 127 ]; From wilde at ci.uchicago.edu Thu Jan 3 18:22:18 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 3 Jan 2013 18:22:18 -0600 (CST) Subject: [Swift-commit] r6121 - SwiftApps/SciColSim/bin Message-ID: <20130104002218.5CCD59CCE5@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-03 18:22:18 -0600 (Thu, 03 Jan 2013) New Revision: 6121 Modified: SwiftApps/SciColSim/bin/showbest.sh Log: Updated for latest output format for best.*.txt file Modified: SwiftApps/SciColSim/bin/showbest.sh =================================================================== --- SwiftApps/SciColSim/bin/showbest.sh 2013-01-04 00:20:09 UTC (rev 6120) +++ SwiftApps/SciColSim/bin/showbest.sh 2013-01-04 00:22:18 UTC (rev 6121) @@ -1,8 +1,23 @@ #! /bin/sh -awk '{ - printf( "N %.2f %2d %2d %10.5f %5.2f | %5.2f %10.5f [ %5.2f %5.2f %10.5f %10.5f %10.5f ] %10.5f\n", - $2, $3, $4, $5, $7, $8, $10, $11, $12, $13, $14, $16, $17); +awk ' + +BEGIN { + printf " AvgTime i j dx Rej TgtInno AvgLoss alpha_i alpha_m beta gamma delta LossSdev\n" + printf " ---- -- -- ------ ---- | ------- --------- [ -------- --------- ------- ------- ------ ] --------\n" +# N 2.38 0 0 2.3000 0.00 | 67 2196.6891 [ -0.1586 0.0000 0.0000 0.0000 0.0000 ] 922.2443 +# N 2.32 0 1 2.3000 0.00 | 67 2147.4971 [ -0.1586 -0.5617 0.0000 0.0000 0.0000 ] 942.2637 + +} + +{ + printf( "N %6.2f %2d %2d %10.4f %5.2f | %7d %10.4f [ %9.4f %9.4f %9.4f %9.4f %9.4f ] %10.4f\n", + $2, $3, $4, $5, $6, $8, $9, $11, $12, $13, $14, $15, $17); }' +#N, 2.528571s, 3, 4, 2.3, 4.0, |, 67, 2105.204186, [, 1.5225260194167338, -1.3900387949548139, -0.05373482546917286, 1.8853191190715326, -1.9217664739947533, ], 960.585884 + +#1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 + +#N 2.38 0 0 2.30000 0.00 | 67.00 0.00000 [ -0.16 0.00 0.00000 0.00000 0.00000 ] 922.24425 From wilde at ci.uchicago.edu Thu Jan 3 18:23:22 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 3 Jan 2013 18:23:22 -0600 (CST) Subject: [Swift-commit] r6122 - SwiftApps/SciColSim Message-ID: <20130104002322.0D4B69CCE5@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-03 18:23:21 -0600 (Thu, 03 Jan 2013) New Revision: 6122 Modified: SwiftApps/SciColSim/gengraphs.sh Log: Temp change to gen .05% graph Modified: SwiftApps/SciColSim/gengraphs.sh =================================================================== --- SwiftApps/SciColSim/gengraphs.sh 2013-01-04 00:22:18 UTC (rev 6121) +++ SwiftApps/SciColSim/gengraphs.sh 2013-01-04 00:23:21 UTC (rev 6122) @@ -2,7 +2,7 @@ mkdir -p graph -for p in .001 .002 .003 .004 .005 .006 .007 .008 .009 .010; do +for p in .0005 ; do # .001 .002 .003 .004 .005 .006 .007 .008 .009 .010; do for ((i=0;i<10;i++)); do ./samplegraph.sh $p > g/bg${p}.${i} done From wilde at ci.uchicago.edu Thu Jan 3 18:24:00 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 3 Jan 2013 18:24:00 -0600 (CST) Subject: [Swift-commit] r6123 - SwiftApps/SciColSim Message-ID: <20130104002400.603499CCE5@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-03 18:24:00 -0600 (Thu, 03 Jan 2013) New Revision: 6123 Modified: SwiftApps/SciColSim/sumloss.sh Log: Corrections to std dev calculation. Modified: SwiftApps/SciColSim/sumloss.sh =================================================================== --- SwiftApps/SciColSim/sumloss.sh 2013-01-04 00:23:21 UTC (rev 6122) +++ SwiftApps/SciColSim/sumloss.sh 2013-01-04 00:24:00 UTC (rev 6123) @@ -16,30 +16,35 @@ { for(i=1;i Author: wilde Date: 2013-01-03 18:26:38 -0600 (Thu, 03 Jan 2013) New Revision: 6124 Modified: SwiftApps/SciColSim/optimizer.cpp Log: Removed obsolete code, adjusted timing calculations, removed RNG priming loop, and minor code adjustments . Modified: SwiftApps/SciColSim/optimizer.cpp =================================================================== --- SwiftApps/SciColSim/optimizer.cpp 2013-01-04 00:24:00 UTC (rev 6123) +++ SwiftApps/SciColSim/optimizer.cpp 2013-01-04 00:26:38 UTC (rev 6124) @@ -11,7 +11,7 @@ #define MAXNworkers 24 int Nworkers=MAXNworkers; -#define NUniverse 24 +#define NUniverse 240 // Add operation code to enable existing code to be used at lower level from Swift scripts: @@ -20,6 +20,8 @@ unsigned initSeed = 0; +int verbose_level = 2; + #include #include #include @@ -66,6 +68,8 @@ #define FIX_VARIABLES 1 +#define DB if(verbose_level > 3) + using namespace boost; using namespace std; using namespace boost::numeric::ublas; @@ -230,6 +234,26 @@ return a; } +timeval startTime, endTime; +double elapsedTime; +char timenow[100]; + +char *now() +{ + timeval endTime; + double elapsedTime; + + gettimeofday(&endTime, NULL); + elapsedTime = (endTime.tv_sec - startTime.tv_sec) * 1000.0; // sec to ms + elapsedTime += (endTime.tv_usec - startTime.tv_usec) / 1000.0; // us to ms + elapsedTime /= 1000.; + sprintf(timenow, "%10.3f", elapsedTime); + return(timenow); +} + + + + //================================================ class Universe { @@ -260,7 +284,7 @@ double current_novelty; int mode_identify_failed; - int verbose_level; // 0 is silent, higher is more + //int verbose_level; // 0 is silent, higher is more double k_max; @@ -273,9 +297,9 @@ double **EdgeIndex; double *Rank; - base_generator_type generator; - boost::uniform_real<> uni_dist; - boost::geometric_distribution geo; + base_generator_type generator; +// boost::uniform_real<> uni_dist; +// boost::geometric_distribution geo; public: @@ -292,12 +316,13 @@ //------------------------------- - base_generator_type gene(42u); - generator = gene; +// base_generator_type gene(42u); +// generator = gene; // generator.seed(static_cast(std::time(0))); if ( initSeed != 0.0 ) { + generator.seed(static_cast(initSeed)); } else { @@ -310,8 +335,8 @@ - boost::uniform_real<> uni_d(0,1); - uni_dist = uni_d; +// boost::uniform_real<> uni_d(0,1); +// uni_dist = uni_d; //-------------------------------- @@ -336,7 +361,7 @@ id = idd; - verbose_level = 1; + //verbose_level = 1; mode_identify_failed = identify_failed; @@ -689,7 +714,7 @@ std::vector d(num_edges(Full_g)); edge_descriptor s; boost::graph_traits::vertex_descriptor u, v; - cout << "reset_world: start id=" << id << endl; + DB cout << now() << " reset_world: start id=" << id << endl; for (int i=0; i NUniverse ) { @@ -1222,7 +1194,7 @@ #pragma omp parallel for private (i) for(i=0; ievolve_to_target_and_save(i, i+1, Results, Counters); + un[i]->evolve_to_target_and_save(i, Results, Counters); } /* @@ -1266,220 +1238,41 @@ //============================================================ -void multi_annealing(Universe* un[], - double T_start, double T_end, - double Target_rejection, - int Annealing_repeats, - double starting_jump, + +void no_annealing(Universe* un[], + //double T_start, double T_end, + // double Target_rejection, + // int Annealing_repeats, + // double starting_jump, double* Results, int* Counters, - double* params0, - double annealing_cycles){ - - //................................. - // re-implement annealing - - double dx[5]={0.,0.,0.,0.,0}; + double* params0 /*, + double annealing_cycles*/ ) +{ double x[5]={0.,0.,0.,0.,0}; - double rejection[5]={0., 0., 0., 0., 0.}; - double curr_x, curr_err, x_tmp; - double temperature; - double ratio, r; - int cycle=10; - //boost::variate_generator > uni(generator, uni_dist); - // set up parameter for annealing - x[0]=params0[0]; x[1]=params0[1]; x[2]=params0[2]; x[3]=params0[3]; x[4]=params0[4]; - for(int i=0;i<5;i++){ - dx[i] = starting_jump; - } - - // establish the current value std::pairRes; - Res = multi_loss( un, Results, Counters, x); + Res = multi_loss(un, Results, Counters, x); std::cout << "Returned from initial multi_loss:" << std::endl; std::cout << Res.first << " +- " << Res.second << std::endl; - if ( operation == 'm' ) { - FILE *f; - int N = un[0]->get_reruns(); - - f = fopen("multi_loss.data","w"); - for(int i=0; iget_reruns(); + + f = fopen("multi_loss.data","w"); + for(int i=0; iset_parameter(x_tmp, j); - } - - Res = multi_loss( un, Results, Counters, x); - - std::cout << Res.first << " +- " << Res.second << std::endl; - - ratio = min(1.,exp(-(Res.first-curr_x)/temperature)); - r = rand()/(double)(pow(2.,31)-1.); - std::cout << r << " vs " << ratio << std::endl; - - double ALOT=100000000000.; - - if (Res.first < ALOT) - { - ofstream filestr; - - filestr.open ("best_opt_some.txt", ofstream::app); - - // >> i/o operations here << - filestr - - << "N, " << i << ", " << j << ", " << dx[j] << ", " << rejection[j] << ", |, " // FIXME: MW-DEBUGGING - - << un[0]->get_target() << ", " - << Res.first - << ", " << un[0]->get_parameter(0) - << ", " << un[0]->get_parameter(1) - << ", " << un[0]->get_parameter(2) - << ", " << un[0]->get_parameter(3) - << ", " << un[0]->get_parameter(4) << ", " << Res.second << ",\n"; - - filestr.close(); - - - filestr.open ("max_dist.txt", ofstream::app); - - // >> i/o operations here << - filestr << max_dist << ",\n"; - - filestr.close(); - - FILE *bf; - bf = fopen("bestdb.txt","a"); - fprintf(bf, "N %2d %2d %10.5f %5.2f | %5.2f %10.5f [ %5.2f %5.2f %10.5f %10.5f %10.5f ] %10.5f\n", i, j, dx[j], rejection[j], - un[0]->get_target(), - Res.first, - un[0]->get_parameter(0), - un[0]->get_parameter(1), - un[0]->get_parameter(2), - un[0]->get_parameter(3), - un[0]->get_parameter(4), - Res.second); - fclose(bf); - } - - - if (r > ratio){ - - std::cout << " "<< (i+1) << ","<< (j) - <<" "<< (i+1) << " Did not accept " - << x_tmp << "(" << j << ")" << std::endl; - std::cout << un[0]->get_parameter(0) - << " " << un[0]->get_parameter(1) - << " " << un[0]->get_parameter(2) - << " " << un[0]->get_parameter(3) - << " " << un[0]->get_parameter(4) << " " << std::endl; - - x[j]=x_hold; - for(int w=0; wset_parameter(x[j], j); - } - - - //set_parameter(x[j], j); - rejection[j]+=1.; - } - - else { - - curr_x = Res.first; - curr_err = Res.second; - x[j] = x_tmp; - - for(int w=0; wset_parameter(x[j], j); - } - - std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8) - << wrap_double(rejection[0],2) << " " - << wrap_double(rejection[1],7) << " " - << wrap_double(rejection[2],5) << " " - << wrap_double(rejection[3],9) << " " - << wrap_double(rejection[4],6) << " " - << std::endl << std::endl; - - std::cout << " "<< (i+1) <<","<< (j) - <<" " - << string_wrap((string) "***** Did accept! ", 3) - << wrap_double(un[0]->get_parameter(0),2) << " " - << wrap_double(un[0]->get_parameter(1),7) << " " - << wrap_double(un[0]->get_parameter(2),5) << " " - << wrap_double(un[0]->get_parameter(3),9) << " " - << wrap_double(un[0]->get_parameter(4),6) << " " - << std::endl << std::endl; - - - - } - //........................................................ - - } - } - - } - + fclose(f); + exit(0); } @@ -1488,7 +1281,6 @@ int main(int argc, char* argv[]) { - double params0[6] = {0., 0., 0., 0., 0., 0.2}, target=50., range; string par_names0[6] = {"alpha_i", "alpha_m", "beta", "gamma", "delta", "target"}; string par_names1[4] = {"n_epochs", "n_steps", "n_reruns", "range"}; @@ -1502,26 +1294,22 @@ // temperature_start, temperature_end, annealing_steps target_rejection Starting_jump double params2[5] = {1, 0.001, 100, 0.3, 1.5}; - int verbose_level = 2; +// int verbose_level = 2; const std::string one="one", two="two"; + static Universe* un[NUniverse]; static double* Results; static int* Counters; timeval t1, t2; double elapsedTime; - // start timer - gettimeofday(&t1, NULL); + gettimeofday(&t1, NULL); // start timer if (argc < 8) { std::cout << "Usage: super_optimizer alpha_i alpha_m beta gamma delta target_innov [n_epochs n_steps n_reruns] [range] [verbose_level]\n"; std::cout << " [T_start T_end Annealing_steps Target_rejection Starting_jump]\n"; std::cout << " [FREEZE_alpha_i FREEZE_alpha_m FREEZE_beta FREEZE_gamma FREEZE_delta]\n"; - - system("pwd"); - - return(1); } else { @@ -1560,14 +1348,10 @@ initSeed = atoi(argv[nArg]); std::cout << par_names4[2] << ": " << initSeed << std::endl; } - - } - } for (int j=0; j<5; j++){ - cout << j << " | " << var_fixed[j] << " (fixed) \n"; } @@ -1577,31 +1361,19 @@ char* filename= (char *)"movie_graph.txt"; int n_ep=params1[0], n_st=params1[1], n_rep=params1[2]; - //............................... - for(int i=0; i 0){ - + if (n_rep > 0) { Results = new double[n_rep]; Counters = new int[n_rep]; - - }else { - - Results = NULL; - Counters = NULL; + } else { std::cout << " Number of reruns should be positive! " << std::endl; - return 0; - + return 1; } - //............................... - //srand(time(0)); - //srandomdev(); if ( initSeed != 0.0 ) { srand(initSeed); @@ -1612,74 +1384,42 @@ srand(t.tv_usec); } +#ifdef notdef { double r=0; for (int j=0; j<100; j++){ - - - r = rand()/(double)(pow(2.,31)-1.); std::cout << r << " "; } std::cout << "\n"; } - //random initiation of starting parameters +#endif - if (range > 0.){ - - for (int i=0; i < 5; i++){ - - if (params0[i]==-100.){ - - double r1 = (rand()/(double)(pow(2.,31)-1.)); - double r2 = (rand()/(double)(pow(2.,31)-1.)); - double sign = 1.; - - if(r1 > 0.5){ - sign=-1.; - } - - params0[i] = sign*r2*range; - - std::cout << par_names0[i] << ": " << params0[i] << std::endl; - } - } - - } - - double T_start=params2[0], T_end=params2[1], Target_rejection=params2[3], starting_jump=params2[4]; int Annealing_repeats = (int) params2[2]; +#ifdef notdef multi_annealing( un, T_start, T_end, Target_rejection, Annealing_repeats, starting_jump, Results, Counters, params0, Annealing_repeats); +#endif - // stop timer - gettimeofday(&t2, NULL); + no_annealing(un, Results, Counters, params0); + gettimeofday(&t2, NULL); // stop timer + // compute and print the elapsed time in millisec + elapsedTime = (t2.tv_sec - t1.tv_sec) * 1000.0; // sec to ms elapsedTime += (t2.tv_usec - t1.tv_usec) / 1000.0; // us to ms elapsedTime /= 1000.; cout << "\n*** optimizer completed, elapsed time=" << elapsedTime << " seconds " << elapsedTime/60. << " minutes)\n\n"; - //..................... - for(int i=0; i 0){ - delete [] Results; delete [] Counters; - } - return 0; - - - } - From wilde at ci.uchicago.edu Thu Jan 3 18:27:39 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 3 Jan 2013 18:27:39 -0600 (CST) Subject: [Swift-commit] r6125 - SwiftApps/SciColSim Message-ID: <20130104002739.5044B9CCE5@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-03 18:27:39 -0600 (Thu, 03 Jan 2013) New Revision: 6125 Modified: SwiftApps/SciColSim/swiftopt.sh Log: Added startingjump parameter. Modified: SwiftApps/SciColSim/swiftopt.sh =================================================================== --- SwiftApps/SciColSim/swiftopt.sh 2013-01-04 00:26:38 UTC (rev 6124) +++ SwiftApps/SciColSim/swiftopt.sh 2013-01-04 00:27:39 UTC (rev 6125) @@ -27,6 +27,8 @@ -gamma=$gamma \ -delta=$delta \ \ + -startingjump=$starting_jump \ + \ -nworkers=$nworkers \ -rerunsperapp=$reruns_per_opt_invocation \ -wget=$wget \ @@ -165,7 +167,7 @@ # @arg("tend", "0.01") FIXME # @arg("trejection", "0.3") FIXME # @arg("evoreruns", "100") -# @arg("startingjump", "2.3") FIXME +# @arg("startingjump", "2.3") # @arg("alphai", "0.0") # @arg("alpham", "0.0") # @arg("beta", "4.0") From wilde at ci.uchicago.edu Thu Jan 3 18:30:40 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 3 Jan 2013 18:30:40 -0600 (CST) Subject: [Swift-commit] r6126 - in SwiftApps/SciColSim: conf params Message-ID: <20130104003040.14C049CCE5@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-03 18:30:39 -0600 (Thu, 03 Jan 2013) New Revision: 6126 Added: SwiftApps/SciColSim/conf/beagle04.cf SwiftApps/SciColSim/conf/beagle04.xml SwiftApps/SciColSim/conf/beagle05.cf SwiftApps/SciColSim/conf/beagle05.xml SwiftApps/SciColSim/conf/beagle06.cf SwiftApps/SciColSim/conf/beagle06.xml SwiftApps/SciColSim/params/ARtest04 SwiftApps/SciColSim/params/ARtest05 SwiftApps/SciColSim/params/ARtest06 Modified: SwiftApps/SciColSim/conf/beagle03.xml SwiftApps/SciColSim/params/ARtest03 Log: config and param additions/changes. Modified: SwiftApps/SciColSim/conf/beagle03.xml =================================================================== --- SwiftApps/SciColSim/conf/beagle03.xml 2013-01-04 00:27:39 UTC (rev 6125) +++ SwiftApps/SciColSim/conf/beagle03.xml 2013-01-04 00:30:39 UTC (rev 6126) @@ -20,11 +20,12 @@ 100 100 pbs.aprun;pbs.mpp;depth=24 - 1800 + 36000 + 01:00:00 1 - 2 - 2 - scalability + 4 + 4 + batch 12.00 10000 Added: SwiftApps/SciColSim/conf/beagle04.cf =================================================================== --- SwiftApps/SciColSim/conf/beagle04.cf (rev 0) +++ SwiftApps/SciColSim/conf/beagle04.cf 2013-01-04 00:30:39 UTC (rev 6126) @@ -0,0 +1,11 @@ +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=4 +lazy.errors=true +status.mode=provider +use.provider.staging=false +use.wrapper.staging=false +provider.staging.pin.swiftfiles=false + +#app local sumloss=$PWD/../sumloss.sh +#app beagle04 evolve=$PWD/../evolve.sh Added: SwiftApps/SciColSim/conf/beagle04.xml =================================================================== --- SwiftApps/SciColSim/conf/beagle04.xml (rev 0) +++ SwiftApps/SciColSim/conf/beagle04.xml 2013-01-04 00:30:39 UTC (rev 6126) @@ -0,0 +1,40 @@ + + + + + 0.09 + 10000 + + _WORK_/local + + + + + + + KEEP + + 1 + DEBUG + _WORK_/workers + 100 + 100 + pbs.aprun;pbs.mpp;depth=24 + 144000 + 01:00:00 + true + 1 + 420 + 420 + batch + + 12.00 + 10000 + + + _WORK_/beagle + + + + + Added: SwiftApps/SciColSim/conf/beagle05.cf =================================================================== --- SwiftApps/SciColSim/conf/beagle05.cf (rev 0) +++ SwiftApps/SciColSim/conf/beagle05.cf 2013-01-04 00:30:39 UTC (rev 6126) @@ -0,0 +1,11 @@ +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=4 +lazy.errors=true +status.mode=provider +use.provider.staging=false +use.wrapper.staging=false +provider.staging.pin.swiftfiles=false + +#app local sumloss=$PWD/../sumloss.sh +#app beagle05 evolve=$PWD/../evolve.sh Added: SwiftApps/SciColSim/conf/beagle05.xml =================================================================== --- SwiftApps/SciColSim/conf/beagle05.xml (rev 0) +++ SwiftApps/SciColSim/conf/beagle05.xml 2013-01-04 00:30:39 UTC (rev 6126) @@ -0,0 +1,40 @@ + + + + + 0.09 + 10000 + + _WORK_/local + + + + + + + KEEP + + 1 + DEBUG + _WORK_/workers + 100 + 100 + pbs.aprun;pbs.mpp;depth=24 + 144000 + 01:00:00 + true + 4 + 32 + 50 + batch + + 12.00 + 10000 + + + _WORK_/beagle + + + + + Added: SwiftApps/SciColSim/conf/beagle06.cf =================================================================== --- SwiftApps/SciColSim/conf/beagle06.cf (rev 0) +++ SwiftApps/SciColSim/conf/beagle06.cf 2013-01-04 00:30:39 UTC (rev 6126) @@ -0,0 +1,11 @@ +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=4 +lazy.errors=true +status.mode=provider +use.provider.staging=false +use.wrapper.staging=false +provider.staging.pin.swiftfiles=false + +#app local sumloss=$PWD/../sumloss.sh +#app beagle06 evolve=$PWD/../evolve.sh Added: SwiftApps/SciColSim/conf/beagle06.xml =================================================================== --- SwiftApps/SciColSim/conf/beagle06.xml (rev 0) +++ SwiftApps/SciColSim/conf/beagle06.xml 2013-01-04 00:30:39 UTC (rev 6126) @@ -0,0 +1,40 @@ + + + + + 0.09 + 10000 + + _WORK_/local + + + + + + + KEEP + + 1 + DEBUG + _WORK_/workers + 100 + 100 + pbs.aprun;pbs.mpp;depth=24 + 144000 + 01:00:00 + true + 20 + 18 + 18 + batch + + 12.00 + 10000 + + + _WORK_/beagle + + + + + Modified: SwiftApps/SciColSim/params/ARtest03 =================================================================== --- SwiftApps/SciColSim/params/ARtest03 2013-01-04 00:27:39 UTC (rev 6125) +++ SwiftApps/SciColSim/params/ARtest03 2013-01-04 00:30:39 UTC (rev 6126) @@ -1,12 +1,12 @@ -min_target_innovation 135 # starting target innovation value to try -max_target_innovation 136 # stops before this target innovation value +min_target_innovation 50 # starting target innovation value to try +max_target_innovation 51 # stops before this target innovation value target_innovation_increment 135 # increment target innovation by this amount annealing_repeats 1 # times to repeate the entire optimization process for each target_innovation -annealing_cycles 5 # times to perform the simulated annealing loop for all non-fixed parameters +annealing_cycles 25 # times to perform the simulated annealing loop for all non-fixed parameters -evolve_reruns 48 # times to perform evolve_to_target_and_save() (objective function - is batched) +evolve_reruns 96 # times to perform evolve_to_target_and_save() (objective function - is batched) nworkers 24 # number of parallel threads (cores) to use per invocation of C++ optimizer reruns_per_opt_invocation 24 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!! Added: SwiftApps/SciColSim/params/ARtest04 =================================================================== --- SwiftApps/SciColSim/params/ARtest04 (rev 0) +++ SwiftApps/SciColSim/params/ARtest04 2013-01-04 00:30:39 UTC (rev 6126) @@ -0,0 +1,18 @@ + +min_target_innovation 270 # starting target innovation value to try +max_target_innovation 1357 # stops before this target innovation value +target_innovation_increment 540 # increment target innovation by this amount + +annealing_repeats 1 # times to repeate the entire optimization process for each target_innovation +annealing_cycles 100 # times to perform the simulated annealing loop for all non-fixed parameters + +evolve_reruns 10080 # times to perform evolve_to_target_and_save() (objective function - is batched) +nworkers 24 # number of parallel threads (cores) to use per invocation of C++ optimizer +reruns_per_opt_invocation 24 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!! + +alpha_i 0.0 # 5 evolve() parameters: must be decimal values !!! +alpha_m 0.0 # alpha_i and alpha_m are fixed for now +beta 0.0 # rest are varied by the optimization process +gamma 0.0 +delta 0.0 + Added: SwiftApps/SciColSim/params/ARtest05 =================================================================== --- SwiftApps/SciColSim/params/ARtest05 (rev 0) +++ SwiftApps/SciColSim/params/ARtest05 2013-01-04 00:30:39 UTC (rev 6126) @@ -0,0 +1,17 @@ + +min_target_innovation 67 # starting target innovation value to try: 67 134 201 268 335 402 469 536 603 670 +max_target_innovation 672 # stops before this target innovation value +target_innovation_increment 67 # increment target innovation by this amount + +annealing_repeats 1 # times to repeate the entire optimization process for each target_innovation +annealing_cycles 100 # times to perform the simulated annealing loop for all non-fixed parameters + +evolve_reruns 5040 # times to perform evolve_to_target_and_save() (objective function - is batched) +nworkers 24 # number of parallel threads (cores) to use per invocation of C++ optimizer +reruns_per_opt_invocation 240 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!! + +alpha_i 0.0 # 5 evolve() parameters: must be decimal values !!! +alpha_m 0.0 # alpha_i and alpha_m are fixed for now +beta 0.0 # rest are varied by the optimization process +gamma 0.0 +delta 0.0 Added: SwiftApps/SciColSim/params/ARtest06 =================================================================== --- SwiftApps/SciColSim/params/ARtest06 (rev 0) +++ SwiftApps/SciColSim/params/ARtest06 2013-01-04 00:30:39 UTC (rev 6126) @@ -0,0 +1,19 @@ + +min_target_innovation 67 # starting target innovation value to try: 67 134 201 268 335 402 469 536 603 670 +max_target_innovation 672 # stops before this target innovation value +target_innovation_increment 67 # increment target innovation by this amount + +annealing_repeats 1 # times to repeate the entire optimization process for each target_innovation +annealing_cycles 100 # times to perform the simulated annealing loop for all non-fixed parameters + +evolve_reruns 5040 # times to perform evolve_to_target_and_save() (objective function - is batched) +nworkers 24 # number of parallel threads (cores) to use per invocation of C++ optimizer +reruns_per_opt_invocation 120 # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!! + +alpha_i 0.0 # 5 evolve() parameters: must be decimal values !!! +alpha_m 0.0 # alpha_i and alpha_m are fixed for now +beta 0.0 # rest are varied by the optimization process +gamma 0.0 +delta 0.0 + +starting_jump 0.3 From wilde at ci.uchicago.edu Thu Jan 3 18:31:38 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 3 Jan 2013 18:31:38 -0600 (CST) Subject: [Swift-commit] r6127 - SwiftApps/SciColSim Message-ID: <20130104003138.A90C59CCE5@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-03 18:31:38 -0600 (Thu, 03 Jan 2013) New Revision: 6127 Added: SwiftApps/SciColSim/Makefile.pgi Log: Initial makefile for PGI compiler on Cray. Does not yet produce runnable code. Added: SwiftApps/SciColSim/Makefile.pgi =================================================================== --- SwiftApps/SciColSim/Makefile.pgi (rev 0) +++ SwiftApps/SciColSim/Makefile.pgi 2013-01-04 00:31:38 UTC (rev 6127) @@ -0,0 +1,50 @@ +UNAME := $(shell uname) + +ifeq ($(UNAME), Linux) + +all: optimizer-pgi openmp-optimizer-pgi + +openmp-optimizer-pgi: optimizer.cpp + pgCC -g -traceback -DP_OPENMP -DBOOST_UBLAS_UNSUPPORTED_COMPILER=0 -Bstatic -O2 -mp -I boost_1_47_0 -o openmp-optimizer-pgi optimizer.cpp + +optimizer-pgi: optimizer.cpp + pgCC -g -traceback -Bstatic -O2 -DBOOST_UBLAS_UNSUPPORTED_COMPILER=0 -I boost_1_47_0 -o optimizer-pgi optimizer.cpp + +graphsim: graphsim.cpp + g++ -DP_OPENMP -static -pg -O2 -fopenmp -I boost_1_47_0 -o graphsim graphsim.cpp + +openmptest: openmptest.cpp + g++ -DP_OPENMP -static -O -fopenmp -o openmptest openmptest.cpp + +clean: + @rm -rvf openmp-optimizer + +endif + +ifeq ($(UNAME), Darwin) + +all: optimizer openmp-optimizer dispatch-optimizer orig-optimizer graphsim openmptest + +openmp-optimizer: optimizer.cpp + g++ -DP_OPENMP -fopenmp -I boost_1_47_0 -o openmp-optimizer optimizer.cpp + +optimizer: optimizer.cpp + g++ -I boost_1_47_0 -o optimizer optimizer.cpp + +graphsim: graphsim.cpp + g++ -DP_OPENMP -fopenmp -I boost_1_47_0 -g -o graphsim graphsim.cpp + +openmptest: openmptest.cpp + g++ -DP_OPENMP -fopenmp -g -o openmptest openmptest.cpp + +dispatch-optimizer: optimizer.cpp + g++ -DP_DISPATCH -I boost_1_47_0 -o dispatch-optimizer optimizer.cpp + +orig-optimizer: optimizer.orig-mac.cpp + g++ -I boost_1_47_0 -o orig-optimizer optimizer.orig-mac.cpp + +clean: + @rm -rvf openmp-optimizer dispatch-optimizer orig-optimizer + +endif + From wilde at ci.uchicago.edu Thu Jan 3 18:32:10 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 3 Jan 2013 18:32:10 -0600 (CST) Subject: [Swift-commit] r6128 - SwiftApps/SciColSim Message-ID: <20130104003210.5CDB49CCE5@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-03 18:32:10 -0600 (Thu, 03 Jan 2013) New Revision: 6128 Modified: SwiftApps/SciColSim/testopt.py Log: Added mw3 param set for testing Modified: SwiftApps/SciColSim/testopt.py =================================================================== --- SwiftApps/SciColSim/testopt.py 2013-01-04 00:31:38 UTC (rev 6127) +++ SwiftApps/SciColSim/testopt.py 2013-01-04 00:32:10 UTC (rev 6128) @@ -13,6 +13,12 @@ paramset="mw3" verboseLevel = "1" +alpha_i = "0.0" +alpha_m = "0.0" +beta = "0.0" +gamma = "0.0" +delta = "0.0" + print("Running with parameter set " + paramset + "\n\n"); if paramset == "beta": @@ -133,23 +139,44 @@ elif paramset == "mw3": # Timing TEST PARAMETERS by Mike - startTarget = 135 - endTarget = 136 - incrTarget = 135 + startTarget = 67 + endTarget = 68 + incrTarget = 134 optimizerRepeats = 1 evolveReruns = 1 - evolveReruns = 2 + evolveReruns = 240 annealingSteps = 1 - NWorkers = "1" + + alpha_i = "0.0" + alpha_m = "0.0" + beta = "0.0" + gamma = "0.0" + delta = "0.0" + + alpha_i = "0.0" + alpha_m = "-0.673572334121749" + beta = "-2.0320313590203978" + beta = "-1.2320313590203978" + gamma = "0.0" + delta = "0.0" + + alpha_i = "0.0" + alpha_m = "0.0" + beta = "0.0" + gamma = "0.0" + delta = "0.0" + + openmp = "" + NWorkers = "24" openmp = "OMP_NUM_THREADS=" + NWorkers - openmp = "" operation = "m" # n=normal, m=manual (runs 1 multi_loss call) seed = "123456" + seed = "" app = "./graphsim" + app = "./optimizer" app = "./openmp-optimizer" - app = "./optimizer" - verboseLevel = "99" + verboseLevel = "0" # Ensure we dont pass new parameters to original optimizer versions # (but they would be ignored) @@ -164,7 +191,7 @@ for target in range(startTarget,endTarget,incrTarget): for i in range(optimizerRepeats): args = "rm -f bestdb.txt; " + \ - openmp + " " + "/usr/bin/time -v " + app + " 0 -0.673572334121749 -2.0320313590203978 0 0 " + str(target) + " 40000 20 " + str(evolveReruns) + \ + openmp + " " + "/usr/bin/time -v " + app + " " + alpha_i + " " + alpha_m + " " + beta + " " + gamma + " " + delta + " " + str(target) + " 40000 20 " + str(evolveReruns) + \ " 2 " + verboseLevel + " 2. 0.01 " + str(annealingSteps) + " 0.3 2.3 0 0 0 0 0 " + operation + " " + NWorkers + " " + seed \ # + " >& out.T"+str(target)+".i"+str(i) + "; #mv bestdb.txt best.T" + str(target) + ".i" + str(i) print("\n**** Calling optimizer: "+args+"\n") From wilde at ci.uchicago.edu Thu Jan 3 18:34:31 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 3 Jan 2013 18:34:31 -0600 (CST) Subject: [Swift-commit] r6129 - SwiftApps/SciColSim Message-ID: <20130104003431.E4C119CCE5@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-03 18:34:31 -0600 (Thu, 03 Jan 2013) New Revision: 6129 Added: SwiftApps/SciColSim/monitor_run.sh Log: New script to monitor run status and place app-level progress stats into a web page. Added: SwiftApps/SciColSim/monitor_run.sh =================================================================== --- SwiftApps/SciColSim/monitor_run.sh (rev 0) +++ SwiftApps/SciColSim/monitor_run.sh 2013-01-04 00:34:31 UTC (rev 6129) @@ -0,0 +1,42 @@ +#! /bin/sh + +interval=120 # seconds between snapshots +statusdir=$HOME/public_html/scicol/status + +mkdir -p $statusdir + +if [ _$(basename $PWD | sed -e 's/run...$/run/') = _run ]; then + runid=$(basename $PWD) +else + runid=$(/bin/ls -1td run??? | head -1) + cd $runid +fi + +echo monitoring run id $runid + +source ./params.annealing + +min=$min_target_innovation +inc=$target_innovation_increment +max=$max_target_innovation + +while true; do + + ../bin/convertbest.sh + + (echo status of run $PWD at $(date); + echo -e "\nParameters:\n" + cat paramfile + echo -e "\nLatest status of all targets:\n" + for s in $(seq $min $inc $max); do tail -1 best.T$s.R1.fmt; done + echo -e "\nLatest results from each target:\n" + more $(for s in $(seq $min $inc $max); do echo best.T$s.R1.fmt; done) + echo -e "\nResults above are from files last changed at these times:\n" + ls -lt best.*.txt + echo -e "\nBeagle job status:\n" + xtnodestat | grep $USER + ) >$statusdir/$runid + + sleep $interval + +done Property changes on: SwiftApps/SciColSim/monitor_run.sh ___________________________________________________________________ Added: svn:executable + * From wilde at ci.uchicago.edu Thu Jan 3 19:16:18 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 3 Jan 2013 19:16:18 -0600 (CST) Subject: [Swift-commit] r6130 - SwiftApps/SciColSim Message-ID: <20130104011618.EBB8E9CCE5@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-03 19:16:18 -0600 (Thu, 03 Jan 2013) New Revision: 6130 Modified: SwiftApps/SciColSim/annealing.swift Log: Adjust some formatting back to K&R-like swift. Modified: SwiftApps/SciColSim/annealing.swift =================================================================== --- SwiftApps/SciColSim/annealing.swift 2013-01-04 00:34:31 UTC (rev 6129) +++ SwiftApps/SciColSim/annealing.swift 2013-01-04 01:16:18 UTC (rev 6130) @@ -24,27 +24,25 @@ ( float nx ) newx( float x, float dx ) { - float r = (random()); + float r = (random()); - if (r > 0.5) - { - nx = x + (random())*dx; //Java already returns a float between [0-1] - } - else - { - nx = x - (random())*dx; - } - // tracef("newx(%f,%f)=%f\n",x,dx,nx); + if (r > 0.5) { + nx = x + (random())*dx; //Java already returns a float between [0-1] + } + else { + nx = x - (random())*dx; + } + // tracef("newx(%f,%f)=%f\n",x,dx,nx); } app (/*file outfile,*/ file loss ) evolve ( string args[], file graph ) { - evolve @loss args /*stdout=@outfile*/; // FIXME: graph is passed implicitly + evolve @loss args /*stdout=@outfile*/; // FIXME: graph is passed implicitly } app ( file x ) sumloss( file loss[] ) { - sumloss @filenames(loss) stdout=@x; + sumloss @filenames(loss) stdout=@x; } /* From swift at ci.uchicago.edu Fri Jan 4 02:00:07 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 4 Jan 2013 02:00:07 -0600 (CST) Subject: [Swift-commit] cog r3541 Message-ID: <20130104080007.ED3788D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3541 | hategan | 2013-01-04 01:58:07 -0600 (Fri, 04 Jan 2013) | 1 line removed stack dump ------------------------------------------------------------------------ Index: modules/util/src/org/globus/cog/util/concurrent/FileLock.java =================================================================== --- modules/util/src/org/globus/cog/util/concurrent/FileLock.java (revision 3540) +++ modules/util/src/org/globus/cog/util/concurrent/FileLock.java (working copy) @@ -94,7 +94,6 @@ return (Integer) pid_method.invoke(mgmt); } catch (Exception e) { - e.printStackTrace(); return -1; } } From wilde at ci.uchicago.edu Fri Jan 4 14:01:38 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Fri, 4 Jan 2013 14:01:38 -0600 (CST) Subject: [Swift-commit] r6131 - SwiftApps/SciColSim Message-ID: <20130104200138.D30BD9D0F5@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-04 14:01:38 -0600 (Fri, 04 Jan 2013) New Revision: 6131 Modified: SwiftApps/SciColSim/testopt.py Log: Adjust time command for Mac OS X differences. Modified: SwiftApps/SciColSim/testopt.py =================================================================== --- SwiftApps/SciColSim/testopt.py 2013-01-04 01:16:18 UTC (rev 6130) +++ SwiftApps/SciColSim/testopt.py 2013-01-04 20:01:38 UTC (rev 6131) @@ -144,7 +144,7 @@ incrTarget = 134 optimizerRepeats = 1 evolveReruns = 1 - evolveReruns = 240 + evolveReruns = 20 annealingSteps = 1 alpha_i = "0.0" @@ -166,8 +166,8 @@ gamma = "0.0" delta = "0.0" + NWorkers = "2" openmp = "" - NWorkers = "24" openmp = "OMP_NUM_THREADS=" + NWorkers operation = "m" # n=normal, m=manual (runs 1 multi_loss call) @@ -186,12 +186,17 @@ operation = "" seed = "" +if sys.platform == "darwin": + timecmd = "/usr/bin/time " +else: + timecmd = "/usr/bin/time -v " + # Run the test for target in range(startTarget,endTarget,incrTarget): for i in range(optimizerRepeats): args = "rm -f bestdb.txt; " + \ - openmp + " " + "/usr/bin/time -v " + app + " " + alpha_i + " " + alpha_m + " " + beta + " " + gamma + " " + delta + " " + str(target) + " 40000 20 " + str(evolveReruns) + \ + openmp + " " + timecmd + app + " " + alpha_i + " " + alpha_m + " " + beta + " " + gamma + " " + delta + " " + str(target) + " 40000 20 " + str(evolveReruns) + \ " 2 " + verboseLevel + " 2. 0.01 " + str(annealingSteps) + " 0.3 2.3 0 0 0 0 0 " + operation + " " + NWorkers + " " + seed \ # + " >& out.T"+str(target)+".i"+str(i) + "; #mv bestdb.txt best.T" + str(target) + ".i" + str(i) print("\n**** Calling optimizer: "+args+"\n") From hategan at ci.uchicago.edu Sun Jan 6 01:46:02 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sun, 6 Jan 2013 01:46:02 -0600 (CST) Subject: [Swift-commit] r6132 - in branches/release-0.94: . resources src/org/griphyn/vdl/engine Message-ID: <20130106074602.3D3429CD07@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-06 01:45:59 -0600 (Sun, 06 Jan 2013) New Revision: 6132 Modified: branches/release-0.94/ branches/release-0.94/resources/swiftscript.g branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java Log: merged escaping of "<", ">", quote, and "&" from trunk Property changes on: branches/release-0.94 ___________________________________________________________________ Modified: svn:mergeinfo - /branches/release-0.93:4761-5122 + /branches/release-0.93:4761-5122 /trunk:6103-6104 Modified: branches/release-0.94/resources/swiftscript.g =================================================================== --- branches/release-0.94/resources/swiftscript.g 2013-01-04 20:01:38 UTC (rev 6131) +++ branches/release-0.94/resources/swiftscript.g 2013-01-06 07:45:59 UTC (rev 6132) @@ -1147,7 +1147,7 @@ | s:STRING_LITERAL { code=template("sConst"); - code.setAttribute("value",quote(s.getText())); + code.setAttribute("value",quote(escape(s.getText()))); } | t:"true" { Modified: branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java 2013-01-04 20:01:38 UTC (rev 6131) +++ branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java 2013-01-06 07:45:59 UTC (rev 6132) @@ -1606,7 +1606,7 @@ for (String key : stringInternMap.keySet()) { String variableName = stringInternMap.get(key); StringTemplate st = template("sConst"); - st.setAttribute("innervalue",escapeQuotes(key)); + st.setAttribute("innervalue",escapeQuotesAndMarkup(key)); StringTemplate vt = template("globalConstant"); vt.setAttribute("name",variableName); vt.setAttribute("expr",st); @@ -1638,6 +1638,30 @@ return in.replaceAll("\"", """); } + String escapeQuotesAndMarkup(String in) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < in.length(); i++) { + char c = in.charAt(i); + switch (c) { + case '<': + sb.append("<"); + break; + case '>': + sb.append(">"); + break; + case '"': + sb.append("""); + break; + case '&': + sb.append("&"); + break; + default: + sb.append(c); + } + } + return sb.toString(); + } + public static String normalize(String type) { return org.griphyn.vdl.type.Types.normalize(type); } From hategan at ci.uchicago.edu Sun Jan 6 01:52:47 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sun, 6 Jan 2013 01:52:47 -0600 (CST) Subject: [Swift-commit] r6133 - trunk/src/org/griphyn/vdl/util Message-ID: <20130106075247.E2BBA9CCFE@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-06 01:52:45 -0600 (Sun, 06 Jan 2013) New Revision: 6133 Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java Log: default value for wrapper staging Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java =================================================================== --- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2013-01-06 07:45:59 UTC (rev 6132) +++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2013-01-06 07:52:45 UTC (rev 6133) @@ -115,6 +115,7 @@ put("cdm.broadcast.mode", "file"); put("use.provider.staging", "false"); + put("use.wrapper.staging", "false"); put("ticker.date.format", ""); put("ticker.prefix", "Progress: time:"); From hategan at ci.uchicago.edu Sun Jan 6 01:54:36 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sun, 6 Jan 2013 01:54:36 -0600 (CST) Subject: [Swift-commit] r6134 - in branches/release-0.94: . src/org/griphyn/vdl/util Message-ID: <20130106075436.E116E9CCFE@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-06 01:54:35 -0600 (Sun, 06 Jan 2013) New Revision: 6134 Modified: branches/release-0.94/ branches/release-0.94/src/org/griphyn/vdl/util/VDL2Config.java Log: merged default value fro wrapper staging from trunk Property changes on: branches/release-0.94 ___________________________________________________________________ Modified: svn:mergeinfo - /branches/release-0.93:4761-5122 /trunk:6103-6104 + /branches/release-0.93:4761-5122 /trunk:6103-6104,6133 Modified: branches/release-0.94/src/org/griphyn/vdl/util/VDL2Config.java =================================================================== --- branches/release-0.94/src/org/griphyn/vdl/util/VDL2Config.java 2013-01-06 07:52:45 UTC (rev 6133) +++ branches/release-0.94/src/org/griphyn/vdl/util/VDL2Config.java 2013-01-06 07:54:35 UTC (rev 6134) @@ -115,6 +115,7 @@ put("cdm.broadcast.mode", "file"); put("use.provider.staging", "false"); + put("use.wrapper.staging", "false"); put("ticker.date.format", ""); put("ticker.prefix", "Progress: time:"); From swift at ci.uchicago.edu Sun Jan 6 15:20:11 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Sun, 6 Jan 2013 15:20:11 -0600 (CST) Subject: [Swift-commit] cog r3542 Message-ID: <20130106212011.1B27B8D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3542 | hategan | 2013-01-06 15:17:57 -0600 (Sun, 06 Jan 2013) | 1 line removed WS provider; never did much ------------------------------------------------------------------------ Index: modules/provider-ws/project.properties =================================================================== --- modules/provider-ws/project.properties (revision 3541) +++ modules/provider-ws/project.properties (working copy) @@ -1,7 +0,0 @@ -module.name = provider-ws -long.name = Generic Web services provider for abstractions -version = 0.1 -project = Java CoG Kit -lib.deps = wsif.jar, wsdl4j.jar -debug = true - Index: modules/provider-ws/launchers.xml =================================================================== --- modules/provider-ws/launchers.xml (revision 3541) +++ modules/provider-ws/launchers.xml (working copy) @@ -1,6 +0,0 @@ - - - - - - Index: modules/provider-ws/dependencies.xml =================================================================== --- modules/provider-ws/dependencies.xml (revision 3541) +++ modules/provider-ws/dependencies.xml (working copy) @@ -1,8 +0,0 @@ - - - - - - - - Index: modules/provider-ws/lib/wsdl4j.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: modules/provider-ws/lib/wsif.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: modules/provider-ws/meta/description.txt =================================================================== --- modules/provider-ws/meta/description.txt (revision 3541) +++ modules/provider-ws/meta/description.txt (working copy) @@ -1,4 +0,0 @@ -The provider-ws module is a component of the abstractions framework. -It provides the implementation for the execution task handler -supporting invocations of generic Web services using the Web services -invication framework (WSIF). Index: modules/provider-ws/src/org/globus/cog/abstraction/impl/invocation/ws/InvocationThread.java =================================================================== --- modules/provider-ws/src/org/globus/cog/abstraction/impl/invocation/ws/InvocationThread.java (revision 3541) +++ modules/provider-ws/src/org/globus/cog/abstraction/impl/invocation/ws/InvocationThread.java (working copy) @@ -1,270 +0,0 @@ -// ---------------------------------------------------------------------- -//This code is developed as part of the Java CoG Kit project -//The terms of the license can be found at http://www.cogkit.org/license -//This message may not be removed or altered. -//---------------------------------------------------------------------- - -package org.globus.cog.abstraction.impl.invocation.ws; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import javax.wsdl.Definition; -import javax.wsdl.Input; -import javax.wsdl.Operation; -import javax.wsdl.Output; -import javax.wsdl.Part; -import javax.wsdl.PortType; -import javax.wsdl.Service; -import javax.xml.namespace.QName; - -import org.apache.log4j.Logger; -import org.apache.wsif.WSIFException; -import org.apache.wsif.WSIFMessage; -import org.apache.wsif.WSIFOperation; -import org.apache.wsif.WSIFPort; -import org.apache.wsif.WSIFService; -import org.apache.wsif.WSIFServiceFactory; -import org.apache.wsif.util.WSIFUtils; -import org.globus.cog.abstraction.impl.common.StatusImpl; -import org.globus.cog.abstraction.interfaces.Status; -import org.globus.cog.abstraction.interfaces.Task; -import org.globus.cog.abstraction.interfaces.WSInvocationSpecification; - -public class InvocationThread extends Thread { - static Logger logger = Logger.getLogger(InvocationThread.class.getName()); - private Task task = null; - - public InvocationThread(Task task) { - this.task = task; - } - - public void run() { - try { - this.task.setStatus(Status.ACTIVE); - org.globus.cog.abstraction.interfaces.Service service = this.task - .getService(0); - WSInvocationSpecification spec = (WSInvocationSpecification) this.task - .getSpecification(); - String method = spec.getMethod(); - String args[] = spec.getArgumentsAsArray(); - HashMap map = invokeMethod(service.getServiceContact().getContact() - + "?wsdl", method, null, null, null, "", args, 0); - String output = null; - for (Iterator iterator = map.keySet().iterator(); iterator - .hasNext();) { - String name = (String) iterator.next(); - if (output == null) { - output = map.get(name) + "\n"; - } else { - output += map.get(name) + "\n"; - } - } - this.task.setStdOutput(output); - this.task.setStatus(Status.COMPLETED); - } catch (Exception e) { - Status newStatus = new StatusImpl(); - Status oldStatus = this.task.getStatus(); - newStatus.setPrevStatusCode(oldStatus.getStatusCode()); - newStatus.setStatusCode(Status.FAILED); - newStatus.setException(e); - this.task.setStatus(newStatus); - } - } - - private HashMap invokeMethod(String wsdlLocation, String operationName, - String inputName, String outputName, String portName, - String protocol, String[] args, int argShift) throws Exception { - String serviceNS = null; - String serviceName = null; - String portTypeNS = null; - String portTypeName = null; - - logger.debug("Reading WSDL document from '" + wsdlLocation + "'"); - Definition def = WSIFUtils.readWSDL(null, wsdlLocation); - - Service service = WSIFUtils.selectService(def, serviceNS, serviceName); - - Map portTypes = WSIFUtils.getAllItems(def, "PortType"); - if (portTypes.size() > 1 && portName != null) { - for (Iterator i = portTypes.keySet().iterator(); i.hasNext();) { - QName qn = (QName) i.next(); - if (portName.equals(qn.getLocalPart())) { - portTypeName = qn.getLocalPart(); - portTypeNS = qn.getNamespaceURI(); - break; - } - } - } - PortType portType = WSIFUtils.selectPortType(def, portTypeNS, - portTypeName); - - WSIFServiceFactory factory = WSIFServiceFactory.newInstance(); - WSIFService dpf = factory.getService(def, service, portType); - WSIFPort port = null; - if (portName == null) - port = dpf.getPort(); - else - port = dpf.getPort(portName); - - if (inputName == null && outputName == null) { - // retrieve list of operations - List operationList = portType.getOperations(); - - // try to find input and output names for the operation specified - boolean found = false; - for (Iterator i = operationList.iterator(); i.hasNext();) { - Operation op = (Operation) i.next(); - String name = op.getName(); - if (!name.equals(operationName)) { - continue; - } - if (found) { - throw new RuntimeException( - "Operation '" - + operationName - + "' is overloaded. " - + "Please specify the operation in the form " - + "'operationName:inputMessageName:outputMesssageName' to distinguish it"); - } - found = true; - Input opInput = op.getInput(); - inputName = (opInput.getName() == null) ? null : opInput - .getName(); - Output opOutput = op.getOutput(); - outputName = (opOutput.getName() == null) ? null : opOutput - .getName(); - } - } - - WSIFOperation operation = port.createOperation(operationName, - inputName, outputName); - WSIFMessage input = operation.createInputMessage(); - WSIFMessage output = operation.createOutputMessage(); - WSIFMessage fault = operation.createFaultMessage(); - - // retrieve list of names and types for input and names for output - List operationList = portType.getOperations(); - - // find portType operation to prepare in/out message w/ parts - boolean found = false; - String[] outNames = new String[0]; - Class[] outTypes = new Class[0]; - for (Iterator i = operationList.iterator(); i.hasNext();) { - Operation op = (Operation) i.next(); - String name = op.getName(); - if (!name.equals(operationName)) { - continue; - } - if (found) { - throw new RuntimeException( - "overloaded operations are not supported"); - } - found = true; - Input opInput = op.getInput(); - String[] inNames = new String[0]; - Class[] inTypes = new Class[0]; - if (opInput != null) { - List parts = opInput.getMessage().getOrderedParts(null); - unWrapIfWrappedDocLit(parts, name, def); - int count = parts.size(); - inNames = new String[count]; - inTypes = new Class[count]; - retrieveSignature(parts, inNames, inTypes); - } - for (int pos = 0; pos < inNames.length; ++pos) { - String arg = args[pos + argShift]; - Object value = null; - Class c = inTypes[pos]; - if (c.equals(String.class)) { - value = arg; - } else if (c.equals(Double.TYPE)) { - value = new Double(arg); - } else if (c.equals(Float.TYPE)) { - value = new Float(arg); - } else if (c.equals(Integer.TYPE)) { - value = new Integer(arg); - } else if (c.equals(Boolean.TYPE)) { - value = new Boolean(arg); - } else { - throw new RuntimeException("Cannot convert '" + arg - + "' to " + c); - } - - input.setObjectPart(inNames[pos], value); - } - - Output opOutput = op.getOutput(); - if (opOutput != null) { - List parts = opOutput.getMessage().getOrderedParts(null); - unWrapIfWrappedDocLit(parts, name + "Response", def); - int count = parts.size(); - outNames = new String[count]; - outTypes = new Class[count]; - retrieveSignature(parts, outNames, outTypes); - } - - } - if (!found) { - throw new RuntimeException("no operation " + operationName - + " was found in port type " + portType.getQName()); - } - - logger.debug("Executing operation " + operationName); - operation.executeRequestResponseOperation(input, output, fault); - - HashMap map = new HashMap(); - for (int pos = 0; pos < outNames.length; ++pos) { - String name = outNames[pos]; - map.put(name, output.getObjectPart(name)); - } - - return map; - } - - private void retrieveSignature(List parts, String[] names, Class[] types) { - // get parts in correct order - for (int i = 0; i < names.length; ++i) { - Part part = (Part) parts.get(i); - names[i] = part.getName(); - QName partType = part.getTypeName(); - if (partType == null) { - partType = part.getElementName(); - } - if (partType == null) { - throw new RuntimeException("part " + names[i] - + " must have type name declared"); - } - // only limited number of types is supported - // cheerfully ignoring schema namespace ... - String s = partType.getLocalPart(); - if ("string".equals(s)) { - types[i] = String.class; - } else if ("double".equals(s)) { - types[i] = Integer.TYPE; - } else if ("float".equals(s)) { - types[i] = Float.TYPE; - } else if ("int".equals(s)) { - types[i] = Integer.TYPE; - } else if ("boolean".equals(s)) { - types[i] = Boolean.TYPE; - } else { - throw new RuntimeException("part type " + partType - + " not supported in this sample"); - } - } - } - - private void unWrapIfWrappedDocLit(List parts, String operationName, - Definition def) throws WSIFException { - Part p = WSIFUtils.getWrappedDocLiteralPart(parts, operationName); - if (p != null) { - List unWrappedParts = WSIFUtils.unWrapPart(p, def); - parts.remove(p); - parts.addAll(unWrappedParts); - } - } - -} \ No newline at end of file Index: modules/provider-ws/src/org/globus/cog/abstraction/impl/invocation/ws/TaskHandlerImpl.java =================================================================== --- modules/provider-ws/src/org/globus/cog/abstraction/impl/invocation/ws/TaskHandlerImpl.java (revision 3541) +++ modules/provider-ws/src/org/globus/cog/abstraction/impl/invocation/ws/TaskHandlerImpl.java (working copy) @@ -1,247 +0,0 @@ -// ---------------------------------------------------------------------- -// This code is developed as part of the Java CoG Kit project -// The terms of the license can be found at http://www.cogkit.org/license -// This message may not be removed or altered. -// ---------------------------------------------------------------------- - -package org.globus.cog.abstraction.impl.invocation.ws; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Hashtable; -import java.util.Vector; - -import org.globus.cog.abstraction.impl.common.StatusEvent; -import org.globus.cog.abstraction.impl.common.task.ActiveTaskException; -import org.globus.cog.abstraction.impl.common.task.IllegalSpecException; -import org.globus.cog.abstraction.impl.common.task.InvalidSecurityContextException; -import org.globus.cog.abstraction.impl.common.task.InvalidServiceContactException; -import org.globus.cog.abstraction.impl.common.task.TaskSubmissionException; -import org.globus.cog.abstraction.interfaces.DelegatedTaskHandler; -import org.globus.cog.abstraction.interfaces.Status; -import org.globus.cog.abstraction.interfaces.StatusListener; -import org.globus.cog.abstraction.interfaces.Task; -import org.globus.cog.abstraction.interfaces.TaskHandler; - -/** - * Provides generic TaskHandler s for job submission task. - */ -public class TaskHandlerImpl implements TaskHandler, StatusListener { - private Vector submittedList = null; - private Vector activeList = null; - private Vector suspendedList = null; - private Vector resumedList = null; - private Vector failedList = null; - private Vector canceledList = null; - private Vector completedList = null; - private Hashtable handleMap = null; - private int type; - - public TaskHandlerImpl() { - this.submittedList = new Vector(); - this.activeList = new Vector(); - this.suspendedList = new Vector(); - this.resumedList = new Vector(); - this.failedList = new Vector(); - this.canceledList = new Vector(); - this.completedList = new Vector(); - this.handleMap = new Hashtable(); - this.type = TaskHandler.EXECUTION; - } - - public void setType(int type) { - this.type = type; - } - - public int getType() { - return this.type; - } - - public void submit(Task task) throws IllegalSpecException, - InvalidSecurityContextException, InvalidServiceContactException, - TaskSubmissionException { - if (task.getStatus().getStatusCode() != Status.UNSUBMITTED) { - throw new TaskSubmissionException( - "TaskHandler can only handle unsubmitted tasks"); - } - if (task.getType() != Task.WS_INVOCATION) { - throw new TaskSubmissionException( - getName() - + " execution task handler can only handle WS invocation tasks"); - } - int type = task.getType(); - DelegatedTaskHandler dth = newDelegatedTaskHandler(); - task.addStatusListener(this); - this.handleMap.put(task, dth); - dth.submit(task); - } - - protected DelegatedTaskHandler newDelegatedTaskHandler() { - return new InvocationTaskHandler(); - } - - protected String getName() { - return "Invocation"; - } - - public void suspend(Task task) throws InvalidSecurityContextException, - TaskSubmissionException { - DelegatedTaskHandler dth = (DelegatedTaskHandler) this.handleMap - .get(task); - if (dth != null) { - dth.suspend(); - } - } - - public void resume(Task task) throws InvalidSecurityContextException, - TaskSubmissionException { - DelegatedTaskHandler dth = (DelegatedTaskHandler) this.handleMap - .get(task); - if (dth != null) { - dth.resume(); - } - } - - public void cancel(Task task) throws InvalidSecurityContextException, - TaskSubmissionException { - DelegatedTaskHandler dth = (DelegatedTaskHandler) this.handleMap - .get(task); - if (dth != null) { - dth.cancel(); - } else { - task.setStatus(Status.CANCELED); - } - } - - public void remove(Task task) throws ActiveTaskException { - if (!handleMap.containsKey(task)) { - return; - } - int status = task.getStatus().getStatusCode(); - if ((status == Status.ACTIVE)) { - throw new ActiveTaskException( - "Cannot remove an active or suspended Task"); - } else { - // might cause problems - // task.removeStatusListener(this); - this.failedList.remove(task); - this.canceledList.remove(task); - this.completedList.remove(task); - this.suspendedList.remove(task); - this.submittedList.remove(task); - this.handleMap.remove(task); - - /* - * this is required because statusChange listeners are invoked in a - * single thread So if one listener that wants to remove a task from - * the handler is invoked before the listener implemented by this - * object, the task will be be in the activeList and will not be - * actually removed. - * - * To solve this we remove tasks from active list too - */ - this.activeList.remove(task); - } - } - - /** return a collection of all tasks submitted to the handler */ - public Collection getAllTasks() { - try { - return new ArrayList(handleMap.keySet()); - } catch (Exception e) { - return null; - } - } - - /** return a collection of active tasks */ - public Collection getActiveTasks() { - return new ArrayList(this.activeList); - } - - /** return a collection of failed tasks */ - public Collection getFailedTasks() { - return new ArrayList(this.failedList); - } - - /** return a collection of completed tasks */ - public Collection getCompletedTasks() { - return new ArrayList(this.completedList); - } - - /** return a collection of suspended tasks */ - public Collection getSuspendedTasks() { - return new ArrayList(this.suspendedList); - } - - /** return a collection of resumed tasks */ - public Collection getResumedTasks() { - return new ArrayList(this.resumedList); - } - - /** return a collection of canceled tasks */ - public Collection getCanceledTasks() { - return new ArrayList(this.canceledList); - } - - public void statusChanged(StatusEvent event) { - Task task = (Task) event.getSource(); - Status status = event.getStatus(); - int prevStatus = status.getPrevStatusCode(); - int curStatus = status.getStatusCode(); - switch (prevStatus) { - case Status.SUBMITTED: - this.submittedList.remove(task); - break; - case Status.ACTIVE: - this.activeList.remove(task); - break; - case Status.SUSPENDED: - this.suspendedList.remove(task); - break; - case Status.RESUMED: - this.resumedList.remove(task); - break; - case Status.FAILED: - this.failedList.remove(task); - break; - case Status.CANCELED: - this.canceledList.remove(task); - break; - case Status.COMPLETED: - this.completedList.remove(task); - break; - default: - break; - } - if (task != null) { - switch (curStatus) { - case Status.SUBMITTED: - this.submittedList.add(task); - break; - case Status.ACTIVE: - this.activeList.add(task); - break; - case Status.SUSPENDED: - this.suspendedList.add(task); - break; - case Status.RESUMED: - this.resumedList.add(task); - break; - case Status.FAILED: - this.failedList.add(task); - this.handleMap.remove(task); - break; - case Status.CANCELED: - this.canceledList.add(task); - this.handleMap.remove(task); - break; - case Status.COMPLETED: - this.completedList.add(task); - this.handleMap.remove(task); - break; - default: - break; - } - } - } -} \ No newline at end of file Index: modules/provider-ws/src/org/globus/cog/abstraction/impl/invocation/ws/InvocationTaskHandler.java =================================================================== --- modules/provider-ws/src/org/globus/cog/abstraction/impl/invocation/ws/InvocationTaskHandler.java (revision 3541) +++ modules/provider-ws/src/org/globus/cog/abstraction/impl/invocation/ws/InvocationTaskHandler.java (working copy) @@ -1,67 +0,0 @@ -// ---------------------------------------------------------------------- -//This code is developed as part of the Java CoG Kit project -//The terms of the license can be found at http://www.cogkit.org/license -//This message may not be removed or altered. -//---------------------------------------------------------------------- - -package org.globus.cog.abstraction.impl.invocation.ws; - -import org.apache.log4j.Logger; -import org.globus.cog.abstraction.impl.common.StatusImpl; -import org.globus.cog.abstraction.impl.common.task.IllegalSpecException; -import org.globus.cog.abstraction.impl.common.task.InvalidSecurityContextException; -import org.globus.cog.abstraction.impl.common.task.InvalidServiceContactException; -import org.globus.cog.abstraction.impl.common.task.TaskSubmissionException; -import org.globus.cog.abstraction.interfaces.DelegatedTaskHandler; -import org.globus.cog.abstraction.interfaces.Status; -import org.globus.cog.abstraction.interfaces.Task; - -public class InvocationTaskHandler implements DelegatedTaskHandler { - private static Logger logger = Logger - .getLogger(InvocationTaskHandler.class); - - private Task task = null; - - public void submit(Task task) throws IllegalSpecException, - InvalidSecurityContextException, InvalidServiceContactException, - TaskSubmissionException { - if (this.task != null) { - throw new TaskSubmissionException( - "InvocationTaskHandler cannot handle two active tasks simultaneously"); - } else { - this.task = task; - InvocationThread thread = new InvocationThread(this.task); - try { - // check if the task has not been canceled after it was - // submitted for execution - if (this.task.getStatus().getStatusCode() == Status.UNSUBMITTED) { - this.task.setStatus(Status.SUBMITTED); - thread.start(); - } - } catch (Exception e) { - e.printStackTrace(); - Status newStatus = new StatusImpl(); - Status oldStatus = this.task.getStatus(); - newStatus.setPrevStatusCode(oldStatus.getStatusCode()); - newStatus.setStatusCode(Status.FAILED); - newStatus.setException(e); - this.task.setStatus(newStatus); - } - - } - } - - public void suspend() throws InvalidSecurityContextException, - TaskSubmissionException { - } - - public void resume() throws InvalidSecurityContextException, - TaskSubmissionException { - } - - public void cancel() throws InvalidSecurityContextException, - TaskSubmissionException { - - this.task.setStatus(Status.CANCELED); - } -} \ No newline at end of file Index: modules/provider-ws/resources/cog-provider.properties =================================================================== --- modules/provider-ws/resources/cog-provider.properties (revision 3541) +++ modules/provider-ws/resources/cog-provider.properties (working copy) @@ -1,5 +0,0 @@ -provider=ws -sandbox=false -executionTaskHandler=org.globus.cog.abstraction.impl.invocation.ws.TaskHandlerImpl -securityContext=org.globus.cog.abstraction.impl.common.task.SecurityContextImpl - Index: modules/provider-ws/etc/MANIFEST.MF.tail =================================================================== --- modules/provider-ws/etc/MANIFEST.MF.tail (revision 3541) +++ modules/provider-ws/etc/MANIFEST.MF.tail (working copy) @@ -1 +0,0 @@ - Index: modules/provider-ws/etc/MANIFEST.MF.head =================================================================== --- modules/provider-ws/etc/MANIFEST.MF.head (revision 3541) +++ modules/provider-ws/etc/MANIFEST.MF.head (working copy) @@ -1 +0,0 @@ -Manifest-Version: 1.0 Index: modules/provider-ws/build.xml =================================================================== --- modules/provider-ws/build.xml (revision 3541) +++ modules/provider-ws/build.xml (working copy) @@ -1,152 +0,0 @@ - - - - - - - - - - - - - - - - Available targets: - help: - prints out this help message - - dist: - creates a distribution directory of the - ${project} ${long.name} - - jar: - creates a jar file for the ${project} ${long.name} - named ${jar.filename} - - javadoc: - creates the documentation - - clean: - removes the compiled classes - - distclean: - deletes the distribution directory - - all: - dist and javadoc - - deploy.webstart: - deploys the module as a webstart application - - dist.joint: - builds everything into one jar file. Should only - be used globally (from all) - - fixeol: - change newlines to the unix standard - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From wilde at ci.uchicago.edu Sun Jan 6 22:24:07 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Sun, 6 Jan 2013 22:24:07 -0600 (CST) Subject: [Swift-commit] r6135 - in SwiftApps/SciColSim: . conf params Message-ID: <20130107042407.A79939CCFE@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-06 22:24:06 -0600 (Sun, 06 Jan 2013) New Revision: 6135 Added: SwiftApps/SciColSim/genrec.sh Modified: SwiftApps/SciColSim/annealing.swift SwiftApps/SciColSim/conf/local.xml SwiftApps/SciColSim/params/MWtest02 SwiftApps/SciColSim/sumloss.sh SwiftApps/SciColSim/swiftopt.sh Log: Add restart mechanism to annealing.swift and additional logging to sumloss (logs to runNNN/sumloss). Modified: SwiftApps/SciColSim/annealing.swift =================================================================== --- SwiftApps/SciColSim/annealing.swift 2013-01-06 07:54:35 UTC (rev 6134) +++ SwiftApps/SciColSim/annealing.swift 2013-01-07 04:24:06 UTC (rev 6135) @@ -15,6 +15,25 @@ float tsdev; } +type Checkpoint +{ + int i; + int j; + float dx0, dx1, dx2, dx3, dx4; + float rej0, rej1, rej2, rej3, rej4; + float target_innov; + float loss; + float alpha_i; + float alpha_m; + float beta; + float gamma; + float delta; + float sdev; +} + +global boolean restart = (@arg("restart","false") == "true"); +tracef("Restart flag is %b\n", restart); + global int iterate_adjust = @toint(@arg("iterate_adjust","1")); // FIXME!!! : 1 for 0.93, 0 for 0.94 global boolean FIX_VARIABLES = true; @@ -40,9 +59,9 @@ evolve @loss args /*stdout=@outfile*/; // FIXME: graph is passed implicitly } -app ( file x ) sumloss( file loss[] ) +app (file outfile, file logfile ) sumloss( file loss[] ) { - sumloss @filenames(loss) stdout=@x; + sumloss @filename(logfile) @filenames(loss) stdout=@filename(outfile); } /* @@ -57,7 +76,7 @@ sumloss() */ -(file bestfile, file maxfile) multi_annealing (string some_out_filename, +(file bestfile, file maxfile) multi_annealing (string best_filename, float T_start, float T_end, float Target_rejection, @@ -77,18 +96,29 @@ Res mlres[][]; - mlres[0][0] = multi_loss(T_start, T_end, annealing_cycles, Target_rejection, - starting_jump, 0, 0, params0, target_innov, evolve_reruns ); // FIXME: serves for all evolve-params ??? - tracef( "multi_annealing: AR: initial: %f +- %f\n", mlres[0][0].loss, mlres[0][0].sdev ); + file ckptFile ; + Checkpoint ckpt; + int restartIndex; - foreach j in [0:NEVOPARAMS-1] { - x[0][j] = params0[j]; - dx[0][j] = starting_jump; - rejection[0][j] = 0.0; - curr_loss[j] = mlres[0][0].loss; - curr_sdev[j] = mlres[0][0].sdev; + if (restart) { + ckpt = readData(ckptFile); + restartIndex = ckpt.i; } - + else { + restartIndex = -1; + mlres[0][0] = multi_loss(T_start, T_end, annealing_cycles, Target_rejection, + starting_jump, 0, 0, params0, target_innov, evolve_reruns ); // FIXME: serves for all evolve-params ??? + tracef( "multi_annealing: AR: initial: %f +- %f\n", mlres[0][0].loss, mlres[0][0].sdev ); + + foreach j in [0:NEVOPARAMS-1] { + x[0][j] = params0[j]; + dx[0][j] = starting_jump; + rejection[0][j] = 0.0; + curr_loss[j] = mlres[0][0].loss; + curr_sdev[j] = mlres[0][0].sdev; + } + } + iterate iter_i { // number of annealing cycles int i = iter_i + 1; // i ranges [1:n] in the swift script so that [0] can be the initial condition @@ -97,130 +127,147 @@ tracef( @strcat( "multi_annealing: AR: i=%i ....T = ", color( 3, "%f" ),"\n" ), i, temperature ); - // On each new "major" cycle within the annealing_cycles (other than the first) set new rejection and dx values - - if (i %% cycle == 1 && i > 1) { - tracef( "multi_annealing: new cycle at i=%i\n", i ); - tracef( color( Pink, "multi_annealing: AR: New cycle at %i: prev dx[0-4]=[%f %f %f %f %f]\n" ), - i, dx[i-1][0], dx[i-1][1], dx[i-1][2], dx[i-1][3], dx[i-1][4] ); - foreach k in [0:NEVOPARAMS-1] { - float newrejection = rejection[i-1][k] / @tofloat( cycle ); - if (newrejection > 0.0) { - dx[i][k] = dx[i-1][k] / ( newrejection / Target_rejection ); - // FIXME: re-enable: rejection[i][k]=0.0; - trejection[i][k]=0.0; - } - else { - dx[i][k] = dx[i-1][k] * 2.0; - // FIXME: re-enable: rejection[i][k]=rejection[i-1][k]; - trejection[i][k]=newrejection; - } - // FIXME: HANGS? : tracef(color(Red,"Recomputed rejection: i=%d k=%d dx[i][k]=%f\n"), i, k, dx[i][k]); - } - tracef( color( Blue, "multi_annealing: AR: New cycle at %i: dx[0-4]=[%f %f %f %f %f]\n" ), - i, dx[i][0], dx[i][1], dx[i][2], dx[i][3], dx[i][4] ); + if ( i < restartIndex ) { + tracef( "skipping index %i - less than restart index %i\n", i, restartIndex ); } - else { // If not new cycle, set dx[i][*] from previous dx ([i-1]). rejection[i][j] is set later. - foreach k in [0:NEVOPARAMS-1] { - dx[i][k] = dx[i-1][k]; - } + else { if ( i == restartIndex ) { // reset variables from the restart file + dx[i] = [ckpt.dx0, ckpt.dx1, ckpt.dx2, ckpt.dx3, ckpt.dx4]; + rejection[i] = [ckpt.rej0, ckpt.rej1, ckpt.rej2, ckpt.rej3, ckpt.rej4]; + x[i][0] = ckpt.alpha_i; + x[i][1] = ckpt.alpha_m; + x[i][2] = ckpt.beta; + x[i][3] = ckpt.gamma; + x[i][4] = ckpt.delta; + curr_loss[((i+1)*NEVOPARAMS)-1] = ckpt.loss; + curr_sdev[((i+1)*NEVOPARAMS)-1] = ckpt.sdev; } - iterate j { // Try a new value for each non-fixed param; then write results and accept or reject - int curr = ( i * NEVOPARAMS ) + j; - int prev = curr-1; + else { // i > restartIndex: proceed as normal whether restarting or not - if ( /* (!FIX_VARIABLES) || */ ( var_fixed[j] == 0 ) ) { // Adjustable vars - // fixed=1,1,0,0,0: FIXME: FIX_VARIABLES flag has faulty logic but OK when TRUE - float try_x[]; - foreach k in [0:NEVOPARAMS-1] { // Select the evolve params to try - if (k < j) { - try_x[k] = x[i][k]; // already set x[i][k] + // On each new "major" cycle within the annealing_cycles (other than the first) set new rejection and dx values + + if (i %% cycle == 1 && i > 1) { + tracef( "multi_annealing: new cycle at i=%i\n", i ); + tracef( color( Pink, "multi_annealing: AR: New cycle at %i: prev dx[0-4]=[%f %f %f %f %f]\n" ), + i, dx[i-1][0], dx[i-1][1], dx[i-1][2], dx[i-1][3], dx[i-1][4] ); + foreach k in [0:NEVOPARAMS-1] { + float newrejection = rejection[i-1][k] / @tofloat( cycle ); + if (newrejection > 0.0) { + dx[i][k] = dx[i-1][k] / ( newrejection / Target_rejection ); + // FIXME: re-enable: rejection[i][k]=0.0; + trejection[i][k]=0.0; } else { - if (k == j) { - try_x[k] = newx(x[i-1][j], dx[i-1][j]); // permute x[i-1][j] - } - else { // k > j - try_x[k] = x[i-1][k]; // use x[i-1][k] (from prior cycle) - } + dx[i][k] = dx[i-1][k] * 2.0; + // FIXME: re-enable: rejection[i][k]=rejection[i-1][k]; + trejection[i][k]=newrejection; } + // FIXME: HANGS? : tracef(color(Red,"Recomputed rejection: i=%d k=%d dx[i][k]=%f\n"), i, k, dx[i][k]); } - tracef( @strcat( "multi_annealing: AR: ", color( 10,"%f" ), " ", color( 9,"%i" ),"\n" ), try_x[j], j ); - mlres[i][j] = multi_loss( T_start, T_end, annealing_cycles, Target_rejection, starting_jump, - i, j, try_x, target_innov, evolve_reruns ); // do the N evolve()'s, N=evolve_rerusn - tracef( "multi_annealing: AR: %f +- %f\n", mlres[i][j].loss, mlres[i][j].sdev ); - // Beyond this point, x[] and dx[] are being set for this i,j - float ALOT = 100000000000.0; // 100,000,000,000. = 10^11 - if (mlres[i][j].loss < ALOT) { - fprintf( some_out_filename, "N, %fs, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n", - mlres[i][j].tavg, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss, - try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev ); - // Note i-1 field: print that way to match with C++ output - - // fprintf( "max_dist_swift.txt", color( Red,"multi_annealing: AF: max_dist.txt - tbd\n" ) ); - // FIXME: max_dist is global set in evolve() + tracef( color( Blue, "multi_annealing: AR: New cycle at %i: dx[0-4]=[%f %f %f %f %f]\n" ), + i, dx[i][0], dx[i][1], dx[i][2], dx[i][3], dx[i][4] ); + } + else { // If not new cycle, set dx[i][*] from previous dx ([i-1]). rejection[i][j] is set later. + foreach k in [0:NEVOPARAMS-1] { + dx[i][k] = dx[i-1][k]; } - else { // does this ever occur? if so did we want to still do the ratio computation above??? - fprintf( some_out_filename, "A, %fs, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n", - mlres[i][j].tavg, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss, - try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev ); - // Note i-1 field: print that way to match with C++ output - //tracef( "multi_annealing: Loss %f > ALOT at [i][j] = [%d][%d]\n", mlres[i][j].loss, i ,j ); - } - float ratio = min( 1.0, exp( -( mlres[i][j].loss - curr_loss[prev] ) / temperature ) ); - float r = (random()); //Java already returns a random float between [0.0-1.0] - tracef("multi_annealing: AR: %f vs %f\n", r, ratio); - if (r > ratio) { // Reject new parameter - x[i][j] = x[i-1][j]; - if (i %% cycle == 1 && i > 1) { - rejection[i][j] = trejection[i][j] + 1.0; // FIXME: triple-check this! + } + iterate j { // Try a new value for each non-fixed param; then write results and accept or reject + int curr = ( i * NEVOPARAMS ) + j; + int prev = curr-1; + + if ( /* (!FIX_VARIABLES) || */ ( var_fixed[j] == 0 ) ) { // Adjustable vars + // fixed=1,1,0,0,0: FIXME: FIX_VARIABLES flag has faulty logic but OK when TRUE + float try_x[]; + foreach k in [0:NEVOPARAMS-1] { // Select the evolve params to try + if (k < j) { + try_x[k] = x[i][k]; // already set x[i][k] + } + else { + if (k == j) { + try_x[k] = newx(x[i-1][j], dx[i-1][j]); // permute x[i-1][j] + } + else { // k > j + try_x[k] = x[i-1][k]; // use x[i-1][k] (from prior cycle) + } + } } - else { - rejection[i][j] = rejection[i-1][j] + 1.0; // FIXME: AR: Is this correct? incr rejection? + tracef( @strcat( "multi_annealing: AR: ", color( 10,"%f" ), " ", color( 9,"%i" ),"\n" ), try_x[j], j ); + mlres[i][j] = multi_loss( T_start, T_end, annealing_cycles, Target_rejection, starting_jump, + i, j, try_x, target_innov, evolve_reruns ); // do the N evolve()'s, N=evolve_rerusn + tracef( "multi_annealing: AR: %f +- %f\n", mlres[i][j].loss, mlres[i][j].sdev ); + // Beyond this point, x[] and dx[] are being set for this i,j + float ALOT = 100000000000.0; // 100,000,000,000. = 10^11 + if (mlres[i][j].loss < ALOT) { + fprintf( best_filename, "N, %fs, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n", + mlres[i][j].tavg, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss, + try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev ); + // Note i-1 field: print that way to match with C++ output + + // fprintf( "max_dist_swift.txt", color( Red,"multi_annealing: AF: max_dist.txt - tbd\n" ) ); + // FIXME: max_dist is global set in evolve() } - curr_loss[curr] = curr_loss[prev]; - curr_sdev[curr] = curr_sdev[prev]; - // FIXME: AR: the following prints seem to replicate values in the .cpp version - please clarify. - tracef( "multi_annealing: AR: %i,%i %i Did not accept: %f (%i)\n", i, j, i, try_x[j], j ); - tracef( "multi_annealing: AR: %f %f %f %f %f\n", try_x[0],try_x[1],try_x[2],try_x[3],try_x[4] ); - } - else { // Accept new parameter - tracef( "multi_annealing: Accepting try_x[j], i=%i j=%i\n",i,j ); - x[i][j] = try_x[j]; - if (i %% cycle == 1 && i > 1) { - rejection[i][j] = trejection[i][j]; // FIXME: triple-check this! + else { // does this ever occur? if so did we want to still do the ratio computation above??? + fprintf( best_filename, "A, %fs, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n", + mlres[i][j].tavg, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss, + try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev ); + // Note i-1 field: print that way to match with C++ output + //tracef( "multi_annealing: Loss %f > ALOT at [i][j] = [%d][%d]\n", mlres[i][j].loss, i ,j ); } - else { - rejection[i][j] = rejection[i-1][j]; // FIXME: AR: Is this correct? no incr of rejection? + float ratio = min( 1.0, exp( -( mlres[i][j].loss - curr_loss[prev] ) / temperature ) ); + float r = (random()); //Java already returns a random float between [0.0-1.0] + tracef("multi_annealing: AR: %f vs %f\n", r, ratio); + if (r > ratio) { // Reject new parameter + x[i][j] = x[i-1][j]; + if (i %% cycle == 1 && i > 1) { + rejection[i][j] = trejection[i][j] + 1.0; // FIXME: triple-check this! + } + else { + rejection[i][j] = rejection[i-1][j] + 1.0; // FIXME: AR: Is this correct? incr rejection? + } + curr_loss[curr] = curr_loss[prev]; + curr_sdev[curr] = curr_sdev[prev]; + // FIXME: AR: the following prints seem to replicate values in the .cpp version - please clarify. + tracef( "multi_annealing: AR: %i,%i %i Did not accept: %f (%i)\n", i, j, i, try_x[j], j ); + tracef( "multi_annealing: AR: %f %f %f %f %f\n", try_x[0],try_x[1],try_x[2],try_x[3],try_x[4] ); } - curr_loss[curr] = mlres[i][j].loss; - curr_sdev[curr] = mlres[i][j].sdev; - tracef( "multi_annealing: Accepting try_x[j], i=%i j=%i try_x[j]=%f\n", i, j, try_x[j] ); - float rj[]; - foreach k in [0:NEVOPARAMS-1] { - if (k <= j) { - rj[k] = rejection[i][k]; // Was either set from previous j or just set for this j + else { // Accept new parameter + tracef( "multi_annealing: Accepting try_x[j], i=%i j=%i\n",i,j ); + x[i][j] = try_x[j]; + if (i %% cycle == 1 && i > 1) { + rejection[i][j] = trejection[i][j]; // FIXME: triple-check this! } else { - rj[k] = rejection[i-1][k]; // Not yet set, use previous + rejection[i][j] = rejection[i-1][j]; // FIXME: AR: Is this correct? no incr of rejection? } + curr_loss[curr] = mlres[i][j].loss; + curr_sdev[curr] = mlres[i][j].sdev; + tracef( "multi_annealing: Accepting try_x[j], i=%i j=%i try_x[j]=%f\n", i, j, try_x[j] ); + float rj[]; + foreach k in [0:NEVOPARAMS-1] { + if (k <= j) { + rj[k] = rejection[i][k]; // Was either set from previous j or just set for this j + } + else { + rj[k] = rejection[i-1][k]; // Not yet set, use previous + } + } + tracef(@strcat("multi_annealing: AR: [%i][%i] ", color(8,"Rejection counts: "), + color(1,"%f"), " ", color(7,"%f"), " ", color(5,"%f"), " ", color(9,"%f"), " ", color(6,"%f"), "\n\n"), + i, j, rj[0], rj[1], rj[2], rj[3], rj[4]); + tracef(@strcat("multi_annealing: AR: %i ", color(8,"***** Did accept! "), + color(1,"%f"), " ", color(7,"%f"), " ", color(5,"%f"), " ", color(9,"%f"), " ", color(6,"%f"), "\n\n"), + i, try_x[0], try_x[1], try_x[2], try_x[3], try_x[4]); + } } - tracef(@strcat("multi_annealing: AR: [%i][%i] ", color(8,"Rejection counts: "), - color(1,"%f"), " ", color(7,"%f"), " ", color(5,"%f"), " ", color(9,"%f"), " ", color(6,"%f"), "\n\n"), - i, j, rj[0], rj[1], rj[2], rj[3], rj[4]); - tracef(@strcat("multi_annealing: AR: %i ", color(8,"***** Did accept! "), - color(1,"%f"), " ", color(7,"%f"), " ", color(5,"%f"), " ", color(9,"%f"), " ", color(6,"%f"), "\n\n"), - i, try_x[0], try_x[1], try_x[2], try_x[3], try_x[4]); + else {// Fixed Vars + x[i][j] = x[i-1][j]; + rejection[i][j] = rejection[i-1][j]; + curr_loss[curr] = curr_loss[prev]; + curr_sdev[curr] = curr_sdev[prev]; + // dx[i][j] not set for fixed vars } - } - else {// Fixed Vars - x[i][j] = x[i-1][j]; - rejection[i][j] = rejection[i-1][j]; - curr_loss[curr] = curr_loss[prev]; - curr_sdev[curr] = curr_sdev[prev]; - // dx[i][j] not set for fixed vars - } - } until( j == (NEVOPARAMS-iterate_adjust) ); + } until( j == (NEVOPARAMS-iterate_adjust) ); + }} // of if/else } until( iter_i == (annealing_cycles-iterate_adjust) ); } @@ -260,7 +307,19 @@ tracef("multi_loss: i=%i calling evolve, args=%q\n", i, args); // tracef("multi_loss: after evolve: i=%i %k %k\n", i, outfile, rfile[i]); } - file sumfile = sumloss(rfile); + + string anneal_cycle; + if(ci==0) { + anneal_cycle="init"; + } + else { + anneal_cycle=@strcat(ci-1); + } + + file sumfile; + file logfile; + + (sumfile, logfile) = sumloss(rfile); r = readData(sumfile); tracef("multi_loss: returning: ci=%i cj=%i r.loss=%f r.sdev=%f\n",ci,cj,r.loss,r.sdev); @@ -287,11 +346,11 @@ int target_innov = minrange + (i*rangeinc); foreach rep in [1:nreps] { - string some_out_filename = @strcat( "best.T", @strcat(target_innov), ".R", @strcat(rep), ".txt" ); + string best_filename = @strcat( "best.T", @strcat(target_innov), ".R", @strcat(rep), ".txt" ); file outfile; // ; file lossfile; // ; - (outfile,lossfile) = multi_annealing(some_out_filename, + (outfile,lossfile) = multi_annealing(best_filename, @tofloat(@arg("tstart", "2.0")), @tofloat(@arg("tend", "0.01")), @tofloat(@arg("trejection", "0.3")), @@ -326,11 +385,11 @@ foreach target_innov in [minrange:maxrange:rangeinc] { foreach rep in [1:nreps] { - string some_out_filename = @strcat( "best.T", @strcat(target_innov), ".R", @strcat(rep), ".txt" ); + string best_filename = @strcat( "best.T", @strcat(target_innov), ".R", @strcat(rep), ".txt" ); file outfile; // ; file lossfile; // ; - (outfile,lossfile) = multi_annealing(some_out_filename, + (outfile,lossfile) = multi_annealing(best_filename, @tofloat(@arg("tstart", "2.0")), @tofloat(@arg("tend", "0.01")), @tofloat(@arg("trejection", "0.3")), Modified: SwiftApps/SciColSim/conf/local.xml =================================================================== --- SwiftApps/SciColSim/conf/local.xml 2013-01-06 07:54:35 UTC (rev 6134) +++ SwiftApps/SciColSim/conf/local.xml 2013-01-07 04:24:06 UTC (rev 6135) @@ -1,7 +1,7 @@ - .03 + .01 10000 _WORK_/local Added: SwiftApps/SciColSim/genrec.sh =================================================================== --- SwiftApps/SciColSim/genrec.sh (rev 0) +++ SwiftApps/SciColSim/genrec.sh 2013-01-07 04:24:06 UTC (rev 6135) @@ -0,0 +1,28 @@ +#! /bin/sh + +for f in $1/best.T*.R1.txt; do + ti=$(echo $f | sed -e 's/^.*T//' -e 's/\..*//') + ( + echo "i j dx0 dx1 dx2 dx3 dx4 rej0 rej1 rej2 rej3 rej4 target_innov loss alpha_i alpha_m beta gamma delta sdev" + cycle=$(sed -e 's/\]//g' -e 's/[,|\[]//g' $f | awk '$4 == 4 {print $3}' | tail -1) + echo 1>&2 ti=$ti will restart at cycle $cycle + sed -e 's/\]//g' -e 's/[,|\[]//g' $f | + awk ' + $3 == cycle { dx[$4] = $5; rej[$4] = $6 } + + $3 == cycle && $4 == 4 { + print $3, $4, + dx[0], dx[1], dx[2], dx[3], dx[4], + rej[0], rej[1], rej[2], rej[3], rej[4], + $7, $8, $9, $10, $11, $12, $13, $14 + } + ' cycle=$cycle | + tail -1 + ) > ckpt.$ti + if [ $(wc -l 50000 ) { + printf("sumloss warning: high sdev_loss=%f\n", sdev_loss) > sdev_warning + } +} ' $* 2>&1 | tee $outfile + + +if [ _$logfile != _ -a _$logfile != "_none" ]; then +( echo sumloss: $0 $* + if [ -f sdev_warning ]; then + cat sdve_warning + fi + echo + for f in $*; do + echo -e "\ninput file $f:\n" + cat $f + done + echo -e "\nsumloss output: \n" + cat $outfile + +) >$logfile +fi + +rm -f $outfile Modified: SwiftApps/SciColSim/swiftopt.sh =================================================================== --- SwiftApps/SciColSim/swiftopt.sh 2013-01-06 07:54:35 UTC (rev 6134) +++ SwiftApps/SciColSim/swiftopt.sh 2013-01-07 04:24:06 UTC (rev 6135) @@ -32,14 +32,15 @@ -nworkers=$nworkers \ -rerunsperapp=$reruns_per_opt_invocation \ -wget=$wget \ - -longruns=$longruns + -longruns=$longruns \ + -restart=$restart } # Default settings execsite=local paramfile=Fast01 -ram=2048M +ram=6000M dryrun= escapecode=$(printf '\033') variable_params=5 # Currently CONSTANT, will want to have this vary @@ -49,11 +50,12 @@ # Process command line arguments while [ $# -gt 0 ]; do case $1 in - -l) longruns=true; shift 1;; + -l) longruns=true; shift 1;; # FIXME -n) dryrun=true; shift 1;; -p) paramfile=$2; shift 2;; -s) execsite=$2; shift 2;; - -w) wget=true; shift 1;; + -w) wget=true; shift 1;; # FIXME + -r) resdir=$2; shift 2;; *) echo $usage 1>&2 exit 1;; esac @@ -71,6 +73,18 @@ mkdir $rundir fi +if [ _$resdir != _ ]; then + if [ -d resdir ]; then + restart=true + cp $resdir/ckpt.* $rundir + else + echo $0: no restart directory $resdir found. + exit 1 + fi +else + restart=false +fi + # Get optimization parameters if [ ! -f params/$paramfile ]; then From davidk at ci.uchicago.edu Mon Jan 7 11:20:43 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 7 Jan 2013 11:20:43 -0600 (CST) Subject: [Swift-commit] r6136 - trunk/bin Message-ID: <20130107172043.817459CCA8@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-07 11:20:43 -0600 (Mon, 07 Jan 2013) New Revision: 6136 Modified: trunk/bin/swiftrun Log: Start of a bash based swiftrun Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2013-01-07 04:24:06 UTC (rev 6135) +++ trunk/bin/swiftrun 2013-01-07 17:20:43 UTC (rev 6136) @@ -1,230 +1,54 @@ -#!/usr/bin/perl -w +#!/bin/bash -use strict; -use FindBin qw($Bin); -use lib "$FindBin::Bin/../lib/perl"; -use Cwd; -use Cwd qw(abs_path); -use File::Copy; -use File::Path; -use Getopt::Long qw(:config permute pass_through); -use File::Which qw(which where); -use XML::Simple; -use Pod::Usage; -use SwiftConfig; - -# Command line processing -my @option_sites; # Site name to execute upon -my $option_script; # Script name to run -my @option_inputs; # Name of data to include with execution -my @option_groups; # Name of group/groups to run script on -my $option_certificate; # Use a default certificate instead of your own - -GetOptions( - 'site|sites=s{,}' => \@option_sites, - 'script=s' => \$option_script, - 'input|inputs=s{,}' => \@option_inputs, - 'help|man|h' => sub { pod2usage(-verbose => 2); }, - 'group|groups=s{,}' => \@option_groups, - 'certificate' => \$option_certificate, -); - -# Check arguments -initialize_swiftconfig(); -if(!$option_script || (!@option_sites && !@option_groups)) { - print "Incorrect usage. Try swiftrun -help\n"; - exit; +# Verify an argument is not null +verify_not_null() +{ + argname=$1; shift + if [ _$1 != _ ]; then + return 0; + else + echo $0: value for $argname can not be null + exit 1 + fi } +# Parse command line arguments +args="" +script="" +while [ $# -gt 0 ] +do + case "$1" in + -site|-sites|-s) SITES=$2; verify_not_null sites $SITES; shift ;; + *) if [ -z "$args" ]; then script="$1" + else args+="$1" + fi ;; + esac + shift +done + # Create run directory -my $script_filename = strip_directory($option_script); -my $dotswift_directory = "$ENV{'HOME'}/.swift"; -my @run_directories = ; -if(!@run_directories) { - $run_directories[0] = "run.0000"; -} -my ($run_number) = $run_directories[-1] =~ m{(\d+)}; -$run_number++; -my $run_directory = getcwd() . "/" . sprintf "run.%04s", $run_number; -create_directory($run_directory); +rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' ) +mkdir $rundir -# Copy over needed files -copy_file( "$option_script", "$run_directory" ); +# Create sites.xml +IFS="," +echo "" > $rundir/sites.xml +for site in $SITES +do + gensites -l $site >> $rundir/sites.xml +done +echo "" >> $rundir/sites.xml -# Determine list of sites. Store each swiftconfig directory we need in @sites -my @sites = (); +# tc.data +if [ -f "tc.data" ]; then + cp tc.data $rundir +fi -# Check for -sites all -foreach my $option_site (@option_sites) { - if(lc($option_site) eq 'all') { - @sites = glob("$dotswift_directory/sites/*"); - } -} +# Swift script +if [ -f "$script" ]; then + cp $script $rundir +fi -# Everything else specified by -sites -foreach my $option_site (@option_sites) { - if(lc($option_site) eq 'all') { - next; - } - if(-d "$dotswift_directory/sites/$option_site") { - push(@sites, "$dotswift_directory/sites/$option_site"); - } - else { - die "Unable to find configuration for $option_site\n"; - } -} - -# Groups -foreach my $group(@option_groups) { - open(GROUPFILE, "$dotswift_directory/groups/$group.group") - || die "Unable to open file for $group\n"; - while() { - chomp($_); - push(@sites, "$dotswift_directory/sites/$_"); - } - close(GROUPFILE); -} - -# Create configuration files for run -my @sites_xml; -my @apps_data; -foreach my $site (@sites) { - if ( !-d "$site" ) { - die "Unable to find configuration for $site\n"; - } - - # Sites.xml - if ( !-e "$site/sites.xml" ) { - die "Unable to find sites.xml for $site\n"; - } - open(CURRENT_SITE, "$site/sites.xml") - || die "Unable to open $site/sites.xml"; - push(@sites_xml, ); - close(CURRENT_SITE); - - # Properties - if( -e "$site/swift.properties") { - copy_file( "$site/swift.properties", "$run_directory"); - } - - # Applications - if(-e "$site/apps") { - open(APPFILE, "$site/apps") || die "Unable to open $site/apps"; - while() { - chomp($_); - if(-e "$dotswift_directory/apps/$_.apps") { - open(APPDATA, "$dotswift_directory/apps/$_.apps") - || die "Unable to open $dotswift_directory/apps/$_.apps"; - my @temp_app_data = ; - close(APPDATA); - foreach(@temp_app_data) { - my $simple_site_name = strip_directory($site); - s/SWIFTCONFIGURATION/$simple_site_name/ig; - } - push(@apps_data, @temp_app_data); - } - } - } -} - -# Save configuration information -open(TC, ">>$run_directory/tc.data") - || die "Unable to write to $run_directory/tc.data\n"; -print TC @apps_data; -close(TC); - -open( SITEFILE, ">$run_directory/sites.xml" ) - || die "Unable to open $run_directory/sites.xml for writing\n"; -print SITEFILE "\n at sites_xml\n"; -close(SITEFILE); - -# Link input data -foreach my $input_file (@option_inputs) { - # Check for relative path - my $first_char = substr $input_file, 0, 1; - if( $first_char ne '/') { - $input_file = getcwd() . "/$input_file"; - } - - if ( !-e $input_file ) { - die "Unable to find requested input file: $input_file\n"; - } - my $new_path = "$run_directory/" . strip_directory($input_file); - symlink $input_file, $new_path - || die "Unable to create symlink to $new_path\n"; -} - -# Handle certificates if needed -if($option_certificate) { - $ENV{'X509_CERT_DIR'} = "$FindBin::Bin/../etc/certificate"; - $ENV{'X509_CADIR'} = "$FindBin::Bin/../etc/certificate"; - $ENV{'X509_USER_PROXY'} = "$FindBin::Bin/../etc/certificate/x509up_swift"; -} - - -# Run swift -chdir("$run_directory") || die "Unable to chdir to $run_directory\n"; -print "Executing script $option_script\n"; -print "Check $run_directory for output and debugging information\n"; -my $system_command = - 'swift ' - . "-sites.file $run_directory/sites.xml " - . "-tc.file $run_directory/tc.data " - . "$option_script "; -if(-e "$run_directory/swift.properties") { - $system_command .= "-config $run_directory/swift.properties "; -} - -# Add unknown options -foreach my $unknown_argument (@ARGV) { - $system_command .= "$unknown_argument "; -} - -# Execute -system($system_command); - -my $sec = (time - $^T); -my $hours = ($sec/(60*60))%24; -my $mins = ($sec/60)%60; -my $secs = $sec%60; -my $datestring = sprintf("%dh:%dm:%ds", $hours, $mins, $secs); -print "Script $option_script finished in $datestring\n"; - -__END__ - -=head1 NAME - -swiftrun - Run swift using configurations generated by swiftconfig - -=head1 SYNOPSIS - -swiftrun [B<-option> value] - -=head1 OVERVIEW - -The swiftrun program allows you to execute swift using configurations -generated by swiftconfig. - -=head1 DESCRIPTION - -General operations: - -site Execute swift on given sitename(s) - -script file.swift Specify the swift script to run - -input Link to data required for execution - -groups Run script on a site grouping -=head1 EXAMPLES - -Run a script on a single site - swiftrun -site local -script my.script - -Run a script on multiple sites - swiftrun -site host1 host2 -script my.script - -Run a script on all available configurations - swiftrun -site all -script my.script - -Link multiple files - swiftrun -site local -script my.script -input /foo/file* /foo2/file* - -=cut - +# Run +cd $rundir +time swift -sites.file sites.xml -tc.file tc.data $script $args 2>&1 | tee swift.out From davidk at ci.uchicago.edu Mon Jan 7 14:53:40 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 7 Jan 2013 14:53:40 -0600 (CST) Subject: [Swift-commit] r6137 - trunk/bin Message-ID: <20130107205340.98BCE9CCC8@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-07 14:53:40 -0600 (Mon, 07 Jan 2013) New Revision: 6137 Modified: trunk/bin/gensites Log: Allow gensites to work with any tags defined in a template (like _JOBSPERNODE_), don't require an explicit list of what is allowed Modified: trunk/bin/gensites =================================================================== --- trunk/bin/gensites 2013-01-07 17:20:43 UTC (rev 6136) +++ trunk/bin/gensites 2013-01-07 20:53:40 UTC (rev 6137) @@ -10,13 +10,20 @@ # get_value: Return the right hand side of an equals # statment in swift.properties format: #site sitename option=value -# will return bar get_value() { for last; do true; done - eval echo $last |cut -d "=" -f2 + eval echo $last |cut -d'=' -f2 } +# get_variable: Return the left hand side of an equals +# statement in swift.properties format: #site sitename option=value +get_variable() +{ + for last; do true; done + eval echo $last | cut -d'=' -f1 +} + # Verify an argument is not null verify_not_null() { @@ -156,42 +163,11 @@ while read line do case "$line" in - "#site $TEMPLATE work="*|'#site work='*) - WORK=`get_value $line` + "#site $TEMPLATE "*) + var=$( get_variable $line | tr '[a-z]' '[A-Z]' ) + val=$( get_value $line ) + eval $( echo $var )=$val ;; - "#site $TEMPLATE globus_hostname="*|'#site globus_hostname=') - GLOBUS_HOSTNAME=`get_value $line` - ;; - "#site $TEMPLATE nodes="*|'#site nodes='*) - NODES=`get_value $line` - ;; - "#site $TEMPLATE project="*|'#site project='*) - PROJECT=`get_value $line` - ;; - "#site $TEMPLATE queue="*|'#site queue='*) - QUEUE=`get_value $line` - ;; - "#site $TEMPLATE n_gran="*|'#site n_gran='*) - N_GRAN=`get_value $line` - ;; - "#site $TEMPLATE n_max="*|'#site n_max='*) - MAXTIME=`get_value $line` - ;; - "#site $TEMPLATE slots="*|'#site slots='*) - SLOTS=`get_value $line` - ;; - "#site $TEMPLATE execution_url="*|'#site execution_url='*) - EXECUTION_URL=`get_value $line` - ;; - "#site $TEMPLATE internalhostname="*|'#site internalhostname='*) - INTERNALHOSTNAME=`get_value $line` - ;; - "#site $TEMPLATE jobs_per_node="*|'#site jobs_per_node='*) - JOBS_PER_NODE=`get_value $line` - ;; - "#site $TEMPLATE job_throttle="*|'#site job_throttle='*) - JOB_THROTTLE=`get_value $line` - ;; '#app'*) if [ `echo $line | wc -w` == 2 ]; then for HOST in $HOSTS @@ -226,41 +202,14 @@ done < $PROPERTIES_FILE fi -# Verify that the variables by the template are defined -for TOKEN in NODES HOST WORK PROJECT QUEUE N_GRAN N_MAX SLOTS INTERNALHOSTNAME MAXTIME EXECUTION_URL JOBS_PER_NODE JOB_THROTTLE +# Replace values +VALUES_TO_REPLACE=$( sed -n 's/.*_\(.*\)_.*/\1/p' $TEMPLATE_PATH ) +SED_COMMAND="sed " +for value in $VALUES_TO_REPLACE do - # Test for HOST/GLOBUS_HOSTNAME - the only values which don't match - if [ "$TOKEN" == "HOST" ]; then - if [ -z "$GLOBUS_HOSTNAME" ]; then - GLOBUS_HOSTNAME=$( hostname -f ) - fi - elif grep _${TOKEN}_ $TEMPLATE_PATH > /dev/null; then - if [ -z "${!TOKEN}" ]; then - crash "Not specified: ${TOKEN}" - fi + if [ -z "${!value}" ]; then + crash "$value not defined" fi + SED_COMMAND+="-e s@"\_$value\_"@${!value}@g " done - -# Replace values -SEDFILE=`mktemp tmp.XXXXXXXXXX` -{ - echo "s/_NODES_/${NODES}/" - echo "s/_HOST_/${GLOBUS_HOSTNAME}/" - echo "s at _WORK_@${WORK}@" - echo "s/_PROJECT_/${PROJECT}/" - echo "s/_QUEUE_/${QUEUE}/" - echo "s/_N_GRAN_/${N_GRAN}/" - echo "s/_N_MAX_/${N_MAX}/" - echo "s/_SLOTS_/${SLOTS}/" - echo "s/_INTERNALHOSTNAME_/${INTERNALHOSTNAME}/" - echo "s/_MAXTIME_/${MAXTIME}/" - echo "s at _EXECUTION_URL_@${EXECUTION_URL}@" - echo "s at _SERVICE_COASTERS_@${SERVICE_COASTERS:-NO_URL_GIVEN}@" - echo "s at _SERVICE_PORT_@${SERVICE_PORT:-NO_PORT_GIVEN}@" - echo "s at _JOBS_PER_NODE_@${JOBS_PER_NODE}@" - echo "s at _JOB_THROTTLE_@${JOB_THROTTLE}@" -} > $SEDFILE - - -sed -f $SEDFILE < $TEMPLATE_PATH -rm $SEDFILE +$SED_COMMAND $TEMPLATE_PATH From davidk at ci.uchicago.edu Mon Jan 7 16:07:34 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 7 Jan 2013 16:07:34 -0600 (CST) Subject: [Swift-commit] r6138 - trunk/bin Message-ID: <20130107220734.19CBB9CCC8@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-07 16:07:33 -0600 (Mon, 07 Jan 2013) New Revision: 6138 Added: trunk/bin/swiftstat Modified: trunk/bin/swiftrun Log: Have swiftrun create .swift structure and keep track of run status First simple version of swiftstat Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2013-01-07 20:53:40 UTC (rev 6137) +++ trunk/bin/swiftrun 2013-01-07 22:07:33 UTC (rev 6138) @@ -12,6 +12,34 @@ fi } +# crash: Report a problem and exit +crash() +{ + MSG=$1 + echo ${MSG} >&2 + exit 1 +} + +# Create .swift structure +create_dotswift() +{ + if [ ! -d "$HOME/.swift" ]; then + mkdir $HOME/.swift || crash "Unable to mkdir $HOME/.swift" + fi + + if [ ! -d "$HOME/.swift/runs" ]; then + mkdir $HOME/.swift/runs || crash "Unable to mkdir $HOME/.swift/runs" + fi + + if [ ! -d "$HOME/.swift/runs/current" ]; then + mkdir $HOME/.swift/runs/current || crash "Unable to mkdir $HOME/.swift/runs/current" + fi + + if [ ! -d "$HOME/.swift/runs/completed" ]; then + mkdir $HOME/.swift/runs/completed || crash "Unable to mkdir $HOME/.swift/runs/completed" + fi +} + # Parse command line arguments args="" script="" @@ -27,6 +55,7 @@ done # Create run directory +create_dotswift rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' ) mkdir $rundir @@ -51,4 +80,7 @@ # Run cd $rundir +timestamp=$( date +%s ) +ln -s $PWD $HOME/.swift/runs/current/$rundir.$timestamp time swift -sites.file sites.xml -tc.file tc.data $script $args 2>&1 | tee swift.out +mv $HOME/.swift/runs/current/$rundir.$timestamp $HOME/.swift/runs/completed Added: trunk/bin/swiftstat =================================================================== --- trunk/bin/swiftstat (rev 0) +++ trunk/bin/swiftstat 2013-01-07 22:07:33 UTC (rev 6138) @@ -0,0 +1,3 @@ +#!/bin/bash + +watch --no-title -n 5 "find -L $HOME/.swift/runs/current -name swift.out -exec echo \; -exec dirname {} \; -exec tail -1 {} \;" Property changes on: trunk/bin/swiftstat ___________________________________________________________________ Added: svn:executable + * From davidk at ci.uchicago.edu Mon Jan 7 16:11:52 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 7 Jan 2013 16:11:52 -0600 (CST) Subject: [Swift-commit] r6139 - trunk/etc/sites Message-ID: <20130107221152.810689CCC8@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-07 16:11:52 -0600 (Mon, 07 Jan 2013) New Revision: 6139 Added: trunk/etc/sites/midway Log: Site template for midway Added: trunk/etc/sites/midway =================================================================== --- trunk/etc/sites/midway (rev 0) +++ trunk/etc/sites/midway 2013-01-07 22:11:52 UTC (rev 6139) @@ -0,0 +1,19 @@ + + + 16 + 24:00:00 + sandyb + 10000 + + /scratch/midway/{env.USER} + + + + + 12 + 24:00:00 + westmere + 10000 + + /scratch/midway/{env.USER} + From davidk at ci.uchicago.edu Mon Jan 7 17:35:17 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 7 Jan 2013 17:35:17 -0600 (CST) Subject: [Swift-commit] r6140 - trunk/bin Message-ID: <20130107233517.0B51D9CCA8@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-07 17:35:16 -0600 (Mon, 07 Jan 2013) New Revision: 6140 Modified: trunk/bin/gensites trunk/bin/swiftrun Log: Specify a swift.properties/cf to swiftrun for auto tc.data generation Have gensites just cat the template when no replacements need to happen Modified: trunk/bin/gensites =================================================================== --- trunk/bin/gensites 2013-01-07 22:11:52 UTC (rev 6139) +++ trunk/bin/gensites 2013-01-07 23:35:16 UTC (rev 6140) @@ -212,4 +212,10 @@ fi SED_COMMAND+="-e s@"\_$value\_"@${!value}@g " done -$SED_COMMAND $TEMPLATE_PATH +len=$( echo $SED_COMMAND |wc -w ) +if [ $len -gt 1 ]; then + $SED_COMMAND $TEMPLATE_PATH +else + cat $TEMPLATE_PATH +fi + Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2013-01-07 22:11:52 UTC (rev 6139) +++ trunk/bin/swiftrun 2013-01-07 23:35:16 UTC (rev 6140) @@ -47,6 +47,7 @@ do case "$1" in -site|-sites|-s) SITES=$2; verify_not_null sites $SITES; shift ;; + -config|-c) CONFIG=$2; verify_nul_null config $CONFIG; shift ;; *) if [ -z "$args" ]; then script="$1" else args+="$1" fi ;; @@ -64,7 +65,11 @@ echo "" > $rundir/sites.xml for site in $SITES do - gensites -l $site >> $rundir/sites.xml + if [ -f "$CONFIG" ]; then + gensites -p $CONFIG -l $site >> $rundir/sites.xml + else + gensites -l $site >> $rundir/sites.xml + fi done echo "" >> $rundir/sites.xml From davidk at ci.uchicago.edu Mon Jan 7 17:46:39 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 7 Jan 2013 17:46:39 -0600 (CST) Subject: [Swift-commit] r6141 - trunk/bin Message-ID: <20130107234639.BDB979CCA8@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-07 17:46:39 -0600 (Mon, 07 Jan 2013) New Revision: 6141 Modified: trunk/bin/swiftrun Log: Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2013-01-07 23:35:16 UTC (rev 6140) +++ trunk/bin/swiftrun 2013-01-07 23:46:39 UTC (rev 6141) @@ -47,7 +47,7 @@ do case "$1" in -site|-sites|-s) SITES=$2; verify_not_null sites $SITES; shift ;; - -config|-c) CONFIG=$2; verify_nul_null config $CONFIG; shift ;; + -config|-c) CONFIG=$2; verify_not_null config $CONFIG; shift ;; *) if [ -z "$args" ]; then script="$1" else args+="$1" fi ;; From davidk at ci.uchicago.edu Mon Jan 7 18:01:44 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 7 Jan 2013 18:01:44 -0600 (CST) Subject: [Swift-commit] r6142 - trunk/bin Message-ID: <20130108000144.8CBFD9CCA8@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-07 18:01:44 -0600 (Mon, 07 Jan 2013) New Revision: 6142 Modified: trunk/bin/swiftrun Log: Fix the gensites parameters that swiftrun uses Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2013-01-07 23:46:39 UTC (rev 6141) +++ trunk/bin/swiftrun 2013-01-08 00:01:44 UTC (rev 6142) @@ -60,31 +60,27 @@ rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' ) mkdir $rundir +# Swift script +if [ -f "$script" ]; then + cp $script $rundir +fi + # Create sites.xml +cd $rundir IFS="," -echo "" > $rundir/sites.xml +echo "" > sites.xml for site in $SITES do if [ -f "$CONFIG" ]; then - gensites -p $CONFIG -l $site >> $rundir/sites.xml + gensites -p $CONFIG $site >> sites.xml else - gensites -l $site >> $rundir/sites.xml + gensites $site >> sites.xml fi done -echo "" >> $rundir/sites.xml +echo "" >> sites.xml -# tc.data -if [ -f "tc.data" ]; then - cp tc.data $rundir -fi -# Swift script -if [ -f "$script" ]; then - cp $script $rundir -fi - # Run -cd $rundir timestamp=$( date +%s ) ln -s $PWD $HOME/.swift/runs/current/$rundir.$timestamp time swift -sites.file sites.xml -tc.file tc.data $script $args 2>&1 | tee swift.out From hategan at ci.uchicago.edu Tue Jan 8 13:43:26 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Tue, 8 Jan 2013 13:43:26 -0600 (CST) Subject: [Swift-commit] r6143 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20130108194326.420919CCA8@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-08 13:43:26 -0600 (Tue, 08 Jan 2013) New Revision: 6143 Modified: trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java Log: convert variable names to lower case when handling write references in foreach Modified: trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2013-01-08 00:01:44 UTC (rev 6142) +++ trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2013-01-08 19:43:26 UTC (rev 6143) @@ -117,7 +117,7 @@ } else { int count = Integer.parseInt(st.nextToken()); - l.add(new StaticRefCount(name, count)); + l.add(new StaticRefCount(name.toLowerCase(), count)); } flip = !flip; } From davidk at ci.uchicago.edu Tue Jan 8 13:56:40 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 8 Jan 2013 13:56:40 -0600 (CST) Subject: [Swift-commit] r6144 - branches Message-ID: <20130108195640.AA0B39CCA8@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-08 13:56:40 -0600 (Tue, 08 Jan 2013) New Revision: 6144 Removed: branches/release-0.94/ Log: From davidk at ci.uchicago.edu Tue Jan 8 13:57:07 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 8 Jan 2013 13:57:07 -0600 (CST) Subject: [Swift-commit] r6145 - branches Message-ID: <20130108195707.E4A989CCA8@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-08 13:57:07 -0600 (Tue, 08 Jan 2013) New Revision: 6145 Added: branches/release-0.94/ Log: From davidk at ci.uchicago.edu Tue Jan 8 14:03:31 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 8 Jan 2013 14:03:31 -0600 (CST) Subject: [Swift-commit] r6146 - branches/release-0.94/libexec Message-ID: <20130108200331.23BCD9CCA8@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-08 14:03:30 -0600 (Tue, 08 Jan 2013) New Revision: 6146 Modified: branches/release-0.94/libexec/release.txt Log: 0.94RC3 Modified: branches/release-0.94/libexec/release.txt =================================================================== --- branches/release-0.94/libexec/release.txt 2013-01-08 19:57:07 UTC (rev 6145) +++ branches/release-0.94/libexec/release.txt 2013-01-08 20:03:30 UTC (rev 6146) @@ -1 +1 @@ -trunk +0.94RC3 From swift at ci.uchicago.edu Wed Jan 9 13:05:05 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 9 Jan 2013 13:05:05 -0600 (CST) Subject: [Swift-commit] cog r3545 Message-ID: <20130109190505.6F20E8D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3545 | davidkelly999 | 2013-01-09 13:01:40 -0600 (Wed, 09 Jan 2013) | 4 lines A few tweaks to get the slurm provider working on Stampede Stampede uses 'ibrun' instead of srun, so make srun a property can you can override by setting slurm.properties Remove leading whitespace from squeue output ------------------------------------------------------------------------ Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java (revision 3544) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java (working copy) @@ -9,7 +9,7 @@ import org.apache.log4j.Logger; import org.globus.cog.abstraction.impl.common.execution.WallTime; import org.globus.cog.abstraction.impl.scheduler.common.AbstractExecutor; -import org.globus.cog.abstraction.impl.scheduler.common.AbstractProperties; +import org.globus.cog.abstraction.impl.scheduler.slurm.Properties; import org.globus.cog.abstraction.impl.scheduler.common.AbstractQueuePoller; import org.globus.cog.abstraction.impl.scheduler.common.Job; import org.globus.cog.abstraction.impl.scheduler.common.ProcessListener; @@ -166,7 +166,7 @@ } } - wr.write("srun /bin/bash -c \'"); + wr.write(getProperties().getRunCommand() + " /bin/bash -c \'"); if (spec.getDirectory() != null) { wr.write("cd " + quote(spec.getDirectory()) + " && "); } @@ -199,7 +199,7 @@ } @Override - protected AbstractProperties getProperties() { + protected Properties getProperties() { return Properties.getProperties(); } Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/QueuePoller.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/QueuePoller.java (revision 3544) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/QueuePoller.java (working copy) @@ -84,9 +84,9 @@ do { line = br.readLine(); if(line != null) { - String words[] = line.split("\\s+"); - String jobid = words[0].trim(); - String state = words[4].trim(); + String words[] = line.trim().split("\\s+"); + String jobid = words[0]; + String state = words[4]; if (jobid == null || jobid.equals("") || state == null || state.equals("")) { throw new IOException("Failed to parse squeue line: " + line); } Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/Properties.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/Properties.java (revision 3544) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/Properties.java (working copy) @@ -9,6 +9,7 @@ public static final String POLL_INTERVAL = "poll.interval"; public static final String SBATCH = "sbatch"; + public static final String SRUN = "srun"; public static final String SQUEUE = "squeue"; public static final String SCANCEL = "scancel"; @@ -27,19 +28,29 @@ setSubmitCommand("sbatch"); setPollCommand("squeue"); setRemoveCommand("scancel"); + setRunCommand("srun"); } + public void setRunCommand(String val) { + setProperty(getRunCommandName(), val); + } + public String getRunCommandName() { + return SRUN; + } + + public String getRunCommand() { + return getProperty(getRunCommandName()); + } + public String getPollCommandName() { return SQUEUE; } - public String getRemoveCommandName() { return SCANCEL; } - public String getSubmitCommandName() { return SBATCH; } From davidk at ci.uchicago.edu Wed Jan 9 17:16:09 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 9 Jan 2013 17:16:09 -0600 (CST) Subject: [Swift-commit] r6147 - trunk/docs/siteguide Message-ID: <20130109231609.8EC549CCE5@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-09 17:16:09 -0600 (Wed, 09 Jan 2013) New Revision: 6147 Modified: trunk/docs/siteguide/uc3 Log: Adding information about HDFS and staging in executable Modified: trunk/docs/siteguide/uc3 =================================================================== --- trunk/docs/siteguide/uc3 2013-01-08 20:03:30 UTC (rev 6146) +++ trunk/docs/siteguide/uc3 2013-01-09 23:16:09 UTC (rev 6147) @@ -134,3 +134,137 @@ ----- UidDomain == "osg-gk.mwt2.org" ----- + +Installing Application Scripts on HDFS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +NOTE: This section will only work if the application you want to use is an interpreted script (bash, +python, perl, etc). HDFS does not have the ability to give programs an execution bit and run them. If +the application you want to run on UC3 is a compiled executable, skip this section and read ahead. + +Once your simple echo test is running, you'll want to start using your own applications. One way +to go about doing this is by using the Hadoop filesystem. This filesystem is only available on +the UC3 Seeder Cluster. In order to limit yourself to machines that can access this filesystem, +add the following line to your sites.xml file: + +----- +UidDomain == "osg-gk.mwt2.org" && regexp("uc3-c*", Machine) +----- + +Now you can install your script somewhere under the directory /mnt/hadoop/users/. +Here is an example with putting everything together. + +.sites.xml +----- + + + + 999.99 + 10000 + 1 + 3600 + 1 + 100 + 100 + 1000 + 1 + "group_friends.{env.USER}" + nonshared + UidDomain == "osg-gk.mwt2.org" && regexp("uc3-c*", Machine) + + . + + +----- + +.tc.data +----- +uc3 bash /bin/bash null null null +----- + +.myscript.swift +----- +type file; + +app (file o) myscript () +{ + bash "/mnt/hadoop/users//myscript.sh" stdout=@o; +} + +file out[]; +int ntasks = @toInt(@arg("n","1")); + +foreach n in [1:ntasks] { + out[n] = myscript(); +} +----- + +./mnt/hadoop/users//myscript.sh +----- +#!/bin/bash + +echo This is my script +----- + +.cf +----- +wrapperlog.always.transfer=false +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false +use.wrapper.staging=false +----- + +.Example run +----- +$ swift -sites.file sites.xml -tc.file tc.data -config cf myscript.swift -n=10 +Swift trunk swift-r6146 cog-r3544 + +RunID: 20130109-1657-tf01jpaa +Progress: time: Wed, 09 Jan 2013 16:58:00 -0600 +Progress: time: Wed, 09 Jan 2013 16:58:30 -0600 Submitted:10 +Progress: time: Wed, 09 Jan 2013 16:59:00 -0600 Submitted:10 +Progress: time: Wed, 09 Jan 2013 16:59:12 -0600 Stage in:1 Submitted:9 +Final status: Wed, 09 Jan 2013 16:59:12 -0600 Finished successfully:10 +$ ls outdir/* +outdir/myscript.0001.out outdir/myscript.0003.out outdir/myscript.0005.out outdir/myscript.0007.out outdir/myscript.0009.out +outdir/myscript.0002.out outdir/myscript.0004.out outdir/myscript.0006.out outdir/myscript.0008.out outdir/myscript.0010.out +----- + +Staging in Applications with Coaster Provider Staging +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If you want your application to be as portable as possible, you can use +coaster provider staging to send your application(s) to a remote node. +By removing the condor requirements in the previous section, you will +have more cores available. Here is a simple script that stages in and +executes a shell script. + +----- +type file; + +app (file o) sleep (file script, int delay) +{ + # chmod +x script.sh ; ./script.sh delay + bash "-c" @strcat("chmod +x ./", @script, " ; ./", @script, " ", delay) stdout=@o; +} + +file sleep_script <"sleep.sh">; + +foreach i in [1:5] { + file o ; + o = sleep(sleep_script, 10); +} +----- + +Mapping our script to a file and passing it as an argument to an app function +causes the application to be staged in. The only thing we need on the worker +node is /bin/bash. + +TIP: If the program you are staging in is an executable, statically compiling it +will increase the chances of a successful run. + +TIP: If the application you are staging is more complex, with multiple files, +add everything you need into just one compressed tar file, then extract on the +worker node. From davidk at ci.uchicago.edu Thu Jan 10 15:43:22 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 10 Jan 2013 15:43:22 -0600 (CST) Subject: [Swift-commit] r6149 - trunk/etc/sites Message-ID: <20130110214322.98B689CCE5@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-10 15:43:22 -0600 (Thu, 10 Jan 2013) New Revision: 6149 Added: trunk/etc/sites/uc3 Log: uc3 template Added: trunk/etc/sites/uc3 =================================================================== --- trunk/etc/sites/uc3 (rev 0) +++ trunk/etc/sites/uc3 2013-01-10 21:43:22 UTC (rev 6149) @@ -0,0 +1,17 @@ + + + 999.99 + 10000 + 1 + 3600 + 1 + 100 + 100 + 1000 + 1 + "group_friends.{env.USER}" + nonshared + + . + + From davidk at ci.uchicago.edu Thu Jan 10 16:11:33 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 10 Jan 2013 16:11:33 -0600 (CST) Subject: [Swift-commit] r6150 - trunk/bin Message-ID: <20130110221133.1C7E99CF80@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-10 16:11:32 -0600 (Thu, 10 Jan 2013) New Revision: 6150 Modified: trunk/bin/swiftrun Log: Fix for handling config files Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2013-01-10 21:43:22 UTC (rev 6149) +++ trunk/bin/swiftrun 2013-01-10 22:11:32 UTC (rev 6150) @@ -48,8 +48,8 @@ case "$1" in -site|-sites|-s) SITES=$2; verify_not_null sites $SITES; shift ;; -config|-c) CONFIG=$2; verify_not_null config $CONFIG; shift ;; - *) if [ -z "$args" ]; then script="$1" - else args+="$1" + *) if [ -z "$script" ]; then script="$1" + else args+="$1 " fi ;; esac shift @@ -83,5 +83,9 @@ # Run timestamp=$( date +%s ) ln -s $PWD $HOME/.swift/runs/current/$rundir.$timestamp -time swift -sites.file sites.xml -tc.file tc.data $script $args 2>&1 | tee swift.out +if [ -n "$CONFIG" ]; then + eval time swift -sites.file sites.xml -tc.file tc.data -config $CONFIG $script $args 2>&1 | tee swift.out +else + eval time swift -sites.file sites.xml -tc.file tc.data $script $args 2>&1 | tee swift.out +fi mv $HOME/.swift/runs/current/$rundir.$timestamp $HOME/.swift/runs/completed From swift at ci.uchicago.edu Thu Jan 10 18:40:05 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Thu, 10 Jan 2013 18:40:05 -0600 (CST) Subject: [Swift-commit] cog r3546 Message-ID: <20130111004005.B29008D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3546 | hategan | 2013-01-10 18:35:52 -0600 (Thu, 10 Jan 2013) | 1 line added initial flag to distinguish between a valid request and a timed-out request sending spurious data ------------------------------------------------------------------------ Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/KarajanChannel.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/KarajanChannel.java (revision 3545) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/KarajanChannel.java (working copy) @@ -26,6 +26,7 @@ public static final int ERROR_FLAG = 0x00000004; public static final int COMPRESSED_FLAG = 0x00000008; public static final int SIGNAL_FLAG = 0x00000010; + public static final int INITIAL_FLAG = 0x00000020; void sendTaggedData(int i, boolean fin, byte[] bytes); Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java (revision 3545) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java (working copy) @@ -479,9 +479,16 @@ return; } try { - handler = getRequestManager().handleInitialRequest(tag, data); - handler.setId(tag); - registerHandler(handler, tag); + if (flagIsSet(flags, INITIAL_FLAG)) { + handler = getRequestManager().handleInitialRequest(tag, data); + handler.setId(tag); + registerHandler(handler, tag); + } + else { + if (logger.isInfoEnabled()) { + logger.info("Received spurious request data, tag: " + tag + ", len: " + len); + } + } } catch (NoSuchHandlerException e) { if (!getChannelContext().isIgnoredRequest(tag)) { Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/commands/Command.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/commands/Command.java (revision 3545) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/commands/Command.java (working copy) @@ -110,8 +110,7 @@ if (channel == null) { throw new ProtocolException("Unregistered command"); } - boolean fin = (outData == null) || (outData.size() == 0); - + if (logger.isDebugEnabled()) { logger.debug(ppOutData("CMD")); } @@ -123,7 +122,16 @@ if (id == NOID) { logger.warn("Command has NOID: " + this, new Throwable()); } - channel.sendTaggedData(id, fin, getOutCmd().getBytes(), fin ? this : null); + int flags; + boolean fin = (outData == null) || (outData.size() == 0); + if (fin) { + flags = KarajanChannel.FINAL_FLAG + KarajanChannel.INITIAL_FLAG;; + } + else { + flags = KarajanChannel.INITIAL_FLAG;; + } + + channel.sendTaggedData(id, flags, getOutCmd().getBytes(), fin ? this : null); if (!fin) { Iterator i = outData.iterator(); while (i.hasNext()) { From swift at ci.uchicago.edu Fri Jan 11 13:20:05 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 11 Jan 2013 13:20:05 -0600 (CST) Subject: [Swift-commit] cog r3547 Message-ID: <20130111192006.059E38D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3547 | davidkelly999 | 2013-01-11 13:16:59 -0600 (Fri, 11 Jan 2013) | 2 lines Regenerate squeue command at every poll (fix for the stampede multijob problem) ------------------------------------------------------------------------ Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/QueuePoller.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/QueuePoller.java (revision 3546) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/QueuePoller.java (working copy) @@ -30,26 +30,24 @@ private static String[] CMDARRAY; protected synchronized String[] getCMDArray() { - if(CMDARRAY == null) { - if (getJobs().size() <= FULL_LIST_THRESHOLD) { - CMDARRAY = new String[4]; - CMDARRAY[0] = getProperties().getPollCommand(); - CMDARRAY[1] = "--noheader"; - CMDARRAY[2] = "--jobs"; - boolean first=true; - for (Job j : getJobs().values()) { - if(first) { - CMDARRAY[3] = j.getJobID(); - first=false; - } else { - CMDARRAY[3] += "," + j.getJobID(); - } + if (getJobs().size() <= FULL_LIST_THRESHOLD) { + CMDARRAY = new String[4]; + CMDARRAY[0] = getProperties().getPollCommand(); + CMDARRAY[1] = "--noheader"; + CMDARRAY[2] = "--jobs"; + boolean first = true; + for (Job j : getJobs().values()) { + if (first) { + CMDARRAY[3] = j.getJobID(); + first = false; + } else { + CMDARRAY[3] += "," + j.getJobID(); } - } else { - CMDARRAY = new String[] { getProperties().getPollCommand(), "--noheader" }; - } + } + } else { + CMDARRAY = new String[] { getProperties().getPollCommand(), + "--noheader" }; } - return CMDARRAY; } From swift at ci.uchicago.edu Fri Jan 11 14:25:04 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 11 Jan 2013 14:25:04 -0600 (CST) Subject: [Swift-commit] cog r3548 Message-ID: <20130111202505.178258D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3548 | hategan | 2013-01-11 14:24:34 -0600 (Fri, 11 Jan 2013) | 1 line return after spurious data ------------------------------------------------------------------------ Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java (revision 3547) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java (working copy) @@ -488,6 +488,7 @@ if (logger.isInfoEnabled()) { logger.info("Received spurious request data, tag: " + tag + ", len: " + len); } + return; } } catch (NoSuchHandlerException e) { From swift at ci.uchicago.edu Fri Jan 11 14:30:06 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 11 Jan 2013 14:30:06 -0600 (CST) Subject: [Swift-commit] cog r3550 Message-ID: <20130111203006.8319C8D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3550 | hategan | 2013-01-11 14:27:11 -0600 (Fri, 11 Jan 2013) | 1 line added initial flag to C client ------------------------------------------------------------------------ Index: modules/provider-coaster-c-client/src/Flags.h =================================================================== --- modules/provider-coaster-c-client/src/Flags.h (revision 3549) +++ modules/provider-coaster-c-client/src/Flags.h (working copy) @@ -12,5 +12,6 @@ #define FLAG_REPLY 0x01 #define FLAG_SIGNAL 0x10 #define FLAG_FINAL 0x02 +#define FLAG_INITIAL 0x20 #endif /* CONSTANTS_H_ */ Index: modules/provider-coaster-c-client/src/Command.cpp =================================================================== --- modules/provider-coaster-c-client/src/Command.cpp (revision 3549) +++ modules/provider-coaster-c-client/src/Command.cpp (working copy) @@ -24,7 +24,7 @@ list* od = getOutData(); - channel->send(tag, Buffer::wrap(name), od->empty() ? FLAG_FINAL : 0, this); + channel->send(tag, Buffer::wrap(name), od->empty() ? FLAG_FINAL + FLAG_INITIAL : FLAG_INITIAL, this); while (od->size() > 0) { Buffer* b = od->front(); From davidk at ci.uchicago.edu Fri Jan 11 15:04:00 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Fri, 11 Jan 2013 15:04:00 -0600 (CST) Subject: [Swift-commit] r6151 - trunk/etc/sites Message-ID: <20130111210400.1A4509CD06@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-11 15:03:59 -0600 (Fri, 11 Jan 2013) New Revision: 6151 Added: trunk/etc/sites/stampede Log: Stampede template Added: trunk/etc/sites/stampede =================================================================== --- trunk/etc/sites/stampede (rev 0) +++ trunk/etc/sites/stampede 2013-01-11 21:03:59 UTC (rev 6151) @@ -0,0 +1,18 @@ + + + + + 16 + 16 + 43200 + 00:05:00 + 100 + 100 + normal + 1 + TG-ASC090068 + 100 + 10000 + {env.SCRATCH} + + From swift at ci.uchicago.edu Fri Jan 11 18:25:05 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 11 Jan 2013 18:25:05 -0600 (CST) Subject: [Swift-commit] cog r3551 Message-ID: <20130112002506.396DB8D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3551 | davidkelly999 | 2013-01-11 18:20:56 -0600 (Fri, 11 Jan 2013) | 2 lines Use ibrun is available (TACC), otherwise use srun ------------------------------------------------------------------------ Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java =================================================================== --- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java (revision 3550) +++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java (working copy) @@ -165,8 +165,9 @@ logger.debug("Wrapper after variable substitution: " + wrapper); } } - - wr.write(getProperties().getRunCommand() + " /bin/bash -c \'"); + + wr.write("RUNCOMMAND=$( command -v ibrun || command -v srun )\n"); + wr.write("$RUNCOMMAND /bin/bash -c \'"); if (spec.getDirectory() != null) { wr.write("cd " + quote(spec.getDirectory()) + " && "); } @@ -234,4 +235,4 @@ return outArray[outArray.length-1].trim(); } -} \ No newline at end of file +} From swift at ci.uchicago.edu Fri Jan 11 19:40:06 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 11 Jan 2013 19:40:06 -0600 (CST) Subject: [Swift-commit] cog r3552 Message-ID: <20130112014007.13DF98D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3552 | hategan | 2013-01-11 19:36:16 -0600 (Fri, 11 Jan 2013) | 1 line add initial flag to PUT command ------------------------------------------------------------------------ Index: modules/provider-coaster/src/org/globus/cog/abstraction/impl/file/coaster/commands/PutFileCommand.java =================================================================== --- modules/provider-coaster/src/org/globus/cog/abstraction/impl/file/coaster/commands/PutFileCommand.java (revision 3551) +++ modules/provider-coaster/src/org/globus/cog/abstraction/impl/file/coaster/commands/PutFileCommand.java (working copy) @@ -72,7 +72,7 @@ if (logger.isDebugEnabled()) { logger.debug(this + ", src: " + src + ", dest: " + dest + ", size: " + size); } - channel.sendTaggedData(getId(), false, getOutCmd().getBytes()); + channel.sendTaggedData(getId(), KarajanChannel.INITIAL_FLAG, getOutCmd().getBytes()); channel.sendTaggedData(getId(), false, pack(size)); channel.sendTaggedData(getId(), false, src.getBytes()); channel.sendTaggedData(getId(), size == 0, dest.getBytes()); From davidk at ci.uchicago.edu Mon Jan 14 02:42:13 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 14 Jan 2013 02:42:13 -0600 (CST) Subject: [Swift-commit] r6152 - SwiftApps Message-ID: <20130114084213.996AF9D0D9@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-14 02:42:09 -0600 (Mon, 14 Jan 2013) New Revision: 6152 Added: SwiftApps/activityplot/ Log: Activity plot script From davidk at ci.uchicago.edu Mon Jan 14 02:45:50 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 14 Jan 2013 02:45:50 -0600 (CST) Subject: [Swift-commit] r6153 - SwiftApps/activityplot Message-ID: <20130114084550.D040D9D0D9@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-14 02:45:50 -0600 (Mon, 14 Jan 2013) New Revision: 6153 Added: SwiftApps/activityplot/README SwiftApps/activityplot/activityplot.gp SwiftApps/activityplot/activityplot.pl SwiftApps/activityplot/verdana.ttf Log: mmm, plotty. Added: SwiftApps/activityplot/README =================================================================== --- SwiftApps/activityplot/README (rev 0) +++ SwiftApps/activityplot/README 2013-01-14 08:45:50 UTC (rev 6153) @@ -0,0 +1 @@ +Usage: ./activityplot.pl swift.out Added: SwiftApps/activityplot/activityplot.gp =================================================================== --- SwiftApps/activityplot/activityplot.gp (rev 0) +++ SwiftApps/activityplot/activityplot.gp 2013-01-14 08:45:50 UTC (rev 6153) @@ -0,0 +1,19 @@ +set term png enhanced font "./verdana.ttf" 18 size 2000,2000 +set output "activityplot.png" +set key outside + +set multiplot +set grid +set size 1.0,0.5 +set origin 0.0,0.5 +plot 'activityplot.dat' using 1:5:(0) title 'Tasks completed' with lines lw 4 +set yrange [0:160] +set xlabel "Time in seconds" +set size 1.0,0.5 +set origin 0,0 +plot 'activityplot.dat' using 1:2:(0) title 'Staging in' with filledcurves, \ + '' using 1:3:(0) title 'Active' with filledcurves, \ + '' using 1:3:(0) title "" w lines lt 2 lc rgb "black", \ + '' using 1:4:(0) title 'Staging out' with filledcurves, \ + '' using 1:4:(0) title "" w lines lt 2 lc rgb "black" +unset multiplot Added: SwiftApps/activityplot/activityplot.pl =================================================================== --- SwiftApps/activityplot/activityplot.pl (rev 0) +++ SwiftApps/activityplot/activityplot.pl 2013-01-14 08:45:50 UTC (rev 6153) @@ -0,0 +1,65 @@ +#!/usr/bin/perl -w +use strict; +use Time::Local; + +# Usage: ./activityplot.pl swift.out +my $first_timestamp=0; +my %mon2num = qw( + jan 1 feb 2 mar 3 apr 4 may 5 jun 6 + jul 7 aug 8 sep 9 oct 10 nov 11 dec 12 +); + +# Read file +my $filename=shift; +open(FILE, "$filename") || die "Unable to open $filename\n"; +my @filedata = ; +close(FILE); + +# Open file for writing +open(FILE, ">activityplot.dat") || die "Unable too open activityplot.dat for writing\n"; + +sub get_value +{ + my $statement = $_[0]; + my $value = (split /:/, $statement)[1]; + return $value; +} + +sub timestamp_to_seconds +{ + (my $progress, my $tme, my $dayname, my $monthday, my $month, my $year, my $hhmmss, my @junk) = split('\s+', $_[0]); + (my $hh, my $mm, my $ss) = split(':', $hhmmss); + my $time = timelocal($ss, $mm, $hh, $monthday, $mon2num{lc substr($month, 0, 3)}, $year); + return $time; +} + +# Progress: time: Sat, 12 Jan 2013 05:26:32 +0000 Stage in:913 Submitting:5 Submitted:443 Active:678 Stage out:9 Finished successfully:82 + +foreach my $line(@filedata) { + if($line !~ /^Progress/) { next; } + my @words = split('\s+', $line); + my $timestamp = timestamp_to_seconds($line); + + if($first_timestamp == 0) { $first_timestamp = $timestamp; $timestamp=0; } + else { $timestamp = $timestamp - $first_timestamp; } + + my $stagein=0, my $submitting=0, my $submitted=0; + my $active=0, my $stageout=0, my $finished=0; + + foreach my $word(@words) { + if ($word =~ /^in:/) { $stagein = get_value($word); } + elsif ($word =~ /^Submitting:/) { $submitting = get_value($word); } + elsif ($word =~ /^Submitted:/) { $submitted = get_value($word); } + elsif ($word =~ /^Active:/) { $active = get_value($word); } + elsif ($word =~ /^out:/) { $stageout = get_value($word); } + elsif ($word =~ /^successfully/) { $finished = get_value($word); } + } + + # Adjust values for plotting + $stagein = $stagein + $active + $stageout; + $active = $active + $stageout; + print FILE "$timestamp $stagein $active $stageout $finished\n"; +} + +close(FILE); +system("gnuplot activityplot.gp"); Property changes on: SwiftApps/activityplot/activityplot.pl ___________________________________________________________________ Added: svn:executable + * Added: SwiftApps/activityplot/verdana.ttf =================================================================== (Binary files differ) Property changes on: SwiftApps/activityplot/verdana.ttf ___________________________________________________________________ Added: svn:mime-type + application/octet-stream From davidk at ci.uchicago.edu Mon Jan 14 02:47:43 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 14 Jan 2013 02:47:43 -0600 (CST) Subject: [Swift-commit] r6154 - SwiftApps/activityplot Message-ID: <20130114084743.1E7579D0D9@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-14 02:47:41 -0600 (Mon, 14 Jan 2013) New Revision: 6154 Modified: SwiftApps/activityplot/activityplot.gp Log: Roops. Modified: SwiftApps/activityplot/activityplot.gp =================================================================== --- SwiftApps/activityplot/activityplot.gp 2013-01-14 08:45:50 UTC (rev 6153) +++ SwiftApps/activityplot/activityplot.gp 2013-01-14 08:47:41 UTC (rev 6154) @@ -7,7 +7,6 @@ set size 1.0,0.5 set origin 0.0,0.5 plot 'activityplot.dat' using 1:5:(0) title 'Tasks completed' with lines lw 4 -set yrange [0:160] set xlabel "Time in seconds" set size 1.0,0.5 set origin 0,0 From davidk at ci.uchicago.edu Tue Jan 15 16:27:07 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 15 Jan 2013 16:27:07 -0600 (CST) Subject: [Swift-commit] r6155 - www/usage Message-ID: <20130115222707.C90749CD06@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-15 16:27:07 -0600 (Tue, 15 Jan 2013) New Revision: 6155 Modified: www/usage/update_data.php Log: Write data as tabular text for gnuplot rather thanml for flash graphs Modified: www/usage/update_data.php =================================================================== --- www/usage/update_data.php 2013-01-14 08:47:41 UTC (rev 6154) +++ www/usage/update_data.php 2013-01-15 22:27:07 UTC (rev 6155) @@ -115,72 +115,38 @@ } # Total runs -$usageFile = fopen("usage.xml", 'w'); -fwrite($usageFile, "\n\n\n\n\n\n\n"); +$usageFile = fopen("usage.txt", 'w'); foreach($months as $key => $value) { - fwrite($usageFile, "$key\n"); + fwrite($usageFile, "$key " . $value['runs'] . "\n"); } -fwrite($usageFile, "\n"); -fwrite($usageFile, "\nSwift runs\n"); -foreach($months as $key => $value) { - fwrite($usageFile, "" . $value['runs'] . "\n"); -} -fwrite($usageFile, "\n\n3d column\n\n\n"); fclose($usageFile); # Total runs longer than 1 minute -$longUsage1File = fopen("longUsage1.xml", 'w'); -fwrite($longUsage1File, "\n\n\n\n\n\n\n"); +$longUsage1File = fopen("longUsage1.txt", 'w'); foreach($months as $key => $value) { - fwrite($longUsage1File, "$key\n"); + fwrite($longUsage1File, "$key " . $value['longruns1'] . "\n"); } -fwrite($longUsage1File, "\n"); -fwrite($longUsage1File, "\nSwift runs (longer than 1 minute)\n"); -foreach($months as $key => $value) { - fwrite($longUsage1File, "" . $value['longruns1'] . "\n"); -} -fwrite($longUsage1File, "\n\n3d column\n\n\n"); fclose($longUsage1File); # Total runs longer than 5 minutes -$longUsage5File = fopen("longUsage5.xml", 'w'); -fwrite($longUsage5File, "\n\n\n\n\n\n\n"); +$longUsage5File = fopen("longUsage5.txt", 'w'); foreach($months as $key => $value) { - fwrite($longUsage5File, "$key\n"); + fwrite($longUsage5File, "$key " . $value['longruns5'] . "\n"); } -fwrite($longUsage5File, "\n"); -fwrite($longUsage5File, "\nSwift runs (longer than 5 minutes)\n"); -foreach($months as $key => $value) { - fwrite($longUsage5File, "" . $value['longruns5'] . "\n"); -} -fwrite($longUsage5File, "\n\n3d column\n\n\n"); fclose($longUsage5File); - # Unique users per month -$users_per_month_file = fopen("users_per_month.xml", 'w'); -fwrite($users_per_month_file, "\n\n\n\n\n\n\n"); +$users_per_month_file = fopen("users_per_month.txt", 'w'); foreach($months as $key => $value) { - fwrite($users_per_month_file, "$key\n"); + fwrite($users_per_month_file, "$key " . count($value['users']) . "\n"); } -fwrite($users_per_month_file, "\n\nUnique Users Per Month\n"); -foreach($months as $key => $value) { - fwrite($users_per_month_file, "" . count($value['users']) . "\n"); -} -fwrite($users_per_month_file, "\n\n3d column\n\n\n"); fclose($users_per_month_file); # New users per month -$new_users_per_month_file = fopen("new_users_per_month.xml", 'w'); -fwrite($new_users_per_month_file, "\n\n\n\n\n\n\n"); +$new_users_per_month_file = fopen("new_users_per_month.txt", 'w'); foreach($months as $key => $value) { - fwrite($new_users_per_month_file, "$key\n"); + fwrite($new_users_per_month_file, "$key " . count($value['newusers']) . "\n"); } -fwrite($new_users_per_month_file, "\n\nNew Users Per Month\n"); -foreach($months as $key => $value) { - fwrite($new_users_per_month_file, "" . count($value['newusers']) . "\n"); -} -fwrite($new_users_per_month_file, "\n\n3d column\n\n\n"); fclose($new_users_per_month_file); mysql_free_result($result); From swift at ci.uchicago.edu Wed Jan 16 01:30:07 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 16 Jan 2013 01:30:07 -0600 (CST) Subject: [Swift-commit] cog r3553 Message-ID: <20130116073007.6A8448D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3553 | hategan | 2013-01-16 01:28:22 -0600 (Wed, 16 Jan 2013) | 1 line removed command timeouts and revert to channel timeouts ------------------------------------------------------------------------ Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/RequestReply.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/RequestReply.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/RequestReply.java (working copy) @@ -33,10 +33,7 @@ public abstract class RequestReply { public static final Logger logger = Logger.getLogger(RequestReply.class); - - public static final int DEFAULT_TIMEOUT = 120 * 1000; - private int timeout = DEFAULT_TIMEOUT; - + public static final int NOID = -1; private int id; private String outCmd; @@ -45,10 +42,7 @@ private List inData; private boolean inDataReceived; private KarajanChannel channel; - private long lastTime = Long.MAX_VALUE; - - // private static final byte[] NO_EXCEPTION = new byte[0]; - + protected String getInCmd() { return inCmd; } @@ -152,7 +146,6 @@ public abstract void send(boolean err) throws ProtocolException; protected void dataReceived(boolean fin, boolean error, byte[] data) throws ProtocolException { - setLastTime(System.currentTimeMillis()); } protected synchronized void addInData(boolean fin, boolean err, byte[] data) { @@ -405,28 +398,7 @@ protected Object getInObject(int index) { return deserialize(getInData(index)); } - - public long getLastTime() { - return lastTime; - } - - public void setLastTime(long lastTime) { - this.lastTime = lastTime; - } - public int getTimeout() { - return timeout; - } - - public void setTimeout(int timeout) { - this.timeout = timeout; - } - - public void handleTimeout() { - logger.warn("Unhandled timeout", new Throwable()); - setLastTime(Long.MAX_VALUE); - } - public void handleSignal(byte[] data) { } Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/ChannelContext.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/ChannelContext.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/ChannelContext.java (working copy) @@ -15,7 +15,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Timer; import org.apache.log4j.Logger; import org.globus.cog.karajan.workflow.events.EventBus; @@ -42,8 +41,6 @@ private int cmdseq; private TagTable activeSenders; private TagTable activeReceivers; - private TagTable ignoredRequests; - private static Timer timer; private ServiceContext serviceContext; private int reconnectionAttempts; private long lastHeartBeat; @@ -224,7 +221,7 @@ public void notifyRegisteredCommandsAndHandlers(Exception e) { if (logger.isInfoEnabled()) { - logger.info("Notifying commands and handlers about exception"); + logger.info("Notifying commands and handlers about exception", e); } notifyListeners(activeReceivers, e); notifyListeners(activeSenders, e); @@ -243,15 +240,6 @@ } } - public Timer getTimer() { - synchronized (ChannelContext.class) { - if (timer == null) { - timer = new Timer(); - } - return timer; - } - } - public void reexecute(final Command command) { //TODO } @@ -334,37 +322,5 @@ } } } - - public synchronized void ignoreRequest(int tag, int timeout) { - if (ignoredRequests == null) { - ignoredRequests = new TagTable(); - } - ignoredRequests.put(tag, System.currentTimeMillis() + timeout); - } - - public synchronized boolean isIgnoredRequest(int tag) { - if (ignoredRequests == null) { - return false; - } - else { - return ignoredRequests.containsKey(tag); - } - } - - public synchronized void removeOldIgnoredRequests() { - if (ignoredRequests == null) { - return; - } - long now = System.currentTimeMillis(); - for (Integer i : ignoredRequests.keys()) { - if (ignoredRequests.get(i) < now) { - ignoredRequests.remove(i); - } - } - if (ignoredRequests.isEmpty()) { - ignoredRequests = null; - } - } - } Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractPipedChannel.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractPipedChannel.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractPipedChannel.java (working copy) @@ -46,6 +46,11 @@ // no heart beat for these } + @Override + public void configureTimeoutChecks() { + // no timeouts either + } + public boolean isOffline() { return false; } Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/KarajanChannel.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/KarajanChannel.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/KarajanChannel.java (working copy) @@ -99,8 +99,6 @@ void flush() throws IOException; SelectableChannel getNIOChannel(); - - void ignoreRequest(RequestHandler h, int timeout); boolean handleChannelException(Exception e); } Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/MetaChannel.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/MetaChannel.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/MetaChannel.java (working copy) @@ -10,7 +10,6 @@ package org.globus.cog.karajan.workflow.service.channels; import java.io.IOException; -import java.util.Timer; import java.util.TimerTask; import org.apache.log4j.Logger; @@ -39,6 +38,11 @@ public synchronized void sendTaggedData(int tag, int flags, byte[] data, SendCallback cb) { current.sendTaggedData(tag, flags, data, cb); } + + @Override + public void configureTimeoutChecks() { + // only for the actual channels + } public void registerCommand(Command cmd) throws ProtocolException { current.registerCommand(cmd); @@ -112,13 +116,9 @@ ChannelManager.getManager().unregisterChannel(MetaChannel.this); } }; - getTimer().schedule(deactivator, (long) seconds * 1000); + Timer.schedule(deactivator, (long) seconds * 1000); } - public Timer getTimer() { - return getChannelContext().getTimer(); - } - public synchronized void poll(final int seconds) { if (poller != null) { return; @@ -152,7 +152,7 @@ } }; long interval = (long) seconds * 1000; - getTimer().schedule(poller, interval, interval); + Timer.schedule(poller, interval, interval); } public String toString() { Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/Timer.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/Timer.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/Timer.java (working copy) @@ -21,4 +21,16 @@ } }, interval, interval); } + + public static void every(final long interval, final TimerTask task) { + TIMER.scheduleAtFixedRate(task, interval, interval); + } + + public static void schedule(TimerTask task, long delay, long period) { + TIMER.schedule(task, delay, period); + } + + public static void schedule(TimerTask task, long delay) { + TIMER.schedule(task, delay); + } } Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java (working copy) @@ -15,7 +15,6 @@ import java.net.URI; import java.nio.ByteBuffer; import java.nio.channels.SelectableChannel; -import java.util.Collection; import java.util.TimerTask; import org.apache.log4j.Logger; @@ -24,8 +23,8 @@ import org.globus.cog.karajan.workflow.service.RemoteConfiguration; import org.globus.cog.karajan.workflow.service.RemoteConfiguration.Entry; import org.globus.cog.karajan.workflow.service.RequestManager; -import org.globus.cog.karajan.workflow.service.RequestReply; import org.globus.cog.karajan.workflow.service.Service; +import org.globus.cog.karajan.workflow.service.TimeoutException; import org.globus.cog.karajan.workflow.service.UserContext; import org.globus.cog.karajan.workflow.service.commands.Command; import org.globus.cog.karajan.workflow.service.handlers.RequestHandler; @@ -38,7 +37,8 @@ public static final int DEFAULT_HBI_SPREAD = 10; public static final int TIMEOUT_CHECK_INTERVAL = 1; - + public static final int TIMEOUT = 120; + private ChannelContext context; private volatile int usageCount, longTermUsageCount; private RequestManager requestManager; @@ -47,6 +47,7 @@ private String name; private Service callbackService; private final boolean client; + private long lastTime; protected AbstractKarajanChannel(RequestManager requestManager, ChannelContext channelContext, boolean client) { @@ -58,6 +59,7 @@ this.client = client; configureHeartBeat(); configureTimeoutChecks(); + updateLastTime(); } protected void configureHeartBeat() { @@ -99,47 +101,45 @@ public void scheduleHeartbeats(int heartBeatInterval) { TimerTask heartBeatTask; heartBeatTask = new HeartBeatTask(this); - context.getTimer().schedule(heartBeatTask, - heartBeatInterval + (int) (Math.random() * DEFAULT_HBI_INITIAL_SPREAD * 1000), - heartBeatInterval + (int) (Math.random() * DEFAULT_HBI_SPREAD * 1000)); + Timer.schedule(heartBeatTask, + heartBeatInterval + (int) (Math.random() * DEFAULT_HBI_INITIAL_SPREAD * 1000), + heartBeatInterval + (int) (Math.random() * DEFAULT_HBI_SPREAD * 1000)); } public void scheduleHeartbeatCheck(int heartBeatInterval) { TimerTask heartBeatTask; int mult = 2; heartBeatTask = new HeartBeatCheckTask(this, heartBeatInterval, mult); - context.getTimer().schedule(heartBeatTask, mult * heartBeatInterval, - mult * heartBeatInterval); + Timer.every(mult * heartBeatInterval, heartBeatTask); } public void configureTimeoutChecks() { - context.getTimer().schedule(new TimerTask() { + Timer.every(TIMEOUT_CHECK_INTERVAL * 1000, new TimerTask() { public void run() { checkTimeouts(); - }}, - TIMEOUT_CHECK_INTERVAL * 1000, TIMEOUT_CHECK_INTERVAL * 1000); + }} + ); } protected void checkTimeouts() { - checkTimeouts(context.getActiveCommands()); - checkTimeouts(context.getActiveHandlers()); - getChannelContext().removeOldIgnoredRequests(); + long now = System.currentTimeMillis(); + long lastTime = getLastTime(); + if (now - lastTime > TIMEOUT * 1000) { + lastTime = Long.MAX_VALUE; + TimeoutException e = new TimeoutException(this, "Channel timed out", lastTime); + context.notifyRegisteredCommandsAndHandlers(e); + handleChannelException(e); + } } - private void checkTimeouts(Collection l) { - long now = System.currentTimeMillis(); - for (RequestReply r : l) { - if (now - r.getLastTime() > r.getTimeout()) { - try { - r.handleTimeout(); - } - catch (Exception e) { - logger.warn("Error handling timeout", e); - } - } - } + protected synchronized void updateLastTime() { + lastTime = System.currentTimeMillis(); } - + + protected synchronized long getLastTime() { + return lastTime; + } + protected boolean clientControlsHeartbeats() { return true; } @@ -386,6 +386,7 @@ flagIsSet(flags, FINAL_FLAG) + ", err = " + flagIsSet(flags, ERROR_FLAG) + ", datalen = " + len + ", data = " + ppByteBuf(data)); } + updateLastTime(); Command cmd = getChannelContext().getRegisteredCommand(tag); if (cmd != null) { try { @@ -446,6 +447,7 @@ flagIsSet(flags, FINAL_FLAG) + ", err = " + flagIsSet(flags, ERROR_FLAG) + ", datalen = " + len + ", data = " + ppByteBuf(data)); } + updateLastTime(); RequestHandler handler = getChannelContext().getRegisteredHandler(tag); boolean fin = finalFlagIsSet(flags); boolean err = errorFlagIsSet(flags); @@ -492,9 +494,7 @@ } } catch (NoSuchHandlerException e) { - if (!getChannelContext().isIgnoredRequest(tag)) { - logger.warn(getName() + "Could not handle request", e); - } + logger.warn(getName() + "Could not handle request", e); } } @@ -547,11 +547,6 @@ return null; } - @Override - public void ignoreRequest(RequestHandler h, int timeout) { - getChannelContext().ignoreRequest(h.getId(), timeout); - } - public synchronized boolean handleChannelException(Exception e) { logger.info("Channel config: " + getChannelContext().getConfiguration()); if (!ChannelManager.getManager().handleChannelException(this, e)) { Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/Client.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/Client.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/Client.java (working copy) @@ -128,7 +128,7 @@ } } - public void execute(Command command) throws IOException, ProtocolException { + public void execute(Command command) throws IOException, ProtocolException, InterruptedException { command.execute(channel); } Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/handlers/RequestHandler.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/handlers/RequestHandler.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/handlers/RequestHandler.java (working copy) @@ -15,7 +15,6 @@ import org.apache.log4j.Logger; import org.globus.cog.karajan.workflow.service.ProtocolException; import org.globus.cog.karajan.workflow.service.RequestReply; -import org.globus.cog.karajan.workflow.service.TimeoutException; import org.globus.cog.karajan.workflow.service.channels.KarajanChannel; public abstract class RequestHandler extends RequestReply { @@ -35,7 +34,6 @@ } protected void sendReply() throws ProtocolException { - setLastTime(System.currentTimeMillis()); send(); replySent = true; } @@ -118,14 +116,4 @@ public String toString() { return "Handler(tag: " + getId() + ", " + getInCmd() + ")"; } - - public void handleTimeout() { - if (isInDataReceived()) { - return; - } - setLastTime(Long.MAX_VALUE); - TimeoutException t = new TimeoutException(this, "Timed out receiving request"); - logger.info(t.getMessage()); - errorReceived(t.getMessage(), t); - } } Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/TimeoutException.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/TimeoutException.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/TimeoutException.java (working copy) @@ -13,39 +13,18 @@ import java.text.SimpleDateFormat; import java.util.Date; -import org.globus.cog.karajan.workflow.service.commands.Command; -import org.globus.cog.karajan.workflow.service.handlers.RequestHandler; +import org.globus.cog.karajan.workflow.service.channels.KarajanChannel; -public class TimeoutException extends ProtocolException { - private static final long serialVersionUID = -6781619140427115780L; - +public class TimeoutException extends Exception { public static final DateFormat DF = new SimpleDateFormat("yyMMdd-HHmmss.SSS"); - public TimeoutException(Command c, String msg) { - super(c + " " + msg + ". sendReqTime=" - + DF.format(new Date(c.getSendReqTime())) + ", lastSendTime=" + DF.format(new Date(c.getSendTime())) - + ", now=" + DF.format(new Date()) + ", channel=" + c.getChannel()); + public TimeoutException(String msg) { + super(msg); } - public TimeoutException(RequestHandler h, String msg) { - super(h + " " + msg + ". lastTime=" - + DF.format(new Date(h.getLastTime())) - + ", now=" + DF.format(new Date()) + ", channel=" + h.getChannel()); + public TimeoutException(KarajanChannel channel, String msg, long lastTime) { + super(msg + ". lastTime=" + + DF.format(new Date(lastTime)) + + ", now=" + DF.format(new Date()) + ", channel=" + channel); } - - public TimeoutException() { - super(); - } - - public TimeoutException(String message) { - super(message); - } - - public TimeoutException(String message, Throwable cause) { - super(message, cause); - } - - public TimeoutException(Throwable cause) { - super(cause); - } } Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/commands/Command.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/commands/Command.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/commands/Command.java (working copy) @@ -39,8 +39,6 @@ private String errorMsg; private Exception exception; private int retries; - private long sendTime; - private long sendReqTime; public Command() { setId(NOID); @@ -55,23 +53,10 @@ this.cb = cb; } - public void waitForReply() throws TimeoutException { + public void waitForReply() throws InterruptedException { synchronized (this) { - if (!this.isInDataReceived()) { - long left = getTimeout(); - while (!this.isInDataReceived()) { - if (left <= 0) { - throw new TimeoutException(); - } - try { - wait(left); - } - catch (InterruptedException e) { - e.printStackTrace(); - } - left = sendTime == 0 ? 1000 : getTimeout() - - (System.currentTimeMillis() - sendTime); - } + while (!this.isInDataReceived()) { + wait(0); } } } @@ -139,8 +124,6 @@ channel.sendTaggedData(id, !i.hasNext(), buf, !i.hasNext() ? this : null); } } - sendReqTime = System.currentTimeMillis(); - setLastTime(sendReqTime); } catch (ChannelIOException e) { reexecute(e.getMessage(), e); @@ -148,14 +131,11 @@ } public void dataSent() { - sendTime = System.currentTimeMillis(); - //when using the piped channels the reply will arrive before this method is called - setLastTime(sendTime); } private static boolean shutdownMsg; - public byte[] execute(KarajanChannel channel) throws ProtocolException, IOException { + public byte[] execute(KarajanChannel channel) throws ProtocolException, IOException, InterruptedException { send(channel); waitForReply(); if (errorMsg != null) { @@ -256,29 +236,7 @@ } } } - - public void handleTimeout() { - if (isInDataReceived()) { - return; - } - TimeoutException t = new TimeoutException(this, "Reply timeout"); - logger.warn(t.getMessage()); - errorReceived(t.getMessage(), t); - getChannel().unregisterCommand(this); - } - public long getSendReqTime() { - return sendReqTime; - } - - public long getSendTime() { - return sendTime; - } - - protected void setSendReqTime(long sendReqTime) { - this.sendReqTime = sendReqTime; - } - public String toString() { return "Command(tag: " + this.getId() + ", " + this.getOutCmd() + ")"; } Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/commands/HeartBeatCommand.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/commands/HeartBeatCommand.java (revision 3552) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/commands/HeartBeatCommand.java (working copy) @@ -11,8 +11,6 @@ import org.apache.log4j.Logger; import org.globus.cog.karajan.workflow.service.ProtocolException; -import org.globus.cog.karajan.workflow.service.TimeoutException; -import org.globus.cog.karajan.workflow.service.channels.ChannelManager; import org.globus.cog.karajan.workflow.service.handlers.HeartBeatHandler; @@ -42,13 +40,4 @@ logger.info(getChannel() + " up latency: " + (now - rst) + "ms, rtt: " + (now - start) + "ms"); } } - - @Override - public void handleTimeout() { - if (logger.isInfoEnabled()) { - logger.info("Heartbeat timed out. Closing channel."); - } - ChannelManager.getManager().handleChannelException(getChannel(), new TimeoutException(this, "Heartbeat timed out")); - unregister(); - } } From swift at ci.uchicago.edu Wed Jan 16 01:35:09 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 16 Jan 2013 01:35:09 -0600 (CST) Subject: [Swift-commit] cog r3555 Message-ID: <20130116073510.1AD278D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3555 | hategan | 2013-01-16 01:31:00 -0600 (Wed, 16 Jan 2013) | 1 line removed command timeouts ------------------------------------------------------------------------ Index: modules/provider-coaster/resources/worker.pl =================================================================== --- modules/provider-coaster/resources/worker.pl (revision 3554) +++ modules/provider-coaster/resources/worker.pl (working copy) @@ -120,7 +120,8 @@ my $TAG = int(rand(10000)); use constant RETRIES => 3; -use constant REPLYTIMEOUT => 180; +use constant CHANNEL_TIMEOUT => 180; +use constant HEARTBEAT_INTERVAL => 60; use constant MAXFRAGS => 16; # TODO: Make this configurable (#537) use constant MAX_RECONNECT_ATTEMPTS => 3; @@ -136,11 +137,6 @@ use constant IOBUFSZ => 32768; use constant IOBLOCKSZ => 8; -# 60 seconds by default. Note that since there is no configuration handshake -# this would have to match the default interval in the service in order to avoid -# "lost heartbeats". -use constant HEARTBEAT_INTERVAL => 30; - # If true, enable a profile result that is written to the log my $PROFILE = 0; # Contains tuples (EVENT, PID, TIMESTAMP) (flattened) @@ -196,6 +192,8 @@ my %SUSPENDED_TRANSFERS = (); +my $LAST_RECEIVE_TIME = 0; + # partial message being sent when # writing to the socket would have blocked my %partialSend; @@ -244,7 +242,6 @@ # # # -# time: last communication time (used to determine timeouts) # my $LOG = logfilename($LOGDIR, $BLOCKID); @@ -867,38 +864,11 @@ return 1; } -sub checkTimeouts2 { - my ($hash) = @_; - - my $now = time(); - my @del = (); - - my $k; - my $v; - - while (($k, $v) = each(%$hash)) { - if ($now - $$v[1] > REPLYTIMEOUT) { - push(@del, $k); - my $cont = $$v[0]; - $$cont{"dataIn"}($cont, $k, 1, 0, 0, "Reply timeout"); - } - } - - foreach $k (@del) { - delete $$hash{$k}; - } -} - -my $LASTTIMEOUTCHECK = 0; - sub checkTimeouts { my $time = time(); - if ($time - $LASTTIMEOUTCHECK < 1) { - return; + if ($time - $LAST_RECEIVE_TIME > CHANNEL_TIMEOUT) { + crash("Channel timed out. Last receive time: $LAST_RECEIVE_TIME, now: $time"); } - $LASTTIMEOUTCHECK = $time; - checkTimeouts2(\%REQUESTS); - checkTimeouts2(\%REPLIES); } my $DATA = ""; @@ -909,6 +879,7 @@ my $buf; $SOCK->recv($buf, 20 - length($DATA)); if (length($buf) > 0) { + $LAST_RECEIVE_TIME = time(); $DATA = $DATA . $buf; if (length($DATA) == 20) { # wlog DEBUG, "Received " . unpackData($DATA) . "\n"; From davidk at ci.uchicago.edu Thu Jan 17 16:11:11 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 17 Jan 2013 16:11:11 -0600 (CST) Subject: [Swift-commit] r6156 - trunk/bin Message-ID: <20130117221111.580499CCAA@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-17 16:11:11 -0600 (Thu, 17 Jan 2013) New Revision: 6156 Modified: trunk/bin/start-coaster-service Log: Some changes to work with the updated gensites Slurm Modified: trunk/bin/start-coaster-service =================================================================== --- trunk/bin/start-coaster-service 2013-01-15 22:27:07 UTC (rev 6155) +++ trunk/bin/start-coaster-service 2013-01-17 22:11:11 UTC (rev 6156) @@ -13,7 +13,7 @@ { # Setup environment PORT=$1 - EXECUTION_URL=http://localhost:$PORT + EXECUTIONURL=http://localhost:$PORT export EC2_ACCESS_KEY=$FUTUREGRID_IAAS_ACCESS_KEY export EC2_SECRET_KEY=$FUTUREGRID_IAAS_SECRET_KEY @@ -73,7 +73,7 @@ echo $! >> $PID_FILE else echo "Starting worker on $MACHINE" - ssh $WORKER_USERNAME@$MACHINE "$WORKER_LOCATION/$WORKER $EXECUTION_URL $MACHINE $LOG_DIR" & + ssh $WORKER_USERNAME@$MACHINE "$WORKER_LOCATION/$WORKER $EXECUTIONURL $MACHINE $LOG_DIR" & echo $! >> $PID_FILE fi @@ -84,7 +84,7 @@ start-workers-gp() { PORT=$1 - EXECUTION_URL=http://localhost:$PORT + EXECUTIONURL=http://localhost:$PORT if [ ! -d "$SWIFTVMBOOT_DIR" ] || [ ! -x "$SWIFTVMBOOT_DIR/bin/gp-instance-create" ]; then crash "SWIFTVMBOOT_DIR incorrectly defined in coaster-service.conf" @@ -129,7 +129,7 @@ # Copy and start worker script scp -q -o StrictHostKeyChecking=no $SWIFT_BIN/$WORKER $WORKER_USERNAME@$MACHINE:$WORKER_LOCATION > /dev/null 2>&1 echo "Starting worker on $MACHINE" - ssh -q -o StrictHostKeyChecking=no $WORKER_USERNAME@$MACHINE "$WORKER_LOCATION/$WORKER $EXECUTION_URL $MACHINE $LOG_DIR" & + ssh -q -o StrictHostKeyChecking=no $WORKER_USERNAME@$MACHINE "$WORKER_LOCATION/$WORKER $EXECUTIONURL $MACHINE $LOG_DIR" & echo $! >> $PID_FILE done } @@ -138,7 +138,7 @@ start-workers-ec2() { PORT=$1 - EXECUTION_URL=http://localhost:$PORT + EXECUTIONURL=http://localhost:$PORT if [ ! -d "$SWIFTVMBOOT_DIR" ] || [ ! -x "$SWIFTVMBOOT_DIR/ec2/bin/ec2-run-instances" ]; then crash "SWIFTVMBOOT_DIR incorrectly defined in coaster-service.conf" fi @@ -193,7 +193,7 @@ echo $! >> $PID_FILE else echo "Starting worker on $MACHINE" - ssh $SSH_OPTS $WORKER_USERNAME@$MACHINE "$WORKER_LOCATION/$WORKER $EXECUTION_URL $MACHINE $LOG_DIR" > /dev/null 2>&1 & + ssh $SSH_OPTS $WORKER_USERNAME@$MACHINE "$WORKER_LOCATION/$WORKER $EXECUTIONURL $MACHINE $LOG_DIR" > /dev/null 2>&1 & echo $! >> $PID_FILE fi @@ -207,7 +207,7 @@ start-workers-ssh() { PORT=$1 - EXECUTION_URL=http://$IPADDR:$PORT + EXECUTIONURL=http://$IPADDR:$PORT if [ -z "$PORT" ]; then crash "start-workers-ssh: Port number not specified, giving up" fi @@ -229,14 +229,14 @@ ssh -A $WORKER_USERNAME@$WORKER_RELAY_HOST ssh $MACHINE mkdir -p $WORKER_LOCATION > /dev/null 2>&1 ssh -A $WORKER_USERNAME@$WORKER_RELAY_HOST "scp /tmp/$WORKER $WORKER_USERNAME@$MACHINE:$WORKER_LOCATION" > /dev/null 2>&1 echo Starting worker on $MACHINE - ssh -A $WORKER_USERNAME@$WORKER_RELAY_HOST ssh $WORKER_USERNAME@$MACHINE "WORKER_LOGGING_LEVEL=$WORKER_LOGGING_LEVEL $WORKER_LOCATION/$WORKER $EXECUTION_URL $MACHINE $WORKER_LOG_DIR" & + ssh -A $WORKER_USERNAME@$WORKER_RELAY_HOST ssh $WORKER_USERNAME@$MACHINE "WORKER_LOGGING_LEVEL=$WORKER_LOGGING_LEVEL $WORKER_LOCATION/$WORKER $EXECUTIONURL $MACHINE $WORKER_LOG_DIR" & echo $! >> $PID_FILE # Connect directly else ssh $WORKER_USERNAME@$MACHINE mkdir -p $WORKER_LOCATION > /dev/null 2>&1 scp $SWIFT_BIN/$WORKER $WORKER_USERNAME@$MACHINE:$WORKER_LOCATION > /dev/null 2>&1 echo Starting worker on $MACHINE - ssh $WORKER_USERNAME@$MACHINE "WORKER_LOGGING_LEVEL=$WORKER_LOGGING_LEVEL $WORKER_LOCATION/$WORKER $EXECUTION_URL $MACHINE $WORKER_LOG_DIR" & + ssh $WORKER_USERNAME@$MACHINE "WORKER_LOGGING_LEVEL=$WORKER_LOGGING_LEVEL $WORKER_LOCATION/$WORKER $EXECUTIONURL $MACHINE $WORKER_LOG_DIR" & echo $! >> $PID_FILE fi done @@ -247,12 +247,12 @@ start-workers-local() { PORT=$1 - EXECUTION_URL=http://$IPADDR:$PORT + EXECUTIONURL=http://$IPADDR:$PORT if [ -z "$PORT" ]; then crash "start-workers-local: Port number not specified, giving up" fi echo Starting worker on local machine - $WORKER $EXECUTION_URL LOCAL $LOG_DIR & + $WORKER $EXECUTIONURL LOCAL $LOG_DIR & echo $! >> $PID_FILE return 0 } @@ -261,7 +261,7 @@ start-workers-condor() { PORT=$1 - EXECUTION_URL=http://$IPADDR:$PORT + EXECUTIONURL=http://$IPADDR:$PORT if [ -z "$PORT" ]; then crash "start-workers-local: Port number not specified, giving up" fi @@ -279,7 +279,7 @@ output = workers.stdout error = workers.stderr executable = $WORKER_LOCATION/$WORKER -arguments = $EXECUTION_URL node . +arguments = $EXECUTIONURL node . notification = Never leave_in_queue = FALSE machine_count = $CONDOR_MACHINE_COUNT @@ -296,11 +296,51 @@ fi } +# Start condor workers +start-workers-slurm() +{ + PORT=$1 + EXECUTIONURL=http://$IPADDR:$PORT + if [ -z "$PORT" ]; then + crash "start-workers-local: Port number not specified, giving up" + fi + echo Starting workers + + if [ -z "$MACHINE_COUNT" ]; then + export MACHINE_COUNT=1 + fi + +cat < start_workers.submit +#!/bin/bash + +#SBATCH --output=start_workers.stdout +#SBATCH --error=start_workers.stderr +#SBATCH --nodes=$MACHINE_COUNT +#SBATCH --exclusive +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=12 +#SBATCH --partition=normal +#SBATCH --time=11:59:00 +#SBATCH -A TG-ASC090068 +export WORKER_LOGGING_LEVEL=NONE +ibrun $WORKER_LOCATION/$WORKER $EXECUTIONURL node . +EOF + + if [ -n "$WORKER_RELAY_HOST" ]; then + scp start_workers.submit $WORKER_USERNAME@$WORKER_RELAY_HOST: + scp $SWIFT_BIN/$WORKER $WORKER_USERNAME@$WORKER_RELAY_HOST:$WORKER_LOCATION > /dev/null 2>&1 + ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "sbatch start_workers.submit" + else + sbatch start_workers.submit + fi +} + + # Start GWMS workers start-workers-gwms() { PORT=$1 - EXECUTION_URL=http://$IPADDR:$PORT + EXECUTIONURL=http://$IPADDR:$PORT if [ -z "$PORT" ]; then crash "start-workers-local: Port number not specified, giving up" @@ -318,10 +358,10 @@ if [ -n "$WORKER_RELAY_HOST" ]; then ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "mkdir -p condor" ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "grid-proxy-init" - ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "WORKER_INIT_CMD='$WORKER_INIT_CMD' WORKER_LOGGING_LEVEL=$WORKER_LOGGING_LEVEL run-gwms-workers $EXECUTION_URL $CONDOR_WORKERS" + ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "WORKER_INIT_CMD='$WORKER_INIT_CMD' WORKER_LOGGING_LEVEL=$WORKER_LOGGING_LEVEL run-gwms-workers $EXECUTIONURL $CONDOR_WORKERS" else mkdir -p condor - run-gwms-workers $EXECUTION_URL $CONDOR_WORKERS + run-gwms-workers $EXECUTIONURL $CONDOR_WORKERS fi } @@ -332,7 +372,7 @@ if [ -z "$PORT" ]; then crash "start-workers-cobalt: Port number not specified, giving up" fi - EXECUTION_URL=http://$IPADDR:$PORT + EXECUTIONURL=http://$IPADDR:$PORT TIMESTAMP=$(date "+%Y.%m%d.%H%M%S") R=${RANDOM} ID="${TIMESTAMP}.${R}" @@ -346,7 +386,7 @@ -E cobalt.${$}.stderr \ -o cobalt.${$}.stdout \ -e "WORKER_LOGGING_LEVEL=DEBUG:ZOID_ENABLE_NAT=true" \ - $SWIFT_BIN/$WORKER $EXECUTION_URL $ID $PWD/$LOG_DIR + $SWIFT_BIN/$WORKER $EXECUTIONURL $ID $PWD/$LOG_DIR echo $! >> $PID_FILE return 0 @@ -477,7 +517,7 @@ echo Local port: $LOCAL_PORT # Generate sites.xml -export EXECUTION_URL="http://$IPADDR:$SERVICE_PORT" +export EXECUTIONURL="http://$IPADDR:$SERVICE_PORT" echo Generating sites.xml if [ -f "gensites.template" ]; then gensites `cat gensites.template` -p $CONFIG_FILE > $RUN_DIR/sites.xml @@ -518,6 +558,9 @@ gwms) start-workers-gwms $LOCAL_PORT ;; + slurm) + start-workers-slurm $LOCAL_PORT + ;; *) crash "Unknown WORKER_MODE. Please modify coaster-service.conf" ;; From davidk at ci.uchicago.edu Thu Jan 17 21:12:18 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 17 Jan 2013 21:12:18 -0600 (CST) Subject: [Swift-commit] r6157 - branches Message-ID: <20130118031218.E66679CCAA@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-17 21:12:18 -0600 (Thu, 17 Jan 2013) New Revision: 6157 Removed: branches/release-0.94/ Log: From davidk at ci.uchicago.edu Thu Jan 17 21:25:12 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 17 Jan 2013 21:25:12 -0600 (CST) Subject: [Swift-commit] r6158 - branches Message-ID: <20130118032512.8304B9CCAA@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-17 21:25:12 -0600 (Thu, 17 Jan 2013) New Revision: 6158 Added: branches/release-0.94/ Log: From davidk at ci.uchicago.edu Thu Jan 17 21:36:50 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 17 Jan 2013 21:36:50 -0600 (CST) Subject: [Swift-commit] r6159 - branches/release-0.94/libexec Message-ID: <20130118033650.5FA699CCAA@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-17 21:36:50 -0600 (Thu, 17 Jan 2013) New Revision: 6159 Modified: branches/release-0.94/libexec/release.txt Log: 0.94RC3 Modified: branches/release-0.94/libexec/release.txt =================================================================== --- branches/release-0.94/libexec/release.txt 2013-01-18 03:25:12 UTC (rev 6158) +++ branches/release-0.94/libexec/release.txt 2013-01-18 03:36:50 UTC (rev 6159) @@ -1 +1 @@ -trunk +0.94RC3 From davidk at ci.uchicago.edu Thu Jan 17 23:05:08 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 17 Jan 2013 23:05:08 -0600 (CST) Subject: [Swift-commit] r6160 - trunk/tests/language-behaviour/math Message-ID: <20130118050508.8AB989CCC5@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-17 23:05:08 -0600 (Thu, 17 Jan 2013) New Revision: 6160 Added: trunk/tests/language-behaviour/math/intdivision.out trunk/tests/language-behaviour/math/intdivision.out.expected trunk/tests/language-behaviour/math/intdivision.swift Log: Integer division test Added: trunk/tests/language-behaviour/math/intdivision.out =================================================================== --- trunk/tests/language-behaviour/math/intdivision.out (rev 0) +++ trunk/tests/language-behaviour/math/intdivision.out 2013-01-18 05:05:08 UTC (rev 6160) @@ -0,0 +1 @@ +x = 1, y = 1, z = 1 Added: trunk/tests/language-behaviour/math/intdivision.out.expected =================================================================== --- trunk/tests/language-behaviour/math/intdivision.out.expected (rev 0) +++ trunk/tests/language-behaviour/math/intdivision.out.expected 2013-01-18 05:05:08 UTC (rev 6160) @@ -0,0 +1,3 @@ +n = 1 +x = 1.0 +x = 1 Added: trunk/tests/language-behaviour/math/intdivision.swift =================================================================== --- trunk/tests/language-behaviour/math/intdivision.swift (rev 0) +++ trunk/tests/language-behaviour/math/intdivision.swift 2013-01-18 05:05:08 UTC (rev 6160) @@ -0,0 +1,13 @@ +type file; +file outfile <"intdivision.out">; + +app (file o) echo (string s) { + echo s stdout=@o; +} + +int x = 7 %/ 5; // expect x = 1 +float y = 1.0*@toFloat(x); // expect y = 1.0 +string z = @strcat(y); + +string msg = @strcat("x = ", x, ", y = ", y, ", z = ", z); +outfile = echo(msg); From davidk at ci.uchicago.edu Tue Jan 22 11:53:21 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 22 Jan 2013 11:53:21 -0600 (CST) Subject: [Swift-commit] r6161 - trunk/bin Message-ID: <20130122175321.A47DE9CFCC@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-22 11:53:21 -0600 (Tue, 22 Jan 2013) New Revision: 6161 Modified: trunk/bin/stop-coaster-service Log: Remove old jobs when usernames differ Modified: trunk/bin/stop-coaster-service =================================================================== --- trunk/bin/stop-coaster-service 2013-01-18 05:05:08 UTC (rev 6160) +++ trunk/bin/stop-coaster-service 2013-01-22 17:53:21 UTC (rev 6161) @@ -84,9 +84,9 @@ if [ "$WORKER_MODE" == "condor" ] || [ "$WORKER_MODE" == "gwms" ]; then if [ -n "$WORKER_RELAY_HOST" ]; then - ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "condor_rm $USER" + ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "condor_rm $WORKER_USERNAME" else - condor_rm $USER + condor_rm $CONDOR_USERNAME fi fi From davidk at ci.uchicago.edu Tue Jan 22 13:11:26 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 22 Jan 2013 13:11:26 -0600 (CST) Subject: [Swift-commit] r6162 - in trunk: bin etc/sites Message-ID: <20130122191126.6504B9CFCC@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-22 13:11:26 -0600 (Tue, 22 Jan 2013) New Revision: 6162 Modified: trunk/bin/start-coaster-service trunk/etc/sites/persistent-coasters Log: Use $GLOBUS_HOSTNAME before trying ifconfig Ignore junk from ifconfig Fix for persistent-coaster template for new gensites Modified: trunk/bin/start-coaster-service =================================================================== --- trunk/bin/start-coaster-service 2013-01-22 17:53:21 UTC (rev 6161) +++ trunk/bin/start-coaster-service 2013-01-22 19:11:26 UTC (rev 6162) @@ -428,8 +428,10 @@ if [ -z "$IPADDR" ]; then if [ "$SSH_TUNNELING" == "yes" ]; then IPADDR=localhost + elif [ -n "$GLOBUS_HOSTNAME" ]; then + IPADDR=$GLOBUS_HOSTNAME elif [ -x "/sbin/ifconfig" ]; then - IPADDR=$( /sbin/ifconfig | grep 'inet addr' | grep -v 127.0.0.1 | cut -d ':' -f 2 | awk '{print $1}' |head -1) + IPADDR=$( /sbin/ifconfig 2>/dev/null | grep 'inet addr' | grep -v 127.0.0.1 | cut -d ':' -f 2 | awk '{print $1}' |head -1) else crash "Unable to determine IP address of system. Please add to coaster-service.conf" fi Modified: trunk/etc/sites/persistent-coasters =================================================================== --- trunk/etc/sites/persistent-coasters 2013-01-22 17:53:21 UTC (rev 6161) +++ trunk/etc/sites/persistent-coasters 2013-01-22 19:11:26 UTC (rev 6162) @@ -1,11 +1,11 @@ passive - _JOBS_PER_NODE_ - _JOB_THROTTLE_ + _JOBSPERNODE_ + _JOBTHROTTLE_ 10000 _WORK_ From davidk at ci.uchicago.edu Tue Jan 22 15:12:11 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 22 Jan 2013 15:12:11 -0600 (CST) Subject: [Swift-commit] r6163 - trunk/bin Message-ID: <20130122211211.F04AD9CCC5@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-22 15:12:11 -0600 (Tue, 22 Jan 2013) New Revision: 6163 Modified: trunk/bin/start-coaster-service Log: Re-enable bug #467 fix by default Modified: trunk/bin/start-coaster-service =================================================================== --- trunk/bin/start-coaster-service 2013-01-22 19:11:26 UTC (rev 6162) +++ trunk/bin/start-coaster-service 2013-01-22 21:12:11 UTC (rev 6163) @@ -528,11 +528,11 @@ fi # For evil bug #467 -#echo "app echo (string i) { echo i; }" > hi.swift -#echo "echo(\"hi\");" >> hi.swift -#swift -sites.file sites.xml -tc.file tc.data -config cf hi.swift > /dev/null 2>&1 & -#DUMMYPID=$! -#sleep 15 +echo "app echo (string i) { echo i; }" > hi.swift +echo "echo(\"hi\");" >> hi.swift +swift -sites.file sites.xml -tc.file tc.data -config cf hi.swift > /dev/null 2>&1 & +DUMMYPID=$! +sleep 15 # Start workers case $WORKER_MODE in @@ -569,7 +569,7 @@ esac # Wait for dummy script to finish -#wait $DUMMYPID +wait $DUMMYPID # Generate config file if [ "$SHARED_FILESYSTEM" == "no" ]; then From hategan at ci.uchicago.edu Wed Jan 23 18:31:35 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 23 Jan 2013 18:31:35 -0600 (CST) Subject: [Swift-commit] r6164 - trunk/resources Message-ID: <20130124003135.3C8CF9CC90@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-23 18:31:35 -0600 (Wed, 23 Jan 2013) New Revision: 6164 Modified: trunk/resources/Karajan.stg Log: fixed range provenance Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2013-01-22 21:12:11 UTC (rev 6163) +++ trunk/resources/Karajan.stg 2013-01-24 00:31:35 UTC (rev 6164) @@ -422,18 +422,21 @@ range(from, to, step, datatype) ::= << - - $from$ - $to$ + + + $from$ + $to$ $if(step)$ - $step$ + $step$ $endif$ - - $range_log()$ + + + $range_log(from=from, to=to, step=step)$ + swift#rangeout >> -range_log() ::= << +range_log(from, to, step) ::= << provenance.logtrue @@ -441,10 +444,10 @@ ARRAYRANGE thread={#thread} array= - from= - to= + from= $from$ + to= $to$ $if(step)$ - step= + step= $step$ $else$ step=none $endif$ From hategan at ci.uchicago.edu Wed Jan 23 20:29:30 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 23 Jan 2013 20:29:30 -0600 (CST) Subject: [Swift-commit] r6165 - trunk/resources Message-ID: <20130124022930.900059CC90@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-23 20:29:30 -0600 (Wed, 23 Jan 2013) New Revision: 6165 Modified: trunk/resources/Karajan-no-provenance.stg Log: updated range template to conform with the provenance version Modified: trunk/resources/Karajan-no-provenance.stg =================================================================== --- trunk/resources/Karajan-no-provenance.stg 2013-01-24 00:31:35 UTC (rev 6164) +++ trunk/resources/Karajan-no-provenance.stg 2013-01-24 02:29:30 UTC (rev 6165) @@ -10,6 +10,6 @@ variable_log() ::= << >> -range_log() ::= << >> +range_log(from, to, step) ::= << >> cs() ::= << >> From swift at ci.uchicago.edu Wed Jan 23 22:30:04 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 23 Jan 2013 22:30:04 -0600 (CST) Subject: [Swift-commit] cog r3561 Message-ID: <20130124043004.E78658D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3561 | hategan | 2013-01-23 22:25:19 -0600 (Wed, 23 Jan 2013) | 1 line synchronizing on the channel when updating the last used time prevents the channel configuration from completing, so use another lock ------------------------------------------------------------------------ Index: modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java (revision 3560) +++ modules/karajan/src/org/globus/cog/karajan/workflow/service/channels/AbstractKarajanChannel.java (working copy) @@ -48,6 +48,9 @@ private Service callbackService; private final boolean client; private long lastTime; + private final Object lastTimeLock = new Object(); + + private TimerTask timeoutCheckTask; protected AbstractKarajanChannel(RequestManager requestManager, ChannelContext channelContext, boolean client) { @@ -114,7 +117,7 @@ } public void configureTimeoutChecks() { - Timer.every(TIMEOUT_CHECK_INTERVAL * 1000, new TimerTask() { + Timer.every(TIMEOUT_CHECK_INTERVAL * 1000, timeoutCheckTask = new TimerTask() { public void run() { checkTimeouts(); }} @@ -129,15 +132,20 @@ TimeoutException e = new TimeoutException(this, "Channel timed out", lastTime); context.notifyRegisteredCommandsAndHandlers(e); handleChannelException(e); + timeoutCheckTask.cancel(); } } - protected synchronized void updateLastTime() { - lastTime = System.currentTimeMillis(); + protected void updateLastTime() { + synchronized(lastTimeLock) { + lastTime = System.currentTimeMillis(); + } } - protected synchronized long getLastTime() { - return lastTime; + protected long getLastTime() { + synchronized(lastTimeLock) { + return lastTime; + } } protected boolean clientControlsHeartbeats() { From hategan at ci.uchicago.edu Thu Jan 24 16:13:05 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 24 Jan 2013 16:13:05 -0600 (CST) Subject: [Swift-commit] r6166 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20130124221305.9B58C9D0A9@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-24 16:13:05 -0600 (Thu, 24 Jan 2013) New Revision: 6166 Modified: trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java Log: made default foreach throttle very large rather than small Modified: trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2013-01-24 02:29:30 UTC (rev 6165) +++ trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2013-01-24 22:13:05 UTC (rev 6166) @@ -46,7 +46,7 @@ public class ThrottledParallelFor extends AbstractParallelIterator { public static final Logger logger = Logger.getLogger(ThrottledParallelFor.class); - public static final int DEFAULT_MAX_THREADS = 1024; + public static final int DEFAULT_MAX_THREADS = 10000000; public static final Arg A_NAME = new Arg.Positional("name"); public static final Arg A_IN = new Arg.Positional("in"); From lgadelha at ci.uchicago.edu Mon Jan 28 07:03:46 2013 From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu) Date: Mon, 28 Jan 2013 07:03:46 -0600 (CST) Subject: [Swift-commit] r6167 - trunk/libexec/log-processing Message-ID: <20130128130348.155D59CCB3@svn.ci.uchicago.edu> Author: lgadelha Date: 2013-01-28 07:03:17 -0600 (Mon, 28 Jan 2013) New Revision: 6167 Modified: trunk/libexec/log-processing/create-everylog-vs-versions-data Log: Added version of CoG to output Modified: trunk/libexec/log-processing/create-everylog-vs-versions-data =================================================================== --- trunk/libexec/log-processing/create-everylog-vs-versions-data 2013-01-24 22:13:05 UTC (rev 6166) +++ trunk/libexec/log-processing/create-everylog-vs-versions-data 2013-01-28 13:03:17 UTC (rev 6167) @@ -6,21 +6,19 @@ for logfile in `find $1 -name \*.log -and -not -name cluster-\* -and -not -name swift.log $EXTRALOGRESTRICTION`; do - VERSION=$(grep -m 1 -E 'Swift v[^ ]* r[0-9]*' $logfile | sed 's/.*Swift v[^ ]* r\([0-9]*\).*/\1/') - if [ "X$VERSION" == "X" ]; then -# try newer format version info -#2008-05-06 09:30:42,365-0500 INFO unknown Swift svn swift-r1800 cog-r1963 (CoG modified locally) - VERSION=$(grep -m 1 -E 'Swift .* swift-r[0-9]*' $logfile | sed 's/.*Swift .* swift-r\([0-9]*\).*/\1/') - fi - START=$(head -n1 < $logfile | iso-to-secs | cut -f 1 -d ' ') - if [ "X$VERSION" == "X" ]; then - VERSION=1262 - fi - if [ "X$START" != "X" ]; then - echo $START $VERSION $logfile >> everylog-vs-versions.data - fi -# echo $START $VERSION $logfile - + SWIFT_VERSION=$(grep -m 1 -E 'Swift .* swift-r[0-9]*' $logfile | sed 's/.*Swift .* swift-r\([0-9]*\).*/\1/') + COG_VERSION=$(grep -m 1 -E 'Swift .* swift-r[0-9]*' $logfile | sed 's/.*Swift .* cog-r\([0-9]*\).*/\1/') + START=$(head -n1 < $logfile | iso-to-secs | cut -f 1 -d ' ') + if [ "X$SWIFT_VERSION" == "X" ]; then + SWIFT_VERSION=na + COG_VERSION=na + fi + if [ "X$START" != "X" ]; then + echo $START $SWIFT_VERSION $logfile $COG_VERSION >> everylog-vs-versions.data + fi done echo Finished creating log/version data file + + + From lgadelha at ci.uchicago.edu Mon Jan 28 07:48:26 2013 From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu) Date: Mon, 28 Jan 2013 07:48:26 -0600 (CST) Subject: [Swift-commit] r6168 - provenancedb Message-ID: <20130128134826.5B67E9CCBE@svn.ci.uchicago.edu> Author: lgadelha Date: 2013-01-28 07:48:25 -0600 (Mon, 28 Jan 2013) New Revision: 6168 Modified: provenancedb/swift-prov-import-all-logs Log: Minor fixes Modified: provenancedb/swift-prov-import-all-logs =================================================================== --- provenancedb/swift-prov-import-all-logs 2013-01-28 13:03:17 UTC (rev 6167) +++ provenancedb/swift-prov-import-all-logs 2013-01-28 13:48:25 UTC (rev 6168) @@ -31,7 +31,7 @@ $SQLCMD < $PROVDIR/prov-init.sql fi -while read start version cogversion filename; do +while read start version filename cogversion; do export IDIR=$(echo $filename | sed 's/\.log$/.d/') COG_VERSION=$(grep -m 1 -E 'Swift .* swift-r[0-9]*' $filename | sed 's/.*Swift .* cog-r\([0-9]*\).*/\1/') From hategan at ci.uchicago.edu Mon Jan 28 22:30:59 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Mon, 28 Jan 2013 22:30:59 -0600 (CST) Subject: [Swift-commit] r6169 - branches Message-ID: <20130129043059.6F4329CCBE@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-28 22:30:59 -0600 (Mon, 28 Jan 2013) New Revision: 6169 Added: branches/faster/ Log: branched faster From hategan at ci.uchicago.edu Tue Jan 29 02:03:43 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Tue, 29 Jan 2013 02:03:43 -0600 (CST) Subject: [Swift-commit] r6171 - branches/faster/src/org/griphyn/vdl/karajan Message-ID: <20130129080343.60C249CCB3@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-29 02:03:42 -0600 (Tue, 29 Jan 2013) New Revision: 6171 Removed: branches/faster/src/org/griphyn/vdl/karajan/ScalabilityTest.java Modified: branches/faster/src/org/griphyn/vdl/karajan/HangChecker.java branches/faster/src/org/griphyn/vdl/karajan/Loader.java branches/faster/src/org/griphyn/vdl/karajan/Monitor.java Log: initial updates of faster branch Modified: branches/faster/src/org/griphyn/vdl/karajan/HangChecker.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/HangChecker.java 2013-01-29 07:31:09 UTC (rev 6170) +++ branches/faster/src/org/griphyn/vdl/karajan/HangChecker.java 2013-01-29 08:03:42 UTC (rev 6171) @@ -33,6 +33,7 @@ import k.rt.Context; import k.rt.ExecutionException; +import k.rt.Stack; import k.thr.LWThread; import k.thr.Scheduler; @@ -42,8 +43,6 @@ import org.globus.cog.karajan.scheduler.WeightedHostScoreScheduler; import org.griphyn.vdl.mapping.DSHandle; -import com.sun.org.apache.xpath.internal.VariableStack; - public class HangChecker extends TimerTask { public static final Logger logger = Logger.getLogger(HangChecker.class); @@ -202,7 +201,7 @@ } Object prev = c.getLast(); for (Object o : c) { - if (o instanceof VariableStack) { + if (o instanceof Stack) { if (prev != null) { ps.println("\t" + Monitor.varWithLine((DSHandle) prev) + " is needed by: "); } Modified: branches/faster/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/Loader.java 2013-01-29 07:31:09 UTC (rev 6170) +++ branches/faster/src/org/griphyn/vdl/karajan/Loader.java 2013-01-29 08:03:42 UTC (rev 6171) @@ -47,7 +47,6 @@ import org.apache.log4j.FileAppender; import org.apache.log4j.Level; import org.apache.log4j.Logger; -import org.globus.cog.abstraction.impl.execution.fake.JobSubmissionTaskHandler; import org.globus.cog.karajan.compiled.nodes.Main; import org.globus.cog.karajan.compiled.nodes.grid.AbstractGridNode; import org.globus.cog.karajan.parser.WrapperNode; @@ -217,7 +216,7 @@ ec.start(context); ec.waitFor(); long end = System.currentTimeMillis(); - System.out.println(JobSubmissionTaskHandler.jobsRun + " jobs, " + JobSubmissionTaskHandler.jobsRun * 1000 / (end - start) + " j/s"); + //System.out.println(JobSubmissionTaskHandler.jobsRun + " jobs, " + JobSubmissionTaskHandler.jobsRun * 1000 / (end - start) + " j/s"); if (ec.isFailed()) { runerror = true; } Modified: branches/faster/src/org/griphyn/vdl/karajan/Monitor.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/Monitor.java 2013-01-29 07:31:09 UTC (rev 6170) +++ branches/faster/src/org/griphyn/vdl/karajan/Monitor.java 2013-01-29 08:03:42 UTC (rev 6171) @@ -54,6 +54,7 @@ import k.rt.Future; import k.rt.FutureListener; import k.rt.FutureValue; +import k.rt.Stack; import k.thr.LWThread; import org.globus.cog.karajan.analyzer.VariableNotFoundException; @@ -66,8 +67,6 @@ import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.Path; -import com.sun.org.apache.xpath.internal.VariableStack; - public class Monitor implements ActionListener, MouseListener { public static final int VARS = 0; public static final int THREADS = 1; @@ -469,7 +468,7 @@ } else if (crtdisp == THREADS) { Object o = wt.get(row); - if (o instanceof VariableStack) { + if (o instanceof Stack) { displayPopup("Stack trace for " + t.getValueAt(row, 0), " N/A"); } } Deleted: branches/faster/src/org/griphyn/vdl/karajan/ScalabilityTest.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/ScalabilityTest.java 2013-01-29 07:31:09 UTC (rev 6170) +++ branches/faster/src/org/griphyn/vdl/karajan/ScalabilityTest.java 2013-01-29 08:03:42 UTC (rev 6171) @@ -1,181 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Jun 20, 2006 - */ -package org.griphyn.vdl.karajan; - -import java.awt.GridLayout; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - -import javax.swing.JFrame; -import javax.swing.JLabel; -import javax.swing.JPanel; - -import org.globus.cog.karajan.Loader; -import org.globus.cog.karajan.arguments.AbstractWriteOnlyVariableArguments; -import org.globus.cog.karajan.arguments.VariableArguments; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ElementTree; -import org.globus.cog.karajan.workflow.ExecutionContext; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.events.EventListener; - -public class ScalabilityTest { - private static volatile int jobsSubmitted, jobsFailed, jobsCompleted, workflowsStarted, - workflowsFailed, workflowsCompleted; - - private static int sleepTime; - - private static long startTime, endTime; - - public static void main(String[] args) { - if (args.length != 3) { - System.out.println("Usage: my-name "); - System.exit(1); - } - startTime = System.currentTimeMillis(); - sleepTime = Integer.parseInt(args[2]); - int instanceCount = Integer.parseInt(args[0]); - for (int i = 0; i < instanceCount; i++) { - try { - ElementTree tree = Loader.load("scalability.k"); - ExecutionContext ec = new ExecutionContext(tree); - ec.setStdout(new Stdout()); - List l = new LinkedList(); - l.add(args[1]); - l.add(args[2]); - ec.setArguments(l); - ec.addEventListener(new Listener()); - ec.start(); - workflowsStarted++; - } - catch (Exception e) { - e.printStackTrace(); - } - } - Monitor m = new Monitor(); - while (workflowsStarted > workflowsCompleted + workflowsFailed) { - try { - m.update(); - Thread.sleep(1000); - } - catch (InterruptedException e) { - break; - } - } - m.update(); - endTime = System.currentTimeMillis(); - System.out.println("Workflows started: " + workflowsStarted); - System.out.println("Workflows completed: " + workflowsCompleted); - System.out.println("Workflows failed: " + workflowsFailed); - System.out.println("Jobs started: " + jobsSubmitted); - System.out.println("Jobs completed: " + jobsCompleted); - System.out.println("Jobs failed: " + jobsFailed); - System.out.println("Total time: " + (endTime - startTime) / 1000 + "s"); - System.out.println("Total time excluding sleep time: " - + ((endTime - startTime) / 1000 - sleepTime) + "s"); - } - - public static class Stdout extends AbstractWriteOnlyVariableArguments { - public void merge(VariableArguments args) { - appendAll(args.getAll()); - } - - public void append(Object value) { - if (".".equals(value)) { - jobsSubmitted++; - } - else if ("O".equals(value)) { - jobsCompleted++; - } - else if ("-".equals(value)) { - jobsFailed++; - } - System.out.print(value); - } - - public void appendAll(List args) { - Iterator i = args.iterator(); - while (i.hasNext()) { - append(i.next()); - } - } - - public boolean isCommutative() { - return true; - } - } - - public static class Listener implements EventListener { - public void completed(VariableStack stack) throws ExecutionException { - workflowsCompleted++; - } - - public void failed(VariableStack stack, ExecutionException e) - throws ExecutionException { - workflowsFailed++; - } - } - - public static class Monitor { - private JFrame frame; - private JLabel time, js, jf, jc, ws, wf, wc; - - public Monitor() { - frame = new JFrame(); - JPanel p = new JPanel(); - p.setLayout(new GridLayout(0, 2)); - p.add(new JLabel("Time: ")); - p.add(time = new JLabel()); - - p.add(new JLabel("Workflows started: ")); - p.add(ws = new JLabel()); - p.add(new JLabel("Workflows failed: ")); - p.add(wf = new JLabel()); - p.add(new JLabel("Workflows completed: ")); - p.add(wc = new JLabel()); - - p.add(new JLabel("Jobs started: ")); - p.add(js = new JLabel()); - p.add(new JLabel("Jobs failed: ")); - p.add(jf = new JLabel()); - p.add(new JLabel("Jobs completed: ")); - p.add(jc = new JLabel()); - - frame.getContentPane().add(p); - frame.setSize(300, 200); - frame.setVisible(true); - } - - public void update() { - time.setText(String.valueOf((System.currentTimeMillis() - startTime) / 1000) + "s"); - - ws.setText(String.valueOf(workflowsStarted)); - wf.setText(String.valueOf(workflowsFailed)); - wc.setText(String.valueOf(workflowsCompleted)); - - js.setText(String.valueOf(jobsSubmitted)); - jf.setText(String.valueOf(jobsFailed)); - jc.setText(String.valueOf(jobsCompleted)); - } - } - -} From davidk at ci.uchicago.edu Tue Jan 29 10:25:37 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 29 Jan 2013 10:25:37 -0600 (CST) Subject: [Swift-commit] r6172 - trunk/libexec Message-ID: <20130129162537.913989CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-29 10:25:37 -0600 (Tue, 29 Jan 2013) New Revision: 6172 Modified: trunk/libexec/_swiftwrap Log: Fix for _swiftwrap on mac, and also better handle situations where /bin/time isn't found Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2013-01-29 08:03:42 UTC (rev 6171) +++ trunk/libexec/_swiftwrap 2013-01-29 16:25:37 UTC (rev 6172) @@ -479,19 +479,27 @@ TIMEARGS=(-o swiftapp.resources -f APP_RESOURCES=real_secs:%e,kernel_secs:%S,user_secs:%U,percent_cpu:%P,max_rss:%M,avg_rss:%t,avg_tot_vm:%K,avg_priv_data:%D,avg_priv_stack:%p,avg_shared_text:%X,page_size:%Z,major_pgfaults:%F,minor_pgfaults:%R,swaps:%W,invol_context_switches:%c,vol_waits:%w,fs_reads:%I,fs_writes:%O,sock_recv:%r,sock_send:%s,signals:%k,exit_status:%x) - if [ -x /usr/bin/time ]; then + if [[ "$OSTYPE" == *darwin* ]]; then + TIMECMD= + TIMEARGS= + elif [ -x /usr/bin/time ]; then TIMECMD="/usr/bin/time" elif [ -x $HOME/swift.time ]; then TIMECMD="$HOME/swift.time" else TIMECMD="" - TIMEARGS="" + TIMEARGS="" fi if [ "$STDIN" == "" ]; then if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then genScripts fi - "$TIMECMD" "${TIMEARGS[@]}" "$EXEC" "${CMDARGS[@]}" 1>>"$STDOUT" 2>>"$STDERR" + + if [ -n "$TIMECMD" ] && [ -n "$TIMEARGS" ]; then + "$TIMECMD" "${TIMEARGS[@]}" "$EXEC" "${CMDARGS[@]}" 1>>"$STDOUT" 2>>"$STDERR" + else + "$EXEC" "${CMDARGS[@]}" 1>>"$STDOUT" 2>>"$STDERR" + fi else if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then genScripts From davidk at ci.uchicago.edu Tue Jan 29 10:29:25 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 29 Jan 2013 10:29:25 -0600 (CST) Subject: [Swift-commit] r6173 - SwiftApps/modis Message-ID: <20130129162925.0829C9CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-29 10:29:24 -0600 (Tue, 29 Jan 2013) New Revision: 6173 Added: SwiftApps/modis/clean.sh Modified: SwiftApps/modis/modis.swift Log: Script to clean logs toint fix to avoid nag messages Added: SwiftApps/modis/clean.sh =================================================================== --- SwiftApps/modis/clean.sh (rev 0) +++ SwiftApps/modis/clean.sh 2013-01-29 16:29:24 UTC (rev 6173) @@ -0,0 +1,3 @@ +#!/bin/bash + +rm -rf *.log *.rlog *.d *.kml *.swiftx modis-20* Property changes on: SwiftApps/modis/clean.sh ___________________________________________________________________ Added: svn:executable + * Modified: SwiftApps/modis/modis.swift =================================================================== --- SwiftApps/modis/modis.swift 2013-01-29 16:25:37 UTC (rev 6172) +++ SwiftApps/modis/modis.swift 2013-01-29 16:29:24 UTC (rev 6173) @@ -33,8 +33,8 @@ # Constants and command line arguments -int nFiles = @toint(@arg("nfiles","1000")); -int nSelect = @toint(@arg("nselect","12")); +int nFiles = @toInt(@arg("nfiles","1000")); +int nSelect = @toInt(@arg("nselect","12")); string landType = @arg("landtype","urban"); string runID = @arg("runid","modis-run"); string MODISdir= @arg("modisdir","/home/wilde/bigdata/data/modis/2002"); From davidk at ci.uchicago.edu Tue Jan 29 11:42:41 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 29 Jan 2013 11:42:41 -0600 (CST) Subject: [Swift-commit] r6174 - SwiftApps/modis/bin Message-ID: <20130129174241.5981E9CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-29 11:42:41 -0600 (Tue, 29 Jan 2013) New Revision: 6174 Modified: SwiftApps/modis/bin/assemble2.sh Log: Mac doesn't have seq, use bash {} and printf instead Modified: SwiftApps/modis/bin/assemble2.sh =================================================================== --- SwiftApps/modis/bin/assemble2.sh 2013-01-29 16:29:24 UTC (rev 6173) +++ SwiftApps/modis/bin/assemble2.sh 2013-01-29 17:42:41 UTC (rev 6174) @@ -13,8 +13,10 @@ # we reverse this to form the final image cd $indir -for h in `seq -w 00 35`; do - for v in `seq -w 00 17`; do +for h in {0..10}; do + h=$( printf %02d $h ); + for v in {0..17}; do + v=$( printf %02d $v ); hv=h${h}v${v} vh=v${v}h${h} if [ ! -f $hv*png ]; then From davidk at ci.uchicago.edu Tue Jan 29 11:44:56 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 29 Jan 2013 11:44:56 -0600 (CST) Subject: [Swift-commit] r6175 - SwiftApps/modis/bin Message-ID: <20130129174456.160C49CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-29 11:44:55 -0600 (Tue, 29 Jan 2013) New Revision: 6175 Modified: SwiftApps/modis/bin/assemble2.sh Log: Fix range Modified: SwiftApps/modis/bin/assemble2.sh =================================================================== --- SwiftApps/modis/bin/assemble2.sh 2013-01-29 17:42:41 UTC (rev 6174) +++ SwiftApps/modis/bin/assemble2.sh 2013-01-29 17:44:55 UTC (rev 6175) @@ -13,7 +13,8 @@ # we reverse this to form the final image cd $indir -for h in {0..10}; do + +for h in {0..35}; do h=$( printf %02d $h ); for v in {0..17}; do v=$( printf %02d $v ); From davidk at ci.uchicago.edu Tue Jan 29 12:10:02 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 29 Jan 2013 12:10:02 -0600 (CST) Subject: [Swift-commit] r6176 - SwiftApps/modis Message-ID: <20130129181002.D0CD19CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-29 12:10:02 -0600 (Tue, 29 Jan 2013) New Revision: 6176 Added: SwiftApps/modis/cf.midway SwiftApps/modis/demo.midway SwiftApps/modis/sites.midway.xml SwiftApps/modis/tc.midway Log: Working midway configs Added: SwiftApps/modis/cf.midway =================================================================== --- SwiftApps/modis/cf.midway (rev 0) +++ SwiftApps/modis/cf.midway 2013-01-29 18:10:02 UTC (rev 6176) @@ -0,0 +1,7 @@ +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=false +provider.staging.pin.swiftfiles=false Added: SwiftApps/modis/demo.midway =================================================================== --- SwiftApps/modis/demo.midway (rev 0) +++ SwiftApps/modis/demo.midway 2013-01-29 18:10:02 UTC (rev 6176) @@ -0,0 +1,15 @@ +# demo4 forest 50 5 pbs + +landtype=${1:-urban} +nfiles=${2:-999} +nselect=${3:-10} +site=${4:-local} + +runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect + +echo runid=$runid + +swift -tc.file tc.midway \ + -sites.file sites.midway.xml \ + modis.swift -config cf.midway -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \ + -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo Property changes on: SwiftApps/modis/demo.midway ___________________________________________________________________ Added: svn:executable + * Added: SwiftApps/modis/sites.midway.xml =================================================================== --- SwiftApps/modis/sites.midway.xml (rev 0) +++ SwiftApps/modis/sites.midway.xml 2013-01-29 18:10:02 UTC (rev 6176) @@ -0,0 +1,31 @@ + + + + + + /scratch/midway/{env.USER}/work + 3600 + 16 + 10 + 1 + 25 + sandyb + 1000 + 10000 + + + + + + /scratch/midway/{env.USER}/work + 3600 + 12 + 20 + 1 + 25 + westmere + 1000 + 10000 + + + Added: SwiftApps/modis/tc.midway =================================================================== --- SwiftApps/modis/tc.midway (rev 0) +++ SwiftApps/modis/tc.midway 2013-01-29 18:10:02 UTC (rev 6176) @@ -0,0 +1,31 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# site transformation path obsolete fields for compatibility + +localhost echo /bin/echo null null null +localhost cat /bin/cat null null null +localhost ls /bin/ls null null null +localhost grep /bin/grep null null null +localhost sort /bin/sort null null null +localhost paste /bin/paste null null null +localhost pwd /bin/pwd null null null + +# For cluster usage +midway-sandyb getlanduse /scratch/midway/davidkelly999/modis/bin/getlanduse.sh null null null +midway-sandyb analyzelanduse /scratch/midway/davidkelly999/modis/bin/analyzelanduse2.sh null null null +midway-sandyb colormodis /scratch/midway/davidkelly999/modis/bin/colormodis.sh null null null +midway-sandyb assemble /scratch/midway/davidkelly999/modis/bin/assemble2.sh null null null +midway-sandyb markmap /scratch/midway/davidkelly999/modis/bin/markmap.sh null null null +midway-westmere getlanduse /scratch/midway/davidkelly999/modis/bin/getlanduse.sh null null null +midway-westmere analyzelanduse /scratch/midway/davidkelly999/modis/bin/analyzelanduse2.sh null null null +midway-westmere colormodis /scratch/midway/davidkelly999/modis/bin/colormodis.sh null null null +midway-westmere assemble /scratch/midway/davidkelly999/modis/bin/assemble2.sh null null null +midway-westmere markmap /scratch/midway/davidkelly999/modis/bin/markmap.sh null null null From davidk at ci.uchicago.edu Tue Jan 29 14:39:26 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 29 Jan 2013 14:39:26 -0600 (CST) Subject: [Swift-commit] r6177 - trunk/bin Message-ID: <20130129203926.6C885178884@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-29 14:39:26 -0600 (Tue, 29 Jan 2013) New Revision: 6177 Modified: trunk/bin/swiftrun Log: Make swiftrun smarter about finding config files for gensites Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2013-01-29 18:10:02 UTC (rev 6176) +++ trunk/bin/swiftrun 2013-01-29 20:39:26 UTC (rev 6177) @@ -65,17 +65,30 @@ cp $script $rundir fi +# Copy config file if specified +if [ -f "$CONFIG" ]; then + cp $CONFIG $rundir + CONFIG=$( basename $CONFIG ) +fi + # Create sites.xml cd $rundir IFS="," echo "" > sites.xml for site in $SITES do + # If config is not defined, but there is a config in conf/ that matches, use that + if [ -z "$CONFIG" ] && [ -f "../conf/$site.cf" ]; then + cp ../conf/$site.cf . + CONFIG=$site.cf + fi + if [ -f "$CONFIG" ]; then gensites -p $CONFIG $site >> sites.xml else gensites $site >> sites.xml fi + done echo "" >> sites.xml From davidk at ci.uchicago.edu Tue Jan 29 14:46:54 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 29 Jan 2013 14:46:54 -0600 (CST) Subject: [Swift-commit] r6178 - in SwiftApps/modis: . conf Message-ID: <20130129204654.BDED1178884@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-29 14:46:54 -0600 (Tue, 29 Jan 2013) New Revision: 6178 Added: SwiftApps/modis/conf/ SwiftApps/modis/conf/local.cf SwiftApps/modis/conf/midway.cf SwiftApps/modis/demo Removed: SwiftApps/modis/cf.local SwiftApps/modis/cf.midway SwiftApps/modis/cf.pads SwiftApps/modis/demo.local SwiftApps/modis/demo.midway SwiftApps/modis/demo.pads SwiftApps/modis/demo.pads-pbs SwiftApps/modis/scripts/ SwiftApps/modis/sites.pads-pbs.xml Log: Some cleanups and reorginzations Deleted: SwiftApps/modis/cf.local =================================================================== --- SwiftApps/modis/cf.local 2013-01-29 20:39:26 UTC (rev 6177) +++ SwiftApps/modis/cf.local 2013-01-29 20:46:54 UTC (rev 6178) @@ -1,9 +0,0 @@ -wrapperlog.always.transfer=true -sitedir.keep=true -execution.retries=0 -lazy.errors=false -status.mode=provider -use.provider.staging=false -provider.staging.pin.swiftfiles=false -#throttle.transfers=16 -#throttle.file.operations=16 Deleted: SwiftApps/modis/cf.midway =================================================================== --- SwiftApps/modis/cf.midway 2013-01-29 20:39:26 UTC (rev 6177) +++ SwiftApps/modis/cf.midway 2013-01-29 20:46:54 UTC (rev 6178) @@ -1,7 +0,0 @@ -wrapperlog.always.transfer=true -sitedir.keep=true -execution.retries=0 -lazy.errors=false -status.mode=provider -use.provider.staging=false -provider.staging.pin.swiftfiles=false Deleted: SwiftApps/modis/cf.pads =================================================================== --- SwiftApps/modis/cf.pads 2013-01-29 20:39:26 UTC (rev 6177) +++ SwiftApps/modis/cf.pads 2013-01-29 20:46:54 UTC (rev 6178) @@ -1,7 +0,0 @@ -wrapperlog.always.transfer=true -sitedir.keep=true -execution.retries=0 -lazy.errors=false -status.mode=provider -use.provider.staging=false -provider.staging.pin.swiftfiles=false Copied: SwiftApps/modis/conf/local.cf (from rev 6177, SwiftApps/modis/cf.local) =================================================================== --- SwiftApps/modis/conf/local.cf (rev 0) +++ SwiftApps/modis/conf/local.cf 2013-01-29 20:46:54 UTC (rev 6178) @@ -0,0 +1,9 @@ +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=false +provider.staging.pin.swiftfiles=false +#throttle.transfers=16 +#throttle.file.operations=16 Copied: SwiftApps/modis/conf/midway.cf (from rev 6177, SwiftApps/modis/cf.midway) =================================================================== --- SwiftApps/modis/conf/midway.cf (rev 0) +++ SwiftApps/modis/conf/midway.cf 2013-01-29 20:46:54 UTC (rev 6178) @@ -0,0 +1,7 @@ +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=false +provider.staging.pin.swiftfiles=false Copied: SwiftApps/modis/demo (from rev 6177, SwiftApps/modis/demo.local) =================================================================== --- SwiftApps/modis/demo (rev 0) +++ SwiftApps/modis/demo 2013-01-29 20:46:54 UTC (rev 6178) @@ -0,0 +1,15 @@ +# demo4 forest 50 5 pbs + +landtype=${1:-urban} +nfiles=${2:-999} +nselect=${3:-10} +site=${4:-local} + +runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect + +echo runid=$runid + +swift -tc.file tc.local \ + -sites.file sites.local.xml \ + modis.swift -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \ + -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo Deleted: SwiftApps/modis/demo.local =================================================================== --- SwiftApps/modis/demo.local 2013-01-29 20:39:26 UTC (rev 6177) +++ SwiftApps/modis/demo.local 2013-01-29 20:46:54 UTC (rev 6178) @@ -1,15 +0,0 @@ -# demo4 forest 50 5 pbs - -landtype=${1:-urban} -nfiles=${2:-999} -nselect=${3:-10} -site=${4:-local} - -runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect - -echo runid=$runid - -swift -tc.file tc.local \ - -sites.file sites.local.xml \ - modis.swift -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \ - -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo Deleted: SwiftApps/modis/demo.midway =================================================================== --- SwiftApps/modis/demo.midway 2013-01-29 20:39:26 UTC (rev 6177) +++ SwiftApps/modis/demo.midway 2013-01-29 20:46:54 UTC (rev 6178) @@ -1,15 +0,0 @@ -# demo4 forest 50 5 pbs - -landtype=${1:-urban} -nfiles=${2:-999} -nselect=${3:-10} -site=${4:-local} - -runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect - -echo runid=$runid - -swift -tc.file tc.midway \ - -sites.file sites.midway.xml \ - modis.swift -config cf.midway -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \ - -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo Deleted: SwiftApps/modis/demo.pads =================================================================== --- SwiftApps/modis/demo.pads 2013-01-29 20:39:26 UTC (rev 6177) +++ SwiftApps/modis/demo.pads 2013-01-29 20:46:54 UTC (rev 6178) @@ -1,15 +0,0 @@ -# demo4 forest 50 5 pbs - -landtype=${1:-urban} -nfiles=${2:-999} -nselect=${3:-10} -site=${4:-local} - -runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect - -echo runid=$runid - -swift -tc.file tc.pads \ - -sites.file sites.pads.xml \ - modis.swift -config cf.pads -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \ - -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo Deleted: SwiftApps/modis/demo.pads-pbs =================================================================== --- SwiftApps/modis/demo.pads-pbs 2013-01-29 20:39:26 UTC (rev 6177) +++ SwiftApps/modis/demo.pads-pbs 2013-01-29 20:46:54 UTC (rev 6178) @@ -1,15 +0,0 @@ -# demo4 forest 50 5 pbs - -landtype=${1:-urban} -nfiles=${2:-999} -nselect=${3:-10} -site=${4:-local} - -runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect - -echo runid=$runid - -swift -tc.file tc.pads \ - -sites.file sites.pads-pbs.xml \ - modis.swift -config cf.pads -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \ - -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo Deleted: SwiftApps/modis/sites.pads-pbs.xml =================================================================== --- SwiftApps/modis/sites.pads-pbs.xml 2013-01-29 20:39:26 UTC (rev 6177) +++ SwiftApps/modis/sites.pads-pbs.xml 2013-01-29 20:46:54 UTC (rev 6178) @@ -1,16 +0,0 @@ - - - - - /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/swiftwork - 3600 - 1 - 64 - 1 - 1 - fast - .6399 - 10000 - KEEP - - From hategan at ci.uchicago.edu Tue Jan 29 15:07:17 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Tue, 29 Jan 2013 15:07:17 -0600 (CST) Subject: [Swift-commit] r6179 - branches/faster/resources Message-ID: <20130129210717.B2852178884@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-29 15:07:17 -0600 (Tue, 29 Jan 2013) New Revision: 6179 Added: branches/faster/resources/swift-sites-1.0.xsd Log: added sites file schema Added: branches/faster/resources/swift-sites-1.0.xsd =================================================================== --- branches/faster/resources/swift-sites-1.0.xsd (rev 0) +++ branches/faster/resources/swift-sites-1.0.xsd 2013-01-29 21:07:17 UTC (rev 6179) @@ -0,0 +1,155 @@ + + + + + + defines the legal namespaces of execution profiles. + + + + + + + + + + + + A set of attributes to define a Globus version. + + + + + + + + root element aggregating all pool information there is. + + + + + + Describes a single pool. + + + + + + Administrative profile defaults associated with a pool. + + + + + + + + + + + + + Each pool may report to multiple LRCs. + + + + + + + + Each pool may have multiple gridftp servers. + + + + + + Stores the band width informaion related to each gridftp server. + + + + + + + + + + + + + + + + < xs:documentation>The URL (actually, it may be more a URI, but hey, so what) is the access URL to the gridftp server. Each pool may have multiple gridftp servers, or run multiple versions of Globus on different ports. + + + + + This element is the storage mount point prefix. Of course, this may get turned over into other things, augmented by user and system requirements etc. I believe that default works quite well for default Globus setups. + + + + + + + + + + Each pool supports various (usually two) jobmanagers. + + + + + The universe name is actually the primary key for the jobmanager identification. + + + + + The contact string is the secondary key for any job manager. + + + + + Any pool may have multiple versions of Globus installed, and these versions may have multiple jobmanagers l istening on different ports. + + + + + + + + + + + + + + + + + + + Each pool may have one and only one work directory mount point. + + + + + + + + + + + + + + + This is the path to grid launch. As attributes are optional, each pool can have at most one grid launch path. More does not make sense, unless there are different launchers for different remote schedulers, and a pool supports more than two (fork + one other) remote schedulers. + + + + + + + + + + \ No newline at end of file From hategan at ci.uchicago.edu Tue Jan 29 15:31:49 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Tue, 29 Jan 2013 15:31:49 -0600 (CST) Subject: [Swift-commit] r6180 - branches/faster/libexec Message-ID: <20130129213149.BBC9C9CCB3@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-29 15:31:49 -0600 (Tue, 29 Jan 2013) New Revision: 6180 Removed: branches/faster/libexec/execute-default.k branches/faster/libexec/execute-dryrun.k branches/faster/libexec/execute-typecheck.k Log: removed dryrun and typecheck modes Deleted: branches/faster/libexec/execute-default.k =================================================================== --- branches/faster/libexec/execute-default.k 2013-01-29 21:07:17 UTC (rev 6179) +++ branches/faster/libexec/execute-default.k 2013-01-29 21:31:49 UTC (rev 6180) @@ -1,75 +0,0 @@ -element([tr, optional(arguments, stdin, stdout, stderr, attributes, deperror, mdeperror), channel(stagein), channel(stageout), channel(restartout)] - - vdl:initprogressstate() - - done := isDone(restartout) - derr := try(deperror, false) - merr := try(mdeperror, false) - - if( - derr == false then( - if( - sys:not(done) try( - throttled( - log(LOG:INFO, "START thread={#thread} tr={tr}") - vdl:setprogress("Selecting site") - restartOnError(".*", vdl:configProperty("execution.retries"), - replicationChannel := channel() - discard(append(replicationChannel, true)) //trigger the first job - replicationGroup := uid() - parallelFor(i, replicationChannel - ignoreErrors(match = "^Abort$" - execute2( - tr, maybe(arguments=unwrapClosedList(arguments)), - maybe(stdin=stdin), maybe(stdout=stdout), maybe(stderr=stderr), - maybe(attributes=attributes), - stagein, stageout, restartout, replicationGroup, replicationChannel - ) - ) - ) - ) - mark(restartout, err=false, mapping=false) - log(LOG:INFO, "END_SUCCESS thread={#thread} tr={tr}") - vdl:setprogress("Finished successfully") - ) - catch(".*" - log(LOG:INFO, "END_FAILURE thread={#thread} tr={tr}") - vdl:setprogress("Failed") - if( - vdl:configProperty("lazy.errors") == "false" then( - throw(exception) - ) - else ( - to(errors, exception) - log(LOG:INFO, exception) - echo(exception) - mark(restartout, err=true, mapping=false) - graphStuff(tr, stagein, stageout, err=true, maybe(args=arguments)) - ) - ) - ) - ) - else ( - vdl:setprogress("Finished in previous run") - ) - ) - ) - else ( - if( - merr == true then( - exception := exception(concat("VDL2: Application ", str:quote(tr), - " not executed due to errors in mapping dependencies")) - to(errors, exception) - log(LOG:INFO, exception) - ) - derr == true then( - exception := exception("VDL2: Application {tr} not executed due to errors in dependencies") - to(errors, exception) - log(LOG:INFO, exception) - ) - ) - mark(restartout, err=true, mapping=merr) - graphStuff(tr, stagein, stageout, err=true, maybe(args=arguments)) - ) - ) -) Deleted: branches/faster/libexec/execute-dryrun.k =================================================================== --- branches/faster/libexec/execute-dryrun.k 2013-01-29 21:07:17 UTC (rev 6179) +++ branches/faster/libexec/execute-dryrun.k 2013-01-29 21:31:49 UTC (rev 6180) @@ -1,12 +0,0 @@ -element([tr, optional(arguments, stdin, stdout, stderr, deperror), channel(stagein), channel(stageout), channel(restartout)] - - done := isDone(stageout) - - if( - sys:not(done) sequential( - mark(stageout, err=false) - graphStuff(tr, stagein, stageout, err=false, maybe(args=arguments)) - ) - ) -) - Deleted: branches/faster/libexec/execute-typecheck.k =================================================================== --- branches/faster/libexec/execute-typecheck.k 2013-01-29 21:07:17 UTC (rev 6179) +++ branches/faster/libexec/execute-typecheck.k 2013-01-29 21:31:49 UTC (rev 6180) @@ -1,3 +0,0 @@ -element([tr, optional(arguments, stdin, stdout, stderr), channel(stagein), channel(stageout), channel(restartout)] - mark(stageout, err=false) -) From davidk at ci.uchicago.edu Tue Jan 29 16:04:12 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 29 Jan 2013 16:04:12 -0600 (CST) Subject: [Swift-commit] r6181 - in SwiftApps/modis: . conf Message-ID: <20130129220412.6E4B89CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-29 16:04:12 -0600 (Tue, 29 Jan 2013) New Revision: 6181 Modified: SwiftApps/modis/clean.sh SwiftApps/modis/conf/midway.cf SwiftApps/modis/demo SwiftApps/modis/modis.swift Log: Demo script and some path fixes Modified: SwiftApps/modis/clean.sh =================================================================== --- SwiftApps/modis/clean.sh 2013-01-29 21:31:49 UTC (rev 6180) +++ SwiftApps/modis/clean.sh 2013-01-29 22:04:12 UTC (rev 6181) @@ -1,3 +1,3 @@ #!/bin/bash -rm -rf *.log *.rlog *.d *.kml *.swiftx modis-20* +rm -rf run0* Modified: SwiftApps/modis/conf/midway.cf =================================================================== --- SwiftApps/modis/conf/midway.cf 2013-01-29 21:31:49 UTC (rev 6180) +++ SwiftApps/modis/conf/midway.cf 2013-01-29 22:04:12 UTC (rev 6181) @@ -5,3 +5,10 @@ status.mode=provider use.provider.staging=false provider.staging.pin.swiftfiles=false + +#app getlanduse=$PWD/../bin/getlanduse.sh +#app analyzelanduse=$PWD/../bin/analyzelanduse2.sh +#app colormodis=$PWD/../bin/colormodis.sh +#app assemble=$PWD/../bin/assemble2.sh +#app markmap=$PWD/../bin/markmap.sh + Modified: SwiftApps/modis/demo =================================================================== --- SwiftApps/modis/demo 2013-01-29 21:31:49 UTC (rev 6180) +++ SwiftApps/modis/demo 2013-01-29 22:04:12 UTC (rev 6181) @@ -1,15 +1,43 @@ -# demo4 forest 50 5 pbs +#!/bin/bash -landtype=${1:-urban} -nfiles=${2:-999} -nselect=${3:-10} -site=${4:-local} +# Defaults +LANDTYPE=urban +NFILES=1000 +NSELECT=12 +SITE=local +WEBDIR=$HOME/public_html/geo -runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect +# Verify an argument is not null +verify_not_null() +{ + argname=$1; shift + if [ _$1 != _ ]; then + return 0; + else + echo $0: value for $argname can not be null + exit 1 + fi +} -echo runid=$runid +# Parse command line arguments +while [ $# -gt 0 ] +do + case "$1" in + -landtype) LANDTYPE=$2; verify_not_null landtype $LANDTYPE; shift ;; + -nfiles) NFILES=$2; verify_not_null nfiles $NFILES; shift ;; + -nselect) NSELECT=$2; verify_not_null nselect $NSELECT; shift ;; + -site) SITE=$2; verify_not_null site $SITE; shift ;; + -webdir) WEBDIR=$2; verify_not_null webdir $WEBDIR; shift ;; + *) ;; + esac + shift +done -swift -tc.file tc.local \ - -sites.file sites.local.xml \ - modis.swift -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \ - -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo +echo landtype=$LANDTYPE +echo nfiles=$NFILES +echo nselect=$NSELECT +echo webdir=$WEBDIR + +eval swiftrun -site $SITE modis.swift -landtype=$LANDTYPE -nfiles=$NFILES \ + -nselect=$NSELECT -modisdir=$PWD/data/modis/2002 \ + -webdir=$WEBDIR Modified: SwiftApps/modis/modis.swift =================================================================== --- SwiftApps/modis/modis.swift 2013-01-29 21:31:49 UTC (rev 6180) +++ SwiftApps/modis/modis.swift 2013-01-29 22:04:12 UTC (rev 6181) @@ -36,7 +36,6 @@ int nFiles = @toInt(@arg("nfiles","1000")); int nSelect = @toInt(@arg("nselect","12")); string landType = @arg("landtype","urban"); -string runID = @arg("runid","modis-run"); string MODISdir= @arg("modisdir","/home/wilde/bigdata/data/modis/2002"); string webDir = @arg("webdir","/home/wilde/public_html/geo/"); @@ -44,13 +43,14 @@ # Input Dataset -imagefile geos[] ; # site=site + # Compute the land use summary of each MODIS tile landuse land[] ; + transform=@strcat("\\1.landuse.byfreq")>; foreach g,i in geos { land[i] = getLandUse(g,1); @@ -79,7 +79,7 @@ # Assemble a montage of the top selected areas -imagefile montage ; # @arg +imagefile montage ; # @arg montage = assemble(selectedTiles,colorImage,webDir); # future args: From davidk at ci.uchicago.edu Tue Jan 29 16:08:03 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 29 Jan 2013 16:08:03 -0600 (CST) Subject: [Swift-commit] r6182 - trunk/etc/sites Message-ID: <20130129220803.96C309CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-29 16:08:03 -0600 (Tue, 29 Jan 2013) New Revision: 6182 Modified: trunk/etc/sites/midway Log: Make walltime variable for midway template Modified: trunk/etc/sites/midway =================================================================== --- trunk/etc/sites/midway 2013-01-29 22:04:12 UTC (rev 6181) +++ trunk/etc/sites/midway 2013-01-29 22:08:03 UTC (rev 6182) @@ -1,7 +1,9 @@ 16 - 24:00:00 + _WALLTIME_ + 100 + 100 sandyb 10000 @@ -11,7 +13,9 @@ 12 - 24:00:00 + _WALLTIME_ + 100 + 100 westmere 10000 From davidk at ci.uchicago.edu Tue Jan 29 16:17:02 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 29 Jan 2013 16:17:02 -0600 (CST) Subject: [Swift-commit] r6183 - in SwiftApps/modis: . conf Message-ID: <20130129221702.70CD1178884@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-29 16:17:02 -0600 (Tue, 29 Jan 2013) New Revision: 6183 Removed: SwiftApps/modis/old/ Modified: SwiftApps/modis/conf/midway.cf Log: Set walltime via gensites and remove some older unused scripts Modified: SwiftApps/modis/conf/midway.cf =================================================================== --- SwiftApps/modis/conf/midway.cf 2013-01-29 22:08:03 UTC (rev 6182) +++ SwiftApps/modis/conf/midway.cf 2013-01-29 22:17:02 UTC (rev 6183) @@ -6,9 +6,10 @@ use.provider.staging=false provider.staging.pin.swiftfiles=false +#site WALLTIME=00:05:00 + #app getlanduse=$PWD/../bin/getlanduse.sh #app analyzelanduse=$PWD/../bin/analyzelanduse2.sh #app colormodis=$PWD/../bin/colormodis.sh #app assemble=$PWD/../bin/assemble2.sh #app markmap=$PWD/../bin/markmap.sh - From davidk at ci.uchicago.edu Tue Jan 29 16:20:23 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 29 Jan 2013 16:20:23 -0600 (CST) Subject: [Swift-commit] r6184 - in SwiftApps/modis: . bin Message-ID: <20130129222023.5CAAB178884@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-29 16:20:22 -0600 (Tue, 29 Jan 2013) New Revision: 6184 Added: SwiftApps/modis/bin/modis.mapper Removed: SwiftApps/modis/modis.mapper SwiftApps/modis/sites.local.xml SwiftApps/modis/sites.midway.xml SwiftApps/modis/sites.pads.xml SwiftApps/modis/sites.xml SwiftApps/modis/tc.local SwiftApps/modis/tc.midway SwiftApps/modis/tc.pads Log: Remove more of the older sites.xml files, move mapper Copied: SwiftApps/modis/bin/modis.mapper (from rev 6178, SwiftApps/modis/modis.mapper) =================================================================== --- SwiftApps/modis/bin/modis.mapper (rev 0) +++ SwiftApps/modis/bin/modis.mapper 2013-01-29 22:20:22 UTC (rev 6184) @@ -0,0 +1,20 @@ +#!/bin/bash + +# Usage: $0 -location dir -suffix .tif -site gsiftp://gridftp.pads.ci.uchicago.edu/ -n=999 + +while [ $# -gt 0 ]; do + case $1 in + -location) location=$2;; + -suffix) suffix=$2;; + -site) site=$2;; + -n) n=$2;; + *) echo "$0: bad mapper args" 1>&2 + exit 1;; + esac + shift 2 +done + +find $location -type f -name 'h??v??'${suffix} -print | head -$n | +awk '{printf "[%d] %s\n", NR-1, $1}' + + Deleted: SwiftApps/modis/modis.mapper =================================================================== --- SwiftApps/modis/modis.mapper 2013-01-29 22:17:02 UTC (rev 6183) +++ SwiftApps/modis/modis.mapper 2013-01-29 22:20:22 UTC (rev 6184) @@ -1,20 +0,0 @@ -#!/bin/bash - -# Usage: $0 -location dir -suffix .tif -site gsiftp://gridftp.pads.ci.uchicago.edu/ -n=999 - -while [ $# -gt 0 ]; do - case $1 in - -location) location=$2;; - -suffix) suffix=$2;; - -site) site=$2;; - -n) n=$2;; - *) echo "$0: bad mapper args" 1>&2 - exit 1;; - esac - shift 2 -done - -find $location -type f -name 'h??v??'${suffix} -print | head -$n | -awk '{printf "[%d] %s\n", NR-1, $1}' - - Deleted: SwiftApps/modis/sites.local.xml =================================================================== --- SwiftApps/modis/sites.local.xml 2013-01-29 22:17:02 UTC (rev 6183) +++ SwiftApps/modis/sites.local.xml 2013-01-29 22:20:22 UTC (rev 6184) @@ -1,11 +0,0 @@ - - - - - .31 - 10000 - - {env.HOME}/swiftwork - - - Deleted: SwiftApps/modis/sites.midway.xml =================================================================== --- SwiftApps/modis/sites.midway.xml 2013-01-29 22:17:02 UTC (rev 6183) +++ SwiftApps/modis/sites.midway.xml 2013-01-29 22:20:22 UTC (rev 6184) @@ -1,31 +0,0 @@ - - - - - - /scratch/midway/{env.USER}/work - 3600 - 16 - 10 - 1 - 25 - sandyb - 1000 - 10000 - - - - - - /scratch/midway/{env.USER}/work - 3600 - 12 - 20 - 1 - 25 - westmere - 1000 - 10000 - - - Deleted: SwiftApps/modis/sites.pads.xml =================================================================== --- SwiftApps/modis/sites.pads.xml 2013-01-29 22:17:02 UTC (rev 6183) +++ SwiftApps/modis/sites.pads.xml 2013-01-29 22:20:22 UTC (rev 6184) @@ -1,16 +0,0 @@ - - - - - /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/swiftwork - 3600 - 1 - 64 - 1 - 1 - fast - .6399 - 10000 - KEEP - - Deleted: SwiftApps/modis/sites.xml =================================================================== --- SwiftApps/modis/sites.xml 2013-01-29 22:17:02 UTC (rev 6183) +++ SwiftApps/modis/sites.xml 2013-01-29 22:20:22 UTC (rev 6184) @@ -1,38 +0,0 @@ - - - - - .10 - 10000 - - /scratch/local/wilde/swift/demo/work - - - - - - /home/wilde/swiftwork - - - - - - 1 - 3600 - 00:02:00 - 64 - 1 - 1 - fast - - .63 - 10000 - - - /home/wilde/swift/lab/go/work - /scratch/local/wilde - - - - - Deleted: SwiftApps/modis/tc.local =================================================================== --- SwiftApps/modis/tc.local 2013-01-29 22:17:02 UTC (rev 6183) +++ SwiftApps/modis/tc.local 2013-01-29 22:20:22 UTC (rev 6184) @@ -1,34 +0,0 @@ -#This is the transformation catalog. -# -#It comes pre-configured with a number of simple transformations with -#paths that are likely to work on a linux box. However, on some systems, -#the paths to these executables will be different (for example, sometimes -#some of these programs are found in /usr/bin rather than in /bin) -# -#NOTE WELL: fields in this file must be separated by tabs, not spaces; and -#there must be no trailing whitespace at the end of each line. -# -# site transformation path obsolete fields for compatibility - -localhost echo /bin/echo null null null -localhost cat /bin/cat null null null -localhost ls /bin/ls null null null -localhost grep /bin/grep null null null -localhost sort /bin/sort null null null -localhost paste /bin/paste null null null -localhost pwd /bin/pwd null null null - -# For cluster usage - -#pbs getlanduse /home/wilde/swift/demo/modis/bin/getlanduse.sh null null null -#pbs analyzelanduse /home/wilde/swift/demo/modis/bin/analyzelanduse.sh null null null -#pbs colormodis /home/wilde/swift/demo/modis/bin/colormodis.sh null null null -#pbs assemble /home/wilde/swift/demo/modis/bin/assemble.sh null null null - -# For localhost testing - -localhost getlanduse /home/david/modis/bin/getlanduse.sh null null null -localhost analyzelanduse /home/david/modis/bin/analyzelanduse2.sh null null null -localhost colormodis /home/david/modis/bin/colormodis.sh null null null -localhost assemble /home/david/modis/bin/assemble2.sh null null null -localhost markmap /home/david/modis/bin/markmap.sh null null null Deleted: SwiftApps/modis/tc.midway =================================================================== --- SwiftApps/modis/tc.midway 2013-01-29 22:17:02 UTC (rev 6183) +++ SwiftApps/modis/tc.midway 2013-01-29 22:20:22 UTC (rev 6184) @@ -1,31 +0,0 @@ -#This is the transformation catalog. -# -#It comes pre-configured with a number of simple transformations with -#paths that are likely to work on a linux box. However, on some systems, -#the paths to these executables will be different (for example, sometimes -#some of these programs are found in /usr/bin rather than in /bin) -# -#NOTE WELL: fields in this file must be separated by tabs, not spaces; and -#there must be no trailing whitespace at the end of each line. -# -# site transformation path obsolete fields for compatibility - -localhost echo /bin/echo null null null -localhost cat /bin/cat null null null -localhost ls /bin/ls null null null -localhost grep /bin/grep null null null -localhost sort /bin/sort null null null -localhost paste /bin/paste null null null -localhost pwd /bin/pwd null null null - -# For cluster usage -midway-sandyb getlanduse /scratch/midway/davidkelly999/modis/bin/getlanduse.sh null null null -midway-sandyb analyzelanduse /scratch/midway/davidkelly999/modis/bin/analyzelanduse2.sh null null null -midway-sandyb colormodis /scratch/midway/davidkelly999/modis/bin/colormodis.sh null null null -midway-sandyb assemble /scratch/midway/davidkelly999/modis/bin/assemble2.sh null null null -midway-sandyb markmap /scratch/midway/davidkelly999/modis/bin/markmap.sh null null null -midway-westmere getlanduse /scratch/midway/davidkelly999/modis/bin/getlanduse.sh null null null -midway-westmere analyzelanduse /scratch/midway/davidkelly999/modis/bin/analyzelanduse2.sh null null null -midway-westmere colormodis /scratch/midway/davidkelly999/modis/bin/colormodis.sh null null null -midway-westmere assemble /scratch/midway/davidkelly999/modis/bin/assemble2.sh null null null -midway-westmere markmap /scratch/midway/davidkelly999/modis/bin/markmap.sh null null null Deleted: SwiftApps/modis/tc.pads =================================================================== --- SwiftApps/modis/tc.pads 2013-01-29 22:17:02 UTC (rev 6183) +++ SwiftApps/modis/tc.pads 2013-01-29 22:20:22 UTC (rev 6184) @@ -1,26 +0,0 @@ -#This is the transformation catalog. -# -#It comes pre-configured with a number of simple transformations with -#paths that are likely to work on a linux box. However, on some systems, -#the paths to these executables will be different (for example, sometimes -#some of these programs are found in /usr/bin rather than in /bin) -# -#NOTE WELL: fields in this file must be separated by tabs, not spaces; and -#there must be no trailing whitespace at the end of each line. -# -# site transformation path obsolete fields for compatibility - -localhost echo /bin/echo null null null -localhost cat /bin/cat null null null -localhost ls /bin/ls null null null -localhost grep /bin/grep null null null -localhost sort /bin/sort null null null -localhost paste /bin/paste null null null -localhost pwd /bin/pwd null null null - -# For cluster usage -pads getlanduse /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/getlanduse.sh null null null -pads analyzelanduse /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/analyzelanduse2.sh null null null -pads colormodis /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/colormodis.sh null null null -pads assemble /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/assemble2.sh null null null -pads markmap /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/markmap.sh null null null From hategan at ci.uchicago.edu Tue Jan 29 16:24:14 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Tue, 29 Jan 2013 16:24:14 -0600 (CST) Subject: [Swift-commit] r6185 - branches/faster/src/org/griphyn/vdl/karajan/lib Message-ID: <20130129222414.5A036178884@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-29 16:24:14 -0600 (Tue, 29 Jan 2013) New Revision: 6185 Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Log.java branches/faster/src/org/griphyn/vdl/karajan/lib/UnitStart.java Log: Don't compile log statements if they would produce no output; avoid useless string concatenation in unitStart if logging is not enabled Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Log.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Log.java 2013-01-29 22:20:22 UTC (rev 6184) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Log.java 2013-01-29 22:24:14 UTC (rev 6185) @@ -31,8 +31,12 @@ import org.apache.log4j.Logger; import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.compiled.nodes.InternalFunction; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.parser.WrapperNode; public class Log extends InternalFunction { private ArgRef level; @@ -57,8 +61,24 @@ public static Level getLevel(String lvl) { return priorities.get(lvl); } - - protected void runBody(LWThread thr) { + + + + @Override + public Node compile(WrapperNode w, Scope scope) throws CompilationException { + Node n = super.compile(w, scope); + String sLvl = this.level.getValue(); + if (sLvl != null) { + // don't compile this if it won't produce output + if (!logger.isEnabledFor(getLevel(sLvl))) { + return null; + } + } + return n; + } + + + protected void runBody(LWThread thr) { Stack stack = thr.getStack(); Level lvl = getLevel(this.level.getValue(stack)); if (logger.isEnabledFor(lvl)) { Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/UnitStart.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/UnitStart.java 2013-01-29 22:20:22 UTC (rev 6184) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/UnitStart.java 2013-01-29 22:24:14 UTC (rev 6185) @@ -33,32 +33,33 @@ import org.griphyn.vdl.mapping.DSHandle; public class UnitStart extends InternalFunction { - public static final Logger uslogger = Logger.getLogger(UnitStart.class); + public static final Logger uslogger = Logger.getLogger(UnitStart.class); // keep compatibility with log() public static final Logger logger = Logger.getLogger("swift"); - + private ArgRef type; private ArgRef name; private ArgRef line; private ArgRef arguments; private ArgRef outputs; - + @Override protected Signature getSignature() { - return new Signature(params("type", optional("name", null), optional("line", null), + return new Signature(params("type", optional("name", null), + optional("line", null), optional("outputs", null), optional("arguments", null))); } - + private static class NamedRef { public final String name; public final VarRef ref; - + public NamedRef(String name, VarRef ref) { this.name = name; this.ref = ref; } } - + private Tracer tracer; private List inputArgs, outputArgs; @@ -84,7 +85,7 @@ if (outs != null && outs.length() > 0) { outputArgs = new ArrayList(); for (String name : outs.split(",")) { - VarRef ref = scope.getVarRef(name); + VarRef ref = scope.getVarRef(name); outputArgs.add(new NamedRef(name, ref)); outNames.add(name); } @@ -106,27 +107,27 @@ else { inputArgs = null; } - - } - - + + } + @Override protected void runBody(LWThread thr) { String type = this.type.getValue(); String name = this.name.getValue(); String line = this.line.getValue(); - + if (tracer != null && tracer.isEnabled()) { - tracer.trace(thr, Karajan.demangle(name) + "(" + formatArguments(thr.getStack()) + ")"); + tracer.trace(thr, Karajan.demangle(name) + "(" + + formatArguments(thr.getStack()) + ")"); } - + log(true, type, thr, name, line); - + if (outputArgs != null) { trackOutputs(thr); } } - + private String formatArguments(Stack stack) { if (inputArgs != null) { StringBuilder sb = new StringBuilder(); @@ -153,40 +154,56 @@ private void trackOutputs(LWThread thr) { Stack stack = thr.getStack(); - if (!outputArgs.isEmpty()) { + if (!outputArgs.isEmpty()) { List l = new LinkedList(); for (NamedRef nr : outputArgs) { l.add(nr.ref.getValue(stack)); } WaitingThreadsMonitor.addOutput(thr, l); - } + } } - protected static void log(boolean start, String type, LWThread thread, String name, String line) { - if (type.equals("COMPOUND")) { - logger.info((start ? "START" : "END") + type + " thread=" + thread.getName() + " name=" + name); - } - else if (type.equals("PROCEDURE")) { - if (start) { - logger.debug("PROCEDURE line=" + line + " thread=" + thread.getName() + " name=" + name); + protected static void log(boolean start, String type, LWThread thread, + String name, String line) { + if (logger.isInfoEnabled()) { + if (type.equals("COMPOUND")) { + logger.info((start ? "START" : "END") + type + " thread=" + + thread.getName() + " name=" + name); } else { - logger.debug("PROCEDURE_END line=" + line + " thread=" + thread.getName() + " name=" + name); + if (logger.isDebugEnabled()) { + if (type.equals("PROCEDURE")) { + if (start) { + logger.debug("PROCEDURE line=" + line + " thread=" + + thread.getName() + " name=" + name); + } + else { + logger.debug("PROCEDURE_END line=" + line + + " thread=" + + thread.getName() + " name=" + name); + } + } + else if (type.equals("FOREACH_IT")) { + logger.debug("FOREACH_IT_" + (start ? "START" : "END") + + " line=" + line + " thread=" + + thread.getName()); + if (start) { + logger.debug("SCOPE thread=" + thread.getName()); + } + } + else if (type.equals("INTERNALPROC")) { + logger.debug("INTERNALPROC_" + + (start ? "START" : "END") + + " thread=" + thread.getName() + " name=" + + name); + } + else if (type.equals("CONDITION_BLOCK")) { + if (start) { + logger.debug("SCOPE thread=" + thread.getName()); + } + } + } } } - else if (type.equals("FOREACH_IT")) { - logger.debug("FOREACH_IT_" + (start ? "START" : "END") + " line=" + line + " thread=" + thread.getName()); - if (start) { - logger.debug("SCOPE thread=" + thread.getName()); - } - } - else if (type.equals("INTERNALPROC")) { - logger.debug("INTERNALPROC_" + (start ? "START" : "END") + " thread=" + thread.getName() + " name=" + name); - } - else if (type.equals("CONDITION_BLOCK")) { - if (start) { - logger.debug("SCOPE thread=" + thread.getName()); - } - } } } From hategan at ci.uchicago.edu Tue Jan 29 23:28:14 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Tue, 29 Jan 2013 23:28:14 -0600 (CST) Subject: [Swift-commit] r6186 - branches/faster/src/org/griphyn/vdl/karajan/functions Message-ID: <20130130052814.5494B9CCB3@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-29 23:28:14 -0600 (Tue, 29 Jan 2013) New Revision: 6186 Modified: branches/faster/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java Log: static evaluation for configProperty Modified: branches/faster/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2013-01-29 22:24:14 UTC (rev 6185) +++ branches/faster/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2013-01-30 05:28:14 UTC (rev 6186) @@ -25,47 +25,65 @@ import k.rt.Context; import k.rt.ExecutionException; import k.rt.Stack; +import k.thr.LWThread; import org.apache.log4j.Logger; import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.CompilationException; -import org.globus.cog.karajan.analyzer.Param; import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.Var; import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.globus.cog.karajan.compiled.nodes.Node; -import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; import org.globus.cog.karajan.parser.WrapperNode; import org.globus.cog.karajan.util.BoundContact; import org.griphyn.vdl.util.VDL2Config; -public class ConfigProperty extends AbstractSingleValuedFunction { +public class ConfigProperty extends InternalFunction { private ArgRef name; private ArgRef instance; private ArgRef host; + private ChannelRef cr_vargs; private VDL2Config instanceConfig; private VarRef context; @Override - protected Param[] getParams() { - return params("name", optional("instance", Boolean.TRUE), optional("host", null)); + protected Signature getSignature() { + return new Signature(params("name", optional("instance", Boolean.TRUE), optional("host", null)), + returns(channel("..."))); } public static final String INSTANCE_CONFIG = "SWIFT_CONFIG"; public static final Logger logger = Logger.getLogger(ConfigProperty.class); + - @Override protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) throws CompilationException { context = scope.getVarRef("#context"); + Context ctx = context.getValue(); + Var.Channel r = scope.parent.lookupChannel("..."); + if (this.name.isStatic() && this.instance.isStatic() && this.host.isStatic() + && ctx != null && this.host.getValue() == null) { + String value = getProperty(this.name.getValue(), this.instance.getValue(), getInstanceConfig(ctx)); + if (r.append(value)) { + return null; + } + } + + r.appendDynamic(); + return super.compileBody(w, argScope, scope); } @Override - public Object function(Stack stack) { + protected void runBody(LWThread thr) { + Stack stack = thr.getStack(); String name = this.name.getValue(stack); boolean instance = this.instance.getValue(stack); BoundContact host = this.host.getValue(stack); @@ -80,13 +98,14 @@ if (logger.isDebugEnabled()) { logger.debug("Found property " + name + " in BoundContact"); } - return prop; + cr_vargs.append(stack, prop); + return; } if (logger.isDebugEnabled()) { logger.debug("Could not find property " + name + " in BoundContact"); } } - return getProperty(name, instance, getInstanceConfig(stack)); + cr_vargs.append(stack, getProperty(name, instance, getInstanceConfig(stack))); } private synchronized VDL2Config getInstanceConfig(Stack stack) { @@ -96,6 +115,10 @@ } return instanceConfig; } + + private synchronized VDL2Config getInstanceConfig(Context ctx) { + return (VDL2Config) ctx.getAttribute("SWIFT:CONFIG"); + } public static String getProperty(String name, VDL2Config instanceConfig) { return getProperty(name, true, instanceConfig); From hategan at ci.uchicago.edu Tue Jan 29 23:28:48 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Tue, 29 Jan 2013 23:28:48 -0600 (CST) Subject: [Swift-commit] r6187 - branches/faster/libexec Message-ID: <20130130052848.0F75D9CCB3@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-29 23:28:47 -0600 (Tue, 29 Jan 2013) New Revision: 6187 Added: branches/faster/libexec/swift-int-staging.k branches/faster/libexec/swift-int-wrapper-staging.k Removed: branches/faster/libexec/vdl-int-staging.k branches/faster/libexec/vdl-int-wrapper-staging.k Modified: branches/faster/libexec/swift-int.k branches/faster/libexec/swift.k Log: selectable swift-int-xxx Copied: branches/faster/libexec/swift-int-staging.k (from rev 6170, branches/faster/libexec/vdl-int-staging.k) =================================================================== --- branches/faster/libexec/swift-int-staging.k (rev 0) +++ branches/faster/libexec/swift-int-staging.k 2013-01-30 05:28:47 UTC (rev 6187) @@ -0,0 +1,171 @@ +import(sys) +import(task) +import('swift-lib') +/* + * Things that are not exposed to the translated file + */ + + +WRAPPER_TRANSFER_MODE := + if (vdl:configProperty("wrapperlog.always.transfer") == "true", + STAGING_MODE:IF_PRESENT, STAGING_MODE:ON_ERROR + STAGING_MODE:IF_PRESENT) + +pinOption := configProperty("provider.staging.pin.swiftfiles") + +PIN := if(pinOption == "true", "pinned:", "") +PROVENANCE_GRAPH_ENABLED := (vdl:configProperty("pgraph") != "false") + +namespace(swift) { + + initDDir := function() { + ddir := "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}" + + once(ddir) { + if(!file:exists(ddir)) { + task:dir:make(ddir) + } + } + ddir + } + + inFiles := function(stageins) { + pathnames(stageins) + } + + fileSizes := function(files) { + math:sum( + for(f, files, file:size(f)) + ) + } + + export(cleanups, + function(cleanup) { + log(LOG:INFO, "START cleanups={cleanup}") + } + ) + + readErrorFile := function(dir, jobid) { + str:strip(file:read("{dir}/{jobid}.error")) + file:remove("{dir}/{jobid}.error") + } + + export(execute2, + function(progress, tr, stagein, stageout, restartout + replicationGroup, replicationChannel + arguments = [], stdin = null, stdout = null, stderr = null, attributes = null) { + + stagein := list(unique(each(stagein))) + stageout := list(unique(each(stageout))) + + allocateHost(rhost, constraints = jobConstraints(tr, stagein = stagein)) { + + ddir := initDDir() + + uid := UID() + jobdir := substring(uid, from=0, to=1) + jobid := "{tr}-{uid}" + + log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread=", currentThread(), " host={rhost} replicationGroup={replicationGroup}") + + wrapper := "_swiftwrap.staging" + wrapfile := "{ddir}/param-{jobid}" + + stdout := if (stdout == null, "stdout.txt", getFieldValue(stdout)) + stderr := if (stderr == null, "stderr.txt", getFieldValue(stderr)) + + wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" + tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid) + cdmfile := cdm:file() + + try { + log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", if (arguments != null, (" arguments=", arguments)), " tmpdir={tmpdir} host={rhost}") + + setProgress(progress, "Submitting") + + swift:execute( + siteProfile(rhost, "swift:wrapperInterpreter"), + list( + siteProfile(rhost, "swift:wrapperInterpreterOptions"), + wrapper, + "-e", vdl:executable(tr, rhost), + "-out", stdout, + "-err", stderr, + "-i", if (stdin != null, getFieldValue(stdin)), + "-d", flatten(unique(outFileDirs(stageout))), + "-if", flatten(infiles(stagein)), + "-of", flatten(outfiles(stageout)), + "-k", + "-cdmfile", cdmfile, + "-status", "provider" + "-a", if (arguments != null, each(arguments)) + ) + directory = tmpdir + redirect = false + host = rhost + + TCProfile(rhost, if (attributes != null, attributes = attributes), tr = tr) + replicationGroup = replicationGroup + replicationChannel = replicationChannel + jobid = jobid + + stagingMethod := siteProfile(rhost, "swift:stagingMethod", default="proxy") + + stageIn("{PIN}{stagingMethod}://localhost/{swift.home}/libexec/{wrapper}", wrapper) + + if (cdmfile != "") { + d := dirname(cdmfile) + file := basename(cdmfile) + dir := if (d == "", "./", str:concat(d,"/")) + loc := "{PIN}{stagingMethod}://localhost/" + stageIn("{loc}{dir}{file}", cdmfile) + stageIn("{loc}{swift.home}/libexec/cdm.pl", "cdm.pl") + stageIn("{loc}{swift.home}/libexec/cdm_lib.sh", "cdm_lib.sh") + } + + appStageins(jobid, stagein, ".", stagingMethod) + + stageOut("wrapper.log", "{stagingMethod}://localhost/{ddir}/{jobid}.info", + mode = WRAPPER_TRANSFER_MODE) + //stageOut("{stdout}", "{stagingMethod}://localhost/{ddir}/{stdout}") + //stageOut("{stderr}", "{stagingMethod}://localhost/{ddir}/{stderr}") + stageOut("wrapper.error", "{stagingMethod}://localhost/{ddir}/{jobid}.error", + mode = STAGING_MODE:IF_PRESENT) + appStageouts(jobid, stageout, ".", stagingMethod) + + task:cleanUp(".") //the whole job directory + ) // execute + doRestartlog(restartout) + log(LOG:DEBUG, "JOB_END jobid={jobid}") + } + else catch(prev) { + if (matches(prev, "^Abort$")) { + log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") + throw(prev) + } + else { + setProgress(progress, "Failed but can retry") + exception := try(exception(readErrorFile(ddir, jobid)), prev) + log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) + + throw( + exception( + concat( + "Exception in {tr}:", + if (arguments != null, "\n Arguments: {arguments}") + "\n Host: {rhost}", + "\n Directory: {tmpdir}" + ) + exception + ) + ) + } + } + } + } + ) +} + +// Local variables: +// tab-width: 4 +// indent-tabs-mode: 1 +// End: Copied: branches/faster/libexec/swift-int-wrapper-staging.k (from rev 6169, branches/faster/libexec/vdl-int-wrapper-staging.k) =================================================================== --- branches/faster/libexec/swift-int-wrapper-staging.k (rev 0) +++ branches/faster/libexec/swift-int-wrapper-staging.k 2013-01-30 05:28:47 UTC (rev 6187) @@ -0,0 +1,329 @@ +import(sys) +import(task) +import('swift-lib') + + +getURLPrefix := def("org.griphyn.vdl.karajan.lib.GetURLPrefix") + +URL_PREFIX := getURLPrefix() + +WRAPPERLOG_ALWAYS_TRANSFER := vdl:configProperty("wrapperlog.always.transfer") +SITEDIR_KEEP := vdl:configProperty("sitedir.keep") + + +namespace(swift) { + + checkJobStatus := function(jobdir, jobid, tr) { + log(LOG:DEBUG, "START jobid={jobid}") + try { + file:remove("{jobdir}/_swift.success") + log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found") + } + else { + throw(checkErrorFile(jobdir, jobid)) + } + } + + checkErrorFile := function(jobdir, jobid) { + if (file:exists("{jobdir}/_swift.error")) { + log(LOG:INFO, "FAILURE jobid={jobid} - Failure file found") + error := str:strip(file:read("{jobdir}/_swift.error")) + file:remove("{jobdir}/_swift.error") + error + } + else { + log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing") + throw("No status file was found") + } + } + + initSharedDir := function(progress, rhost) { + once(list(rhost, "shared")) { + setProgress(progress, "Initializing site shared directory") + + log(LOG:INFO, "START host={rhost} - Initializing shared directory") + + wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" + dir:make(wfdir, host=rhost) + transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftwrap.wrapperstaging", destdir=wfdir, desthost=rhost) + + wfdir + to(cleanup, list(wfdir, rhost)) + log(LOG:INFO, "END host={rhost} - Done initializing shared directory") + } + } + + initDDir := function() { + ddir := "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}" + + once(ddir) { + if(!file:exists(ddir)) { + task:dir:make(ddir) + } + } + ddir + } + + + inFiles := function(stageins) { + pathnames(stageins) + } + + fileDirs := function(stageins, stageouts) { + list( + unique( + inFileDirs(stageins) + outFileDirs(stageouts) + ) + ) + } + + cleanup := function(dir, host) { + log(LOG:INFO, "START dir={dir} host={host}") + if(vdl:configProperty("sitedir.keep") == "false") { + task:execute( + vdl:siteprofile(host, "swift:cleanupCommand"), + arguments = list( + siteProfile(host, "swift:cleanupCommandOptions"), + dir + ) + host=host, batch=true, TCProfile(host) + ) + log(LOG:INFO, "END dir={dir} host={host}") + } + } + + cleanups := function(cleanup) { + log(LOG:INFO, "START cleanups={cleanup}") + parallelFor(i, cleanup) { + (dir, host) := each(i) + try { + vdl:cleanup(dir, host) + } + else catch(exception) { + log(LOG:DEBUG, "EXCEPTION - Exception caught while cleaning up", exception) + to(warnings, exception("Cleanup on {host} failed", exception)) + } + } + log(LOG:INFO, "END cleanups={cleanup}") + } + + stageWrapperParams := function(jobid, wrapfile, dir, host) { + log(LOG:INFO, "START jobid={jobid} - staging in wrapper params") + provider := provider(wrapfile) + srchost := hostname(wrapfile) + srcdir := vdl:dirname(wrapfile) + destdir := dir + filename := basename(wrapfile) + + cacheOn(list(destdir, host) + dir:make(destdir, host=host, provider=provider) + ) + + log(LOG:INFO, "END jobid={jobid}") + } + + + readStandardFiles := function(jobdir, stdout, stderr) { + concat( + for(f, [["_swift.stderr", stderr], ["_swift.stdout", stdout]]) { + (name, file) := each(f) + destfile := "{jobdir}/{file}" + "\n {name}: " + try { + file:read(destfile) + "\n" + } + else { + "" + } + } + ) + } + + + export(execute2, + function(progress, tr, stagein, stageout, restartout + replicationGroup, replicationChannel + arguments = [], stdin = null, stdout = null, stderr = null, attributes = null) { + + stagein := list(unique(each(stagein))) + stageout := list(unique(each(stageout))) + + allocateHost(rhost, constraints=jobConstraints(tr, stagein = stagein)) { + + ddir := initDDir() + wfdir := try { + initSharedDir(rhost) + } + else catch(exception) { + throw(exception("Could not initialize shared directory on {rhost}", exception)) + } + + uid := UID() + jobid := "{tr}-{uid}" + + jobdir := concat(ddir, "/jobs/", substring(uid, from=0, to=1), "/{jobid}/") + + log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") + + statusMode := configProperty("status.mode", host = rhost) + wrapperMode := configProperty("wrapper.parameter.mode", host = rhost) + + wrapfile := "{jobdir}/_paramfile" + + stdout := if (stdout == null, "stdout.txt", getFieldValue(stdout)) + stderr := if (stderr == null, "stderr.txt", getFieldValue(stderr)) + + fileDirs := fileDirs(stagein, stageout) + os := siteProfile(rhost, "SYSINFO:OS") + + scratch := siteProfile(rhost, "scratch") + + if(wrapperMode == "files") { + file:write(wrapfile, + "-e ",vdl:executable(tr, rhost), + "\n-out ", stdout, + "\n-err ", stderr, + "\n-i ", if (stdin != null, getFieldValue(stdin)), + "\n-d ", flatten(each(fileDirs)), + "\n-if ", flatten(inFiles(stagein)), + "\n-of ", flatten(outFiles(stageout)), + "\n-wt", WRAPPERLOG_ALWAYS_TRANSFER, + "\n-sk", SITEDIR_KEEP, + "\n-cdmfile ", cdm:file(), + "\n-status ", statusMode, + for(a, arguments) { + "\n-a ", a + } + ) + } + + + setProgress(progress, "Stage in") + + try { + if (wrapperMode == "files") { + stageWrapperParams(jobid, wrapfile, wfdir, rhost) + } + + log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", if (arguments != null, (" arguments=", arguments)), " host={rhost}") + + setProgress(progress, "Submitting") + + if (wrapperMode == "files") { + swift:execute( + siteProfile(rhost, "swift:wrapperInterpreter"), + list( + siteProfile(rhost, "swift:wrapperInterpreterOptions"), + "_swiftwrap.wrapperstaging", + jobid, + "-urlprefix", URL_PREFIX, + "-jobdir", jobdir, + "-scratch", scratch, + "-p", wrapfile + ) + directory=wfdir + redirect=false + host=rhost + TCProfile(rhost, if (attributes != null, attributes = attributes), tr=tr) + replicationGroup=replicationGroup + replicationChannel=replicationChannel + jobid=jobid + ) + } + if (wrapperMode == "args") { + swift:execute( + siteProfile(rhost, "swift:wrapperInterpreter"), + list( + siteProfile(rhost, "swift:wrapperInterpreterOptions"), + "_swiftwrap.wrapperstaging", + jobid, + "-urlprefix", URL_PREFIX, + "-jobdir", jobdir, + "-scratch", scratch, + "-e", vdl:executable(tr, rhost), + "-out", stdout, + "-err", stderr, + "-i", if (stdin != null, getFieldValue(stdin)), + "-d", flatten(each(fileDirs)), + "-if", flatten(inFiles(stagein)), + "-of", flatten(outFiles(stageout)), + "-wt", WRAPPERLOG_ALWAYS_TRANSFER, + "-sk", SITEDIR_KEEP, + "-cdmfile", cdm:file(), + "-status", statusMode, + "-a", if (arguments != null, each(arguments)) + ) + directory=wfdir + redirect=false + host=rhost + TCProfile(rhost, if(attributes != null, attributes = attributes), tr=tr) + replicationGroup=replicationGroup + replicationChannel=replicationChannel + jobid=jobid + ) + } + + setProgress(progress, "Checking status") + if (statusMode == "files") { + checkJobStatus(jobdir, jobid, tr) + } + + if (wrapperMode == "files") { + file:remove(wrapfile) + } + + log(LOG:DEBUG, "STAGING_OUT jobid={jobid}") + + + /* need to stage the files to upper scratch area in case they are not transfered to another site + before all the files get cleaned out */ + + + setProgress(progress, "Stage out") + doRestartlog(restartout) + + log(LOG:DEBUG, "JOB_END jobid={jobid}") + } + else catch(prev) { + if (matches(prev, "^Abort$")) { + log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") + throw(prev) + } + else { + setProgress(progress, "Failed but can retry") + exception := try(exception(checkErrorFile(jobdir, jobid)), prev) + + log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) + + if (matches(exception,".*executable bit.*")) { + throw(exception) + } + + outs := readStandardFiles(jobdir, stdout, stderr) + + throw( + exception( + concat( + "Exception in {tr}:", + if (arguments != null, "\n Arguments: {arguments}") + "\n Host: {rhost}", + "\n Directory: {tmpdir}", + "{outs}", + ) + exception + ) + ) + } + } + } + } + ) +} + +// Local variables: +// mode: scheme +// tab-width: 4 +// indent-tabs-mode: t +// End: Modified: branches/faster/libexec/swift-int.k =================================================================== --- branches/faster/libexec/swift-int.k 2013-01-30 05:28:14 UTC (rev 6186) +++ branches/faster/libexec/swift-int.k 2013-01-30 05:28:47 UTC (rev 6187) @@ -455,7 +455,7 @@ stageWrapperParams(jobid, jobdir, wrapfile, wfdir, rhost) } - log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}") + log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", if (arguments != null, (" arguments=", arguments)), " tmpdir={tmpdir} host={rhost}") setProgress(progress, "Submitting") @@ -501,7 +501,7 @@ directory = wfdir redirect = false host = rhost - TCProfile(rhost, if (attributes != null, attributes = attributes), tr = tr) + TCProfile(rhost, if(attributes != null, attributes = attributes), tr = tr) replicationGroup = replicationGroup replicationChannel = replicationChannel jobid = jobid @@ -569,7 +569,7 @@ exception( concat( "Exception in {tr}:", - maybe("\n Arguments: ", arguments), + if (arguments != null, "\n Arguments: {arguments}") "\n Host: {rhost}", "\n Directory: {tmpdir}", "{outs}", @@ -582,24 +582,6 @@ } } ) - - export(generateProvenanceGraph, - function(gdata) { - pgraph := configProperty("pgraph") - gname := if(pgraph == "true", "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.dot", pgraph) - file:write(gname) { - "digraph SwiftProvenance \{\n", - " graph [", configProperty("pgraph.graph.options"), "];\n", - " node [", configProperty("pgraph.node.options"), "];\n", - - for(i, gdata) { - " ", i, "\n" - } - "}\n" - } - log(LOG:INFO, "Provenance graph saved in ", gname) - } - ) } // Local variables: Modified: branches/faster/libexec/swift.k =================================================================== --- branches/faster/libexec/swift.k 2013-01-30 05:28:14 UTC (rev 6186) +++ branches/faster/libexec/swift.k 2013-01-30 05:28:47 UTC (rev 6187) @@ -12,8 +12,8 @@ namespace(swift) { - pstaging := configProperty("use.provider.staging") - wstaging := configProperty("use.wrapper.staging") + pstaging := (configProperty("use.provider.staging") == "true") + wstaging := (configProperty("use.wrapper.staging") == "true") import(java) @@ -132,6 +132,53 @@ } ) + graphStuff := function(tr, stagein, stageout, err, args = null) { + if (configProperty("pgraph") != "false") { + errprops := if(err, ",color=lightsalmon", ",color=lightsteelblue1") + tp := currentThread() + to (graph) { + concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]") + } + for (si, stagein) { + si := basename(si) + to(graph) { + concat(str:quote(si), " [shape=parallelogram]") + concat(str:quote(si), " -> ", str:quote(tp)) + } + } + for (pv, stageout) { + (path, var) := each(pv) + file := fileName(getField(var, path=path)) + file := basename(file) + label := niceName(var, path = path) + to(graph) { + concat(str:quote(file), " [shape=parallelogram,label=", str:quote(label), "]") + concat(str:quote(tp), " -> ", str:quote(file)) + } + } + } + } + + generateProvenanceGraph := function(gdata) { + pgraph := configProperty("pgraph") + gname := if(pgraph == "true", "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.dot", pgraph) + file:write(gname) { + "digraph SwiftProvenance \{\n", + " graph [", configProperty("pgraph.graph.options"), "];\n", + " node [", configProperty("pgraph.node.options"), "];\n", + + for(i, gdata) { + " ", i, "\n" + } + "}\n" + } + log(LOG:INFO, "Provenance graph saved in ", gname) + } + + impl := if (pstaging, "swift-int-staging.k", if (wstaging, "swift-int-wrapper-staging.k", "swift-int.k")) + + import(file = impl) + export(execute, function( tr, arguments = null, Deleted: branches/faster/libexec/vdl-int-staging.k =================================================================== --- branches/faster/libexec/vdl-int-staging.k 2013-01-30 05:28:14 UTC (rev 6186) +++ branches/faster/libexec/vdl-int-staging.k 2013-01-30 05:28:47 UTC (rev 6187) @@ -1,212 +0,0 @@ -import("sys.k") -import("task.k") -import("vdl-lib.xml") -/* - * Things that are not exposed to the translated file - */ - -global(LOG:DEBUG, "debug") -global(LOG:INFO, "info") -global(LOG:WARN, "warn") -global(LOG:ERROR, "error") -global(LOG:FATAL, "fatal") - -global(WRAPPER_TRANSFER_MODE, - if(vdl:configProperty("wrapperlog.always.transfer") == "true", - STAGING_MODE:IF_PRESENT, STAGING_MODE:ON_ERROR + STAGING_MODE:IF_PRESENT)) - -pinOption := configProperty("provider.staging.pin.swiftfiles") -global(PIN, if(pinOption == "true", "pinned:", "")) -global(PROVENANCE_GRAPH_ENABLED, vdl:configProperty("pgraph") != "false") - -namespace("vdl") { - export( - function(initDDir) { - ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d" - once(ddir) { - if(sys:not(file:exists(ddir))) { - task:dir:make(ddir) - } - } - ddir - } - - function(inFiles, stageins) { - pathnames(stageins) - } - - function(graphStuff, tr, stagein, stageout, err, optional(args)) { - if(PROVENANCE_GRAPH_ENABLED) { - errprops := if(err ",color=lightsalmon" ",color=lightsteelblue1") - tp := vdl:threadPrefix() - to(graph) { - concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]") - } - for(si, stagein) { - si := basename(si) - to(graph) { - concat(str:quote(si), " [shape=parallelogram]") - concat(str:quote(si), " -> ", str:quote(tp)) - } - } - for(pv, stageout) { - (path, var) := each(pv) - file := vdl:fileName(vdl:getfield(var, path=path)) - file := basename(file) - label := vdl:niceName(var, path = path) - to(graph) { - concat(str:quote(file), " [shape=parallelogram,label=", - str:quote(label), "]") - concat(str:quote(tp), " -> ", str:quote(file)) - } - } - } - } - - function(fileSizes, files) { - math:sum( - for(f, files, file:size(file)) - ) - } - - function(cleanups, cleanup) { - log(LOG:INFO, "START cleanups={cleanup}") - } - - function(readErrorFile, dir, jobid) { - str:strip(file:read("{dir}/{jobid}.error")) - file:remove("{dir}/{jobid}.error") - } - - function(execute2, tr, optional(arguments, stdin, stdout, stderr), stagein, stageout, restartout, - replicationGroup, replicationChannel) { - - stagein := list(unique(each(stagein))) - stageout := list(unique(each(stageout))) - - allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein)) { - - ddir := initDDir() - - uid := uid() - jobdir := substring(uid, from=0, to=1) - jobid := "{tr}-{uid}" - - log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread=", currentThread(), " host={rhost} replicationGroup={replicationGroup}") - - wrapper := "_swiftwrap.staging" - wrapfile := "{ddir}/param-{jobid}" - - stdout := try(getFieldValue(stdout), "stdout.txt") - stderr := try(getFieldValue(stderr), "stderr.txt") - - wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" - tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid) - cdmfile := cdm:file() - - try { - log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}") - - vdl:setprogress("Submitting") - - vdl:execute( - vdl:siteprofile(rhost, "swift:wrapperInterpreter"), - list( - vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"), - wrapper, - "-e", vdl:executable(tr, rhost), - "-out", stdout, - "-err", stderr, - "-i", maybe(getFieldValue(stdin)), - "-d", flatten(unique(outFileDirs(stageout))), - "-if", flatten(infiles(stagein)), - "-of", flatten(outfiles(stageout)), - "-k", - "-cdmfile", cdmfile, - "-status", "provider" - "-a", maybe(each(arguments)) - ) - directory = tmpdir - redirect = false - host = rhost - - vdl:tcprofile(rhost, tr = tr) //this gets various app params from the tc, such as environment, walltime, etc - replicationGroup = replicationGroup - replicationChannel = replicationChannel - jobid = jobid - - stagingMethod := vdl:siteProfile(rhost, "swift:stagingMethod", default="proxy") - - stageIn("{PIN}{stagingMethod}://localhost/{swift.home}/libexec/{wrapper}", wrapper) - - if (cdmfile != "") { - d := vdl:dirname(cdmfile) - file := basename(cdmfile) - dir := if (d == "", "./", str:concat(d,"/")) - loc := "{PIN}{stagingMethod}://localhost/" - stageIn("{loc}{dir}{file}", cdmfile) - stageIn("{loc}{swift.home}/libexec/cdm.pl", "cdm.pl") - stageIn("{loc}{swift.home}/libexec/cdm_lib.sh", "cdm_lib.sh") - } - - appStageins(jobid, stagein, ".", stagingMethod) - - stageOut("wrapper.log", "{stagingMethod}://localhost/{ddir}/{jobid}.info", - mode = WRAPPER_TRANSFER_MODE) - //stageOut("{stdout}", "{stagingMethod}://localhost/{ddir}/{stdout}") - //stageOut("{stderr}", "{stagingMethod}://localhost/{ddir}/{stderr}") - stageOut("wrapper.error", "{stagingMethod}://localhost/{ddir}/{jobid}.error", - mode = STAGING_MODE:IF_PRESENT) - appStageouts(jobid, stageout, ".", stagingMethod) - - task:cleanUp(".") //the whole job directory - ) // execute - doRestartlog(restartout) - log(LOG:DEBUG, "JOB_END jobid={jobid}") - } - else catch(exception, "^Abort$") { - log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") - throw(exception) - } - else catch(prev, "^(?!Abort$).*") { - vdl:setprogress("Failed but can retry") - exception := try(exception(readErrorFile(ddir, jobid)), prev) - log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) - - throw( - exception( - concat( - "Exception in {tr}:", nl(), - maybe(" Arguments: {arguments}", nl()), - " Host: {rhost}", nl(), - " Directory: {tmpdir}", nl() - ) - exception - ) - ) - } - } - } - - function(generateProvenanceGraph, gdata) { - pgraph := vdl:configProperty("pgraph") - gname := if(pgraph == "true" "{VDL:SCRIPTNAME}-{VDL:RUNID}.dot" pgraph) - file:write(gname) { - "digraph SwiftProvenance {{", nl() - " graph [", vdl:configProperty("pgraph.graph.options"), "];", nl() - " node [", vdl:configProperty("pgraph.node.options"), "];", nl() - - for(i, gdata) { - " ", i, nl() - } - "}", nl() - } - log(LOG:INFO, "Provenance graph saved in ", gname) - } - ) -} - -// Local variables: -// tab-width: 4 -// indent-tabs-mode: 1 -// End: Deleted: branches/faster/libexec/vdl-int-wrapper-staging.k =================================================================== --- branches/faster/libexec/vdl-int-wrapper-staging.k 2013-01-30 05:28:14 UTC (rev 6186) +++ branches/faster/libexec/vdl-int-wrapper-staging.k 2013-01-30 05:28:47 UTC (rev 6187) @@ -1,384 +0,0 @@ -import("sys.k") -import("task.k") -import("vdl-lib.xml") -/* - * Things that are not exposed to the translated file - */ - -global(LOG:DEBUG, "debug") -global(LOG:INFO, "info") -global(LOG:WARN, "warn") -global(LOG:ERROR, "error") -global(LOG:FATAL, "fatal") - -global(URL_PREFIX, - elementDef(getURLPrefix, className="org.griphyn.vdl.karajan.lib.GetURLPrefix") - getURLPrefix() -) - -global(WRAPPERLOG_ALWAYS_TRANSFER, vdl:configProperty("wrapperlog.always.transfer")) -global(SITEDIR_KEEP, vdl:configProperty("sitedir.keep")) - - -namespace("vdl" - export( - element(checkJobStatus, [jobdir, jobid, tr] - log(LOG:DEBUG, "START jobid={jobid}") - try( - sequential( - /* - * This is a bit of optimization, but I'm not completely - * sure of its correctness. The goal is to both detect - * the presence of the success file and remove it, all - * in one operation. It relies on file:remove() throwing - * an exception if the file is not there. - */ - file:remove("{jobdir}/_swift.success") - log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found") - ) - throw(checkErrorFile(jobdir, jobid)) - ) - ) - - element(checkErrorFile, [jobdir, jobid] - if ( - file:exists("{jobdir}/_swift.error") then( - log(LOG:INFO, "FAILURE jobid={jobid} - Failure file found") - error := str:strip(file:read("{jobdir}/_swift.error")) - file:remove("{jobdir}/_swift.error") - error - ) - else ( - log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing") - throw("No status file was found") - ) - ) - ) - - element(initSharedDir, [rhost] - once(list(rhost, "shared") - vdl:setprogress("Initializing site shared directory") - - log(LOG:INFO, "START host={rhost} - Initializing shared directory") - - wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" - dir:make(wfdir, host=rhost) - transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftwrap.wrapperstaging", destdir=wfdir, desthost=rhost) - - wfdir - //we send the cleanup data to vdl:main() - to(cleanup, list(wfdir, rhost)) - log(LOG:INFO, "END host={rhost} - Done initializing shared directory") - ) - ) - - element(initDDir, [] - ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d" - once(ddir - if(sys:not(file:exists(ddir)) - task:dir:make(ddir) - ) - ) - ddir - ) - - element(inFiles, [stageins] - stageins - ) - - element(fileDirs, [stageins, stageouts] - list( - unique( - inFileDirs(stageins) - outFileDirs(stageouts) - ) - ) - ) - - element(cleanup, [dir, host] - log(LOG:INFO, "START dir={dir} host={host}") - if(vdl:configProperty("sitedir.keep") == "false" - task:execute( - vdl:siteprofile(host, "swift:cleanupCommand"), - arguments=list( - vdl:siteprofile(host, "swift:cleanupCommandOptions"), - dir - ) - host=host, batch=true, tcprofile(host)) - ) - log(LOG:INFO, "END dir={dir} host={host}") - ) - - element(cleanups, [cleanup] - log(LOG:INFO, "START cleanups={cleanup}") - parallelFor(i, cleanup - [dir, host] := each(i) - try( - vdl:cleanup(dir, host) - catch(".*", - log(LOG:DEBUG, "EXCEPTION - Exception caught while cleaning up", exception) - to(warnings, exception("Cleanup on {host} failed", exception)) - ) - ) - ) - log(LOG:INFO, "END cleanups={cleanup}") - ) - - element(stageWrapperParams, [jobid, wrapfile, dir, host] - log(LOG:INFO, "START jobid={jobid} - staging in wrapper params") - provider := provider(wrapfile) - srchost := hostname(wrapfile) - srcdir := vdl:dirname(wrapfile) - destdir := dir - filename := basename(wrapfile) - - cacheOn(list(destdir, host) - dir:make(destdir, host=host, provider=provider) - ) - - log(LOG:INFO, "END jobid={jobid}") - ) - - element(graphStuff, [tr, stagein, stageout, err, optional(args)] - if( - vdl:configProperty("pgraph") != "false" then( - errprops := if(err ",color=lightsalmon" ",color=lightsteelblue1") - tp := vdl:threadPrefix() - to(graph, - concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]") - ) - for(si, stagein - si := basename(si) - to(graph - concat(str:quote(si), " [shape=parallelogram]") - concat(str:quote(si), " -> ", str:quote(tp)) - ) - ) - for(pv, stageout - [path, var] := each(pv) - file := vdl:fileName(vdl:getfield(var, path=path)) - file := basename(file) - label := vdl:niceName(var, path = path) - to(graph - concat(str:quote(file), " [shape=parallelogram,label=", - str:quote(label), "]") - concat(str:quote(tp), " -> ", str:quote(file)) - ) - ) - ) - ) - ) - - element(readStandardFiles, [jobdir, stdout, stderr] - concat( - for(f, list(list("_swift.stderr", stderr), list("_swift.stdout", stdout)) - [name, file] := each(f) - destfile := "{jobdir}/{file}" - nl() - "{name}: " - try( - file:read(destfile) - nl() - ) - ) - ) - ) - - - element(execute2, [tr, optional(arguments, stdin, stdout, stderr, attributes), stagein, stageout, restartout, - replicationGroup, replicationChannel] - stagein := list(unique(each(stagein))) - stageout := list(unique(each(stageout))) - - allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein) - - ddir := initDDir() - wfdir := try( - initSharedDir(rhost) - throw(exception("Could not initialize shared directory on {rhost}", exception)) - ) - - uid := uid() - jobid := "{tr}-{uid}" - - jobdir := concat(ddir, "/jobs/", substring(uid, from=0, to=1), "/{jobid}/") - - log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") - - statusMode := configProperty("status.mode",host=rhost) - wrapperMode := configProperty("wrapper.parameter.mode",host=rhost) - - wrapfile := "{jobdir}/_paramfile" - - stdout := try(getFieldValue(stdout), "_swift.stdout") - stderr := try(getFieldValue(stderr), "_swift.stderr") - fileDirs := fileDirs(stagein, stageout) - os := vdl:siteprofile(rhost, "SYSINFO:OS") - - scratch := vdl:siteprofile(rhost, "scratch") - - if(wrapperMode == "files" - sequential( - sys:file:write(wrapfile, - "-e ",vdl:executable(tr, rhost), nl(), - "-out ", stdout, nl(), - "-err ", stderr, nl(), - "-i ", maybe(getFieldValue(stdin)), nl(), - "-d ", flatten(each(fileDirs)), nl(), - "-if ", flatten(inFiles(stagein)), nl(), - "-of ", flatten(outFiles(stageout)), nl(), - "-wt", WRAPPERLOG_ALWAYS_TRANSFER, - "-sk", SITEDIR_KEEP, - "-cdmfile ", cdm:file(), nl(), - "-status ", statusMode, nl(), - for(a, arguments, "-a ", a, nl()) - ) - ) - ) - - vdl:setprogress("Stage in") - - try( - sequential( - - if(wrapperMode == "files" - stageWrapperParams(jobid, wrapfile, wfdir, rhost) - ) - - log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " host={rhost}") - - vdl:setprogress("Submitting") - - if(wrapperMode == "files" - vdl:execute( - vdl:siteprofile(rhost, "swift:wrapperInterpreter"), - list( - vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"), - "_swiftwrap.wrapperstaging", - jobid, - "-urlprefix", URL_PREFIX, - "-jobdir", jobdir, - "-scratch", scratch, - "-p", wrapfile - ) - directory=wfdir - redirect=false - host=rhost - vdl:tcprofile(rhost, maybe(attributes=attributes), tr=tr) //this gets various app params from the tc, such as environment, walltime, etc - replicationGroup=replicationGroup - replicationChannel=replicationChannel - jobid=jobid - ) - ) - if(wrapperMode == "args" - vdl:execute( - vdl:siteprofile(rhost, "swift:wrapperInterpreter"), - list( - vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"), - "_swiftwrap.wrapperstaging", - jobid, - "-urlprefix", URL_PREFIX, - "-jobdir", jobdir, - "-scratch", scratch, - "-e", vdl:executable(tr, rhost), - "-out", stdout, - "-err", stderr, - "-i", maybe(getFieldValue(stdin)), - "-d", flatten(each(fileDirs)), - "-if", flatten(inFiles(stagein)), - "-of", flatten(outFiles(stageout)), - "-wt", WRAPPERLOG_ALWAYS_TRANSFER, - "-sk", SITEDIR_KEEP, - "-cdmfile", cdm:file(), - "-status", statusMode, - "-a", maybe(each(arguments)) - ) - directory=wfdir - redirect=false - host=rhost - vdl:tcprofile(rhost, maybe(attributes=attributes), tr=tr) - replicationGroup=replicationGroup - replicationChannel=replicationChannel - jobid=jobid - ) - ) - - vdl:setprogress("Checking status") - if(statusMode == "files" - checkJobStatus(jobdir, jobid, tr) - ) - - if(wrapperMode == "files" - file:remove(wrapfile) - ) - - log(LOG:DEBUG, "STAGING_OUT jobid={jobid}") - - - /* need to stage the files to upper scratch area in case they are not transfered to another site - before all the files get cleaned out */ - - - vdl:setprogress("Stage out") - doRestartlog(restartout) - - log(LOG:DEBUG, "JOB_END jobid={jobid}") - ) - catch("^Abort$" - log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") - throw(exception) - ) - catch("^(?!Abort$).*" - vdl:setprogress("Failed but can retry") - prev := exception - exception := try(exception(checkErrorFile(jobdir, jobid)), prev) - - log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) - - if(matches(exception,".*executable bit.*") - generateError(exception) - ) - - outs := readStandardFiles(jobdir, stdout, stderr) - - throw( - exception( - concat( - "Exception in {tr}:", nl(), - maybe("Arguments: ", arguments, nl()), - "Host: {rhost}", nl(), - "Directory: {scratch}/{jobid}", - "{outs}", nl(), - "----", nl() - ) - exception - ) - ) - ) - ) - ) - ) - - element(generateProvenanceGraph, [gdata] - pgraph := vdl:configProperty("pgraph") - gname := if(pgraph == "true" "{VDL:SCRIPTNAME}-{VDL:RUNID}.dot" pgraph) - file:write(gname - "digraph SwiftProvenance {{", nl() - " graph [", vdl:configProperty("pgraph.graph.options"), "];", nl() - " node [", vdl:configProperty("pgraph.node.options"), "];", nl() - - for(i, gdata - " ", i, nl() - ) - "}", nl() - ) - log(LOG:INFO, "Provenance graph saved in ", gname) - ) - ) -) - -// Local variables: -// mode: scheme -// tab-width: 4 -// indent-tabs-mode: t -// End: From davidk at ci.uchicago.edu Wed Jan 30 09:03:42 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 09:03:42 -0600 (CST) Subject: [Swift-commit] r6188 - in SwiftApps/modis: . bin Message-ID: <20130130150343.5F4249CCBE@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 09:03:31 -0600 (Wed, 30 Jan 2013) New Revision: 6188 Modified: SwiftApps/modis/bin/colormodis.sh SwiftApps/modis/demo Log: Let mktemp create temp files in the work directory instead of /tmp on systems where it's not available Modified: SwiftApps/modis/bin/colormodis.sh =================================================================== --- SwiftApps/modis/bin/colormodis.sh 2013-01-30 05:28:47 UTC (rev 6187) +++ SwiftApps/modis/bin/colormodis.sh 2013-01-30 15:03:31 UTC (rev 6188) @@ -7,10 +7,12 @@ infile=$1 outfile=$2 -tmp=`mktemp /tmp/modis.$USER.XXXXXX` + +# Create temp files in current directory since /tmp may not be available +tmp=`mktemp -p . modis.$USER.XXXXXX` mv $tmp $tmp.tif tmp=$tmp.tif -map=`mktemp /tmp/colormap.$USER.XXXXXX` +map=`mktemp -p . colormap.$USER.XXXXXX` # Create color set Modified: SwiftApps/modis/demo =================================================================== --- SwiftApps/modis/demo 2013-01-30 05:28:47 UTC (rev 6187) +++ SwiftApps/modis/demo 2013-01-30 15:03:31 UTC (rev 6188) @@ -6,6 +6,7 @@ NSELECT=12 SITE=local WEBDIR=$HOME/public_html/geo +MODISDIR=$PWD/data/modis/2002 # Verify an argument is not null verify_not_null() @@ -28,6 +29,7 @@ -nselect) NSELECT=$2; verify_not_null nselect $NSELECT; shift ;; -site) SITE=$2; verify_not_null site $SITE; shift ;; -webdir) WEBDIR=$2; verify_not_null webdir $WEBDIR; shift ;; + -modisdir) MODISDIR=$2; verify_not_null modisdir $MODISDIR; shift ;; *) ;; esac shift @@ -39,5 +41,4 @@ echo webdir=$WEBDIR eval swiftrun -site $SITE modis.swift -landtype=$LANDTYPE -nfiles=$NFILES \ - -nselect=$NSELECT -modisdir=$PWD/data/modis/2002 \ - -webdir=$WEBDIR + -nselect=$NSELECT -modisdir=$MODISDIR -webdir=$WEBDIR From davidk at ci.uchicago.edu Wed Jan 30 11:46:20 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 11:46:20 -0600 (CST) Subject: [Swift-commit] r6189 - trunk/etc/sites Message-ID: <20130130174620.1700E9CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 11:46:19 -0600 (Wed, 30 Jan 2013) New Revision: 6189 Added: trunk/etc/sites/local Log: local template Added: trunk/etc/sites/local =================================================================== --- trunk/etc/sites/local (rev 0) +++ trunk/etc/sites/local 2013-01-30 17:46:19 UTC (rev 6189) @@ -0,0 +1,7 @@ + + + .31 + 10000 + + {env.HOME}/swiftwork + From davidk at ci.uchicago.edu Wed Jan 30 11:54:00 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 11:54:00 -0600 (CST) Subject: [Swift-commit] r6190 - trunk/bin Message-ID: <20130130175400.4B17C9CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 11:54:00 -0600 (Wed, 30 Jan 2013) New Revision: 6190 Modified: trunk/bin/swiftrun Log: Allow passing other swift args more easily Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2013-01-30 17:46:19 UTC (rev 6189) +++ trunk/bin/swiftrun 2013-01-30 17:54:00 UTC (rev 6190) @@ -48,7 +48,7 @@ case "$1" in -site|-sites|-s) SITES=$2; verify_not_null sites $SITES; shift ;; -config|-c) CONFIG=$2; verify_not_null config $CONFIG; shift ;; - *) if [ -z "$script" ]; then script="$1" + *) if [ -z "$script" ] && [[ $1 == *.swift ]]; then script="$1" else args+="$1 " fi ;; esac From davidk at ci.uchicago.edu Wed Jan 30 13:25:42 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 13:25:42 -0600 (CST) Subject: [Swift-commit] r6191 - SwiftApps/modis/conf Message-ID: <20130130192542.9DFA99CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 13:25:42 -0600 (Wed, 30 Jan 2013) New Revision: 6191 Modified: SwiftApps/modis/conf/local.cf Log: Apps for local.cf Modified: SwiftApps/modis/conf/local.cf =================================================================== --- SwiftApps/modis/conf/local.cf 2013-01-30 17:54:00 UTC (rev 6190) +++ SwiftApps/modis/conf/local.cf 2013-01-30 19:25:42 UTC (rev 6191) @@ -7,3 +7,10 @@ provider.staging.pin.swiftfiles=false #throttle.transfers=16 #throttle.file.operations=16 + +#app getlanduse=$PWD/../bin/getlanduse.sh +#app analyzelanduse=$PWD/../bin/analyzelanduse2.sh +#app colormodis=$PWD/../bin/colormodis.sh +#app assemble=$PWD/../bin/assemble2.sh +#app markmap=$PWD/../bin/markmap.sh + From davidk at ci.uchicago.edu Wed Jan 30 15:00:48 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 15:00:48 -0600 (CST) Subject: [Swift-commit] r6192 - SwiftApps/modis/conf Message-ID: <20130130210048.D940B9CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 15:00:48 -0600 (Wed, 30 Jan 2013) New Revision: 6192 Added: SwiftApps/modis/conf/beagle.cf Log: Beagle config Added: SwiftApps/modis/conf/beagle.cf =================================================================== --- SwiftApps/modis/conf/beagle.cf (rev 0) +++ SwiftApps/modis/conf/beagle.cf 2013-01-30 21:00:48 UTC (rev 6192) @@ -0,0 +1,14 @@ +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +status.mode=provider +use.provider.staging=false +provider.staging.pin.swiftfiles=false + +#site beagle WALLTIME=00:05:00 + +#app getlanduse=$PWD/../bin/getlanduse.sh +#app analyzelanduse=$PWD/../bin/analyzelanduse2.sh +#app colormodis=$PWD/../bin/colormodis.sh +#app assemble=$PWD/../bin/assemble2.sh +#app markmap=$PWD/../bin/markmap.sh From davidk at ci.uchicago.edu Wed Jan 30 15:51:41 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 15:51:41 -0600 (CST) Subject: [Swift-commit] r6193 - SwiftApps/modis Message-ID: <20130130215141.975FB9CCBE@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 15:51:41 -0600 (Wed, 30 Jan 2013) New Revision: 6193 Modified: SwiftApps/modis/modis.swift Log: Put the landuse files into their own directory Modified: SwiftApps/modis/modis.swift =================================================================== --- SwiftApps/modis/modis.swift 2013-01-30 21:00:48 UTC (rev 6192) +++ SwiftApps/modis/modis.swift 2013-01-30 21:51:41 UTC (rev 6193) @@ -36,8 +36,8 @@ int nFiles = @toInt(@arg("nfiles","1000")); int nSelect = @toInt(@arg("nselect","12")); string landType = @arg("landtype","urban"); -string MODISdir= @arg("modisdir","/home/wilde/bigdata/data/modis/2002"); -string webDir = @arg("webdir","/home/wilde/public_html/geo/"); +string MODISdir= @arg("modisdir"); +string webDir = @arg("webdir"); string suffix=".tif"; @@ -50,7 +50,7 @@ # Compute the land use summary of each MODIS tile landuse land[] ; + transform=@strcat("landuse\\1.landuse.byfreq")>; foreach g,i in geos { land[i] = getLandUse(g,1); From swift at ci.uchicago.edu Wed Jan 30 16:50:06 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 30 Jan 2013 16:50:06 -0600 (CST) Subject: [Swift-commit] cog r3574 Message-ID: <20130130225006.913548D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3574 | hategan | 2013-01-30 16:48:56 -0600 (Wed, 30 Jan 2013) | 1 line added some debugging info for bug 945 ------------------------------------------------------------------------ Index: modules/provider-coaster/src/org/globus/cog/abstraction/coaster/service/job/manager/BlockQueueProcessor.java =================================================================== --- modules/provider-coaster/src/org/globus/cog/abstraction/coaster/service/job/manager/BlockQueueProcessor.java (revision 3573) +++ modules/provider-coaster/src/org/globus/cog/abstraction/coaster/service/job/manager/BlockQueueProcessor.java (working copy) @@ -583,10 +583,11 @@ h = Math.min(h + lastwalltime, maxt); } - // create the actual block Block b = new Block(width, TimeInterval.fromSeconds(h), this); - + if (logger.isInfoEnabled()) { + logger.info("index: " + index + ", last: " + last + ", holding.size(): " + holding.size()); + } // now add jobs from holding until the size of the jobs exceeds the // size of the block (as given by the metric) int ii = last; @@ -597,10 +598,15 @@ added++; ii++; } + // lastI <-> ii + // indexI <-> index + 1 + // if ii == index + 1 <=> ii - index - 1 == 0 then + // lastI and indexI point to the same element + if (logger.isInfoEnabled()) { - logger.info("Queued: " + (ii-last) + - " jobs to new Block"); + logger.info("Queued: " + (ii-last) + " jobs to new Block"); + logger.info("index: " + index + ", last: " + last + ", ii: " + ii + ", holding.size(): " + holding.size()); } // update index of last added job // skip ii - index - 1 jobs since the iterator and "index" are off by one From davidk at ci.uchicago.edu Wed Jan 30 19:12:30 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 19:12:30 -0600 (CST) Subject: [Swift-commit] r6194 - in SwiftApps/modis: . bin Message-ID: <20130131011230.91EF79CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 19:12:30 -0600 (Wed, 30 Jan 2013) New Revision: 6194 Modified: SwiftApps/modis/bin/modis.mapper SwiftApps/modis/modis.swift Log: Modified: SwiftApps/modis/bin/modis.mapper =================================================================== --- SwiftApps/modis/bin/modis.mapper 2013-01-30 21:51:41 UTC (rev 6193) +++ SwiftApps/modis/bin/modis.mapper 2013-01-31 01:12:30 UTC (rev 6194) @@ -14,7 +14,6 @@ shift 2 done -find $location -type f -name 'h??v??'${suffix} -print | head -$n | -awk '{printf "[%d] %s\n", NR-1, $1}' +find $location -type f -name 'h??v??'${suffix} -print | sort | head -$n | +awk '{printf "[%d] %s\n", NR-1, $1}' - Modified: SwiftApps/modis/modis.swift =================================================================== --- SwiftApps/modis/modis.swift 2013-01-30 21:51:41 UTC (rev 6193) +++ SwiftApps/modis/modis.swift 2013-01-31 01:12:30 UTC (rev 6194) @@ -50,7 +50,7 @@ # Compute the land use summary of each MODIS tile landuse land[] ; + transform=@strcat("landuse/\\1.landuse.byfreq")>; foreach g,i in geos { land[i] = getLandUse(g,1); From davidk at ci.uchicago.edu Wed Jan 30 21:35:59 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 21:35:59 -0600 (CST) Subject: [Swift-commit] r6195 - SwiftApps/modis/data/modis/2002 Message-ID: <20130131033559.4DA019CCBE@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 21:35:58 -0600 (Wed, 30 Jan 2013) New Revision: 6195 Removed: SwiftApps/modis/data/modis/2002/h00v08.tif SwiftApps/modis/data/modis/2002/h00v09.tif SwiftApps/modis/data/modis/2002/h00v10.tif SwiftApps/modis/data/modis/2002/h01v07.tif SwiftApps/modis/data/modis/2002/h01v08.tif SwiftApps/modis/data/modis/2002/h01v09.tif SwiftApps/modis/data/modis/2002/h01v10.tif SwiftApps/modis/data/modis/2002/h01v11.tif SwiftApps/modis/data/modis/2002/h02v06.tif SwiftApps/modis/data/modis/2002/h02v08.tif SwiftApps/modis/data/modis/2002/h02v09.tif SwiftApps/modis/data/modis/2002/h02v10.tif SwiftApps/modis/data/modis/2002/h02v11.tif SwiftApps/modis/data/modis/2002/h03v06.tif SwiftApps/modis/data/modis/2002/h03v07.tif SwiftApps/modis/data/modis/2002/h03v09.tif SwiftApps/modis/data/modis/2002/h03v10.tif SwiftApps/modis/data/modis/2002/h03v11.tif SwiftApps/modis/data/modis/2002/h04v09.tif SwiftApps/modis/data/modis/2002/h04v10.tif SwiftApps/modis/data/modis/2002/h04v11.tif SwiftApps/modis/data/modis/2002/h05v10.tif SwiftApps/modis/data/modis/2002/h05v11.tif SwiftApps/modis/data/modis/2002/h05v13.tif SwiftApps/modis/data/modis/2002/h06v03.tif SwiftApps/modis/data/modis/2002/h06v11.tif SwiftApps/modis/data/modis/2002/h07v03.tif SwiftApps/modis/data/modis/2002/h07v05.tif SwiftApps/modis/data/modis/2002/h07v06.tif SwiftApps/modis/data/modis/2002/h07v07.tif SwiftApps/modis/data/modis/2002/h08v03.tif SwiftApps/modis/data/modis/2002/h08v04.tif SwiftApps/modis/data/modis/2002/h08v05.tif SwiftApps/modis/data/modis/2002/h08v06.tif SwiftApps/modis/data/modis/2002/h08v07.tif SwiftApps/modis/data/modis/2002/h08v08.tif SwiftApps/modis/data/modis/2002/h08v09.tif SwiftApps/modis/data/modis/2002/h08v11.tif SwiftApps/modis/data/modis/2002/h09v02.tif SwiftApps/modis/data/modis/2002/h09v03.tif SwiftApps/modis/data/modis/2002/h09v04.tif SwiftApps/modis/data/modis/2002/h09v05.tif SwiftApps/modis/data/modis/2002/h09v06.tif SwiftApps/modis/data/modis/2002/h09v07.tif SwiftApps/modis/data/modis/2002/h09v08.tif SwiftApps/modis/data/modis/2002/h09v09.tif SwiftApps/modis/data/modis/2002/h10v02.tif SwiftApps/modis/data/modis/2002/h10v03.tif SwiftApps/modis/data/modis/2002/h10v04.tif SwiftApps/modis/data/modis/2002/h10v05.tif SwiftApps/modis/data/modis/2002/h10v06.tif SwiftApps/modis/data/modis/2002/h10v07.tif SwiftApps/modis/data/modis/2002/h10v08.tif SwiftApps/modis/data/modis/2002/h10v09.tif SwiftApps/modis/data/modis/2002/h10v10.tif SwiftApps/modis/data/modis/2002/h10v11.tif SwiftApps/modis/data/modis/2002/h11v02.tif SwiftApps/modis/data/modis/2002/h11v03.tif SwiftApps/modis/data/modis/2002/h11v04.tif SwiftApps/modis/data/modis/2002/h11v05.tif SwiftApps/modis/data/modis/2002/h11v06.tif SwiftApps/modis/data/modis/2002/h11v07.tif SwiftApps/modis/data/modis/2002/h11v08.tif SwiftApps/modis/data/modis/2002/h11v09.tif SwiftApps/modis/data/modis/2002/h11v10.tif SwiftApps/modis/data/modis/2002/h11v11.tif SwiftApps/modis/data/modis/2002/h11v12.tif SwiftApps/modis/data/modis/2002/h12v01.tif SwiftApps/modis/data/modis/2002/h12v02.tif SwiftApps/modis/data/modis/2002/h12v03.tif SwiftApps/modis/data/modis/2002/h12v04.tif SwiftApps/modis/data/modis/2002/h12v05.tif SwiftApps/modis/data/modis/2002/h12v07.tif SwiftApps/modis/data/modis/2002/h12v08.tif SwiftApps/modis/data/modis/2002/h12v09.tif SwiftApps/modis/data/modis/2002/h12v10.tif SwiftApps/modis/data/modis/2002/h12v11.tif SwiftApps/modis/data/modis/2002/h12v12.tif SwiftApps/modis/data/modis/2002/h12v13.tif SwiftApps/modis/data/modis/2002/h13v01.tif SwiftApps/modis/data/modis/2002/h13v02.tif SwiftApps/modis/data/modis/2002/h13v03.tif SwiftApps/modis/data/modis/2002/h13v04.tif SwiftApps/modis/data/modis/2002/h13v08.tif SwiftApps/modis/data/modis/2002/h13v09.tif SwiftApps/modis/data/modis/2002/h13v10.tif SwiftApps/modis/data/modis/2002/h13v11.tif SwiftApps/modis/data/modis/2002/h13v12.tif SwiftApps/modis/data/modis/2002/h13v13.tif SwiftApps/modis/data/modis/2002/h13v14.tif SwiftApps/modis/data/modis/2002/h14v01.tif SwiftApps/modis/data/modis/2002/h14v02.tif SwiftApps/modis/data/modis/2002/h14v03.tif SwiftApps/modis/data/modis/2002/h14v04.tif SwiftApps/modis/data/modis/2002/h14v09.tif SwiftApps/modis/data/modis/2002/h14v10.tif SwiftApps/modis/data/modis/2002/h14v11.tif SwiftApps/modis/data/modis/2002/h14v14.tif SwiftApps/modis/data/modis/2002/h14v16.tif SwiftApps/modis/data/modis/2002/h14v17.tif SwiftApps/modis/data/modis/2002/h15v01.tif SwiftApps/modis/data/modis/2002/h15v02.tif SwiftApps/modis/data/modis/2002/h15v03.tif SwiftApps/modis/data/modis/2002/h15v05.tif SwiftApps/modis/data/modis/2002/h15v07.tif SwiftApps/modis/data/modis/2002/h15v11.tif SwiftApps/modis/data/modis/2002/h15v14.tif SwiftApps/modis/data/modis/2002/h15v15.tif SwiftApps/modis/data/modis/2002/h15v16.tif SwiftApps/modis/data/modis/2002/h15v17.tif SwiftApps/modis/data/modis/2002/h16v00.tif SwiftApps/modis/data/modis/2002/h16v01.tif SwiftApps/modis/data/modis/2002/h16v02.tif SwiftApps/modis/data/modis/2002/h16v05.tif SwiftApps/modis/data/modis/2002/h16v06.tif SwiftApps/modis/data/modis/2002/h16v07.tif SwiftApps/modis/data/modis/2002/h16v08.tif SwiftApps/modis/data/modis/2002/h16v09.tif SwiftApps/modis/data/modis/2002/h16v12.tif SwiftApps/modis/data/modis/2002/h16v14.tif SwiftApps/modis/data/modis/2002/h16v16.tif SwiftApps/modis/data/modis/2002/h16v17.tif SwiftApps/modis/data/modis/2002/h17v00.tif SwiftApps/modis/data/modis/2002/h17v01.tif SwiftApps/modis/data/modis/2002/h17v02.tif SwiftApps/modis/data/modis/2002/h17v03.tif SwiftApps/modis/data/modis/2002/h17v04.tif SwiftApps/modis/data/modis/2002/h17v05.tif SwiftApps/modis/data/modis/2002/h17v06.tif SwiftApps/modis/data/modis/2002/h17v07.tif SwiftApps/modis/data/modis/2002/h17v08.tif SwiftApps/modis/data/modis/2002/h17v10.tif SwiftApps/modis/data/modis/2002/h17v12.tif SwiftApps/modis/data/modis/2002/h17v13.tif SwiftApps/modis/data/modis/2002/h17v15.tif SwiftApps/modis/data/modis/2002/h17v16.tif SwiftApps/modis/data/modis/2002/h17v17.tif SwiftApps/modis/data/modis/2002/h18v00.tif SwiftApps/modis/data/modis/2002/h18v01.tif SwiftApps/modis/data/modis/2002/h18v02.tif SwiftApps/modis/data/modis/2002/h18v03.tif SwiftApps/modis/data/modis/2002/h18v04.tif SwiftApps/modis/data/modis/2002/h18v05.tif SwiftApps/modis/data/modis/2002/h18v06.tif SwiftApps/modis/data/modis/2002/h18v07.tif SwiftApps/modis/data/modis/2002/h18v08.tif SwiftApps/modis/data/modis/2002/h18v09.tif SwiftApps/modis/data/modis/2002/h18v14.tif SwiftApps/modis/data/modis/2002/h18v15.tif SwiftApps/modis/data/modis/2002/h18v16.tif SwiftApps/modis/data/modis/2002/h18v17.tif SwiftApps/modis/data/modis/2002/h19v00.tif SwiftApps/modis/data/modis/2002/h19v01.tif SwiftApps/modis/data/modis/2002/h19v02.tif SwiftApps/modis/data/modis/2002/h19v03.tif SwiftApps/modis/data/modis/2002/h19v04.tif SwiftApps/modis/data/modis/2002/h19v05.tif SwiftApps/modis/data/modis/2002/h19v06.tif SwiftApps/modis/data/modis/2002/h19v07.tif SwiftApps/modis/data/modis/2002/h19v08.tif SwiftApps/modis/data/modis/2002/h19v09.tif SwiftApps/modis/data/modis/2002/h19v10.tif SwiftApps/modis/data/modis/2002/h19v11.tif SwiftApps/modis/data/modis/2002/h19v12.tif SwiftApps/modis/data/modis/2002/h19v15.tif SwiftApps/modis/data/modis/2002/h19v16.tif SwiftApps/modis/data/modis/2002/h19v17.tif SwiftApps/modis/data/modis/2002/h20v01.tif SwiftApps/modis/data/modis/2002/h20v02.tif SwiftApps/modis/data/modis/2002/h20v03.tif SwiftApps/modis/data/modis/2002/h20v04.tif SwiftApps/modis/data/modis/2002/h20v05.tif SwiftApps/modis/data/modis/2002/h20v06.tif SwiftApps/modis/data/modis/2002/h20v07.tif SwiftApps/modis/data/modis/2002/h20v08.tif SwiftApps/modis/data/modis/2002/h20v09.tif SwiftApps/modis/data/modis/2002/h20v10.tif SwiftApps/modis/data/modis/2002/h20v11.tif SwiftApps/modis/data/modis/2002/h20v12.tif SwiftApps/modis/data/modis/2002/h20v13.tif SwiftApps/modis/data/modis/2002/h20v15.tif SwiftApps/modis/data/modis/2002/h20v16.tif SwiftApps/modis/data/modis/2002/h20v17.tif SwiftApps/modis/data/modis/2002/h21v01.tif SwiftApps/modis/data/modis/2002/h21v02.tif SwiftApps/modis/data/modis/2002/h21v03.tif SwiftApps/modis/data/modis/2002/h21v04.tif SwiftApps/modis/data/modis/2002/h21v05.tif SwiftApps/modis/data/modis/2002/h21v06.tif SwiftApps/modis/data/modis/2002/h21v07.tif SwiftApps/modis/data/modis/2002/h21v08.tif SwiftApps/modis/data/modis/2002/h21v09.tif SwiftApps/modis/data/modis/2002/h21v10.tif SwiftApps/modis/data/modis/2002/h21v11.tif SwiftApps/modis/data/modis/2002/h21v13.tif SwiftApps/modis/data/modis/2002/h21v15.tif SwiftApps/modis/data/modis/2002/h21v16.tif SwiftApps/modis/data/modis/2002/h21v17.tif SwiftApps/modis/data/modis/2002/h22v01.tif SwiftApps/modis/data/modis/2002/h22v02.tif SwiftApps/modis/data/modis/2002/h22v03.tif SwiftApps/modis/data/modis/2002/h22v04.tif SwiftApps/modis/data/modis/2002/h22v05.tif SwiftApps/modis/data/modis/2002/h22v06.tif SwiftApps/modis/data/modis/2002/h22v07.tif SwiftApps/modis/data/modis/2002/h22v08.tif SwiftApps/modis/data/modis/2002/h22v09.tif SwiftApps/modis/data/modis/2002/h22v10.tif SwiftApps/modis/data/modis/2002/h22v11.tif SwiftApps/modis/data/modis/2002/h22v13.tif SwiftApps/modis/data/modis/2002/h22v14.tif SwiftApps/modis/data/modis/2002/h22v15.tif SwiftApps/modis/data/modis/2002/h22v16.tif SwiftApps/modis/data/modis/2002/h23v01.tif SwiftApps/modis/data/modis/2002/h23v02.tif SwiftApps/modis/data/modis/2002/h23v03.tif SwiftApps/modis/data/modis/2002/h23v04.tif SwiftApps/modis/data/modis/2002/h23v05.tif SwiftApps/modis/data/modis/2002/h23v06.tif SwiftApps/modis/data/modis/2002/h23v07.tif SwiftApps/modis/data/modis/2002/h23v08.tif SwiftApps/modis/data/modis/2002/h23v09.tif SwiftApps/modis/data/modis/2002/h23v10.tif SwiftApps/modis/data/modis/2002/h23v11.tif SwiftApps/modis/data/modis/2002/h23v15.tif SwiftApps/modis/data/modis/2002/h23v16.tif SwiftApps/modis/data/modis/2002/h24v02.tif SwiftApps/modis/data/modis/2002/h24v03.tif SwiftApps/modis/data/modis/2002/h24v04.tif SwiftApps/modis/data/modis/2002/h24v05.tif SwiftApps/modis/data/modis/2002/h24v06.tif SwiftApps/modis/data/modis/2002/h24v07.tif SwiftApps/modis/data/modis/2002/h24v12.tif SwiftApps/modis/data/modis/2002/h24v15.tif SwiftApps/modis/data/modis/2002/h25v02.tif SwiftApps/modis/data/modis/2002/h25v03.tif SwiftApps/modis/data/modis/2002/h25v04.tif SwiftApps/modis/data/modis/2002/h25v05.tif SwiftApps/modis/data/modis/2002/h25v06.tif SwiftApps/modis/data/modis/2002/h25v07.tif SwiftApps/modis/data/modis/2002/h25v08.tif SwiftApps/modis/data/modis/2002/h25v09.tif SwiftApps/modis/data/modis/2002/h26v02.tif SwiftApps/modis/data/modis/2002/h26v03.tif SwiftApps/modis/data/modis/2002/h26v04.tif SwiftApps/modis/data/modis/2002/h26v05.tif SwiftApps/modis/data/modis/2002/h26v06.tif SwiftApps/modis/data/modis/2002/h26v07.tif SwiftApps/modis/data/modis/2002/h26v08.tif SwiftApps/modis/data/modis/2002/h27v03.tif SwiftApps/modis/data/modis/2002/h27v04.tif SwiftApps/modis/data/modis/2002/h27v05.tif SwiftApps/modis/data/modis/2002/h27v06.tif SwiftApps/modis/data/modis/2002/h27v07.tif SwiftApps/modis/data/modis/2002/h27v08.tif SwiftApps/modis/data/modis/2002/h27v09.tif SwiftApps/modis/data/modis/2002/h27v10.tif SwiftApps/modis/data/modis/2002/h27v11.tif SwiftApps/modis/data/modis/2002/h27v12.tif SwiftApps/modis/data/modis/2002/h27v14.tif SwiftApps/modis/data/modis/2002/h28v03.tif SwiftApps/modis/data/modis/2002/h28v04.tif SwiftApps/modis/data/modis/2002/h28v05.tif SwiftApps/modis/data/modis/2002/h28v06.tif SwiftApps/modis/data/modis/2002/h28v07.tif SwiftApps/modis/data/modis/2002/h28v08.tif SwiftApps/modis/data/modis/2002/h28v09.tif SwiftApps/modis/data/modis/2002/h28v10.tif SwiftApps/modis/data/modis/2002/h28v11.tif SwiftApps/modis/data/modis/2002/h28v12.tif SwiftApps/modis/data/modis/2002/h28v13.tif SwiftApps/modis/data/modis/2002/h28v14.tif SwiftApps/modis/data/modis/2002/h29v03.tif SwiftApps/modis/data/modis/2002/h29v05.tif SwiftApps/modis/data/modis/2002/h29v06.tif SwiftApps/modis/data/modis/2002/h29v07.tif SwiftApps/modis/data/modis/2002/h29v08.tif SwiftApps/modis/data/modis/2002/h29v09.tif SwiftApps/modis/data/modis/2002/h29v10.tif SwiftApps/modis/data/modis/2002/h29v11.tif SwiftApps/modis/data/modis/2002/h29v12.tif SwiftApps/modis/data/modis/2002/h29v13.tif SwiftApps/modis/data/modis/2002/h30v05.tif SwiftApps/modis/data/modis/2002/h30v06.tif SwiftApps/modis/data/modis/2002/h30v07.tif SwiftApps/modis/data/modis/2002/h30v08.tif SwiftApps/modis/data/modis/2002/h30v09.tif SwiftApps/modis/data/modis/2002/h30v10.tif SwiftApps/modis/data/modis/2002/h30v11.tif SwiftApps/modis/data/modis/2002/h30v12.tif SwiftApps/modis/data/modis/2002/h30v13.tif SwiftApps/modis/data/modis/2002/h31v06.tif SwiftApps/modis/data/modis/2002/h31v07.tif SwiftApps/modis/data/modis/2002/h31v08.tif SwiftApps/modis/data/modis/2002/h31v09.tif SwiftApps/modis/data/modis/2002/h31v10.tif SwiftApps/modis/data/modis/2002/h31v11.tif SwiftApps/modis/data/modis/2002/h31v12.tif SwiftApps/modis/data/modis/2002/h31v13.tif SwiftApps/modis/data/modis/2002/h32v07.tif SwiftApps/modis/data/modis/2002/h32v08.tif SwiftApps/modis/data/modis/2002/h32v09.tif SwiftApps/modis/data/modis/2002/h32v10.tif SwiftApps/modis/data/modis/2002/h32v11.tif SwiftApps/modis/data/modis/2002/h32v12.tif SwiftApps/modis/data/modis/2002/h33v07.tif SwiftApps/modis/data/modis/2002/h33v08.tif SwiftApps/modis/data/modis/2002/h33v09.tif SwiftApps/modis/data/modis/2002/h33v10.tif SwiftApps/modis/data/modis/2002/h33v11.tif SwiftApps/modis/data/modis/2002/h34v07.tif SwiftApps/modis/data/modis/2002/h34v08.tif SwiftApps/modis/data/modis/2002/h34v09.tif SwiftApps/modis/data/modis/2002/h34v10.tif SwiftApps/modis/data/modis/2002/h35v08.tif SwiftApps/modis/data/modis/2002/h35v09.tif SwiftApps/modis/data/modis/2002/h35v10.tif Log: Remove the old tif's Deleted: SwiftApps/modis/data/modis/2002/h00v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h00v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h00v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h01v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h01v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h01v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h01v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h01v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h02v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h02v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h02v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h02v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h02v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h03v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h03v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h03v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h03v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h03v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h04v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h04v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h04v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h05v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h05v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h05v13.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h06v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h06v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h07v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h07v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h07v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h07v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h08v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h08v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h08v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h08v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h08v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h08v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h08v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h08v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h09v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h09v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h09v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h09v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h09v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h09v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h09v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h09v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h10v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h10v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h10v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h10v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h10v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h10v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h10v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h10v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h10v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h10v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h11v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h11v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h11v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h11v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h11v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h11v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h11v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h11v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h11v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h11v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h11v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h12v13.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h13v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h13v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h13v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h13v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h13v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h13v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h13v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h13v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h13v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h13v13.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h13v14.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h14v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h14v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h14v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h14v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h14v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h14v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h14v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h14v14.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h14v16.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h14v17.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h15v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h15v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h15v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h15v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h15v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h15v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h15v14.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h15v15.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h15v16.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h15v17.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v00.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v14.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v16.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h16v17.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v00.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v13.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v15.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v16.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h17v17.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v00.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v14.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v15.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v16.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h18v17.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v00.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v15.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v16.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h19v17.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v13.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v15.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v16.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h20v17.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v13.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v15.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v16.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h21v17.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v13.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v14.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v15.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h22v16.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v01.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v15.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h23v16.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h24v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h24v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h24v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h24v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h24v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h24v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h24v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h24v15.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h25v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h25v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h25v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h25v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h25v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h25v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h25v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h25v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h26v02.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h26v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h26v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h26v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h26v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h26v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h26v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h27v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h27v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h27v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h27v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h27v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h27v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h27v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h27v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h27v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h27v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h27v14.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v04.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v13.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h28v14.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h29v03.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h29v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h29v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h29v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h29v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h29v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h29v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h29v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h29v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h29v13.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h30v05.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h30v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h30v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h30v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h30v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h30v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h30v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h30v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h30v13.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h31v06.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h31v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h31v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h31v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h31v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h31v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h31v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h31v13.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h32v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h32v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h32v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h32v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h32v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h32v12.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h33v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h33v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h33v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h33v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h33v11.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h34v07.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h34v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h34v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h34v10.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h35v08.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h35v09.tif =================================================================== (Binary files differ) Deleted: SwiftApps/modis/data/modis/2002/h35v10.tif =================================================================== (Binary files differ) From davidk at ci.uchicago.edu Wed Jan 30 21:41:10 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 21:41:10 -0600 (CST) Subject: [Swift-commit] r6196 - in SwiftApps/modis: . bin Message-ID: <20130131034110.02D119CCBE@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 21:41:09 -0600 (Wed, 30 Jan 2013) New Revision: 6196 Added: SwiftApps/modis/bin/ppmhist.pl Modified: SwiftApps/modis/bin/assemble2.sh SwiftApps/modis/modis.swift Log: script to generate ppm histograms Modified: SwiftApps/modis/bin/assemble2.sh =================================================================== --- SwiftApps/modis/bin/assemble2.sh 2013-01-31 03:35:58 UTC (rev 6195) +++ SwiftApps/modis/bin/assemble2.sh 2013-01-31 03:41:09 UTC (rev 6196) @@ -5,7 +5,7 @@ indir=$(cd $(dirname $3); pwd) # get full pathname of input dir "geos" webdir=$4 -tmp=$(mktemp -d /tmp/modis.assemble.XXXX) +tmp=$(mktemp -p . -d modis.assemble.XXXX) bindir=$(cd $(dirname $0); pwd) graytile=$bindir/gray.png Added: SwiftApps/modis/bin/ppmhist.pl =================================================================== --- SwiftApps/modis/bin/ppmhist.pl (rev 0) +++ SwiftApps/modis/bin/ppmhist.pl 2013-01-31 03:41:09 UTC (rev 6196) @@ -0,0 +1,42 @@ +#!/usr/bin/perl -w + +use strict; + +# Read file +my $ppm_filename = shift; +open(PPMFILE, $ppm_filename) || die "Unable to open $ppm_filename!\n"; +my @ppm_data = ; +close(PPMFILE); + +my %rgbcount = (); +my $count=0; +my $ppmformat="", my $resolution="", my $maxval=""; + +foreach my $line(@ppm_data) { + chomp($line); + if ($count == 0){ $ppmformat=$line; $count++; next; } + if ($count == 1){ $resolution=$line; $count++; next; } + if ($count == 2){ $maxval=$line; $count++; next; } + + my $wordcount=0, my $red="", my $green="", my $blue=""; + foreach my $rgb(split(/\s+/, $line)) { + if($wordcount == 0) { $red=$rgb; $wordcount++; next; } + if($wordcount == 1) { $green=$rgb; $wordcount++; next; } + if($wordcount == 2) { + $blue=$rgb; + if(!defined($rgbcount{"$red $green $blue"})) { + $rgbcount{"$red $green $blue"}=1; + } else { + $rgbcount{"$red $green $blue"}++; + } + $wordcount=0; + } + } + $count++; +} + +while ( my ($key, $value) = each(%rgbcount) ) { + (my $red, my $green, my $blue) = split(/ /, $key); + my $hex = sprintf("%02x",$red); + print "$value $red $hex\n"; +} Property changes on: SwiftApps/modis/bin/ppmhist.pl ___________________________________________________________________ Added: svn:executable + * Modified: SwiftApps/modis/modis.swift =================================================================== --- SwiftApps/modis/modis.swift 2013-01-31 03:35:58 UTC (rev 6195) +++ SwiftApps/modis/modis.swift 2013-01-31 03:41:09 UTC (rev 6196) @@ -4,9 +4,9 @@ # Define application program interfaces -app (landuse output) getLandUse (imagefile input, int sortfield) +app (landuse output) getLandUse (imagefile input, int sortfield, string mpath) { - getlanduse @input sortfield stdout=@output ; + getlanduse @input sortfield mpath stdout=@output; } app (file output, file tilelist) analyzeLandUse @@ -39,12 +39,12 @@ string MODISdir= @arg("modisdir"); string webDir = @arg("webdir"); -string suffix=".tif"; +string suffix=".ppm.gz"; # Input Dataset imagefile geos[] ; # site=site + location=MODISdir, suffix=".ppm.gz", n=nFiles >; # site=site # Compute the land use summary of each MODIS tile @@ -53,7 +53,7 @@ transform=@strcat("landuse/\\1.landuse.byfreq")>; foreach g,i in geos { - land[i] = getLandUse(g,1); + land[i] = getLandUse(g,1,MODISdir); } # Find the top N tiles (by total area of selected landuse types) From davidk at ci.uchicago.edu Wed Jan 30 22:13:57 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 22:13:57 -0600 (CST) Subject: [Swift-commit] r6198 - SwiftApps/modis/bin Message-ID: <20130131041357.7F06A9CCBE@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 22:13:57 -0600 (Wed, 30 Jan 2013) New Revision: 6198 Modified: SwiftApps/modis/bin/getlanduse.sh Log: updated getlanduse Modified: SwiftApps/modis/bin/getlanduse.sh =================================================================== --- SwiftApps/modis/bin/getlanduse.sh 2013-01-31 04:01:16 UTC (rev 6197) +++ SwiftApps/modis/bin/getlanduse.sh 2013-01-31 04:13:57 UTC (rev 6198) @@ -1,3 +1,5 @@ +#!/bin/bash + # # Read a MODIS .tif file and return a histogram of pixel values. # The pixel values signify land use for that region (see legend in notes) @@ -5,10 +7,4 @@ # Usage: getlanduse.sh modisFile histogramFile # -#sleep 1 -convert $1 -format %c histogram:info:- 2>/dev/null | - grep '[0-9]' | - sort -nr | - sed -e 's/[^0-9 ]//g' | - awk '{print $1, $3, sprintf("%02x",$3)}' | - sort -n -k $2 +$3/../../../bin/ppmhist.pl $1 | sort -n -k $2 From davidk at ci.uchicago.edu Wed Jan 30 22:35:02 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 22:35:02 -0600 (CST) Subject: [Swift-commit] r6199 - SwiftApps/modis/bin Message-ID: <20130131043502.3462A9CCBE@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 22:35:01 -0600 (Wed, 30 Jan 2013) New Revision: 6199 Modified: SwiftApps/modis/bin/ppmhist.pl Log: zcat Modified: SwiftApps/modis/bin/ppmhist.pl =================================================================== --- SwiftApps/modis/bin/ppmhist.pl 2013-01-31 04:13:57 UTC (rev 6198) +++ SwiftApps/modis/bin/ppmhist.pl 2013-01-31 04:35:01 UTC (rev 6199) @@ -4,7 +4,7 @@ # Read file my $ppm_filename = shift; -open(PPMFILE, $ppm_filename) || die "Unable to open $ppm_filename!\n"; +open(PPMFILE, "zcat $ppm_filename|") || die "Unable to open $ppm_filename!\n"; my @ppm_data = ; close(PPMFILE); From davidk at ci.uchicago.edu Wed Jan 30 23:09:59 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 23:09:59 -0600 (CST) Subject: [Swift-commit] r6200 - in SwiftApps/modis: . bin Message-ID: <20130131050959.C16BC9CCBE@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 23:09:59 -0600 (Wed, 30 Jan 2013) New Revision: 6200 Removed: SwiftApps/modis/landuse/ Modified: SwiftApps/modis/bin/assemble2.sh SwiftApps/modis/bin/colormodis.sh SwiftApps/modis/modis.swift Log: More changes for compatibility Modified: SwiftApps/modis/bin/assemble2.sh =================================================================== --- SwiftApps/modis/bin/assemble2.sh 2013-01-31 04:35:01 UTC (rev 6199) +++ SwiftApps/modis/bin/assemble2.sh 2013-01-31 05:09:59 UTC (rev 6200) @@ -5,7 +5,7 @@ indir=$(cd $(dirname $3); pwd) # get full pathname of input dir "geos" webdir=$4 -tmp=$(mktemp -p . -d modis.assemble.XXXX) +tmp=$(mktemp -d modis.assemble.XXXX) bindir=$(cd $(dirname $0); pwd) graytile=$bindir/gray.png Modified: SwiftApps/modis/bin/colormodis.sh =================================================================== --- SwiftApps/modis/bin/colormodis.sh 2013-01-31 04:35:01 UTC (rev 6199) +++ SwiftApps/modis/bin/colormodis.sh 2013-01-31 05:09:59 UTC (rev 6200) @@ -9,17 +9,17 @@ outfile=$2 # Create temp files in current directory since /tmp may not be available -tmp=`mktemp -p . modis.$USER.XXXXXX` +tmp=`mktemp modis.$USER.XXXXXX` mv $tmp $tmp.tif tmp=$tmp.tif -map=`mktemp -p . colormap.$USER.XXXXXX` +map=`mktemp colormap.$USER.XXXXXX` # Create color set ( cat | sed -e 's/ c.=/ /g' -e 's/[^0-9 ]//g' | - awk '{printf("#%02x%02x%02x #%02x%02x%02x\n",NR-1,NR-1,NR-1,$1,$2,$3)}') >$map <$map < Modified: SwiftApps/modis/modis.swift =================================================================== --- SwiftApps/modis/modis.swift 2013-01-31 04:35:01 UTC (rev 6199) +++ SwiftApps/modis/modis.swift 2013-01-31 05:09:59 UTC (rev 6200) @@ -20,7 +20,7 @@ colormodis @input @output; } -app (imagefile output) assemble +app (imagefile output, file elog) assemble (file selected, imagefile image[], string webdir) { assemble @output @selected @filename(image[0]) webdir; @@ -50,7 +50,7 @@ # Compute the land use summary of each MODIS tile landuse land[] ; + transform=@strcat("\\1.landuse.byfreq")>; foreach g,i in geos { land[i] = getLandUse(g,1,MODISdir); @@ -71,7 +71,7 @@ imagefile colorImage[] ; + transform="\\1.color.png">; foreach g, i in geos { colorImage[i] = colorMODIS(g); From davidk at ci.uchicago.edu Wed Jan 30 23:29:31 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 30 Jan 2013 23:29:31 -0600 (CST) Subject: [Swift-commit] r6201 - SwiftApps/modis Message-ID: <20130131052931.C89AC9CCB3@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 23:29:31 -0600 (Wed, 30 Jan 2013) New Revision: 6201 Modified: SwiftApps/modis/modis.swift Log: Modified: SwiftApps/modis/modis.swift =================================================================== --- SwiftApps/modis/modis.swift 2013-01-31 05:09:59 UTC (rev 6200) +++ SwiftApps/modis/modis.swift 2013-01-31 05:29:31 UTC (rev 6201) @@ -20,7 +20,7 @@ colormodis @input @output; } -app (imagefile output, file elog) assemble +app (imagefile output) assemble (file selected, imagefile image[], string webdir) { assemble @output @selected @filename(image[0]) webdir; From lgadelha at ci.uchicago.edu Thu Jan 31 06:50:30 2013 From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu) Date: Thu, 31 Jan 2013 06:50:30 -0600 (CST) Subject: [Swift-commit] r6202 - trunk/libexec/log-processing Message-ID: <20130131125031.03A979CF8F@svn.ci.uchicago.edu> Author: lgadelha Date: 2013-01-31 06:49:46 -0600 (Thu, 31 Jan 2013) New Revision: 6202 Modified: trunk/libexec/log-processing/log-to-execute-transitions trunk/libexec/log-processing/log-to-execute2-transitions Log: Fixed extraction scripts to reflect new log format changes. Modified: trunk/libexec/log-processing/log-to-execute-transitions =================================================================== --- trunk/libexec/log-processing/log-to-execute-transitions 2013-01-31 05:29:31 UTC (rev 6201) +++ trunk/libexec/log-processing/log-to-execute-transitions 2013-01-31 12:49:46 UTC (rev 6202) @@ -11,15 +11,16 @@ # then change relevant lines to transition format # then filter out unchanged lines because they're irrelevant +grep ' INFO swift ' | grep thread | grep -v provenanceid | \ +grep -v ARRAYRANGE | iso-to-secs | grep -E '^[^ ]+ +[^ ]+ +swift ' | \ +sed 's/\([^ ]*\) INFO swift Exception in .*://' | \ +sed 's/\([^ ]*\) INFO swift \([^ =]*\) \([^ =]*\) .*thread=\([^ ]*\).*$/\1 \4 \2.\3/' | \ +sed 's/\([^ ]*\) INFO swift \([^ ]*\) .*thread=\([^ ]*\).*tr=\([^ ]*\).*$/\1 \3 \2 \4/' | \ +sed 's/\([^ ]*\) INFO swift \([^ ]*\) .*thread=\([^ ]*\).*$/\1 \3 \2/' | \ +grep -E '^[^$]' -grep ' vdl:execute ' | iso-to-secs | \ -grep -E '^[^ ]+ +[^ ]+ +vdl:execute ' | \ -sed 's/\([^ ]*\) INFO vdl:execute Exception in .*://' | \ -sed 's/\([^ ]*\) INFO vdl:execute \([^ =]*\) \([^ =]*\) .*thread=\([^ ]*\).*$/\1 \4 \2.\3/' |\ -sed 's/\([^ ]*\) INFO vdl:execute \([^ ]*\) .*thread=\([^ ]*\).*tr=\([^ ]*\).*$/\1 \3 \2 \4/' |\ -sed 's/\([^ ]*\) INFO vdl:execute \([^ ]*\) .*thread=\([^ ]*\).*$/\1 \3 \2/' |\ - grep -E '^[^$]' + # force a successful exit, rather than using the exit codes from pipeline # components above (most probably grep?) Modified: trunk/libexec/log-processing/log-to-execute2-transitions =================================================================== --- trunk/libexec/log-processing/log-to-execute2-transitions 2013-01-31 05:29:31 UTC (rev 6201) +++ trunk/libexec/log-processing/log-to-execute2-transitions 2013-01-31 12:49:46 UTC (rev 6202) @@ -12,12 +12,19 @@ # then filter out unchanged lines because they're irrelevant -grep ' vdl:execute2 ' | iso-to-secs | \ -grep -E '^[^ ]+ +[^ ]+ +vdl:execute2 ' | \ -sed 's/^\(.*\) DEBUG vdl:execute2 THREAD_ASSOCIATION jobid=\([^ ]*\) thread=\([0-9\-]*\) host=\([^ ]*\) replicationGroup=\([^ ]*\).*$/\1 \2 ASSOCIATED \3 \4 \5/' | \ +#grep ' vdl:execute2 ' | iso-to-secs | \ +#grep -E '^[^ ]+ +[^ ]+ +vdl:execute2 ' | \ +#sed 's/^\(.*\) DEBUG vdl:execute2 THREAD_ASSOCIATION jobid=\([^ ]*\) thread=\([0-9\-]*\) host=\([^ ]*\) replicationGroup=\([^ ]*\).*$/\1 \2 ASSOCIATED \3 \4 \5/' | \ +#\ +#sed 's/^\([^ ]*\) DEBUG vdl:execute2 \([^ ]*\) jobid=\([^ ]*\).*/\1 \3 \2/' + +grep ' swift ' | iso-to-secs | \ +grep -E '^[^ ]+ +[^ ]+ +swift ' | \ +sed 's/^\(.*\) DEBUG swift THREAD_ASSOCIATION jobid=\([^ ]*\) thread=\([0-9\-]*\) host=\([^ ]*\) replicationGroup=\([^ ]*\).*$/\1 \2 ASSOCIATED \3 \4 \5/' | \ \ -sed 's/^\([^ ]*\) DEBUG vdl:execute2 \([^ ]*\) jobid=\([^ ]*\).*/\1 \3 \2/' +sed 's/^\([^ ]*\) DEBUG swift \([^ ]*\) jobid=\([^ ]*\).*/\1 \3 \2/' | grep -v DEBUG | grep -v INFO + # | \ #sed 's/^\([^ ]*\) DEBUG vdl:execute2 \([^ ]*\) application exception: \(.*\)$/\1 \2 FAILED \3/' | \ #sed 's/^\([^ ]*\) INFO vdl:execute2 Completed job \([^ ]*\) \([^ ]*\) .* on [^ ]*$/\1 \2 EXECCOMPLETED/' | \ From davidk at ci.uchicago.edu Thu Jan 31 07:51:20 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 31 Jan 2013 07:51:20 -0600 (CST) Subject: [Swift-commit] r6203 - trunk/etc/sites Message-ID: <20130131135120.668419CF8F@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-31 07:51:13 -0600 (Thu, 31 Jan 2013) New Revision: 6203 Modified: trunk/etc/sites/local Log: Make job throttle a gensite option for local template Modified: trunk/etc/sites/local =================================================================== --- trunk/etc/sites/local 2013-01-31 12:49:46 UTC (rev 6202) +++ trunk/etc/sites/local 2013-01-31 13:51:13 UTC (rev 6203) @@ -1,6 +1,6 @@ - .31 + _JOBTHROTTLE_ 10000 {env.HOME}/swiftwork From lgadelha at ci.uchicago.edu Thu Jan 31 07:51:53 2013 From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu) Date: Thu, 31 Jan 2013 07:51:53 -0600 (CST) Subject: [Swift-commit] r6204 - provenancedb Message-ID: <20130131135153.187489CD1D@svn.ci.uchicago.edu> Author: lgadelha Date: 2013-01-31 07:51:49 -0600 (Thu, 31 Jan 2013) New Revision: 6204 Modified: provenancedb/README.asciidoc provenancedb/prov-init.sql provenancedb/prov-to-sql.sh Log: Modified: provenancedb/README.asciidoc =================================================================== --- provenancedb/README.asciidoc 2013-01-31 13:51:13 UTC (rev 6203) +++ provenancedb/README.asciidoc 2013-01-31 13:51:49 UTC (rev 6204) @@ -164,10 +164,11 @@ === Swift Configuration -To enable the generation of provenance information in Swift's log files the option +provenance.log+ should be set to true in +etc/swift.properties+: +To enable the generation of provenance information in Swift's log files and to trasfer wrapper logs back to the submitting machine for runtimei behavior information extraction the options +provenance.log+ and wrapperlog.always.transfer=true should be set to true in +etc/swift.properties+: -------------------------------------- provenance.log=true +wrapperlog.always.transfer=true -------------------------------------- If Swift's SVN revision is 3417 or greater, the following options should be set in +etc/log4j.properties+: Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2013-01-31 13:51:13 UTC (rev 6203) +++ provenancedb/prov-init.sql 2013-01-31 13:51:49 UTC (rev 6204) @@ -161,6 +161,12 @@ select id, app_fun_call_id as function_call_id, to_timestamp(start_time) as start_time, duration, final_state, site from app_exec; +create view application_runtime_info as + select id, real_secs as real, kernel_secs as kernel, user_secs as user, percent_cpu as cpu, max_rss as maxrss, + avg_rss as avgrss, avg_tot_vm as avgtotvm, avg_priv_data as avgprdata, avg_priv_stack as avgprvstck, + avg_shared_text as avgshtxt, page_size, major_pgfaults as majpf, minor_pgfaults as minpf, swaps, + invol_context_switches as invcs, vol_waits as volwait, fs_reads as fsread, fs_writes as fswrite, + sock_recv as sockrcv, sock_send as socksnd, signals, exit_status as exit_st from app_exec; -- app execution runtime info extracted from the /proc filesystem (assumes the app executed -- in a Linux host) Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2013-01-31 13:51:13 UTC (rev 6203) +++ provenancedb/prov-to-sql.sh 2013-01-31 13:51:49 UTC (rev 6204) @@ -131,28 +131,28 @@ while read execute2_id runtime; do real_secs=$(echo $runtime | awk -F "," '{print $1}' | awk -F ":" '{print $2}') kernel_secs=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - user_secs=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - percent_cpu=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - max_rss=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - avg_rss=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - avg_tot_vm=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - avg_priv_data=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - avg_priv_stack=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - avg_shared_text=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - page_size=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - major_pgfaults=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - minor_pgfaults=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - swaps=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') + user_secs=$(echo $runtime | awk -F "," '{print $3}' | awk -F ":" '{print $2}') + percent_cpu=$(echo $runtime | awk -F "," '{print $3}' | awk -F ":" '{print $2}') + max_rss=$(echo $runtime | awk -F "," '{print $3}' | awk -F ":" '{print $2}') + avg_rss=$(echo $runtime | awk -F "," '{print $6}' | awk -F ":" '{print $2}') + avg_tot_vm=$(echo $runtime | awk -F "," '{print $7}' | awk -F ":" '{print $2}') + avg_priv_data=$(echo $runtime | awk -F "," '{print $8}' | awk -F ":" '{print $2}') + avg_priv_stack=$(echo $runtime | awk -F "," '{print $9}' | awk -F ":" '{print $2}') + avg_shared_text=$(echo $runtime | awk -F "," '{print $10}' | awk -F ":" '{print $2}') + page_size=$(echo $runtime | awk -F "," '{print $11}' | awk -F ":" '{print $2}') + major_pgfaults=$(echo $runtime | awk -F "," '{print $12}' | awk -F ":" '{print $2}') + minor_pgfaults=$(echo $runtime | awk -F "," '{print $13}' | awk -F ":" '{print $2}') + swaps=$(echo $runtime | awk -F "," '{print $14}' | awk -F ":" '{print $2}') invol_context_switches=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - vol_waits=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - fs_reads=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - fs_writes=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - sock_recv=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - sock_send=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - signals=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') - exit_status=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}') + vol_waits=$(echo $runtime | awk -F "," '{print $15}' | awk -F ":" '{print $2}') + fs_reads=$(echo $runtime | awk -F "," '{print $16}' | awk -F ":" '{print $2}') + fs_writes=$(echo $runtime | awk -F "," '{print $17}' | awk -F ":" '{print $2}') + sock_recv=$(echo $runtime | awk -F "," '{print $18}' | awk -F ":" '{print $2}') + sock_send=$(echo $runtime | awk -F "," '{print $19}' | awk -F ":" '{print $2}') + signals=$(echo $runtime | awk -F "," '{print $20}' | awk -F ":" '{print $2}') + exit_status=$(echo $runtime | awk -F "," '{print $21}' | awk -F ":" '{print $2}') - echo "UPDATE app_exec SET real_secs='$real_secs', kernel_secs='$kernel_secs', user_secs='$user_secs', percent_cpu='$percent_cpu', max_rss='$max_rss', avg_rss='$avg_rss', avg_tot_vm='$avg_tot_vm', avg_priv_data='$avg_priv_data', avg_priv_stack='$avg_priv_stack', avg_priv_stack='$avg_priv_stack', avg_shared_text='$avg_shared_text', page_size='$page_size', major_pgfaults='$major_pgfaults', minor_pgfaults='$minor_pgfaults', swaps='$swaps', invol_context_switches='$invol_context_switches', vol_waits='$vol_waits', fs_reads='$fs_reads', fs_writes='$fs_writes', sock_recv='$sock_recv', sock_send='$sock_send', signals='$signals', exit_status='$exit_status' WHERE app_exec_id='$execute2_id';" + echo "UPDATE app_exec SET real_secs='$real_secs', kernel_secs='$kernel_secs', user_secs='$user_secs', percent_cpu='$percent_cpu', max_rss='$max_rss', avg_rss='$avg_rss', avg_tot_vm='$avg_tot_vm', avg_priv_data='$avg_priv_data', avg_priv_stack='$avg_priv_stack', avg_shared_text='$avg_shared_text', page_size='$page_size', major_pgfaults='$major_pgfaults', minor_pgfaults='$minor_pgfaults', swaps='$swaps', invol_context_switches='$invol_context_switches', vol_waits='$vol_waits', fs_reads='$fs_reads', fs_writes='$fs_writes', sock_recv='$sock_recv', sock_send='$sock_send', signals='$signals', exit_status='$exit_status' WHERE id='$execute2_id';" >> /tmp/$RUNID.sql #echo "INSERT INTO rt_info (app_exec_id, timestamp, cpu_usage, max_phys_mem, max_virt_mem, io_read, io_write) VALUES ('$execute2_id', $timestamp, $cpu_usage, $max_phys_mem, $max_virtual_mem, $io_read_bytes, $io_write_bytes);" >> /tmp/$RUNID.sql # for key in $(echo maxrss walltime systime usertime cpu fsin fsout timesswapped socketrecv socketsent majorpagefaults minorpagefaults contextswitchesinv contextswitchesvol); do From davidk at ci.uchicago.edu Thu Jan 31 11:03:11 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 31 Jan 2013 11:03:11 -0600 (CST) Subject: [Swift-commit] r6205 - in SwiftApps/modis: bin conf Message-ID: <20130131170311.2E17E9CD1D@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-31 11:03:11 -0600 (Thu, 31 Jan 2013) New Revision: 6205 Modified: SwiftApps/modis/bin/assemble2.sh SwiftApps/modis/bin/colormodis.sh SwiftApps/modis/bin/ppmhist.pl SwiftApps/modis/conf/local.cf Log: A few steps closer.. completes successfully on mac Modified: SwiftApps/modis/bin/assemble2.sh =================================================================== --- SwiftApps/modis/bin/assemble2.sh 2013-01-31 13:51:49 UTC (rev 6204) +++ SwiftApps/modis/bin/assemble2.sh 2013-01-31 17:03:11 UTC (rev 6205) @@ -1,11 +1,21 @@ -#! /bin/bash +#!/bin/bash output=$(cd $(dirname $1); echo $(pwd)/$(basename $1)) # get full pathname of output file selected=$(cd $(dirname $2); echo $(pwd)/$(basename $2)) # get full pathname of input file "selected" indir=$(cd $(dirname $3); pwd) # get full pathname of input dir "geos" webdir=$4 -tmp=$(mktemp -d modis.assemble.XXXX) +if [ ! -d $webdir ]; then + mkdir -p $webdir + if [ $? -ne 0 ]; then + echo Unable to create $webdir, quitting + exit 1 + fi +fi + +tmpdir=$PWD/$( basename $( mktemp -d modis.assemble.XXXX ) ) +ls -ld $tmpdir + bindir=$(cd $(dirname $0); pwd) graytile=$bindir/gray.png @@ -15,21 +25,23 @@ cd $indir for h in {0..35}; do - h=$( printf %02d $h ); - for v in {0..17}; do - v=$( printf %02d $v ); - hv=h${h}v${v} - vh=v${v}h${h} - if [ ! -f $hv*png ]; then - cp $graytile $tmp/$vh.png - elif grep $hv $selected >&/dev/null; then -echo "adding border to $hv" - # add red border - convert $hv*png -resize 290x290 $tmp/t.png - convert $tmp/t.png -bordercolor red -border 5x5 $tmp/$vh.png - else - cp $hv*png $tmp/$vh.png - fi + h=$( printf %02d $h ); + + for v in {0..17}; do + v=$( printf %02d $v ); + hv=h${h}v${v} + vh=v${v}h${h} + + if [ ! -f $hv*png ]; then + cp $graytile $tmpdir/$vh.png + elif grep $hv $selected >&/dev/null; then + echo "adding border to $hv" + # add red border + convert $hv*png -resize 290x290 $tmp/t.png + convert $tmp/t.png -bordercolor red -border 5x5 $tmp/$vh.png + else + cp $hv*png $tmp/$vh.png + fi done done Modified: SwiftApps/modis/bin/colormodis.sh =================================================================== --- SwiftApps/modis/bin/colormodis.sh 2013-01-31 13:51:49 UTC (rev 6204) +++ SwiftApps/modis/bin/colormodis.sh 2013-01-31 17:03:11 UTC (rev 6205) @@ -2,16 +2,14 @@ # Return a new modis files with the 0-16 pixel values changed to # colors that reflect the land use of that region. (See legend) # -# usage: colormodis.sh modis.tif recolored.tif +# usage: colormodis.sh modis.ppm.gz recolored.tif # infile=$1 outfile=$2 # Create temp files in current directory since /tmp may not be available -tmp=`mktemp modis.$USER.XXXXXX` -mv $tmp $tmp.tif -tmp=$tmp.tif +tmp=`mktemp modis.$USER.XXXXXX.ppn.gz` map=`mktemp colormap.$USER.XXXXXX` # Create color set @@ -40,6 +38,8 @@ EOF cp $infile $tmp +gunzip $tmp +tmp=$( basename $tmp .gz ) # output logged to stdout/error is ignored by swift for this app() Modified: SwiftApps/modis/bin/ppmhist.pl =================================================================== --- SwiftApps/modis/bin/ppmhist.pl 2013-01-31 13:51:49 UTC (rev 6204) +++ SwiftApps/modis/bin/ppmhist.pl 2013-01-31 17:03:11 UTC (rev 6205) @@ -4,7 +4,7 @@ # Read file my $ppm_filename = shift; -open(PPMFILE, "zcat $ppm_filename|") || die "Unable to open $ppm_filename!\n"; +open(PPMFILE, "gunzip -c $ppm_filename |") || die "Unable to open $ppm_filename!\n"; my @ppm_data = ; close(PPMFILE); Modified: SwiftApps/modis/conf/local.cf =================================================================== --- SwiftApps/modis/conf/local.cf 2013-01-31 13:51:49 UTC (rev 6204) +++ SwiftApps/modis/conf/local.cf 2013-01-31 17:03:11 UTC (rev 6205) @@ -5,8 +5,7 @@ status.mode=provider use.provider.staging=false provider.staging.pin.swiftfiles=false -#throttle.transfers=16 -#throttle.file.operations=16 +#site local JOBTHROTTLE=0.038 #app getlanduse=$PWD/../bin/getlanduse.sh #app analyzelanduse=$PWD/../bin/analyzelanduse2.sh From ketan at ci.uchicago.edu Thu Jan 31 17:56:46 2013 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 31 Jan 2013 17:56:46 -0600 (CST) Subject: [Swift-commit] r6206 - trunk/docs/userguide Message-ID: <20130131235646.2B7FF9CD1D@svn.ci.uchicago.edu> Author: ketan Date: 2013-01-31 17:56:46 -0600 (Thu, 31 Jan 2013) New Revision: 6206 Modified: trunk/docs/userguide/mappers Log: test mod to doc Modified: trunk/docs/userguide/mappers =================================================================== --- trunk/docs/userguide/mappers 2013-01-31 17:03:11 UTC (rev 6205) +++ trunk/docs/userguide/mappers 2013-01-31 23:56:46 UTC (rev 6206) @@ -360,7 +360,7 @@ |======================= |Swift variable|Filename |f|myfile -|f[0]|INVALID +|f [0]|INVALID |f.bar|INVALID |======================= From ketan at ci.uchicago.edu Thu Jan 31 18:43:05 2013 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 31 Jan 2013 18:43:05 -0600 (CST) Subject: [Swift-commit] r6207 - trunk/docs/userguide Message-ID: <20130201004305.77A579CD1D@svn.ci.uchicago.edu> Author: ketan Date: 2013-01-31 18:43:05 -0600 (Thu, 31 Jan 2013) New Revision: 6207 Modified: trunk/docs/userguide/mappers Log: playing with doc Modified: trunk/docs/userguide/mappers =================================================================== --- trunk/docs/userguide/mappers 2013-01-31 23:56:46 UTC (rev 6206) +++ trunk/docs/userguide/mappers 2013-02-01 00:43:05 UTC (rev 6207) @@ -338,7 +338,7 @@ will only process the imports once. Imports may contain anything that is valid in a SwiftScript program, -including code that causes remote execution. +including the code that causes remote execution. Mappers ------- @@ -347,9 +347,9 @@ remote sites for execution or to pass to applications. Swift provides a number of mappers that are useful in common cases. This -section details those standard mappers. For more complex cases, it is +section details those mappers. For more complex cases, it is possible to write application-specific mappers in Java and use them -within a SwiftScript program. +within a Swift script. The single file mapper @@ -488,7 +488,7 @@ concurrent mapper ~~~~~~~~~~~~~~~~~ -concurrent_mapper is almost the same as the simple mapper, except that +The concurrent_mapper is almost the same as the simple mapper, except that it is used to map an output file, and the filename generated will contain an extract sequence that is unique. This mapper is the default mapper for variables when no mapper is specified. @@ -516,12 +516,12 @@ File system mapper ~~~~~~~~~~~~~~~~~~ -filesys_mapper is similar to the simple mapper, but maps a file or a +The filesys_mapper is similar to the simple mapper, but maps a file or a list of files to an array. Each of the filename is mapped as an element in the array. The order of files in the resulting array is not defined. TODO: note on difference between location as a relative vs absolute path -wrt staging to remote location - as mihael said: It's because you +w.r.t. staging to remote location - as mihael said: It's because you specify that location in the mapper. Try location="." instead of location="/sandbox/..." From ketan at ci.uchicago.edu Thu Jan 31 19:04:17 2013 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 31 Jan 2013 19:04:17 -0600 (CST) Subject: [Swift-commit] r6208 - trunk/docs/userguide Message-ID: <20130201010417.1E7919CD1D@svn.ci.uchicago.edu> Author: ketan Date: 2013-01-31 19:04:16 -0600 (Thu, 31 Jan 2013) New Revision: 6208 Modified: trunk/docs/userguide/mappers Log: padding Modified: trunk/docs/userguide/mappers =================================================================== --- trunk/docs/userguide/mappers 2013-02-01 00:43:05 UTC (rev 6207) +++ trunk/docs/userguide/mappers 2013-02-01 01:04:16 UTC (rev 6208) @@ -392,6 +392,7 @@ |location|A directory that the files are located. |prefix|The prefix of the files |suffix|The suffix of the files, for instance: ".txt" +|padding| The number of digits used to uniquely identify the mapped file. This is an optional parameter which defaults to 4. |pattern|A UNIX glob style pattern, for instance: "\*foo*" would match all file names that contain foo. When this mapper is used to specify output filenames, pattern is ignored. @@ -438,7 +439,7 @@ } } -messagefile outfile[] ; +messagefile outfile[] ; outfile[0] = greeting("hello"); outfile[1] = greeting("middle"); @@ -448,9 +449,9 @@ [options="header, autowidth"] |======================= |Swift variable|Filename -|outfile[0]|baz0000.txt -|outfile[1]|baz0001.txt -|outfile[2]|baz0002.txt +|outfile[0]|baz00.txt +|outfile[1]|baz01.txt +|outfile[2]|baz02.txt |======================= simple_mapper can be used to map structures. It will map the name of From wilde at ci.uchicago.edu Tue Jan 1 18:14:04 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Wed, 02 Jan 2013 00:14:04 -0000 Subject: [Swift-commit] r6117 - SwiftApps/SciColSim Message-ID: <20130102001403.C763F9CCFC@svn.ci.uchicago.edu> Author: wilde Date: 2013-01-01 18:14:03 -0600 (Tue, 01 Jan 2013) New Revision: 6117 Added: SwiftApps/SciColSim/optimizer-dispatch.cpp Modified: SwiftApps/SciColSim/optimizer.cpp Log: Remove Grand Central Dispatch code from main optimizer version, and reformat code. Save version with GCD as optimizer-dispatch.cpp. Added: SwiftApps/SciColSim/optimizer-dispatch.cpp =================================================================== --- SwiftApps/SciColSim/optimizer-dispatch.cpp (rev 0) +++ SwiftApps/SciColSim/optimizer-dispatch.cpp 2013-01-02 00:14:03 UTC (rev 6117) @@ -0,0 +1,1755 @@ +// +// main.cpp +// optimizer +// +// Created by Andrey Rzhetsky on 4/11/11. +// Copyright 2011 University of Chicago. All rights reserved. +// + +// Select OpenMP or Grand Central Dispatch for multithreading: + +#define MAXNworkers 24 +int Nworkers=MAXNworkers; + +#define NUniverse 24 + +// Add operation code to enable existing code to be used at lower level from Swift scripts: + +char operation = 'n'; // n: normal; m: do one multi_loss (with n_reruns). + // Not used: a: analyze and generate next annealing parameter set. g: tbd + +unsigned initSeed = 0; + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef P_DISPATCH +#include +#endif + +#include + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define BOOST_MATH_OVERFLOW_ERROR_POLICY ignore_error +#define BOOST_MATH_DISCRETE_QUANTILE_POLICY real +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define INT_INFINITY 2147483647 + +#define FIX_VARIABLES 1 + +using namespace boost; +using namespace std; +using namespace boost::numeric::ublas; + +static int max_dist=0; + +typedef boost::adjacency_matrix Graph; +typedef std::pair Edge; +typedef boost::graph_traits GraphTraits; +typedef boost::numeric::ublas::triangular_matrix prob; +typedef boost::numeric::ublas::triangular_matrix pathlength; +typedef boost::graph_traits::vertex_descriptor vertex_descriptor; + +namespace std { + using ::time; +} + +static int var_fixed[5] = {1, 0, 1, 1, 0}; + +typedef boost::minstd_rand base_generator_type; +typedef adjacency_list < listS, vecS, directedS, +no_property, property < edge_weight_t, int > > graph_t; +typedef graph_traits < graph_t >::vertex_descriptor vertex_descriptor; +typedef graph_traits < graph_t >::edge_descriptor edge_descriptor; + +double max_loss = -1.0; // intialized to "unset"; set by first Universe constructor to run. + +//================================================ +string strDouble(double number) +{ + stringstream ss;//create a stringstream + ss << number;//add number to the stream + return ss.str();//return a string with the contents of the stream +} + +//================================================ + +double gaussian(double sigma) +{ + double GaussNum = 0.0; + int NumInSum = 10; + for(int i = 0; i < NumInSum; i++) + { + GaussNum += ((double)rand()/(double)RAND_MAX - 0.5); + } + GaussNum = GaussNum*sqrt((double)12/(double)NumInSum); + + + return GaussNum; + +} + + + +//================================================= +double diffclock(clock_t clock1,clock_t clock2) +{ + double diffticks=clock1-clock2; + double diffms=(diffticks)/CLOCKS_PER_SEC; + return diffms; +} + +//================================================ +//================================================================ +double get_new_x(double x, double dx){ + + double new_x; + // boost::variate_generator > uni(generator, uni_dist); + double r = rand()/(double)(pow(2.,31)-1.); + + if (r > 0.5){ + new_x = x + rand()*dx/(double)(pow(2.,31)-1.); + } else { + new_x = x - rand()*dx/(double)(pow(2.,31)-1.); + } + + return new_x; + +} + + +//=============================================== +string string_wrap(string ins, int mode){ + + std::ostringstream s; + + switch(mode){ + case 0: + s << "\033[1;29m" << ins << "\033[0m"; + break; + case 1: + s << "\033[1;34m" << ins << "\033[0m"; + break; + case 2: + s << "\033[1;44m" << ins << "\033[0m"; + break; + case 3: + s << "\033[1;35m" << ins << "\033[0m"; + break; + case 4: + s << "\033[1;33;44m" << ins << "\033[0m"; + break; + case 5: + s << "\033[1;47;34m" << ins << "\033[0m"; + break; + case 6: + s << "\033[1;1;31m" << ins << "\033[0m"; + break; + case 7: + s << "\033[1;1;33m" << ins << "\033[0m"; + break; + case 8: + s << "\033[1;1;43;34m" << ins << "\033[0m"; + break; + case 9: + s << "\033[1;1;37m" << ins << "\033[0m"; + break; + case 10: + s << "\033[1;30;47m" << ins << "\033[0m"; + break; + default: + s << ins; + } + + return s.str(); +} + + +//=============================================== +string wrap_double(double val, int mode){ + + std::ostringstream s; + s << string_wrap(strDouble(val),mode); + + return s.str(); +} + + + +//=============================================== +const +string i2string(int i){ + + std::ostringstream s; + s << "worker" + << lexical_cast(i); + + return s.str(); + +} + +//=============================================== +char* i2char(int i){ + + std::ostringstream s; + s << "worker" + << lexical_cast(i); + + char* a=new char[s.str().size()+1]; + memcpy(a,s.str().c_str(), s.str().size()); + + return a; +} + +//================================================ +class Universe { + +private: + + double alpha_i; + double alpha_m; + double beta; + double gamma; + double delta; + + double TargetNovelty; + double CumulativeRelativeLoss; + double CRLsquare; + string id; + + + int N_nodes; + int M_edges; + + int N_epochs; + int N_steps; + int N_repeats; + + int current_epoch; + double current_loss; + int current_repeat; + double current_novelty; + + int mode_identify_failed; + int verbose_level; // 0 is silent, higher is more + + double k_max; + + graph_t Full_g; + + double **Prob; + double **Tried; + double **Dist; + double **Final; + double **EdgeIndex; + double *Rank; + + base_generator_type generator; + boost::uniform_real<> uni_dist; + boost::geometric_distribution geo; + +public: + + + + //====== Constructor ====== + Universe(const std::string FileToOpen, int Epochs, int Steps, int Repeats, int identify_failed, double target, const std::string idd) + { + //typedef array_type2::index index2; + + + std::ifstream inFile; + //string line; + + //------------------------------- + + base_generator_type gene(42u); + generator = gene; +// generator.seed(static_cast(std::time(0))); + + + if ( initSeed != 0.0 ) { + generator.seed(static_cast(initSeed)); + } + else { + timeval t; + gettimeofday(&t, NULL); + generator.seed(static_cast(t.tv_usec)); + } + + + + + + boost::uniform_real<> uni_d(0,1); + uni_dist = uni_d; + + //-------------------------------- + + int i, k; + int x, y; + Edge* edge_array_mine; + int num_arcs_mine, num_nodes_mine; + int* weights_mine; + + TargetNovelty = target; + CumulativeRelativeLoss = 0.; + CRLsquare = 0.; + + + N_epochs = Epochs; + N_steps = Steps; + N_repeats = Repeats; + + current_epoch = 0; + current_loss = 0.; + current_repeat = 0; + + id = idd; + + verbose_level = 1; + + mode_identify_failed = identify_failed; + + + //------------------------------- + // The first pass though file with the graph + inFile.open(FileToOpen.c_str()); + if (inFile.fail()) { + cout << "Unable to open file"; + exit(1); // terminate with error + }else { + + if (verbose_level > 2){ + std::cout << " Opened <" << FileToOpen << ">"<> x; + inFile >> y; + + if (verbose_level > 2){ + std::cout << " x: " << x; + std::cout << " y: " << y << std::endl; + } + + if (i==0){ + N_nodes=x; + M_edges=y; + break; + } + i++; + + + } + inFile.close(); + + if (verbose_level == 2){ + std::cout << N_nodes << " nodes, " << M_edges << " edges"< 2){ + std::cout << " Opened <" << FileToOpen << ">"<> x && inFile >>y) { + if (i > 0) { + Final[x][y]=1.; + Final[y][x]=1.; + + + if (verbose_level == 2){ + std::cout << "."; + } + } + i++; + + } + if (verbose_level == 2){ + std::cout << std::endl; + } + inFile.close(); + + k=0; + for (int i=0; i 0.){ + EdgeIndex[i][j]=k; + k++; + } + } + } + + + + //+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + // create graph -- hopefully, we can keep it, just modifying edge weights + + + edge_array_mine = new Edge[2*M_edges]; + num_arcs_mine = 2*M_edges; + num_nodes_mine = N_nodes; + weights_mine = new int[2*M_edges]; + for (int i=0; i<2*M_edges; i++){ weights_mine[i]=1;} + + k=0; + for(int i=0; i0.){ + edge_array_mine[2*k] =Edge(i,j); + edge_array_mine[2*k+1]=Edge(j,i); + k++; + } + } + } + graph_t g(edge_array_mine, edge_array_mine + num_arcs_mine, weights_mine, num_nodes_mine); + + Full_g = g; + delete edge_array_mine; + delete weights_mine; + + //=========================================================================== + std::vector p(num_edges(Full_g)); + std::vector d(num_edges(Full_g)); + edge_descriptor s; + boost::graph_traits::vertex_descriptor u, v; + + for (int i=0; i 0.){ + u = vertex(i, Full_g); + v = vertex(j, Full_g); + remove_edge(u,v,Full_g); + remove_edge(v,u,Full_g); + + } + } + } + if(max_loss <= 0.0) set_max_loss(N_nodes, M_edges, TargetNovelty); + } + + + //===================================================================== + int sample_failed_number(double pfail){ + + //boost::geometric_distribution geo(pfail); + //boost::variate_generator > geom(generator, geo); + + double r, u, g; + + r=0.; + for(int i=0; i=3){ + std::cout << id << " failed " << r << std::endl; + } + return (int) round(r); // FIXME: Andrey: please verify that round() is correct. + + } + + //============================================= + double get_target(void){ + return TargetNovelty; + } + + //============================================= + void set_target(double target){ + TargetNovelty=target; + } + + //============================================= + int sample(){ + + //boost::variate_generator > uni(generator, uni_dist); + // double r = uni(), Summa = 0.; + + + + double r = rand(), Summa = 0.; + r /= (double)(pow(2.,31)-1.); + int result = 0; + int finished = 0; + + if (verbose_level==4){ + std::cout << id << " sampled " << r << std::endl; + } + + for(int i=0; i r){ + + Tried[i][j]+=1.; + + if (Final[i][j] > 0.){ + result = 1; + } + finished = 1; + } + } + } + + return result; + + } + + //=============================== + void update_current_graph(void){ + + std::vector p(num_edges(Full_g)); + std::vector d(num_edges(Full_g)); + edge_descriptor s; + boost::graph_traits::vertex_descriptor u, v; + + //property_map::type weightmap = get(edge_weight, Full_g); + for (int i=0; i 0. && Tried[i][j]>0){ + //s = edge(i, j, Full_g); + boost::graph_traits::edge_descriptor e1,e2; + bool found1, found2; + u = vertex(i, Full_g); + v = vertex(j, Full_g); + tie(e1, found1) = edge(u, v, Full_g); + tie(e2, found2) = edge(v, u, Full_g); + if (!found1 && !found2){ + add_edge(u,v,1,Full_g); + add_edge(v,u,1,Full_g); + } + + } + } + + } + } + + //=============================== + void update_distances(void){ + // put shortest paths to the *Dist[][] + std::vector p(num_vertices(Full_g)); + std::vector d(num_vertices(Full_g)); + vertex_descriptor s; + + + // put shortest paths to the *Dist[][] + for (int j=0; j 0.){ + s = vertex(j, Full_g); + dijkstra_shortest_paths(Full_g, s, predecessor_map(&p[0]).distance_map(&d[0])); + + //std::cout <<" Vertex "<< j << std::endl; + graph_traits < graph_t >::vertex_iterator vi, vend; + + for (boost::tie(vi, vend) = vertices(Full_g); vi != vend; ++vi) { + + if (p[*vi]!=*vi){ + Dist[*vi][j]=d[*vi]; + Dist[j][*vi]=d[*vi]; + + if ( (int)round(Dist[*vi][j]>max_dist)) { + // FIXME: Andrey: please verify that (int) cast is correct. Do we need to round()? + // also, the indent on this iff statement was way off - + // perhaps due to space v. tab? + max_dist=(int)round(Dist[*vi][j]); + } + + + } else { + Dist[*vi][j]=-1.; + Dist[j][*vi]=-1.; + } + } + } + + } + + + } + + //====================================================== + void update_ranks(void){ + + for(int i=0; i0. && Final[i][j] >0.){ + Rank[i]++; + Rank[j]++; + } + } + } + + } + + //==================================================================== + void set_world(double a_i, double a_m, double b, double g, double d){ + + alpha_i=a_i; + alpha_m=a_m; + gamma=g; + beta=b; + delta=d; + + } + + //==================================================================== + void reset_world(){ + + //==================================================== + std::vector p(num_edges(Full_g)); + std::vector d(num_edges(Full_g)); + edge_descriptor s; + boost::graph_traits::vertex_descriptor u, v; + cout << "reset_world: start id=" << id << endl; + + for (int i=0; i 0. && Tried[i][j] > 0){ + u = vertex(i, Full_g); + v = vertex(j, Full_g); + remove_edge(u,v,Full_g); + remove_edge(v,u,Full_g); + + } + } + } + + //================================================== + + current_loss=0; + current_epoch=0; + current_repeat++; + current_novelty=0; + + for(int i = 0; i < N_nodes; ++i) { + Rank[i]=0.; + for(int j = 0; j < N_nodes; ++j) { + Prob[i][j]=0.; + Dist[i][j]=-1.; + Tried[i][j]=0.; + } + } + cout << "reset_world: end id=" << id << endl; + } + + + //============================================== + void show_parameters(void){ + + std::cout << "Parameters: " + << alpha_i << " " + << alpha_m << " | " + << beta << " " + << gamma << " | " + << delta << std::endl; + + } + + + + //=============================================== + string file_name(){ + + std::ostringstream s; + s << "world_" + << lexical_cast(alpha_i) << "_" + << lexical_cast(alpha_m) << "_" + << lexical_cast(beta) << "_" + << lexical_cast(gamma) << "_" + << lexical_cast(delta) << "_" + << lexical_cast(N_epochs) << "_" + << lexical_cast(N_steps) << "_" + << lexical_cast(N_repeats) << ".txt"; + + return s.str(); + + } + + + + + //================================================= + void set_verbose(int verbose){ + + verbose_level = verbose; + } + + + //============================================================= + void update_probabilities(void){ + + + //========================= + // Compute sampling probabilities + // first pass: \xi_i,j + for(int i=0; i 0.){ + + double k = Dist[i][j]; + if (k >= k_max){ + k = k_max-1; + } + + bg = beta * log(k/k_max) + gamma * log(1. - k/k_max); + + } else { + bg = delta; + } + + Prob[i][j] = exp(Prob[i][j] + bg); + } + } + + + // second pass: sum + double Summa = 0.; + + for(int i=0; i0. && Final[i][j]>0.){ + novel+=1.; + } + } + } + + } + //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + else { + + double pfail=0.; + int n_failed; + //, n_check = 0; + + for(int i=0; i0. && Final[i][j]>0.){ + novel+=1.; + } + } + } + } + + current_novelty = novel; + + + //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + if (verbose_level == 2){ + std::cout << (current_repeat+1) << " epoch=" << (current_epoch+1) + + << " cost=" << cost + << " novel=" << novel + << " rel_loss=" << cost/novel + << std::endl; + } + + current_epoch++; + } + + + //====== Destructor ====== + ~Universe(){ + + delete_2Dmatrix(Final, N_nodes); + delete_2Dmatrix(Dist, N_nodes); + delete_2Dmatrix(Tried, N_nodes); + delete_2Dmatrix(Prob, N_nodes); + delete_2Dmatrix(EdgeIndex, N_nodes); + delete_1Dmatrix(Rank); + } + + //================================================ + // Allocate memory + double** allocate_2Dmatrix(int N, int M) + { + double **pointer; + + if (verbose_level == 2){ + std::cout<< "["< 0){ + + pointer = new double[N]; + + }else { + + pointer = NULL; + } + + return pointer; + + } + + //============================================== + // De-Allocate memory to prevent memory leak + void delete_2Dmatrix(double **pointer, int N){ + + if (pointer != NULL){ + + for (int i = 0; i < N; ++i){ + delete [] pointer[i]; + } + delete [] pointer; + } + } + //==================== + void delete_1Dmatrix(double *pointer){ + + delete [] pointer; + } + + //=========================================== + double get_rel_loss(){ + + return CumulativeRelativeLoss ; + } + + //=========================================== + double get_rel_loss_err(){ + + return CRLsquare ; + } + + + + //================================================================================== + void evolve_to_target_and_save(int istart, int iend, double* storage, int* counters){ + + //double ALOT=100000000000.; + double ALOT= 3.0 * max_loss; // Per Andrey, 2012.12.29 + + double ratio = 0.0; + int check = 0; + + reset_world(); + + for (int k = istart; k < iend; k++){ + cout << "evolve_ttas: k=" << k << endl; + + for(int i=0; i< N_epochs && current_novelty < TargetNovelty; i++){ + + if(ratio < max_loss){ + update_world(); + } + else{ + cout << "k=" << k << " i=" << i << " ratio=" << ratio << " max_loss=" << max_loss << endl; + break; + } + ratio = current_loss/current_novelty; + } + if( ratio < max_loss ) + storage[k]=current_loss/current_novelty; + else + storage[k]=ALOT; + counters[k]=1; + reset_world(); + } + } + + //============================================== + int get_reruns(void){ + return N_repeats; + } + + //============================================== + double get_parameter(int i){ + + switch(i){ + case 0: + return alpha_i; + case 1: + return alpha_m; + case 2: + return beta; + case 3: + return gamma; + case 4: + return delta; + default: + + std::cout << "Erroneous parameter id!!!!\n\n\n"; + return 0.; + } + } + + + //============================================== + void evolve_to_target(){ + + reset_world(); + if (beta < -1. || gamma < -1.){ + CumulativeRelativeLoss = 100000000000.; + CRLsquare = 0.; + return; + } + + + for (int k=0; k< N_repeats; k++){ + + + for(int i=0; i 4) {return 0;} + + else { + + switch(position){ + case 0: + alpha_i=value; + return 1; + case 1: + alpha_m=value; + return 1; + case 2: + beta=value; + return 1; + case 3: + gamma=value; + return 1; + case 4: + delta=value; + return 1; + } + + } + + return 0; + } + +//-------- + +void set_max_loss(long long v, long long e, long long t) { + + cout << "set_max_loss v=" << v << " e=" << e << " t=" << t << " max_loss=" << max_loss << endl; + + // Based on formulas from random_analytical.pdf + + double sum, ERLt, VarRLt; + + // (9): E[RLt] = (1/t) * SUM(i=0 to T of: (V*(V-1) / (2 * (E-i)) + + sum = 0; + for(int i=0;i multi_loss(dispatch_group_t group, + Universe* un[], + dispatch_queue_t* CustomQueues, + double* Results, + int* Counters, + double* params){ +#else +std::pair multi_loss(Universe* un[], + double* Results, + int* Counters, + double* params){ + +#endif + + int N = un[0]->get_reruns(); + int step = (int)floor((double)N/(double)(Nworkers)); // FIXME: Andrey: please check change in cast grouping and use of floor + int istart=0; + int iend = istart+step; + + double Loss=0., LossSquare=0.; + + timeval startTime, endTime; + double elapsedTime; + gettimeofday(&startTime, NULL); + + if ( N > NUniverse ) { + std::cerr << "Error: Number of reruns=" << N << " is greater than max NUniverse=" << NUniverse << "\n"; + exit(1); + } + + for(int i=0; iset_parameter(params[j],j); + } + } + +#ifdef P_DISPATCH + for(int i=0; ievolve_to_target_and_save(istart, iend, Results, Counters); + un[i]->evolve_to_target_and_save(i*step, min((i+1)*step,N), Results, Counters); + }); + + std::cout << "queued: i=" << i << " N=" << N << " istart=" << istart << " iend=" << iend << "\n"; + // istart += step; + // iend = min(istart+step,N); + + } + dispatch_group_wait(group, DISPATCH_TIME_FOREVER); + //dispatch_release(group); +#else + int i; + + // Execute rerun loop in parallel + + #pragma omp parallel for private (i) + for(i=0; ievolve_to_target_and_save(i, i+1, Results, Counters); + } +#endif + +/* + for (int i=0; i > uni(generator, uni_dist); + + // set up parameter for annealing + + x[0]=params0[0]; + x[1]=params0[1]; + x[2]=params0[2]; + x[3]=params0[3]; + x[4]=params0[4]; + + for(int i=0;i<5;i++){ + dx[i] = starting_jump; + } + + // establish the current value + std::pairRes; + +#ifdef P_DISPATCH + Res = multi_loss(group, un, CustomQueues, Results, Counters, x); +#else + Res = multi_loss( un, Results, Counters, x); +#endif + std::cout << "Returned from initial multi_loss:" << std::endl; + std::cout << Res.first << " +- " << Res.second << std::endl; + + if ( operation == 'm' ) { + FILE *f; + int N = un[0]->get_reruns(); + + f = fopen("multi_loss.data","w"); + for(int i=0; iset_parameter(x_tmp, j); + } + +#ifdef P_DISPATCH + Res = multi_loss(group, un, CustomQueues, Results, Counters, x); +#else + Res = multi_loss( un, Results, Counters, x); +#endif + std::cout << Res.first << " +- " << Res.second << std::endl; + + ratio = min(1.,exp(-(Res.first-curr_x)/temperature)); + r = rand()/(double)(pow(2.,31)-1.); + std::cout << r << " vs " << ratio << std::endl; + + double ALOT=100000000000.; + + if (Res.first < ALOT) + { + ofstream filestr; + + filestr.open ("best_opt_some.txt", ofstream::app); + + // >> i/o operations here << + filestr + + << "N, " << i << ", " << j << ", " << dx[j] << ", " << rejection[j] << ", |, " // FIXME: MW-DEBUGGING + + << un[0]->get_target() << ", " + << Res.first + << ", " << un[0]->get_parameter(0) + << ", " << un[0]->get_parameter(1) + << ", " << un[0]->get_parameter(2) + << ", " << un[0]->get_parameter(3) + << ", " << un[0]->get_parameter(4) << ", " << Res.second << ",\n"; + + filestr.close(); + + + filestr.open ("max_dist.txt", ofstream::app); + + // >> i/o operations here << + filestr << max_dist << ",\n"; + + filestr.close(); + + FILE *bf; + bf = fopen("bestdb.txt","a"); + fprintf(bf, "N %2d %2d %10.5f %5.2f | %5.2f %10.5f [ %5.2f %5.2f %10.5f %10.5f %10.5f ] %10.5f\n", i, j, dx[j], rejection[j], + un[0]->get_target(), + Res.first, + un[0]->get_parameter(0), + un[0]->get_parameter(1), + un[0]->get_parameter(2), + un[0]->get_parameter(3), + un[0]->get_parameter(4), + Res.second); + fclose(bf); + } + + + if (r > ratio){ + + std::cout << " "<< (i+1) << ","<< (j) + <<" "<< (i+1) << " Did not accept " + << x_tmp << "(" << j << ")" << std::endl; + std::cout << un[0]->get_parameter(0) + << " " << un[0]->get_parameter(1) + << " " << un[0]->get_parameter(2) + << " " << un[0]->get_parameter(3) + << " " << un[0]->get_parameter(4) << " " << std::endl; + + x[j]=x_hold; + for(int w=0; wset_parameter(x[j], j); + } + + + //set_parameter(x[j], j); + rejection[j]+=1.; + } + + else { + + curr_x = Res.first; + curr_err = Res.second; + x[j] = x_tmp; + + for(int w=0; wset_parameter(x[j], j); + } + + std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8) + << wrap_double(rejection[0],2) << " " + << wrap_double(rejection[1],7) << " " + << wrap_double(rejection[2],5) << " " + << wrap_double(rejection[3],9) << " " + << wrap_double(rejection[4],6) << " " + << std::endl << std::endl; + + std::cout << " "<< (i+1) <<","<< (j) + <<" " + << string_wrap((string) "***** Did accept! ", 3) + << wrap_double(un[0]->get_parameter(0),2) << " " + << wrap_double(un[0]->get_parameter(1),7) << " " + << wrap_double(un[0]->get_parameter(2),5) << " " + << wrap_double(un[0]->get_parameter(3),9) << " " + << wrap_double(un[0]->get_parameter(4),6) << " " + << std::endl << std::endl; + + + + } + //........................................................ + + } + } + + } + +} + + + +//================================================ +int +main(int argc, char* argv[]) +{ + + double params0[6] = {0., 0., 0., 0., 0., 0.2}, target=50., range; + string par_names0[6] = {"alpha_i", "alpha_m", "beta", "gamma", "delta", "target"}; + string par_names1[4] = {"n_epochs", "n_steps", "n_reruns", "range"}; + string par_names2[5] = {"T_start", "T_end", "Annealing_steps","Target_rejection","Starting_jump"}; + string par_names3[5] = {"FREEZE_alpha_i", "FREEZE_alpha_m", "FREEZE_beta", "FREEZE_gamma", "FREEZE_delta"}; + string par_names4[3] = {"Operation", "Nworkers", "initSeed"}; + + int params1[4] = {300, 50, 1000, 10}; + int params3[5] = { 0, 0, 0, 0, 0}; + + // temperature_start, temperature_end, annealing_steps target_rejection Starting_jump + double params2[5] = {1, 0.001, 100, 0.3, 1.5}; + + int verbose_level = 2; + const std::string one="one", two="two"; + static Universe* un[NUniverse]; +#ifdef P_DISPATCH + static dispatch_queue_t CustomQueues[MAXNworkers]; +#endif + static double* Results; + static int* Counters; + + timeval t1, t2; + double elapsedTime; + // start timer + gettimeofday(&t1, NULL); + + + if (argc < 8) { + std::cout << "Usage: super_optimizer alpha_i alpha_m beta gamma delta target_innov [n_epochs n_steps n_reruns] [range] [verbose_level]\n"; + std::cout << " [T_start T_end Annealing_steps Target_rejection Starting_jump]\n"; + std::cout << " [FREEZE_alpha_i FREEZE_alpha_m FREEZE_beta FREEZE_gamma FREEZE_delta]\n"; + + system("pwd"); + + + return(1); + } + else { + for (int nArg=0; nArg < argc; nArg++){ + //std::cout << nArg << " " << argv[nArg] << std::endl; + if (nArg > 0 && nArg < 7){ + params0[nArg-1]= atof(argv[nArg]); + std::cout << par_names0[nArg-1] << ": " << params0[nArg-1] << std::endl; + } + if (nArg > 6 && nArg < 11){ + params1[nArg-7]= atoi(argv[nArg]); + std::cout << par_names1[nArg-7] << ": " << params1[nArg-7] << std::endl; + } + if (nArg == 11){ + verbose_level = atoi(argv[nArg]); + std::cout << "verbose level: " << verbose_level << std::endl; + } + if (nArg > 11 && nArg < 17){ + params2[nArg-12]= atof(argv[nArg]); + std::cout << par_names2[nArg-12] << ": " << params2[nArg-12] << std::endl; + } + if (nArg > 16 && nArg < 22){ + params3[nArg-17]= (int)round(atof(argv[nArg])); // FIXME: Andrey: please verify that round() is correct. + var_fixed[nArg-17]= (int)round(atof(argv[nArg])); // FIXME: ditto + std::cout << par_names3[nArg-17] << ": " << var_fixed[nArg-17] << std::endl; + } + if (nArg == 22 ){ + operation = *argv[nArg]; + std::cout << par_names4[0] << ": " << operation << std::endl; + } + if (nArg == 23 ){ + Nworkers = atoi(argv[nArg]); + std::cout << par_names4[1] << ": " << Nworkers << std::endl; + } + if (nArg == 24 ){ + initSeed = atoi(argv[nArg]); + std::cout << par_names4[2] << ": " << initSeed << std::endl; + } + + + } + + } + + for (int j=0; j<5; j++){ + + cout << j << " | " << var_fixed[j] << " (fixed) \n"; + } + + target=params0[5]; + range = (double)params1[3]; + int identify_failed = 0; + char* filename= (char *)"movie_graph.txt"; + int n_ep=params1[0], n_st=params1[1], n_rep=params1[2]; + + //............................... + + for(int i=0; i 0){ + + Results = new double[n_rep]; + Counters = new int[n_rep]; + + }else { + + Results = NULL; + Counters = NULL; + std::cout << " Number of reruns should be positive! " << std::endl; + return 0; + + } + //............................... + //srand(time(0)); + //srandomdev(); + + if ( initSeed != 0.0 ) { + srand(initSeed); + } + else { + timeval t; + gettimeofday(&t, NULL); + srand(t.tv_usec); + } + + { + double r=0; + for (int j=0; j<100; j++){ + + + + r = rand()/(double)(pow(2.,31)-1.); + std::cout << r << " "; + } + std::cout << "\n"; + } + //random initiation of starting parameters + + if (range > 0.){ + + for (int i=0; i < 5; i++){ + + if (params0[i]==-100.){ + + double r1 = (rand()/(double)(pow(2.,31)-1.)); + double r2 = (rand()/(double)(pow(2.,31)-1.)); + double sign = 1.; + + if(r1 > 0.5){ + sign=-1.; + } + + params0[i] = sign*r2*range; + + std::cout << par_names0[i] << ": " << params0[i] << std::endl; + } + } + + } + + + double T_start=params2[0], T_end=params2[1], Target_rejection=params2[3], starting_jump=params2[4]; + int Annealing_repeats = (int) params2[2]; + +#ifdef P_DISPATCH + dispatch_group_t group = dispatch_group_create(); + + //............................. + multi_annealing(group, un, CustomQueues, T_start, T_end, Target_rejection, Annealing_repeats, + starting_jump, Results, Counters, params0, Annealing_repeats); + + //dispatch_group_wait(group, DISPATCH_TIME_FOREVER); + dispatch_release(group); + //............................. +#else + multi_annealing( un, T_start, T_end, Target_rejection, Annealing_repeats, + starting_jump, Results, Counters, params0, Annealing_repeats); +#endif + + + // stop timer + gettimeofday(&t2, NULL); + + // compute and print the elapsed time in millisec + elapsedTime = (t2.tv_sec - t1.tv_sec) * 1000.0; // sec to ms + elapsedTime += (t2.tv_usec - t1.tv_usec) / 1000.0; // us to ms + elapsedTime /= 1000.; + cout << "\n*** optimizer completed, elapsed time=" << elapsedTime << " seconds " << elapsedTime/60. << " minutes)\n\n"; + + //..................... + + for(int i=0; i 0){ + + delete [] Results; + delete [] Counters; + + } + + return 0; + + + +} + Modified: SwiftApps/SciColSim/optimizer.cpp =================================================================== --- SwiftApps/SciColSim/optimizer.cpp 2013-01-01 23:41:00 UTC (rev 6116) +++ SwiftApps/SciColSim/optimizer.cpp 2013-01-02 00:14:03 UTC (rev 6117) @@ -33,13 +33,8 @@ #include #include -#ifdef P_DISPATCH -#include -#endif - #include - #include #include #include @@ -85,14 +80,14 @@ typedef boost::graph_traits::vertex_descriptor vertex_descriptor; namespace std { - using ::time; + using ::time; } static int var_fixed[5] = {1, 0, 1, 1, 0}; typedef boost::minstd_rand base_generator_type; typedef adjacency_list < listS, vecS, directedS, -no_property, property < edge_weight_t, int > > graph_t; + no_property, property < edge_weight_t, int > > graph_t; typedef graph_traits < graph_t >::vertex_descriptor vertex_descriptor; typedef graph_traits < graph_t >::edge_descriptor edge_descriptor; @@ -101,25 +96,25 @@ //================================================ string strDouble(double number) { - stringstream ss;//create a stringstream - ss << number;//add number to the stream - return ss.str();//return a string with the contents of the stream + stringstream ss;//create a stringstream + ss << number;//add number to the stream + return ss.str();//return a string with the contents of the stream } //================================================ double gaussian(double sigma) { - double GaussNum = 0.0; - int NumInSum = 10; - for(int i = 0; i < NumInSum; i++) - { - GaussNum += ((double)rand()/(double)RAND_MAX - 0.5); - } - GaussNum = GaussNum*sqrt((double)12/(double)NumInSum); + double GaussNum = 0.0; + int NumInSum = 10; + for(int i = 0; i < NumInSum; i++) + { + GaussNum += ((double)rand()/(double)RAND_MAX - 0.5); + } + GaussNum = GaussNum*sqrt((double)12/(double)NumInSum); - return GaussNum; + return GaussNum; } @@ -128,26 +123,26 @@ //================================================= double diffclock(clock_t clock1,clock_t clock2) { - double diffticks=clock1-clock2; - double diffms=(diffticks)/CLOCKS_PER_SEC; - return diffms; + double diffticks=clock1-clock2; + double diffms=(diffticks)/CLOCKS_PER_SEC; + return diffms; } //================================================ //================================================================ double get_new_x(double x, double dx){ - double new_x; - // boost::variate_generator > uni(generator, uni_dist); - double r = rand()/(double)(pow(2.,31)-1.); + double new_x; + // boost::variate_generator > uni(generator, uni_dist); + double r = rand()/(double)(pow(2.,31)-1.); - if (r > 0.5){ - new_x = x + rand()*dx/(double)(pow(2.,31)-1.); - } else { - new_x = x - rand()*dx/(double)(pow(2.,31)-1.); - } + if (r > 0.5){ + new_x = x + rand()*dx/(double)(pow(2.,31)-1.); + } else { + new_x = x - rand()*dx/(double)(pow(2.,31)-1.); + } - return new_x; + return new_x; } @@ -155,57 +150,57 @@ //=============================================== string string_wrap(string ins, int mode){ - std::ostringstream s; + std::ostringstream s; - switch(mode){ - case 0: - s << "\033[1;29m" << ins << "\033[0m"; - break; - case 1: - s << "\033[1;34m" << ins << "\033[0m"; - break; - case 2: - s << "\033[1;44m" << ins << "\033[0m"; - break; - case 3: - s << "\033[1;35m" << ins << "\033[0m"; - break; - case 4: - s << "\033[1;33;44m" << ins << "\033[0m"; - break; - case 5: - s << "\033[1;47;34m" << ins << "\033[0m"; - break; - case 6: - s << "\033[1;1;31m" << ins << "\033[0m"; - break; - case 7: - s << "\033[1;1;33m" << ins << "\033[0m"; - break; - case 8: - s << "\033[1;1;43;34m" << ins << "\033[0m"; - break; - case 9: - s << "\033[1;1;37m" << ins << "\033[0m"; - break; - case 10: - s << "\033[1;30;47m" << ins << "\033[0m"; - break; - default: - s << ins; - } + switch(mode){ + case 0: + s << "\033[1;29m" << ins << "\033[0m"; + break; + case 1: + s << "\033[1;34m" << ins << "\033[0m"; + break; + case 2: + s << "\033[1;44m" << ins << "\033[0m"; + break; + case 3: + s << "\033[1;35m" << ins << "\033[0m"; + break; + case 4: + s << "\033[1;33;44m" << ins << "\033[0m"; + break; + case 5: + s << "\033[1;47;34m" << ins << "\033[0m"; + break; + case 6: + s << "\033[1;1;31m" << ins << "\033[0m"; + break; + case 7: + s << "\033[1;1;33m" << ins << "\033[0m"; + break; + case 8: + s << "\033[1;1;43;34m" << ins << "\033[0m"; + break; + case 9: + s << "\033[1;1;37m" << ins << "\033[0m"; + break; + case 10: + s << "\033[1;30;47m" << ins << "\033[0m"; + break; + default: + s << ins; + } - return s.str(); + return s.str(); } //=============================================== string wrap_double(double val, int mode){ - std::ostringstream s; - s << string_wrap(strDouble(val),mode); + std::ostringstream s; + s << string_wrap(strDouble(val),mode); - return s.str(); + return s.str(); } @@ -214,25 +209,25 @@ const string i2string(int i){ - std::ostringstream s; - s << "worker" + std::ostringstream s; + s << "worker" << lexical_cast(i); - return s.str(); + return s.str(); } //=============================================== char* i2char(int i){ - std::ostringstream s; - s << "worker" + std::ostringstream s; + s << "worker" << lexical_cast(i); - char* a=new char[s.str().size()+1]; - memcpy(a,s.str().c_str(), s.str().size()); + char* a=new char[s.str().size()+1]; + memcpy(a,s.str().c_str(), s.str().size()); - return a; + return a; } //================================================ @@ -240,72 +235,72 @@ private: - double alpha_i; - double alpha_m; - double beta; - double gamma; - double delta; + double alpha_i; + double alpha_m; + double beta; + double gamma; + double delta; - double TargetNovelty; - double CumulativeRelativeLoss; - double CRLsquare; - string id; + double TargetNovelty; + double CumulativeRelativeLoss; + double CRLsquare; + string id; - int N_nodes; - int M_edges; + int N_nodes; + int M_edges; - int N_epochs; - int N_steps; - int N_repeats; + int N_epochs; + int N_steps; + int N_repeats; - int current_epoch; - double current_loss; + int current_epoch; + double current_loss; int current_repeat; double current_novelty; - int mode_identify_failed; - int verbose_level; // 0 is silent, higher is more + int mode_identify_failed; + int verbose_level; // 0 is silent, higher is more - double k_max; + double k_max; - graph_t Full_g; + graph_t Full_g; - double **Prob; - double **Tried; - double **Dist; - double **Final; - double **EdgeIndex; - double *Rank; + double **Prob; + double **Tried; + double **Dist; + double **Final; + double **EdgeIndex; + double *Rank; - base_generator_type generator; - boost::uniform_real<> uni_dist; - boost::geometric_distribution geo; + base_generator_type generator; + boost::uniform_real<> uni_dist; + boost::geometric_distribution geo; public: - //====== Constructor ====== - Universe(const std::string FileToOpen, int Epochs, int Steps, int Repeats, int identify_failed, double target, const std::string idd) - { - //typedef array_type2::index index2; + //====== Constructor ====== + Universe(const std::string FileToOpen, int Epochs, int Steps, int Repeats, int identify_failed, double target, const std::string idd) + { + //typedef array_type2::index index2; - std::ifstream inFile; - //string line; + std::ifstream inFile; + //string line; - //------------------------------- + //------------------------------- - base_generator_type gene(42u); - generator = gene; + base_generator_type gene(42u); + generator = gene; // generator.seed(static_cast(std::time(0))); - if ( initSeed != 0.0 ) { + if ( initSeed != 0.0 ) { generator.seed(static_cast(initSeed)); } - else { + else { timeval t; gettimeofday(&t, NULL); generator.seed(static_cast(t.tv_usec)); @@ -315,1002 +310,962 @@ - boost::uniform_real<> uni_d(0,1); - uni_dist = uni_d; + boost::uniform_real<> uni_d(0,1); + uni_dist = uni_d; - //-------------------------------- + //-------------------------------- - int i, k; - int x, y; - Edge* edge_array_mine; - int num_arcs_mine, num_nodes_mine; - int* weights_mine; + int i, k; + int x, y; + Edge* edge_array_mine; + int num_arcs_mine, num_nodes_mine; + int* weights_mine; - TargetNovelty = target; - CumulativeRelativeLoss = 0.; - CRLsquare = 0.; + TargetNovelty = target; + CumulativeRelativeLoss = 0.; + CRLsquare = 0.; - N_epochs = Epochs; - N_steps = Steps; - N_repeats = Repeats; + N_epochs = Epochs; + N_steps = Steps; + N_repeats = Repeats; - current_epoch = 0; - current_loss = 0.; - current_repeat = 0; + current_epoch = 0; + current_loss = 0.; + current_repeat = 0; - id = idd; + id = idd; - verbose_level = 1; + verbose_level = 1; - mode_identify_failed = identify_failed; + mode_identify_failed = identify_failed; - //------------------------------- - // The first pass though file with the graph - inFile.open(FileToOpen.c_str()); - if (inFile.fail()) { - cout << "Unable to open file"; - exit(1); // terminate with error - }else { + //------------------------------- + // The first pass though file with the graph + inFile.open(FileToOpen.c_str()); + if (inFile.fail()) { + cout << "Unable to open file"; + exit(1); // terminate with error + }else { - if (verbose_level > 2){ - std::cout << " Opened <" << FileToOpen << ">"< 2){ + std::cout << " Opened <" << FileToOpen << ">"<> x; - inFile >> y; + inFile >> x; + inFile >> y; - if (verbose_level > 2){ - std::cout << " x: " << x; - std::cout << " y: " << y << std::endl; - } + if (verbose_level > 2){ + std::cout << " x: " << x; + std::cout << " y: " << y << std::endl; + } - if (i==0){ - N_nodes=x; - M_edges=y; - break; - } - i++; + if (i==0){ + N_nodes=x; + M_edges=y; + break; + } + i++; - } - inFile.close(); + } + inFile.close(); - if (verbose_level == 2){ - std::cout << N_nodes << " nodes, " << M_edges << " edges"< 2){ - std::cout << " Opened <" << FileToOpen << ">"< 2){ + std::cout << " Opened <" << FileToOpen << ">"<> x && inFile >>y) { - if (i > 0) { - Final[x][y]=1.; - Final[y][x]=1.; + i=0; + while (inFile >> x && inFile >>y) { + if (i > 0) { + Final[x][y]=1.; + Final[y][x]=1.; - if (verbose_level == 2){ - std::cout << "."; - } - } - i++; - - } - if (verbose_level == 2){ - std::cout << std::endl; + if (verbose_level == 2){ + std::cout << "."; + } } - inFile.close(); + i++; - k=0; - for (int i=0; i 0.){ - EdgeIndex[i][j]=k; - k++; - } - } + } + if (verbose_level == 2){ + std::cout << std::endl; + } + inFile.close(); + + k=0; + for (int i=0; i 0.){ + EdgeIndex[i][j]=k; + k++; + } } + } - //+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - // create graph -- hopefully, we can keep it, just modifying edge weights + //+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + // create graph -- hopefully, we can keep it, just modifying edge weights - edge_array_mine = new Edge[2*M_edges]; - num_arcs_mine = 2*M_edges; - num_nodes_mine = N_nodes; - weights_mine = new int[2*M_edges]; - for (int i=0; i<2*M_edges; i++){ weights_mine[i]=1;} + edge_array_mine = new Edge[2*M_edges]; + num_arcs_mine = 2*M_edges; + num_nodes_mine = N_nodes; + weights_mine = new int[2*M_edges]; + for (int i=0; i<2*M_edges; i++){ weights_mine[i]=1;} - k=0; - for(int i=0; i0.){ - edge_array_mine[2*k] =Edge(i,j); - edge_array_mine[2*k+1]=Edge(j,i); - k++; - } - } - } - graph_t g(edge_array_mine, edge_array_mine + num_arcs_mine, weights_mine, num_nodes_mine); + k=0; + for(int i=0; i0.){ + edge_array_mine[2*k] =Edge(i,j); + edge_array_mine[2*k+1]=Edge(j,i); + k++; + } + } + } + graph_t g(edge_array_mine, edge_array_mine + num_arcs_mine, weights_mine, num_nodes_mine); - Full_g = g; - delete edge_array_mine; - delete weights_mine; + Full_g = g; + delete edge_array_mine; + delete weights_mine; - //=========================================================================== - std::vector p(num_edges(Full_g)); - std::vector d(num_edges(Full_g)); - edge_descriptor s; - boost::graph_traits::vertex_descriptor u, v; + //=========================================================================== + std::vector p(num_edges(Full_g)); + std::vector d(num_edges(Full_g)); + edge_descriptor s; + boost::graph_traits::vertex_descriptor u, v; - for (int i=0; i 0.){ - u = vertex(i, Full_g); - v = vertex(j, Full_g); - remove_edge(u,v,Full_g); - remove_edge(v,u,Full_g); + for (int i=0; i 0.){ + u = vertex(i, Full_g); + v = vertex(j, Full_g); + remove_edge(u,v,Full_g); + remove_edge(v,u,Full_g); - } - } - } - if(max_loss <= 0.0) set_max_loss(N_nodes, M_edges, TargetNovelty); + } + } + } + if(max_loss <= 0.0) set_max_loss(N_nodes, M_edges, TargetNovelty); } - //===================================================================== - int sample_failed_number(double pfail){ + //===================================================================== + int sample_failed_number(double pfail){ - //boost::geometric_distribution geo(pfail); - //boost::variate_generator > geom(generator, geo); + //boost::geometric_distribution geo(pfail); + //boost::variate_generator > geom(generator, geo); - double r, u, g; + double r, u, g; - r=0.; - for(int i=0; i=3){ - std::cout << id << " failed " << r << std::endl; - } - return (int) round(r); // FIXME: Andrey: please verify that round() is correct. - - } - - //============================================= - double get_target(void){ - return TargetNovelty; + r+=g; } - //============================================= - void set_target(double target){ - TargetNovelty=target; + if (verbose_level>=3){ + std::cout << id << " failed " << r << std::endl; } + return (int) round(r); // FIXME: Andrey: please verify that round() is correct. - //============================================= - int sample(){ + } - //boost::variate_generator > uni(generator, uni_dist); - // double r = uni(), Summa = 0.; + //============================================= + double get_target(void){ + return TargetNovelty; + } + //============================================= + void set_target(double target){ + TargetNovelty=target; + } + //============================================= + int sample(){ - double r = rand(), Summa = 0.; - r /= (double)(pow(2.,31)-1.); - int result = 0; - int finished = 0; + //boost::variate_generator > uni(generator, uni_dist); + // double r = uni(), Summa = 0.; - if (verbose_level==4){ - std::cout << id << " sampled " << r << std::endl; - } - for(int i=0; i r){ + if (verbose_level==4){ + std::cout << id << " sampled " << r << std::endl; + } - Tried[i][j]+=1.; + for(int i=0; i 0.){ - result = 1; - } - finished = 1; - } - } - } + Summa += Prob[i][j]; - return result; + if (Summa > r){ - } + Tried[i][j]+=1.; - //=============================== - void update_current_graph(void){ + if (Final[i][j] > 0.){ + result = 1; + } + finished = 1; + } + } + } - std::vector p(num_edges(Full_g)); - std::vector d(num_edges(Full_g)); - edge_descriptor s; - boost::graph_traits::vertex_descriptor u, v; + return result; - //property_map::type weightmap = get(edge_weight, Full_g); - for (int i=0; i 0. && Tried[i][j]>0){ - //s = edge(i, j, Full_g); - boost::graph_traits::edge_descriptor e1,e2; - bool found1, found2; - u = vertex(i, Full_g); - v = vertex(j, Full_g); - tie(e1, found1) = edge(u, v, Full_g); - tie(e2, found2) = edge(v, u, Full_g); - if (!found1 && !found2){ - add_edge(u,v,1,Full_g); - add_edge(v,u,1,Full_g); - } + } - } - } + //=============================== + void update_current_graph(void){ - } - } + std::vector p(num_edges(Full_g)); + std::vector d(num_edges(Full_g)); + edge_descriptor s; + boost::graph_traits::vertex_descriptor u, v; - //=============================== - void update_distances(void){ - // put shortest paths to the *Dist[][] - std::vector p(num_vertices(Full_g)); - std::vector d(num_vertices(Full_g)); - vertex_descriptor s; + //property_map::type weightmap = get(edge_weight, Full_g); + for (int i=0; i 0. && Tried[i][j]>0){ + //s = edge(i, j, Full_g); + boost::graph_traits::edge_descriptor e1,e2; + bool found1, found2; + u = vertex(i, Full_g); + v = vertex(j, Full_g); + tie(e1, found1) = edge(u, v, Full_g); + tie(e2, found2) = edge(v, u, Full_g); + if (!found1 && !found2){ + add_edge(u,v,1,Full_g); + add_edge(v,u,1,Full_g); + } + } + } - // put shortest paths to the *Dist[][] - for (int j=0; j 0.){ - s = vertex(j, Full_g); - dijkstra_shortest_paths(Full_g, s, predecessor_map(&p[0]).distance_map(&d[0])); + //=============================== + void update_distances(void){ + // put shortest paths to the *Dist[][] + std::vector p(num_vertices(Full_g)); + std::vector d(num_vertices(Full_g)); + vertex_descriptor s; - //std::cout <<" Vertex "<< j << std::endl; - graph_traits < graph_t >::vertex_iterator vi, vend; - for (boost::tie(vi, vend) = vertices(Full_g); vi != vend; ++vi) { + // put shortest paths to the *Dist[][] + for (int j=0; j 0.){ + s = vertex(j, Full_g); + dijkstra_shortest_paths(Full_g, s, predecessor_map(&p[0]).distance_map(&d[0])); - if ( (int)round(Dist[*vi][j]>max_dist)) { - // FIXME: Andrey: please verify that (int) cast is correct. Do we need to round()? - // also, the indent on this iff statement was way off - - // perhaps due to space v. tab? - max_dist=(int)round(Dist[*vi][j]); - } + //std::cout <<" Vertex "<< j << std::endl; + graph_traits < graph_t >::vertex_iterator vi, vend; + for (boost::tie(vi, vend) = vertices(Full_g); vi != vend; ++vi) { - } else { - Dist[*vi][j]=-1.; - Dist[j][*vi]=-1.; - } - } - } + if (p[*vi]!=*vi){ + Dist[*vi][j]=d[*vi]; + Dist[j][*vi]=d[*vi]; - } + if ( (int)round(Dist[*vi][j]>max_dist)) { + // FIXME: Andrey: please verify that (int) cast is correct. Do we need to round()? + // also, the indent on this iff statement was way off - + // perhaps due to space v. tab? + max_dist=(int)round(Dist[*vi][j]); + } - } + } else { + Dist[*vi][j]=-1.; + Dist[j][*vi]=-1.; + } + } + } - //====================================================== - void update_ranks(void){ + } - for(int i=0; i0. && Final[i][j] >0.){ - Rank[i]++; - Rank[j]++; - } - } - } + } - } + //====================================================== + void update_ranks(void){ - //==================================================================== - void set_world(double a_i, double a_m, double b, double g, double d){ + for(int i=0; i0. && Final[i][j] >0.){ + Rank[i]++; + Rank[j]++; + } + } + } - } + } - //==================================================================== - void reset_world(){ + //==================================================================== + void set_world(double a_i, double a_m, double b, double g, double d){ - //==================================================== - std::vector p(num_edges(Full_g)); - std::vector d(num_edges(Full_g)); - edge_descriptor s; - boost::graph_traits::vertex_descriptor u, v; - cout << "reset_world: start id=" << id << endl; + alpha_i=a_i; + alpha_m=a_m; + gamma=g; + beta=b; + delta=d; - for (int i=0; i 0. && Tried[i][j] > 0){ - u = vertex(i, Full_g); - v = vertex(j, Full_g); - remove_edge(u,v,Full_g); - remove_edge(v,u,Full_g); + } - } - } - } + //==================================================================== + void reset_world(){ - //================================================== + //==================================================== + std::vector p(num_edges(Full_g)); + std::vector d(num_edges(Full_g)); + edge_descriptor s; + boost::graph_traits::vertex_descriptor u, v; + cout << "reset_world: start id=" << id << endl; - current_loss=0; - current_epoch=0; - current_repeat++; - current_novelty=0; + for (int i=0; i 0. && Tried[i][j] > 0){ + u = vertex(i, Full_g); + v = vertex(j, Full_g); + remove_edge(u,v,Full_g); + remove_edge(v,u,Full_g); - for(int i = 0; i < N_nodes; ++i) { - Rank[i]=0.; - for(int j = 0; j < N_nodes; ++j) { - Prob[i][j]=0.; - Dist[i][j]=-1.; - Tried[i][j]=0.; - } - } - cout << "reset_world: end id=" << id << endl; - } + } + } + } + //================================================== - //============================================== - void show_parameters(void){ + current_loss=0; + current_epoch=0; + current_repeat++; + current_novelty=0; - std::cout << "Parameters: " - << alpha_i << " " - << alpha_m << " | " - << beta << " " - << gamma << " | " - << delta << std::endl; - + for(int i = 0; i < N_nodes; ++i) { + Rank[i]=0.; + for(int j = 0; j < N_nodes; ++j) { + Prob[i][j]=0.; + Dist[i][j]=-1.; + Tried[i][j]=0.; + } } + cout << "reset_world: end id=" << id << endl; + } + //============================================== + void show_parameters(void){ - //=============================================== - string file_name(){ + std::cout << "Parameters: " + << alpha_i << " " + << alpha_m << " | " + << beta << " " + << gamma << " | " + << delta << std::endl; - std::ostringstream s; - s << "world_" - << lexical_cast(alpha_i) << "_" - << lexical_cast(alpha_m) << "_" - << lexical_cast(beta) << "_" - << lexical_cast(gamma) << "_" - << lexical_cast(delta) << "_" - << lexical_cast(N_epochs) << "_" - << lexical_cast(N_steps) << "_" - << lexical_cast(N_repeats) << ".txt"; + } - return s.str(); - } + //=============================================== + string file_name(){ + std::ostringstream s; + s << "world_" + << lexical_cast(alpha_i) << "_" + << lexical_cast(alpha_m) << "_" + << lexical_cast(beta) << "_" + << lexical_cast(gamma) << "_" + << lexical_cast(delta) << "_" + << lexical_cast(N_epochs) << "_" + << lexical_cast(N_steps) << "_" + << lexical_cast(N_repeats) << ".txt"; + return s.str(); - //================================================= - void set_verbose(int verbose){ + } - verbose_level = verbose; - } - //============================================================= - void update_probabilities(void){ + //================================================= + void set_verbose(int verbose){ - //========================= - // Compute sampling probabilities - // first pass: \xi_i,j - for(int i=0; i 0.){ - double k = Dist[i][j]; - if (k >= k_max){ - k = k_max-1; - } + //========================= + // Compute sampling probabilities + // first pass: \xi_i,j + for(int i=0; i 0.){ + double k = Dist[i][j]; + if (k >= k_max){ + k = k_max-1; + } - // second pass: sum - double Summa = 0.; + bg = beta * log(k/k_max) + gamma * log(1. - k/k_max); - for(int i=0; i0. && Final[i][j]>0.){ - novel+=1.; - } - } - } + while(publishable < N_steps){ - } - //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - else { + result = sample(); + publishable += result; + failed += (1-result); - double pfail=0.; - int n_failed; - //, n_check = 0; + } - for(int i=0; i0. && Final[i][j]>0.){ + novel+=1.; + } + } + } - n_failed = sample_failed_number(pfail); - while(publishable < N_steps){ + } + //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + else { - result = sample(); - publishable += result; - } + double pfail=0.; + int n_failed; + //, n_check = 0; + for(int i=0; i0. && Final[i][j]>0.){ - novel+=1.; - } - } - } - } + n_failed = sample_failed_number(pfail); + while(publishable < N_steps){ - current_novelty = novel; + result = sample(); + publishable += result; + } - //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - if (verbose_level == 2){ - std::cout << (current_repeat+1) << " epoch=" << (current_epoch+1) + current_loss += (n_failed + N_steps); + cost = current_loss; - << " cost=" << cost - << " novel=" << novel - << " rel_loss=" << cost/novel - << std::endl; + for(int i=0; i0. && Final[i][j]>0.){ + novel+=1.; + } } + } + } - current_epoch++; - } + current_novelty = novel; - //====== Destructor ====== - ~Universe(){ + //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + if (verbose_level == 2){ + std::cout << (current_repeat+1) << " epoch=" << (current_epoch+1) - delete_2Dmatrix(Final, N_nodes); - delete_2Dmatrix(Dist, N_nodes); - delete_2Dmatrix(Tried, N_nodes); - delete_2Dmatrix(Prob, N_nodes); - delete_2Dmatrix(EdgeIndex, N_nodes); - delete_1Dmatrix(Rank); - } + << " cost=" << cost + << " novel=" << novel + << " rel_loss=" << cost/novel + << std::endl; + } - //================================================ - // Allocate memory - double** allocate_2Dmatrix(int N, int M) + current_epoch++; + } + + + //====== Destructor ====== + ~Universe(){ + + delete_2Dmatrix(Final, N_nodes); + delete_2Dmatrix(Dist, N_nodes); + delete_2Dmatrix(Tried, N_nodes); + delete_2Dmatrix(Prob, N_nodes); + delete_2Dmatrix(EdgeIndex, N_nodes); + delete_1Dmatrix(Rank); + } + + //================================================ + // Allocate memory + double** allocate_2Dmatrix(int N, int M) { - double **pointer; + double **pointer; - if (verbose_level == 2){ - std::cout<< "["< 0){ + if(N > 0){ - pointer = new double[N]; + pointer = new double[N]; - }else { + }else { - pointer = NULL; - } + pointer = NULL; + } - return pointer; + return pointer; } - //============================================== - // De-Allocate memory to prevent memory leak - void delete_2Dmatrix(double **pointer, int N){ + //============================================== + // De-Allocate memory to prevent memory leak + void delete_2Dmatrix(double **pointer, int N){ - if (pointer != NULL){ + if (pointer != NULL){ - for (int i = 0; i < N; ++i){ - delete [] pointer[i]; - } - delete [] pointer; - } + for (int i = 0; i < N; ++i){ + delete [] pointer[i]; + } + delete [] pointer; } - //==================== - void delete_1Dmatrix(double *pointer){ + } + //==================== + void delete_1Dmatrix(double *pointer){ - delete [] pointer; - } + delete [] pointer; + } - //=========================================== - double get_rel_loss(){ + //=========================================== + double get_rel_loss(){ - return CumulativeRelativeLoss ; - } + return CumulativeRelativeLoss ; + } - //=========================================== - double get_rel_loss_err(){ + //=========================================== + double get_rel_loss_err(){ - return CRLsquare ; - } + return CRLsquare ; + } - //================================================================================== - void evolve_to_target_and_save(int istart, int iend, double* storage, int* counters){ + //================================================================================== + void evolve_to_target_and_save(int istart, int iend, double* storage, int* counters){ - //double ALOT=100000000000.; - double ALOT= 3.0 * max_loss; // Per Andrey, 2012.12.29 + //double ALOT=100000000000.; + double ALOT= 3.0 * max_loss; // Per Andrey, 2012.12.29 - double ratio = 0.0; - int check = 0; + double ratio = 0.0; + int check = 0; - reset_world(); + reset_world(); - for (int k = istart; k < iend; k++){ - cout << "evolve_ttas: k=" << k << endl; + for (int k = istart; k < iend; k++){ + cout << "evolve_ttas: k=" << k << endl; - for(int i=0; i< N_epochs && current_novelty < TargetNovelty; i++){ + for(int i=0; i< N_epochs && current_novelty < TargetNovelty; i++){ - if(ratio < max_loss){ - update_world(); - } - else{ - cout << "k=" << k << " i=" << i << " ratio=" << ratio << " max_loss=" << max_loss << endl; - break; - } - ratio = current_loss/current_novelty; - } - if( ratio < max_loss ) - storage[k]=current_loss/current_novelty; - else - storage[k]=ALOT; - counters[k]=1; - reset_world(); + if(ratio < max_loss){ + update_world(); } + else{ + cout << "k=" << k << " i=" << i << " ratio=" << ratio << " max_loss=" << max_loss << endl; + break; + } + ratio = current_loss/current_novelty; + } + if( ratio < max_loss ) + storage[k]=current_loss/current_novelty; + else + storage[k]=ALOT; + counters[k]=1; + reset_world(); } + } - //============================================== - int get_reruns(void){ - return N_repeats; - } + //============================================== + int get_reruns(void){ + return N_repeats; + } - //============================================== - double get_parameter(int i){ + //============================================== + double get_parameter(int i){ - switch(i){ - case 0: - return alpha_i; - case 1: - return alpha_m; - case 2: - return beta; - case 3: - return gamma; - case 4: - return delta; - default: + switch(i){ + case 0: + return alpha_i; + case 1: + return alpha_m; + case 2: + return beta; + case 3: + return gamma; + case 4: + return delta; + default: - std::cout << "Erroneous parameter id!!!!\n\n\n"; - return 0.; - } + std::cout << "Erroneous parameter id!!!!\n\n\n"; + return 0.; } + } - //============================================== - void evolve_to_target(){ + //============================================== + void evolve_to_target(){ - reset_world(); - if (beta < -1. || gamma < -1.){ - CumulativeRelativeLoss = 100000000000.; - CRLsquare = 0.; - return; - } + reset_world(); + if (beta < -1. || gamma < -1.){ + CumulativeRelativeLoss = 100000000000.; + CRLsquare = 0.; + return; + } - for (int k=0; k< N_repeats; k++){ + for (int k=0; k< N_repeats; k++){ - for(int i=0; i 4) {return 0;} - else { + //================================================================ + int set_parameter(double value, int position){ - switch(position){ - case 0: - alpha_i=value; - return 1; - case 1: - alpha_m=value; - return 1; - case 2: - beta=value; - return 1; - case 3: - gamma=value; - return 1; - case 4: - delta=value; - return 1; - } + if (position < 0 || position > 4) {return 0;} - } + else { - return 0; + switch(position){ + case 0: + alpha_i=value; + return 1; + case 1: + alpha_m=value; + return 1; + case 2: + beta=value; + return 1; + case 3: + gamma=value; + return 1; + case 4: + delta=value; + return 1; + } + } + return 0; + } + //-------- -void set_max_loss(long long v, long long e, long long t) { + void set_max_loss(long long v, long long e, long long t) { - cout << "set_max_loss v=" << v << " e=" << e << " t=" << t << " max_loss=" << max_loss << endl; + cout << "set_max_loss v=" << v << " e=" << e << " t=" << t << " max_loss=" << max_loss << endl; - // Based on formulas from random_analytical.pdf + // Based on formulas from random_analytical.pdf - double sum, ERLt, VarRLt; + double sum, ERLt, VarRLt; - // (9): E[RLt] = (1/t) * SUM(i=0 to T of: (V*(V-1) / (2 * (E-i)) + // (9): E[RLt] = (1/t) * SUM(i=0 to T of: (V*(V-1) / (2 * (E-i)) - sum = 0; - for(int i=0;i multi_loss(dispatch_group_t group, - Universe* un[], - dispatch_queue_t* CustomQueues, - double* Results, - int* Counters, - double* params){ -#else std::pair multi_loss(Universe* un[], double* Results, int* Counters, double* params){ -#endif + int N = un[0]->get_reruns(); + int step = (int)floor((double)N/(double)(Nworkers)); // FIXME: Andrey: please check change in cast grouping and use of floor + int istart=0; + int iend = istart+step; - int N = un[0]->get_reruns(); - int step = (int)floor((double)N/(double)(Nworkers)); // FIXME: Andrey: please check change in cast grouping and use of floor - int istart=0; - int iend = istart+step; + double Loss=0., LossSquare=0.; - double Loss=0., LossSquare=0.; + timeval startTime, endTime; + double elapsedTime; + gettimeofday(&startTime, NULL); - timeval startTime, endTime; - double elapsedTime; - gettimeofday(&startTime, NULL); + if ( N > NUniverse ) { + std::cerr << "Error: Number of reruns=" << N << " is greater than max NUniverse=" << NUniverse << "\n"; + exit(1); + } - if ( N > NUniverse ) { - std::cerr << "Error: Number of reruns=" << N << " is greater than max NUniverse=" << NUniverse << "\n"; - exit(1); + for(int i=0; iset_parameter(params[j],j); } + } - for(int i=0; iset_parameter(params[j],j); - } - } + int i; -#ifdef P_DISPATCH - for(int i=0; ievolve_to_target_and_save(i, i+1, Results, Counters); + } - // un[i]->evolve_to_target_and_save(istart, iend, Results, Counters); - un[i]->evolve_to_target_and_save(i*step, min((i+1)*step,N), Results, Counters); - }); - - std::cout << "queued: i=" << i << " N=" << N << " istart=" << istart << " iend=" << iend << "\n"; - // istart += step; - // iend = min(istart+step,N); - - } - dispatch_group_wait(group, DISPATCH_TIME_FOREVER); - //dispatch_release(group); -#else - int i; - - // Execute rerun loop in parallel - - #pragma omp parallel for private (i) - for(i=0; ievolve_to_target_and_save(i, i+1, Results, Counters); - } -#endif - /* - for (int i=0; i > uni(generator, uni_dist); + //................................. + // re-implement annealing - // set up parameter for annealing + double dx[5]={0.,0.,0.,0.,0}; + double x[5]={0.,0.,0.,0.,0}; + double rejection[5]={0., 0., 0., 0., 0.}; + double curr_x, curr_err, x_tmp; + double temperature; + double ratio, r; + int cycle=10; + //boost::variate_generator > uni(generator, uni_dist); - x[0]=params0[0]; - x[1]=params0[1]; - x[2]=params0[2]; - x[3]=params0[3]; - x[4]=params0[4]; + // set up parameter for annealing - for(int i=0;i<5;i++){ - dx[i] = starting_jump; - } + x[0]=params0[0]; + x[1]=params0[1]; + x[2]=params0[2]; + x[3]=params0[3]; + x[4]=params0[4]; - // establish the current value - std::pairRes; + for(int i=0;i<5;i++){ + dx[i] = starting_jump; + } -#ifdef P_DISPATCH - Res = multi_loss(group, un, CustomQueues, Results, Counters, x); -#else - Res = multi_loss( un, Results, Counters, x); -#endif - std::cout << "Returned from initial multi_loss:" << std::endl; - std::cout << Res.first << " +- " << Res.second << std::endl; + // establish the current value + std::pairRes; - if ( operation == 'm' ) { - FILE *f; - int N = un[0]->get_reruns(); + Res = multi_loss( un, Results, Counters, x); - f = fopen("multi_loss.data","w"); - for(int i=0; iget_reruns(); + + f = fopen("multi_loss.data","w"); + for(int i=0; i 0){ + dx[k] = dx[k]/(rejection[k]/Target_rejection); + rejection[k]=0.; } + else{ + dx[k]*=2.; + } + std::cout << dx[k] << " "; + } + std::cout << std::endl; + } - for (int j=0; j<5; j++){ + for (int j=0; j<5; j++){ - /////////////////////////////// - if (FIX_VARIABLES==0 || var_fixed[j]==0){ + /////////////////////////////// + if (FIX_VARIABLES==0 || var_fixed[j]==0){ - // get new value of x[j] - double x_hold=x[j]; - x_tmp = get_new_x(x[j],dx[j]); - x[j]=x_tmp; + // get new value of x[j] + double x_hold=x[j]; + x_tmp = get_new_x(x[j],dx[j]); + x[j]=x_tmp; - std::cout << wrap_double(x_tmp,10) << " " << wrap_double(j,9) << "\n\n"; - //======================================= - //............................................. - for(int w=0; wset_parameter(x_tmp, j); - } + std::cout << wrap_double(x_tmp,10) << " " << wrap_double(j,9) << "\n\n"; + //======================================= + //............................................. + for(int w=0; wset_parameter(x_tmp, j); + } -#ifdef P_DISPATCH - Res = multi_loss(group, un, CustomQueues, Results, Counters, x); -#else - Res = multi_loss( un, Results, Counters, x); -#endif - std::cout << Res.first << " +- " << Res.second << std::endl; + Res = multi_loss( un, Results, Counters, x); - ratio = min(1.,exp(-(Res.first-curr_x)/temperature)); - r = rand()/(double)(pow(2.,31)-1.); - std::cout << r << " vs " << ratio << std::endl; + std::cout << Res.first << " +- " << Res.second << std::endl; - double ALOT=100000000000.; + ratio = min(1.,exp(-(Res.first-curr_x)/temperature)); + r = rand()/(double)(pow(2.,31)-1.); + std::cout << r << " vs " << ratio << std::endl; - if (Res.first < ALOT) - { - ofstream filestr; + double ALOT=100000000000.; - filestr.open ("best_opt_some.txt", ofstream::app); + if (Res.first < ALOT) + { + ofstream filestr; - // >> i/o operations here << - filestr + filestr.open ("best_opt_some.txt", ofstream::app); - << "N, " << i << ", " << j << ", " << dx[j] << ", " << rejection[j] << ", |, " // FIXME: MW-DEBUGGING + // >> i/o operations here << + filestr - << un[0]->get_target() << ", " - << Res.first - << ", " << un[0]->get_parameter(0) - << ", " << un[0]->get_parameter(1) - << ", " << un[0]->get_parameter(2) - << ", " << un[0]->get_parameter(3) - << ", " << un[0]->get_parameter(4) << ", " << Res.second << ",\n"; + << "N, " << i << ", " << j << ", " << dx[j] << ", " << rejection[j] << ", |, " // FIXME: MW-DEBUGGING - filestr.close(); + << un[0]->get_target() << ", " + << Res.first + << ", " << un[0]->get_parameter(0) + << ", " << un[0]->get_parameter(1) + << ", " << un[0]->get_parameter(2) + << ", " << un[0]->get_parameter(3) + << ", " << un[0]->get_parameter(4) << ", " << Res.second << ",\n"; + filestr.close(); - filestr.open ("max_dist.txt", ofstream::app); - // >> i/o operations here << - filestr << max_dist << ",\n"; + filestr.open ("max_dist.txt", ofstream::app); - filestr.close(); + // >> i/o operations here << + filestr << max_dist << ",\n"; - FILE *bf; - bf = fopen("bestdb.txt","a"); - fprintf(bf, "N %2d %2d %10.5f %5.2f | %5.2f %10.5f [ %5.2f %5.2f %10.5f %10.5f %10.5f ] %10.5f\n", i, j, dx[j], rejection[j], - un[0]->get_target(), - Res.first, - un[0]->get_parameter(0), - un[0]->get_parameter(1), - un[0]->get_parameter(2), - un[0]->get_parameter(3), - un[0]->get_parameter(4), - Res.second); - fclose(bf); - } + filestr.close(); + FILE *bf; + bf = fopen("bestdb.txt","a"); + fprintf(bf, "N %2d %2d %10.5f %5.2f | %5.2f %10.5f [ %5.2f %5.2f %10.5f %10.5f %10.5f ] %10.5f\n", i, j, dx[j], rejection[j], + un[0]->get_target(), + Res.first, + un[0]->get_parameter(0), + un[0]->get_parameter(1), + un[0]->get_parameter(2), + un[0]->get_parameter(3), + un[0]->get_parameter(4), + Res.second); + fclose(bf); + } - if (r > ratio){ - std::cout << " "<< (i+1) << ","<< (j) + if (r > ratio){ + + std::cout << " "<< (i+1) << ","<< (j) <<" "<< (i+1) << " Did not accept " << x_tmp << "(" << j << ")" << std::endl; - std::cout << un[0]->get_parameter(0) + std::cout << un[0]->get_parameter(0) << " " << un[0]->get_parameter(1) << " " << un[0]->get_parameter(2) << " " << un[0]->get_parameter(3) << " " << un[0]->get_parameter(4) << " " << std::endl; - x[j]=x_hold; - for(int w=0; wset_parameter(x[j], j); - } + x[j]=x_hold; + for(int w=0; wset_parameter(x[j], j); + } - //set_parameter(x[j], j); - rejection[j]+=1.; - } + //set_parameter(x[j], j); + rejection[j]+=1.; + } - else { + else { - curr_x = Res.first; - curr_err = Res.second; - x[j] = x_tmp; + curr_x = Res.first; + curr_err = Res.second; + x[j] = x_tmp; - for(int w=0; wset_parameter(x[j], j); - } + for(int w=0; wset_parameter(x[j], j); + } - std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8) + std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8) << wrap_double(rejection[0],2) << " " << wrap_double(rejection[1],7) << " " << wrap_double(rejection[2],5) << " " @@ -1511,7 +1460,7 @@ << wrap_double(rejection[4],6) << " " << std::endl << std::endl; - std::cout << " "<< (i+1) <<","<< (j) + std::cout << " "<< (i+1) <<","<< (j) <<" " << string_wrap((string) "***** Did accept! ", 3) << wrap_double(un[0]->get_parameter(0),2) << " " @@ -1523,14 +1472,14 @@ - } - //........................................................ - - } } + //........................................................ + } } + } + } @@ -1540,216 +1489,197 @@ main(int argc, char* argv[]) { - double params0[6] = {0., 0., 0., 0., 0., 0.2}, target=50., range; - string par_names0[6] = {"alpha_i", "alpha_m", "beta", "gamma", "delta", "target"}; - string par_names1[4] = {"n_epochs", "n_steps", "n_reruns", "range"}; - string par_names2[5] = {"T_start", "T_end", "Annealing_steps","Target_rejection","Starting_jump"}; - string par_names3[5] = {"FREEZE_alpha_i", "FREEZE_alpha_m", "FREEZE_beta", "FREEZE_gamma", "FREEZE_delta"}; - string par_names4[3] = {"Operation", "Nworkers", "initSeed"}; + double params0[6] = {0., 0., 0., 0., 0., 0.2}, target=50., range; + string par_names0[6] = {"alpha_i", "alpha_m", "beta", "gamma", "delta", "target"}; + string par_names1[4] = {"n_epochs", "n_steps", "n_reruns", "range"}; + string par_names2[5] = {"T_start", "T_end", "Annealing_steps","Target_rejection","Starting_jump"}; + string par_names3[5] = {"FREEZE_alpha_i", "FREEZE_alpha_m", "FREEZE_beta", "FREEZE_gamma", "FREEZE_delta"}; + string par_names4[3] = {"Operation", "Nworkers", "initSeed"}; - int params1[4] = {300, 50, 1000, 10}; - int params3[5] = { 0, 0, 0, 0, 0}; + int params1[4] = {300, 50, 1000, 10}; + int params3[5] = { 0, 0, 0, 0, 0}; - // temperature_start, temperature_end, annealing_steps target_rejection Starting_jump - double params2[5] = {1, 0.001, 100, 0.3, 1.5}; + // temperature_start, temperature_end, annealing_steps target_rejection Starting_jump + double params2[5] = {1, 0.001, 100, 0.3, 1.5}; - int verbose_level = 2; - const std::string one="one", two="two"; - static Universe* un[NUniverse]; -#ifdef P_DISPATCH - static dispatch_queue_t CustomQueues[MAXNworkers]; -#endif - static double* Results; - static int* Counters; + int verbose_level = 2; + const std::string one="one", two="two"; + static Universe* un[NUniverse]; + static double* Results; + static int* Counters; - timeval t1, t2; - double elapsedTime; - // start timer - gettimeofday(&t1, NULL); + timeval t1, t2; + double elapsedTime; + // start timer + gettimeofday(&t1, NULL); - if (argc < 8) { - std::cout << "Usage: super_optimizer alpha_i alpha_m beta gamma delta target_innov [n_epochs n_steps n_reruns] [range] [verbose_level]\n"; - std::cout << " [T_start T_end Annealing_steps Target_rejection Starting_jump]\n"; - std::cout << " [FREEZE_alpha_i FREEZE_alpha_m FREEZE_beta FREEZE_gamma FREEZE_delta]\n"; + if (argc < 8) { + std::cout << "Usage: super_optimizer alpha_i alpha_m beta gamma delta target_innov [n_epochs n_steps n_reruns] [range] [verbose_level]\n"; + std::cout << " [T_start T_end Annealing_steps Target_rejection Starting_jump]\n"; + std::cout << " [FREEZE_alpha_i FREEZE_alpha_m FREEZE_beta FREEZE_gamma FREEZE_delta]\n"; - system("pwd"); + system("pwd"); - return(1); - } - else { - for (int nArg=0; nArg < argc; nArg++){ - //std::cout << nArg << " " << argv[nArg] << std::endl; - if (nArg > 0 && nArg < 7){ - params0[nArg-1]= atof(argv[nArg]); - std::cout << par_names0[nArg-1] << ": " << params0[nArg-1] << std::endl; - } - if (nArg > 6 && nArg < 11){ - params1[nArg-7]= atoi(argv[nArg]); - std::cout << par_names1[nArg-7] << ": " << params1[nArg-7] << std::endl; - } - if (nArg == 11){ - verbose_level = atoi(argv[nArg]); - std::cout << "verbose level: " << verbose_level << std::endl; - } - if (nArg > 11 && nArg < 17){ - params2[nArg-12]= atof(argv[nArg]); - std::cout << par_names2[nArg-12] << ": " << params2[nArg-12] << std::endl; - } - if (nArg > 16 && nArg < 22){ - params3[nArg-17]= (int)round(atof(argv[nArg])); // FIXME: Andrey: please verify that round() is correct. - var_fixed[nArg-17]= (int)round(atof(argv[nArg])); // FIXME: ditto - std::cout << par_names3[nArg-17] << ": " << var_fixed[nArg-17] << std::endl; - } - if (nArg == 22 ){ - operation = *argv[nArg]; - std::cout << par_names4[0] << ": " << operation << std::endl; - } - if (nArg == 23 ){ - Nworkers = atoi(argv[nArg]); - std::cout << par_names4[1] << ": " << Nworkers << std::endl; - } - if (nArg == 24 ){ - initSeed = atoi(argv[nArg]); - std::cout << par_names4[2] << ": " << initSeed << std::endl; - } + return(1); + } + else { + for (int nArg=0; nArg < argc; nArg++){ + //std::cout << nArg << " " << argv[nArg] << std::endl; + if (nArg > 0 && nArg < 7){ + params0[nArg-1]= atof(argv[nArg]); + std::cout << par_names0[nArg-1] << ": " << params0[nArg-1] << std::endl; + } + if (nArg > 6 && nArg < 11){ + params1[nArg-7]= atoi(argv[nArg]); + std::cout << par_names1[nArg-7] << ": " << params1[nArg-7] << std::endl; + } + if (nArg == 11){ + verbose_level = atoi(argv[nArg]); + std::cout << "verbose level: " << verbose_level << std::endl; + } + if (nArg > 11 && nArg < 17){ + params2[nArg-12]= atof(argv[nArg]); + std::cout << par_names2[nArg-12] << ": " << params2[nArg-12] << std::endl; + } + if (nArg > 16 && nArg < 22){ + params3[nArg-17]= (int)round(atof(argv[nArg])); // FIXME: Andrey: please verify that round() is correct. + var_fixed[nArg-17]= (int)round(atof(argv[nArg])); // FIXME: ditto + std::cout << par_names3[nArg-17] << ": " << var_fixed[nArg-17] << std::endl; + } + if (nArg == 22 ){ + operation = *argv[nArg]; + std::cout << par_names4[0] << ": " << operation << std::endl; + } + if (nArg == 23 ){ + Nworkers = atoi(argv[nArg]); + std::cout << par_names4[1] << ": " << Nworkers << std::endl; + } + if (nArg == 24 ){ + initSeed = atoi(argv[nArg]); + std::cout << par_names4[2] << ": " << initSeed << std::endl; + } - } - } - for (int j=0; j<5; j++){ + } - cout << j << " | " << var_fixed[j] << " (fixed) \n"; - } + for (int j=0; j<5; j++){ - target=params0[5]; - range = (double)params1[3]; - int identify_failed = 0; - char* filename= (char *)"movie_graph.txt"; - int n_ep=params1[0], n_st=params1[1], n_rep=params1[2]; + cout << j << " | " << var_fixed[j] << " (fixed) \n"; + } - //............................... + target=params0[5]; + range = (double)params1[3]; + int identify_failed = 0; + char* filename= (char *)"movie_graph.txt"; + int n_ep=params1[0], n_st=params1[1], n_rep=params1[2]; - for(int i=0; i 0){ + for(int i=0; i 0){ - }else { + Results = new double[n_rep]; + Counters = new int[n_rep]; - Results = NULL; - Counters = NULL; - std::cout << " Number of reruns should be positive! " << std::endl; - return 0; + }else { - } - //............................... - //srand(time(0)); - //srandomdev(); + Results = NULL; + Counters = NULL; + std::cout << " Number of reruns should be positive! " << std::endl; + return 0; - if ( initSeed != 0.0 ) { - srand(initSeed); - } - else { - timeval t; - gettimeofday(&t, NULL); - srand(t.tv_usec); - } + } + //............................... + //srand(time(0)); + //srandomdev(); - { - double r=0; - for (int j=0; j<100; j++){ + if ( initSeed != 0.0 ) { + srand(initSeed); + } + else { + timeval t; + gettimeofday(&t, NULL); + srand(t.tv_usec); + } + { + double r=0; + for (int j=0; j<100; j++){ - r = rand()/(double)(pow(2.,31)-1.); - std::cout << r << " "; - } - std::cout << "\n"; + + r = rand()/(double)(pow(2.,31)-1.); + std::cout << r << " "; } - //random initiation of starting parameters + std::cout << "\n"; + } + //random initiation of starting parameters - if (range > 0.){ + if (range > 0.){ - for (int i=0; i < 5; i++){ + for (int i=0; i < 5; i++){ - if (params0[i]==-100.){ + if (params0[i]==-100.){ - double r1 = (rand()/(double)(pow(2.,31)-1.)); - double r2 = (rand()/(double)(pow(2.,31)-1.)); - double sign = 1.; + double r1 = (rand()/(double)(pow(2.,31)-1.)); + double r2 = (rand()/(double)(pow(2.,31)-1.)); + double sign = 1.; - if(r1 > 0.5){ - sign=-1.; - } - - params0[i] = sign*r2*range; - - std::cout << par_names0[i] << ": " << params0[i] << std::endl; - } + if(r1 > 0.5){ + sign=-1.; } + params0[i] = sign*r2*range; + + std::cout << par_names0[i] << ": " << params0[i] << std::endl; + } } + } - double T_start=params2[0], T_end=params2[1], Target_rejection=params2[3], starting_jump=params2[4]; - int Annealing_repeats = (int) params2[2]; -#ifdef P_DISPATCH - dispatch_group_t group = dispatch_group_create(); + double T_start=params2[0], T_end=params2[1], Target_rejection=params2[3], starting_jump=params2[4]; + int Annealing_repeats = (int) params2[2]; - //............................. - multi_annealing(group, un, CustomQueues, T_start, T_end, Target_rejection, Annealing_repeats, - starting_jump, Results, Counters, params0, Annealing_repeats); + multi_annealing( un, T_start, T_end, Target_rejection, Annealing_repeats, + starting_jump, Results, Counters, params0, Annealing_repeats); - //dispatch_group_wait(group, DISPATCH_TIME_FOREVER); - dispatch_release(group); - //............................. -#else - multi_annealing( un, T_start, T_end, Target_rejection, Annealing_repeats, - starting_jump, Results, Counters, params0, Annealing_repeats); -#endif + // stop timer + gettimeofday(&t2, NULL); + // compute and print the elapsed time in millisec + elapsedTime = (t2.tv_sec - t1.tv_sec) * 1000.0; // sec to ms + elapsedTime += (t2.tv_usec - t1.tv_usec) / 1000.0; // us to ms + elapsedTime /= 1000.; + cout << "\n*** optimizer completed, elapsed time=" << elapsedTime << " seconds " << elapsedTime/60. << " minutes)\n\n"; - // stop timer - gettimeofday(&t2, NULL); + //..................... - // compute and print the elapsed time in millisec - elapsedTime = (t2.tv_sec - t1.tv_sec) * 1000.0; // sec to ms - elapsedTime += (t2.tv_usec - t1.tv_usec) / 1000.0; // us to ms - elapsedTime /= 1000.; - cout << "\n*** optimizer completed, elapsed time=" << elapsedTime << " seconds " << elapsedTime/60. << " minutes)\n\n"; + for(int i=0; i 0){ - for(int i=0; i 0){ + } - delete [] Results; - delete [] Counters; + return 0; - } - return 0; - - } From davidk at ci.uchicago.edu Wed Jan 30 22:01:17 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 31 Jan 2013 04:01:17 -0000 Subject: [Swift-commit] r6197 - SwiftApps/modis/data/modis/2002 Message-ID: <20130131040116.89C449CCBE@svn.ci.uchicago.edu> Author: davidk Date: 2013-01-30 22:01:16 -0600 (Wed, 30 Jan 2013) New Revision: 6197 Added: SwiftApps/modis/data/modis/2002/h00v08.ppm.gz SwiftApps/modis/data/modis/2002/h00v09.ppm.gz SwiftApps/modis/data/modis/2002/h00v10.ppm.gz SwiftApps/modis/data/modis/2002/h01v07.ppm.gz SwiftApps/modis/data/modis/2002/h01v08.ppm.gz SwiftApps/modis/data/modis/2002/h01v09.ppm.gz SwiftApps/modis/data/modis/2002/h01v10.ppm.gz SwiftApps/modis/data/modis/2002/h01v11.ppm.gz SwiftApps/modis/data/modis/2002/h02v06.ppm.gz SwiftApps/modis/data/modis/2002/h02v08.ppm.gz SwiftApps/modis/data/modis/2002/h02v09.ppm.gz SwiftApps/modis/data/modis/2002/h02v10.ppm.gz SwiftApps/modis/data/modis/2002/h02v11.ppm.gz SwiftApps/modis/data/modis/2002/h03v06.ppm.gz SwiftApps/modis/data/modis/2002/h03v07.ppm.gz SwiftApps/modis/data/modis/2002/h03v09.ppm.gz SwiftApps/modis/data/modis/2002/h03v10.ppm.gz SwiftApps/modis/data/modis/2002/h03v11.ppm.gz SwiftApps/modis/data/modis/2002/h04v09.ppm.gz SwiftApps/modis/data/modis/2002/h04v10.ppm.gz SwiftApps/modis/data/modis/2002/h04v11.ppm.gz SwiftApps/modis/data/modis/2002/h05v10.ppm.gz SwiftApps/modis/data/modis/2002/h05v11.ppm.gz SwiftApps/modis/data/modis/2002/h05v13.ppm.gz SwiftApps/modis/data/modis/2002/h06v03.ppm.gz SwiftApps/modis/data/modis/2002/h06v11.ppm.gz SwiftApps/modis/data/modis/2002/h07v03.ppm.gz SwiftApps/modis/data/modis/2002/h07v05.ppm.gz SwiftApps/modis/data/modis/2002/h07v06.ppm.gz SwiftApps/modis/data/modis/2002/h07v07.ppm.gz SwiftApps/modis/data/modis/2002/h08v03.ppm.gz SwiftApps/modis/data/modis/2002/h08v04.ppm.gz SwiftApps/modis/data/modis/2002/h08v05.ppm.gz SwiftApps/modis/data/modis/2002/h08v06.ppm.gz SwiftApps/modis/data/modis/2002/h08v07.ppm.gz SwiftApps/modis/data/modis/2002/h08v08.ppm.gz SwiftApps/modis/data/modis/2002/h08v09.ppm.gz SwiftApps/modis/data/modis/2002/h08v11.ppm.gz SwiftApps/modis/data/modis/2002/h09v02.ppm.gz SwiftApps/modis/data/modis/2002/h09v03.ppm.gz SwiftApps/modis/data/modis/2002/h09v04.ppm.gz SwiftApps/modis/data/modis/2002/h09v05.ppm.gz SwiftApps/modis/data/modis/2002/h09v06.ppm.gz SwiftApps/modis/data/modis/2002/h09v07.ppm.gz SwiftApps/modis/data/modis/2002/h09v08.ppm.gz SwiftApps/modis/data/modis/2002/h09v09.ppm.gz SwiftApps/modis/data/modis/2002/h10v02.ppm.gz SwiftApps/modis/data/modis/2002/h10v03.ppm.gz SwiftApps/modis/data/modis/2002/h10v04.ppm.gz SwiftApps/modis/data/modis/2002/h10v05.ppm.gz SwiftApps/modis/data/modis/2002/h10v06.ppm.gz SwiftApps/modis/data/modis/2002/h10v07.ppm.gz SwiftApps/modis/data/modis/2002/h10v08.ppm.gz SwiftApps/modis/data/modis/2002/h10v09.ppm.gz SwiftApps/modis/data/modis/2002/h10v10.ppm.gz SwiftApps/modis/data/modis/2002/h10v11.ppm.gz SwiftApps/modis/data/modis/2002/h11v02.ppm.gz SwiftApps/modis/data/modis/2002/h11v03.ppm.gz SwiftApps/modis/data/modis/2002/h11v04.ppm.gz SwiftApps/modis/data/modis/2002/h11v05.ppm.gz SwiftApps/modis/data/modis/2002/h11v06.ppm.gz SwiftApps/modis/data/modis/2002/h11v07.ppm.gz SwiftApps/modis/data/modis/2002/h11v08.ppm.gz SwiftApps/modis/data/modis/2002/h11v09.ppm.gz SwiftApps/modis/data/modis/2002/h11v10.ppm.gz SwiftApps/modis/data/modis/2002/h11v11.ppm.gz SwiftApps/modis/data/modis/2002/h11v12.ppm.gz SwiftApps/modis/data/modis/2002/h12v01.ppm.gz SwiftApps/modis/data/modis/2002/h12v02.ppm.gz SwiftApps/modis/data/modis/2002/h12v03.ppm.gz SwiftApps/modis/data/modis/2002/h12v04.ppm.gz SwiftApps/modis/data/modis/2002/h12v05.ppm.gz SwiftApps/modis/data/modis/2002/h12v07.ppm.gz SwiftApps/modis/data/modis/2002/h12v08.ppm.gz SwiftApps/modis/data/modis/2002/h12v09.ppm.gz SwiftApps/modis/data/modis/2002/h12v10.ppm.gz SwiftApps/modis/data/modis/2002/h12v11.ppm.gz SwiftApps/modis/data/modis/2002/h12v12.ppm.gz SwiftApps/modis/data/modis/2002/h12v13.ppm.gz SwiftApps/modis/data/modis/2002/h13v01.ppm.gz SwiftApps/modis/data/modis/2002/h13v02.ppm.gz SwiftApps/modis/data/modis/2002/h13v03.ppm.gz SwiftApps/modis/data/modis/2002/h13v04.ppm.gz SwiftApps/modis/data/modis/2002/h13v08.ppm.gz SwiftApps/modis/data/modis/2002/h13v09.ppm.gz SwiftApps/modis/data/modis/2002/h13v10.ppm.gz SwiftApps/modis/data/modis/2002/h13v11.ppm.gz SwiftApps/modis/data/modis/2002/h13v12.ppm.gz SwiftApps/modis/data/modis/2002/h13v13.ppm.gz SwiftApps/modis/data/modis/2002/h13v14.ppm.gz SwiftApps/modis/data/modis/2002/h14v01.ppm.gz SwiftApps/modis/data/modis/2002/h14v02.ppm.gz SwiftApps/modis/data/modis/2002/h14v03.ppm.gz SwiftApps/modis/data/modis/2002/h14v04.ppm.gz SwiftApps/modis/data/modis/2002/h14v09.ppm.gz SwiftApps/modis/data/modis/2002/h14v10.ppm.gz SwiftApps/modis/data/modis/2002/h14v11.ppm.gz SwiftApps/modis/data/modis/2002/h14v14.ppm.gz SwiftApps/modis/data/modis/2002/h14v16.ppm.gz SwiftApps/modis/data/modis/2002/h14v17.ppm.gz SwiftApps/modis/data/modis/2002/h15v01.ppm.gz SwiftApps/modis/data/modis/2002/h15v02.ppm.gz SwiftApps/modis/data/modis/2002/h15v03.ppm.gz SwiftApps/modis/data/modis/2002/h15v05.ppm.gz SwiftApps/modis/data/modis/2002/h15v07.ppm.gz SwiftApps/modis/data/modis/2002/h15v11.ppm.gz SwiftApps/modis/data/modis/2002/h15v14.ppm.gz SwiftApps/modis/data/modis/2002/h15v15.ppm.gz SwiftApps/modis/data/modis/2002/h15v16.ppm.gz SwiftApps/modis/data/modis/2002/h15v17.ppm.gz SwiftApps/modis/data/modis/2002/h16v00.ppm.gz SwiftApps/modis/data/modis/2002/h16v01.ppm.gz SwiftApps/modis/data/modis/2002/h16v02.ppm.gz SwiftApps/modis/data/modis/2002/h16v05.ppm.gz SwiftApps/modis/data/modis/2002/h16v06.ppm.gz SwiftApps/modis/data/modis/2002/h16v07.ppm.gz SwiftApps/modis/data/modis/2002/h16v08.ppm.gz SwiftApps/modis/data/modis/2002/h16v09.ppm.gz SwiftApps/modis/data/modis/2002/h16v12.ppm.gz SwiftApps/modis/data/modis/2002/h16v14.ppm.gz SwiftApps/modis/data/modis/2002/h16v16.ppm.gz SwiftApps/modis/data/modis/2002/h16v17.ppm.gz SwiftApps/modis/data/modis/2002/h17v00.ppm.gz SwiftApps/modis/data/modis/2002/h17v01.ppm.gz SwiftApps/modis/data/modis/2002/h17v02.ppm.gz SwiftApps/modis/data/modis/2002/h17v03.ppm.gz SwiftApps/modis/data/modis/2002/h17v04.ppm.gz SwiftApps/modis/data/modis/2002/h17v05.ppm.gz SwiftApps/modis/data/modis/2002/h17v06.ppm.gz SwiftApps/modis/data/modis/2002/h17v07.ppm.gz SwiftApps/modis/data/modis/2002/h17v08.ppm.gz SwiftApps/modis/data/modis/2002/h17v10.ppm.gz SwiftApps/modis/data/modis/2002/h17v12.ppm.gz SwiftApps/modis/data/modis/2002/h17v13.ppm.gz SwiftApps/modis/data/modis/2002/h17v15.ppm.gz SwiftApps/modis/data/modis/2002/h17v16.ppm.gz SwiftApps/modis/data/modis/2002/h17v17.ppm.gz SwiftApps/modis/data/modis/2002/h18v00.ppm.gz SwiftApps/modis/data/modis/2002/h18v01.ppm.gz SwiftApps/modis/data/modis/2002/h18v02.ppm.gz SwiftApps/modis/data/modis/2002/h18v03.ppm.gz SwiftApps/modis/data/modis/2002/h18v04.ppm.gz SwiftApps/modis/data/modis/2002/h18v05.ppm.gz SwiftApps/modis/data/modis/2002/h18v06.ppm.gz SwiftApps/modis/data/modis/2002/h18v07.ppm.gz SwiftApps/modis/data/modis/2002/h18v08.ppm.gz SwiftApps/modis/data/modis/2002/h18v09.ppm.gz SwiftApps/modis/data/modis/2002/h18v14.ppm.gz SwiftApps/modis/data/modis/2002/h18v15.ppm.gz SwiftApps/modis/data/modis/2002/h18v16.ppm.gz SwiftApps/modis/data/modis/2002/h18v17.ppm.gz SwiftApps/modis/data/modis/2002/h19v00.ppm.gz SwiftApps/modis/data/modis/2002/h19v01.ppm.gz SwiftApps/modis/data/modis/2002/h19v02.ppm.gz SwiftApps/modis/data/modis/2002/h19v03.ppm.gz SwiftApps/modis/data/modis/2002/h19v04.ppm.gz SwiftApps/modis/data/modis/2002/h19v05.ppm.gz SwiftApps/modis/data/modis/2002/h19v06.ppm.gz SwiftApps/modis/data/modis/2002/h19v07.ppm.gz SwiftApps/modis/data/modis/2002/h19v08.ppm.gz SwiftApps/modis/data/modis/2002/h19v09.ppm.gz SwiftApps/modis/data/modis/2002/h19v10.ppm.gz SwiftApps/modis/data/modis/2002/h19v11.ppm.gz SwiftApps/modis/data/modis/2002/h19v12.ppm.gz SwiftApps/modis/data/modis/2002/h19v15.ppm.gz SwiftApps/modis/data/modis/2002/h19v16.ppm.gz SwiftApps/modis/data/modis/2002/h19v17.ppm.gz SwiftApps/modis/data/modis/2002/h20v01.ppm.gz SwiftApps/modis/data/modis/2002/h20v02.ppm.gz SwiftApps/modis/data/modis/2002/h20v03.ppm.gz SwiftApps/modis/data/modis/2002/h20v04.ppm.gz SwiftApps/modis/data/modis/2002/h20v05.ppm.gz SwiftApps/modis/data/modis/2002/h20v06.ppm.gz SwiftApps/modis/data/modis/2002/h20v07.ppm.gz SwiftApps/modis/data/modis/2002/h20v08.ppm.gz SwiftApps/modis/data/modis/2002/h20v09.ppm.gz SwiftApps/modis/data/modis/2002/h20v10.ppm.gz SwiftApps/modis/data/modis/2002/h20v11.ppm.gz SwiftApps/modis/data/modis/2002/h20v12.ppm.gz SwiftApps/modis/data/modis/2002/h20v13.ppm.gz SwiftApps/modis/data/modis/2002/h20v15.ppm.gz SwiftApps/modis/data/modis/2002/h20v16.ppm.gz SwiftApps/modis/data/modis/2002/h20v17.ppm.gz SwiftApps/modis/data/modis/2002/h21v01.ppm.gz SwiftApps/modis/data/modis/2002/h21v02.ppm.gz SwiftApps/modis/data/modis/2002/h21v03.ppm.gz SwiftApps/modis/data/modis/2002/h21v04.ppm.gz SwiftApps/modis/data/modis/2002/h21v05.ppm.gz SwiftApps/modis/data/modis/2002/h21v06.ppm.gz SwiftApps/modis/data/modis/2002/h21v07.ppm.gz SwiftApps/modis/data/modis/2002/h21v08.ppm.gz SwiftApps/modis/data/modis/2002/h21v09.ppm.gz SwiftApps/modis/data/modis/2002/h21v10.ppm.gz SwiftApps/modis/data/modis/2002/h21v11.ppm.gz SwiftApps/modis/data/modis/2002/h21v13.ppm.gz SwiftApps/modis/data/modis/2002/h21v15.ppm.gz SwiftApps/modis/data/modis/2002/h21v16.ppm.gz SwiftApps/modis/data/modis/2002/h21v17.ppm.gz SwiftApps/modis/data/modis/2002/h22v01.ppm.gz SwiftApps/modis/data/modis/2002/h22v02.ppm.gz SwiftApps/modis/data/modis/2002/h22v03.ppm.gz SwiftApps/modis/data/modis/2002/h22v04.ppm.gz SwiftApps/modis/data/modis/2002/h22v05.ppm.gz SwiftApps/modis/data/modis/2002/h22v06.ppm.gz SwiftApps/modis/data/modis/2002/h22v07.ppm.gz SwiftApps/modis/data/modis/2002/h22v08.ppm.gz SwiftApps/modis/data/modis/2002/h22v09.ppm.gz SwiftApps/modis/data/modis/2002/h22v10.ppm.gz SwiftApps/modis/data/modis/2002/h22v11.ppm.gz SwiftApps/modis/data/modis/2002/h22v13.ppm.gz SwiftApps/modis/data/modis/2002/h22v14.ppm.gz SwiftApps/modis/data/modis/2002/h22v15.ppm.gz SwiftApps/modis/data/modis/2002/h22v16.ppm.gz SwiftApps/modis/data/modis/2002/h23v01.ppm.gz SwiftApps/modis/data/modis/2002/h23v02.ppm.gz SwiftApps/modis/data/modis/2002/h23v03.ppm.gz SwiftApps/modis/data/modis/2002/h23v04.ppm.gz SwiftApps/modis/data/modis/2002/h23v05.ppm.gz SwiftApps/modis/data/modis/2002/h23v06.ppm.gz SwiftApps/modis/data/modis/2002/h23v07.ppm.gz SwiftApps/modis/data/modis/2002/h23v08.ppm.gz SwiftApps/modis/data/modis/2002/h23v09.ppm.gz SwiftApps/modis/data/modis/2002/h23v10.ppm.gz SwiftApps/modis/data/modis/2002/h23v11.ppm.gz SwiftApps/modis/data/modis/2002/h23v15.ppm.gz SwiftApps/modis/data/modis/2002/h23v16.ppm.gz SwiftApps/modis/data/modis/2002/h24v02.ppm.gz SwiftApps/modis/data/modis/2002/h24v03.ppm.gz SwiftApps/modis/data/modis/2002/h24v04.ppm.gz SwiftApps/modis/data/modis/2002/h24v05.ppm.gz SwiftApps/modis/data/modis/2002/h24v06.ppm.gz SwiftApps/modis/data/modis/2002/h24v07.ppm.gz SwiftApps/modis/data/modis/2002/h24v12.ppm.gz SwiftApps/modis/data/modis/2002/h24v15.ppm.gz SwiftApps/modis/data/modis/2002/h25v02.ppm.gz SwiftApps/modis/data/modis/2002/h25v03.ppm.gz SwiftApps/modis/data/modis/2002/h25v04.ppm.gz SwiftApps/modis/data/modis/2002/h25v05.ppm.gz SwiftApps/modis/data/modis/2002/h25v06.ppm.gz SwiftApps/modis/data/modis/2002/h25v07.ppm.gz SwiftApps/modis/data/modis/2002/h25v08.ppm.gz SwiftApps/modis/data/modis/2002/h25v09.ppm.gz SwiftApps/modis/data/modis/2002/h26v02.ppm.gz SwiftApps/modis/data/modis/2002/h26v03.ppm.gz SwiftApps/modis/data/modis/2002/h26v04.ppm.gz SwiftApps/modis/data/modis/2002/h26v05.ppm.gz SwiftApps/modis/data/modis/2002/h26v06.ppm.gz SwiftApps/modis/data/modis/2002/h26v07.ppm.gz SwiftApps/modis/data/modis/2002/h26v08.ppm.gz SwiftApps/modis/data/modis/2002/h27v03.ppm.gz SwiftApps/modis/data/modis/2002/h27v04.ppm.gz SwiftApps/modis/data/modis/2002/h27v05.ppm.gz SwiftApps/modis/data/modis/2002/h27v06.ppm.gz SwiftApps/modis/data/modis/2002/h27v07.ppm.gz SwiftApps/modis/data/modis/2002/h27v08.ppm.gz SwiftApps/modis/data/modis/2002/h27v09.ppm.gz SwiftApps/modis/data/modis/2002/h27v10.ppm.gz SwiftApps/modis/data/modis/2002/h27v11.ppm.gz SwiftApps/modis/data/modis/2002/h27v12.ppm.gz SwiftApps/modis/data/modis/2002/h27v14.ppm.gz SwiftApps/modis/data/modis/2002/h28v03.ppm.gz SwiftApps/modis/data/modis/2002/h28v04.ppm.gz SwiftApps/modis/data/modis/2002/h28v05.ppm.gz SwiftApps/modis/data/modis/2002/h28v06.ppm.gz SwiftApps/modis/data/modis/2002/h28v07.ppm.gz SwiftApps/modis/data/modis/2002/h28v08.ppm.gz SwiftApps/modis/data/modis/2002/h28v09.ppm.gz SwiftApps/modis/data/modis/2002/h28v10.ppm.gz SwiftApps/modis/data/modis/2002/h28v11.ppm.gz SwiftApps/modis/data/modis/2002/h28v12.ppm.gz SwiftApps/modis/data/modis/2002/h28v13.ppm.gz SwiftApps/modis/data/modis/2002/h28v14.ppm.gz SwiftApps/modis/data/modis/2002/h29v03.ppm.gz SwiftApps/modis/data/modis/2002/h29v05.ppm.gz SwiftApps/modis/data/modis/2002/h29v06.ppm.gz SwiftApps/modis/data/modis/2002/h29v07.ppm.gz SwiftApps/modis/data/modis/2002/h29v08.ppm.gz SwiftApps/modis/data/modis/2002/h29v09.ppm.gz SwiftApps/modis/data/modis/2002/h29v10.ppm.gz SwiftApps/modis/data/modis/2002/h29v11.ppm.gz SwiftApps/modis/data/modis/2002/h29v12.ppm.gz SwiftApps/modis/data/modis/2002/h29v13.ppm.gz SwiftApps/modis/data/modis/2002/h30v05.ppm.gz SwiftApps/modis/data/modis/2002/h30v06.ppm.gz SwiftApps/modis/data/modis/2002/h30v07.ppm.gz SwiftApps/modis/data/modis/2002/h30v08.ppm.gz SwiftApps/modis/data/modis/2002/h30v09.ppm.gz SwiftApps/modis/data/modis/2002/h30v10.ppm.gz SwiftApps/modis/data/modis/2002/h30v11.ppm.gz SwiftApps/modis/data/modis/2002/h30v12.ppm.gz SwiftApps/modis/data/modis/2002/h30v13.ppm.gz SwiftApps/modis/data/modis/2002/h31v06.ppm.gz SwiftApps/modis/data/modis/2002/h31v07.ppm.gz SwiftApps/modis/data/modis/2002/h31v08.ppm.gz SwiftApps/modis/data/modis/2002/h31v09.ppm.gz SwiftApps/modis/data/modis/2002/h31v10.ppm.gz SwiftApps/modis/data/modis/2002/h31v11.ppm.gz SwiftApps/modis/data/modis/2002/h31v12.ppm.gz SwiftApps/modis/data/modis/2002/h31v13.ppm.gz SwiftApps/modis/data/modis/2002/h32v07.ppm.gz SwiftApps/modis/data/modis/2002/h32v08.ppm.gz SwiftApps/modis/data/modis/2002/h32v09.ppm.gz SwiftApps/modis/data/modis/2002/h32v10.ppm.gz SwiftApps/modis/data/modis/2002/h32v11.ppm.gz SwiftApps/modis/data/modis/2002/h32v12.ppm.gz SwiftApps/modis/data/modis/2002/h33v07.ppm.gz SwiftApps/modis/data/modis/2002/h33v08.ppm.gz SwiftApps/modis/data/modis/2002/h33v09.ppm.gz SwiftApps/modis/data/modis/2002/h33v10.ppm.gz SwiftApps/modis/data/modis/2002/h33v11.ppm.gz SwiftApps/modis/data/modis/2002/h34v07.ppm.gz SwiftApps/modis/data/modis/2002/h34v08.ppm.gz SwiftApps/modis/data/modis/2002/h34v09.ppm.gz SwiftApps/modis/data/modis/2002/h34v10.ppm.gz SwiftApps/modis/data/modis/2002/h35v08.ppm.gz SwiftApps/modis/data/modis/2002/h35v09.ppm.gz SwiftApps/modis/data/modis/2002/h35v10.ppm.gz Log: Compressed ppm files Added: SwiftApps/modis/data/modis/2002/h00v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h00v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h00v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h00v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h00v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h00v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h01v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h01v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h01v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h01v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h01v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h01v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h01v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h01v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h01v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h01v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h02v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h02v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h02v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h02v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h02v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h02v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h02v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h02v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h02v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h02v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h03v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h03v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h03v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h03v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h03v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h03v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h03v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h03v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h03v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h03v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h04v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h04v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h04v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h04v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h04v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h04v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h05v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h05v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h05v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h05v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h05v13.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h05v13.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h06v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h06v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h06v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h06v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h07v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h07v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h07v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h07v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h07v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h07v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h07v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h07v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h08v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h08v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h08v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h08v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h08v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h08v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h08v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h08v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h08v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h08v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h08v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h08v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h08v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h08v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h08v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h08v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h09v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h09v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h09v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h09v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h09v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h09v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h09v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h09v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h09v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h09v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h09v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h09v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h09v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h09v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h09v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h09v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h10v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h10v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h10v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h10v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h10v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h10v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h10v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h10v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h10v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h10v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h10v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h10v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h10v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h10v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h10v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h10v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h10v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h10v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h10v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h10v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h11v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h11v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h11v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h11v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h11v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h11v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h11v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h11v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h11v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h11v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h11v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h11v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h11v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h11v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h11v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h11v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h11v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h11v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h11v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h11v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h11v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h11v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h12v13.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h12v13.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h13v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h13v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h13v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h13v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h13v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h13v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h13v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h13v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h13v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h13v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h13v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h13v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h13v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h13v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h13v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h13v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h13v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h13v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h13v13.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h13v13.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h13v14.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h13v14.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h14v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h14v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h14v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h14v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h14v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h14v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h14v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h14v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h14v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h14v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h14v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h14v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h14v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h14v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h14v14.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h14v14.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h14v16.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h14v16.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h14v17.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h14v17.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h15v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h15v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h15v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h15v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h15v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h15v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h15v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h15v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h15v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h15v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h15v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h15v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h15v14.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h15v14.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h15v15.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h15v15.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h15v16.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h15v16.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h15v17.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h15v17.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v00.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v00.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v14.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v14.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v16.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v16.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h16v17.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h16v17.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v00.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v00.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v13.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v13.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v15.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v15.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v16.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v16.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h17v17.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h17v17.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v00.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v00.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v14.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v14.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v15.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v15.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v16.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v16.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h18v17.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h18v17.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v00.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v00.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v15.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v15.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v16.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v16.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h19v17.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h19v17.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v13.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v13.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v15.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v15.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v16.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v16.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h20v17.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h20v17.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v13.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v13.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v15.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v15.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v16.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v16.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h21v17.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h21v17.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v13.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v13.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v14.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v14.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v15.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v15.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h22v16.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h22v16.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v01.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v01.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v15.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v15.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h23v16.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h23v16.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h24v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h24v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h24v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h24v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h24v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h24v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h24v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h24v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h24v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h24v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h24v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h24v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h24v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h24v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h24v15.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h24v15.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h25v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h25v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h25v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h25v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h25v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h25v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h25v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h25v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h25v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h25v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h25v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h25v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h25v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h25v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h25v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h25v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h26v02.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h26v02.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h26v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h26v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h26v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h26v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h26v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h26v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h26v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h26v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h26v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h26v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h26v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h26v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h27v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h27v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h27v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h27v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h27v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h27v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h27v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h27v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h27v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h27v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h27v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h27v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h27v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h27v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h27v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h27v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h27v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h27v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h27v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h27v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h27v14.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h27v14.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v04.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v04.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v13.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v13.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h28v14.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h28v14.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h29v03.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h29v03.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h29v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h29v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h29v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h29v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h29v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h29v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h29v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h29v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h29v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h29v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h29v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h29v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h29v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h29v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h29v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h29v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h29v13.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h29v13.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h30v05.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h30v05.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h30v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h30v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h30v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h30v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h30v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h30v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h30v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h30v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h30v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h30v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h30v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h30v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h30v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h30v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h30v13.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h30v13.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h31v06.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h31v06.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h31v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h31v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h31v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h31v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h31v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h31v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h31v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h31v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h31v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h31v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h31v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h31v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h31v13.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h31v13.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h32v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h32v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h32v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h32v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h32v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h32v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h32v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h32v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h32v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h32v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h32v12.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h32v12.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h33v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h33v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h33v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h33v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h33v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h33v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h33v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h33v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h33v11.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h33v11.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h34v07.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h34v07.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h34v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h34v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h34v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h34v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h34v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h34v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h35v08.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h35v08.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h35v09.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h35v09.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftApps/modis/data/modis/2002/h35v10.ppm.gz =================================================================== (Binary files differ) Property changes on: SwiftApps/modis/data/modis/2002/h35v10.ppm.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream From hategan at ci.uchicago.edu Tue Jan 29 01:31:13 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Tue, 29 Jan 2013 07:31:13 -0000 Subject: [Swift-commit] r6170 - in branches/faster: libexec resources src/org/globus/swift/catalog src/org/globus/swift/catalog/site src/org/globus/swift/catalog/transformation src/org/globus/swift/data src/org/griphyn/vdl/karajan src/org/griphyn/vdl/karajan/functions src/org/griphyn/vdl/karajan/lib src/org/griphyn/vdl/karajan/lib/cache src/org/griphyn/vdl/karajan/lib/swiftscript src/org/griphyn/vdl/karajan/monitor src/org/griphyn/vdl/karajan/monitor/items src/org/griphyn/vdl/karajan/monitor/monitors/ansi src/org/griphyn/vdl/karajan/monitor/processors src/org/griphyn/vdl/mapping src/org/griphyn/vdl/mapping/file src/org/griphyn/vdl/util Message-ID: <20130129073111.241409CCB3@svn.ci.uchicago.edu> Author: hategan Date: 2013-01-29 01:31:09 -0600 (Tue, 29 Jan 2013) New Revision: 6170 Added: branches/faster/libexec/scheduler.k branches/faster/libexec/swift-int.k branches/faster/libexec/swift-lib.k branches/faster/libexec/swift-operators.k branches/faster/libexec/swift-xs.k branches/faster/libexec/swift.k branches/faster/src/org/globus/swift/catalog/site/ branches/faster/src/org/globus/swift/catalog/site/Parser.java branches/faster/src/org/griphyn/vdl/karajan/PairSet.java branches/faster/src/org/griphyn/vdl/karajan/SwiftExecutor.java branches/faster/src/org/griphyn/vdl/karajan/SwiftRootScope.java branches/faster/src/org/griphyn/vdl/karajan/lib/CurrentThread.java branches/faster/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java branches/faster/src/org/griphyn/vdl/mapping/OOBYield.java Removed: branches/faster/libexec/operators.xml branches/faster/libexec/scheduler.xml branches/faster/libexec/vdl-int.k branches/faster/libexec/vdl-lib.xml branches/faster/libexec/vdl-sc.k branches/faster/libexec/vdl-xs.k branches/faster/libexec/vdl.k branches/faster/src/org/griphyn/vdl/karajan/PairIterator.java branches/faster/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java branches/faster/src/org/griphyn/vdl/karajan/VDL2FutureException.java branches/faster/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java branches/faster/src/org/griphyn/vdl/karajan/lib/Kickstart.java branches/faster/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftArg.java branches/faster/src/org/griphyn/vdl/karajan/lib/ThreadPrefix.java branches/faster/src/org/griphyn/vdl/karajan/lib/Typecheck.java branches/faster/src/org/griphyn/vdl/karajan/lib/VDLFunction.java Modified: branches/faster/libexec/vdl-int-staging.k branches/faster/resources/Karajan.stg branches/faster/src/org/globus/swift/catalog/transformation/File.java branches/faster/src/org/globus/swift/data/Action.java branches/faster/src/org/globus/swift/data/Query.java branches/faster/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java branches/faster/src/org/griphyn/vdl/karajan/AssertFailedException.java branches/faster/src/org/griphyn/vdl/karajan/DSHandleFutureWrapper.java branches/faster/src/org/griphyn/vdl/karajan/FuturePairIterator.java branches/faster/src/org/griphyn/vdl/karajan/FutureTracker.java branches/faster/src/org/griphyn/vdl/karajan/FutureWrapper.java branches/faster/src/org/griphyn/vdl/karajan/HangChecker.java branches/faster/src/org/griphyn/vdl/karajan/Loader.java branches/faster/src/org/griphyn/vdl/karajan/Mergeable.java branches/faster/src/org/griphyn/vdl/karajan/Monitor.java branches/faster/src/org/griphyn/vdl/karajan/Pair.java branches/faster/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java branches/faster/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java branches/faster/src/org/griphyn/vdl/karajan/WaitingThreadsMonitor.java branches/faster/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java branches/faster/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java branches/faster/src/org/griphyn/vdl/karajan/lib/AbsFileName.java branches/faster/src/org/griphyn/vdl/karajan/lib/AppStageins.java branches/faster/src/org/griphyn/vdl/karajan/lib/AppStageouts.java branches/faster/src/org/griphyn/vdl/karajan/lib/AppendArray.java branches/faster/src/org/griphyn/vdl/karajan/lib/CacheAddAndLockFile.java branches/faster/src/org/griphyn/vdl/karajan/lib/CacheAddFile.java branches/faster/src/org/griphyn/vdl/karajan/lib/CacheFileRemoved.java branches/faster/src/org/griphyn/vdl/karajan/lib/CacheFunction.java branches/faster/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java branches/faster/src/org/griphyn/vdl/karajan/lib/CleanDataset.java branches/faster/src/org/griphyn/vdl/karajan/lib/CloseDataset.java branches/faster/src/org/griphyn/vdl/karajan/lib/CreateArray.java branches/faster/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java branches/faster/src/org/griphyn/vdl/karajan/lib/Executable.java branches/faster/src/org/griphyn/vdl/karajan/lib/Execute.java branches/faster/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java branches/faster/src/org/griphyn/vdl/karajan/lib/FileCopier.java branches/faster/src/org/griphyn/vdl/karajan/lib/FileName.java branches/faster/src/org/griphyn/vdl/karajan/lib/Flatten.java branches/faster/src/org/griphyn/vdl/karajan/lib/FringePaths.java branches/faster/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java branches/faster/src/org/griphyn/vdl/karajan/lib/GetDatasetProvenanceID.java branches/faster/src/org/griphyn/vdl/karajan/lib/GetField.java branches/faster/src/org/griphyn/vdl/karajan/lib/GetFieldSubscript.java branches/faster/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java branches/faster/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java branches/faster/src/org/griphyn/vdl/karajan/lib/InFileDirs.java branches/faster/src/org/griphyn/vdl/karajan/lib/IsDone.java branches/faster/src/org/griphyn/vdl/karajan/lib/IsFileBound.java branches/faster/src/org/griphyn/vdl/karajan/lib/IsLogged.java branches/faster/src/org/griphyn/vdl/karajan/lib/IsRestartable.java branches/faster/src/org/griphyn/vdl/karajan/lib/JobConstraints.java branches/faster/src/org/griphyn/vdl/karajan/lib/Log.java branches/faster/src/org/griphyn/vdl/karajan/lib/LogVar.java branches/faster/src/org/griphyn/vdl/karajan/lib/Mark.java branches/faster/src/org/griphyn/vdl/karajan/lib/New.java branches/faster/src/org/griphyn/vdl/karajan/lib/NiceName.java branches/faster/src/org/griphyn/vdl/karajan/lib/Operators.java branches/faster/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java branches/faster/src/org/griphyn/vdl/karajan/lib/OutFiles.java branches/faster/src/org/griphyn/vdl/karajan/lib/Parameterlog.java branches/faster/src/org/griphyn/vdl/karajan/lib/PartialCloseDataset.java branches/faster/src/org/griphyn/vdl/karajan/lib/PathUtils.java branches/faster/src/org/griphyn/vdl/karajan/lib/Range.java branches/faster/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java branches/faster/src/org/griphyn/vdl/karajan/lib/SetFutureFault.java branches/faster/src/org/griphyn/vdl/karajan/lib/SetWaitCount.java branches/faster/src/org/griphyn/vdl/karajan/lib/SiteProfile.java branches/faster/src/org/griphyn/vdl/karajan/lib/SliceArray.java branches/faster/src/org/griphyn/vdl/karajan/lib/Stagein.java branches/faster/src/org/griphyn/vdl/karajan/lib/Stageout.java branches/faster/src/org/griphyn/vdl/karajan/lib/TCProfile.java branches/faster/src/org/griphyn/vdl/karajan/lib/Throttled.java branches/faster/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java branches/faster/src/org/griphyn/vdl/karajan/lib/Tracer.java branches/faster/src/org/griphyn/vdl/karajan/lib/UnitEnd.java branches/faster/src/org/griphyn/vdl/karajan/lib/UnitStart.java branches/faster/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java branches/faster/src/org/griphyn/vdl/karajan/lib/WaitFieldValue.java branches/faster/src/org/griphyn/vdl/karajan/lib/cache/CacheReturn.java branches/faster/src/org/griphyn/vdl/karajan/lib/cache/File.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java branches/faster/src/org/griphyn/vdl/karajan/monitor/SystemState.java branches/faster/src/org/griphyn/vdl/karajan/monitor/items/SummaryItem.java branches/faster/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java branches/faster/src/org/griphyn/vdl/karajan/monitor/processors/ExecutionContextProcessor.java branches/faster/src/org/griphyn/vdl/karajan/monitor/processors/TaskProcessor.java branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java branches/faster/src/org/griphyn/vdl/mapping/AbstractMapper.java branches/faster/src/org/griphyn/vdl/mapping/ArrayDataNode.java branches/faster/src/org/griphyn/vdl/mapping/DSHandle.java branches/faster/src/org/griphyn/vdl/mapping/DataDependentException.java branches/faster/src/org/griphyn/vdl/mapping/DependentException.java branches/faster/src/org/griphyn/vdl/mapping/HandleOpenException.java branches/faster/src/org/griphyn/vdl/mapping/Mapper.java branches/faster/src/org/griphyn/vdl/mapping/MappingParam.java branches/faster/src/org/griphyn/vdl/mapping/RootArrayDataNode.java branches/faster/src/org/griphyn/vdl/mapping/RootDataNode.java branches/faster/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java branches/faster/src/org/griphyn/vdl/mapping/file/CSVMapper.java branches/faster/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java branches/faster/src/org/griphyn/vdl/mapping/file/ExternalMapper.java branches/faster/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java branches/faster/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java branches/faster/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java branches/faster/src/org/griphyn/vdl/mapping/file/SimpleFileMapper.java branches/faster/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java branches/faster/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java branches/faster/src/org/griphyn/vdl/mapping/file/TestMapper.java branches/faster/src/org/griphyn/vdl/util/VDL2Config.java Log: initial update of faster branch Deleted: branches/faster/libexec/operators.xml =================================================================== --- branches/faster/libexec/operators.xml 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/libexec/operators.xml 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - - - - Added: branches/faster/libexec/scheduler.k =================================================================== --- branches/faster/libexec/scheduler.k (rev 0) +++ branches/faster/libexec/scheduler.k 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,32 @@ +import(sys) +import(task) +import('swift-lib') + +sites := swift:configProperty("sites.file") +TCFile := swift:configProperty("tc.file") + +log(LOG:INFO, "Using sites file: {sites}") +if (!file:exists(sites)) { + throw("Could not find sites file: {sites}") +} + +log(LOG:INFO, "Using tc.data: {TCFile}") + +scheduler("vds-adaptive", shareID = "swift:scheduler:{sites}" + property("transformationCatalogFile", TCFile) + property("clusteringEnabled", swift:configProperty("clustering.enabled")) + property("clusteringQueueDelay", swift:configProperty("clustering.queue.delay")) + property("clusteringMinTime", swift:configProperty("clustering.min.time")) + + property("hostSubmitThrottle", swift:configProperty("throttle.host.submit")) + property("submitThrottle", swift:configProperty("throttle.submit")) + property("jobsPerCpu", "off") + property("maxTransfers", swift:configProperty("throttle.transfers")) + property("maxFileOperations", swift:configProperty("throttle.file.operations")) + property("jobThrottle", swift:configProperty("throttle.score.job.factor")) + + task:availableHandlers(type = "execution", includeAliases = true) + task:availableHandlers(type = "file", includeAliases = true) + + resources = swift:siteCatalog(sites) +) Deleted: branches/faster/libexec/scheduler.xml =================================================================== --- branches/faster/libexec/scheduler.xml 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/libexec/scheduler.xml 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,53 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - Using sites file: {sites} - - - - - - Could not find sites file: {sites} - - - - Using tc.data: {tcfile} - - - - - - - - - - - - - - - - - - - - Added: branches/faster/libexec/swift-int.k =================================================================== --- branches/faster/libexec/swift-int.k (rev 0) +++ branches/faster/libexec/swift-int.k 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,609 @@ +import(sys) +import(task) +import('swift-lib') +/* + * Things that are not exposed to the translated file + */ + +SWIFT:SCRIPT_NAME := contextAttribute("SWIFT:SCRIPT_NAME") +SWIFT:RUN_ID := contextAttribute("SWIFT:RUN_ID") +SWIFT:HOME := contextAttribute("SWIFT:HOME") + +namespace(swift) { + + rmdir := function(dir, host) { + parallelFor(entry, file:list(dir, host=host)) { + epath := "{dir}/{entry}" + if (file:isDirectory(epath, host=host)) { + rmdir(epath, host) + } + else { + file:remove(epath, host=host) + } + } + dir:remove(dir, host=host) + } + + createdirs := function(path, dir, host) { + dc := dircat(dir, path) + log(LOG:INFO, "START path={path} dir={dir} - Creating directory structure") + + dir:make(dc, host=host) + } + + checkErrorFile := function(rhost, wfdir, jobid, jobdir) { + if (file:exists("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost)) { + log(LOG:INFO, "FAILURE jobid={jobid} - Failure file found") + task:transfer(srchost=rhost, srcdir="{wfdir}/status/{jobdir}", srcfile="{jobid}-error") + error := parallel( + file:remove("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost) + sequential( + str:strip(file:read("{jobid}-error")) + file:remove("{jobid}-error") + ) + ) + error + } + else { + log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Error file missing") + throw("No status file was found. Check the shared filesystem on {rhost}") + } + } + + checkJobStatus := function(rhost, wfdir, jobid, tr, jobdir) { + log(LOG:DEBUG, "START jobid={jobid}") + try { + file:remove("{wfdir}/status/{jobdir}/{jobid}-success", host=rhost) + log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found") + } + else { + try { + msg := checkErrorFile(rhost, wfdir, jobid, jobdir) + } + else { + log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing") + throw("No status file was found. Check the shared filesystem on {rhost}") + } + } + else { + throw(checkErrorFile(rhost, wfdir, jobid, jobdir)) + } + } + + initSharedDir := function(progress, rhost) { + once(list(rhost, "shared")) { + setProgress(progress, "Initializing site shared directory") + log(LOG:INFO, "START host={rhost} - Initializing shared directory") + + wfdir := "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}" + sharedDir := dircat(wfdir, "shared") + + dir:make(sharedDir, host = rhost) + transfer(srcdir="{SWIFT:HOME}/libexec/", srcfile=siteProfile(rhost, "swift:wrapperScript"), destdir=sharedDir, desthost=rhost) + transfer(srcdir="{SWIFT:HOME}/libexec/", srcfile="_swiftseq", destdir=sharedDir, desthost=rhost) + dir:make(dircat(wfdir, "kickstart"), host=rhost) + + statusMode := configProperty("status.mode",host=rhost) + if (statusMode == "files") { + dir:make(dircat(wfdir, "status"), host=rhost) + } + + wrapperMode := configProperty("wrapper.parameter.mode",host=rhost) + if (wrapperMode == "files") { + dir:make(dircat(wfdir, "parameters"), host=rhost) + } + + dir:make(dircat(wfdir, "info"), host=rhost) + wfdir, sharedDir + //we send the cleanup data to vdl:main() + to(cleanup, list(wfdir, rhost)) + log(LOG:INFO, "END host={rhost} - Done initializing shared directory") + } + } + + initDDir := function() { + ddir := "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}" + + once(ddir) { + if(!file:exists(ddir)) { + task:dir:make(ddir) + } + } + ddir + } + + inFiles := function(stageins) { + pathnames(stageins) + } + + fileDirs := function(stageins, stageouts) { + list( + unique( + inFileDirs(stageins) + outFileDirs(stageouts) + ) + ) + } + + createDirSet := function(jobid, destdir, host, dirs) { + /* + * Ideally this would be done by creating a tree of the directories + * to be created and (eventually) exploiting the concurrency in that. + */ + log(LOG:INFO, "START jobid={jobid} host={host} - Initializing directory structure") + for(u, dirs) { + cache(list(u, destdir, host)) { + createdirs(u, destdir, host) + } + } + log(LOG:INFO, "END jobid={jobid} - Done initializing directory structure") + } + + cleanup := function(dir, host) { + log(LOG:INFO, "START dir={dir} host={host}") + cdmfile := cdm:file() + log(LOG:DEBUG, "cdmfile {cdmfile}") + if (cdmfile != "" & cdm:get("GATHER_DIR") != "UNSET") { + log(LOG:INFO, "submitting cdm_cleanup.sh to {dir}") + task:transfer(srcfile="cdm_cleanup.sh", + srcdir="{SWIFT:HOME}/libexec", + desthost=host, destdir=dir) + task:transfer(srcfile="cdm_lib.sh", + srcdir="{SWIFT:HOME}/libexec", + desthost=host, destdir=dir) + log(LOG:INFO, "execute: cdm_cleanup.sh") + task:execute( + executable="/bin/bash", + arguments=list("{dir}/cdm_cleanup.sh", + cdm:get("GATHER_DIR"), cdm:get("GATHER_TARGET"), UID()) + host=host, batch=true, TCProfile(host)) + } + if (swift:configProperty("sitedir.keep") == "false") { + task:execute( + siteProfile(host, "swift:cleanupCommand"), + arguments=list( + siteProfile(host, "swift:cleanupCommandOptions"), + dir + ) + host=host, batch=true, TCProfile(host)) + } + log(LOG:INFO, "END dir={dir} host={host}") + } + + cleanupFiles := function(files, host) { + parallelFor(r, files) { + log(LOG:INFO, "Purging ", r, " on ", host) + file:remove(r, host=host) + cacheFileRemoved(r, host) + } + } + + stageWrapperParams := function(jobid, jobdir, wrapfile, dir, host) { + log(LOG:INFO, "START jobid={jobid} - staging in wrapper params") + provider := provider(wrapfile) + srchost := hostname(wrapfile) + srcdir := swift:dirname(wrapfile) + destdir := dircat(dir, "/parameters/{jobdir}/") + filename := basename(wrapfile) + + cache(list(destdir, host)) { + dir:make(destdir, host=host, provider=provider) + } + + task:transfer(srcprovider=provider, srchost=srchost, srcfile=filename, srcdir=srcdir, desthost=host, destdir=destdir) + log(LOG:INFO, "END jobid={jobid}") + } + + doStageinFile := function(provider, srchost, srcfile, srcdir, desthost, destdir, size, policy) { + cacheAddAndLockFile(srcfile, destdir, desthost, size) { + cleanupFiles(cacheFilesToRemove, desthost) + + log(LOG:DEBUG, "FILE_STAGE_IN_START file={srcfile} ", + "srchost={srchost} srcdir={srcdir} srcname={srcfile} ", + "desthost={desthost} destdir={destdir} provider={provider} ", + "policy={policy}") + if (policy == "DEFAULT") { + restartOnError(2) { + task:transfer(srcprovider=provider, srchost=srchost, srcfile=srcfile, + srcdir=srcdir, desthost=desthost, destdir=destdir, + transferParams(srchost, desthost, provider)) + } + } + else if (policy == "BROADCAST") { + log(LOG:DEBUG, "FILE_STAGE_IN_BROADCAST file={srcfile} policy={policy}") + cdm:broadcast(srcfile=srcfile, srcdir=srcdir) + } + else if (policy == "EXTERNAL") { + log(LOG:DEBUG, "FILE_STAGE_IN_EXTERNAL file={srcfile} policy={policy}") + cdm:external(srcfile=srcfile, srcdir=srcdir, + desthost=desthost, destdir=destdir) + } + else { + log(LOG:DEBUG, "FILE_STAGE_IN_SKIP file={srcfile} policy={policy}") + } + log(LOG:DEBUG, "FILE_STAGE_IN_END file={srcfile} ", + "srchost={srchost} srcdir={srcdir} srcname={srcfile} ", + "desthost={desthost} destdir={destdir} provider={provider}") + } + cdm:wait() + } + + doStagein := function(jobid, files, dir, host) { + log(LOG:INFO, "START jobid={jobid} - Staging in files") + + cdmfile := cdm:file() + libexec := "{SWIFT:HOME}/libexec" + + if (cdmfile != "") { + doStageinFile(provider="file", srchost="localhost", srcfile=basename(cdmfile), + srcdir=swift:dirname(cdmfile), desthost=host, destdir=dir, + size=file:size(cdmfile), policy="DEFAULT") + doStageinFile(provider="file", srchost="localhost", srcfile="cdm.pl", + srcdir=libexec, desthost=host, destdir=dir, + size=file:size("{libexec}/cdm.pl}"), policy="DEFAULT") + doStageinFile(provider="file", srchost="localhost", srcfile="cdm_lib.sh", + srcdir=libexec, desthost=host, destdir=dir, + size=file:size("{libexec}/cdm_lib.sh}"), policy="DEFAULT") + } + + parallelFor(file, files) { + provider := provider(file) + srchost := hostname(file) + srcdir := swift:dirname(file) + destdir := dircat(dir, reldirname(file)) + filename := basename(file) + size := file:size("{srcdir}/{filename}", host=srchost, provider=provider) + + policy := cdm:query(query=file) + log(LOG:DEBUG, "CDM: {file} : {policy}") + + doStageinFile(provider=provider, srchost=srchost, srcfile=filename, + srcdir=srcdir, desthost=host, destdir=destdir, size=size, policy=policy) + } + log(LOG:INFO, "END jobid={jobid} - Staging in finished") + } + + doStageout := function(jobid, stageouts, dir, host) { + log(LOG:INFO, "START jobid={jobid} - Staging out files") + log(LOG:DEBUG, "stageouts: {stageouts}") + done := list( + parallelFor(pv, stageouts) { + (path, var) := each(pv) + file := absFileName(getField(var, path = path)) + provider := provider(file) + dhost := hostname(file) + rdir := dircat(dir, reldirname(file)) + bname := basename(file) + ldir := swift:dirname(file) + fullLocal := dircat(ldir, bname) + fullRemote := dircat(rdir, bname) + + log(LOG:DEBUG, "FILE_STAGE_OUT_START srcname={bname} srcdir={rdir} srchost={host} ", + "destdir={ldir} desthost={dhost} provider={provider}") + //make sure we do have the directory on the client side + dir:make(ldir, host=dhost, provider=provider) + policy := cdm:query(query=file) + log(LOG:DEBUG, "CDM: {file} : {policy}") + if (policy == "DEFAULT" | policy == "BROADCAST") { + restartOnError(2) { + task:transfer(srchost=host, srcfile=bname,srcdir=rdir, + destdir=ldir, desthost=dhost, destprovider=provider, + transferParams(host, dhost, provider)) + } + } + else { + log(LOG:DEBUG, "FILE_STAGE_OUT_SKIP srcname={bname}") + } + log(LOG:DEBUG, "FILE_STAGE_OUT_END srcname={bname} srcdir={rdir} srchost={host} ", + "destdir={ldir} desthost={dhost} provider={provider}") + + list(bname, rdir, host, file:size(fullLocal)) + } + ) + parallelFor(f, done) { + (bname, rdir, host, size) := each(f) + cacheAddFile(bname, rdir, host, size, cleanupFiles(cacheFilesToRemove, host)) + } + log(LOG:INFO, "END jobid={jobid} - Staging out finished") + } + + export(graphStuff, + function(tr, stagein, stageout, err, args = null) { + if (configProperty("pgraph") != "false") { + errprops := if(err, ",color=lightsalmon", ",color=lightsteelblue1") + tp := currentThread() + to (graph) { + concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]") + } + for (si, stagein) { + si := basename(si) + to(graph) { + concat(str:quote(si), " [shape=parallelogram]") + concat(str:quote(si), " -> ", str:quote(tp)) + } + } + for (pv, stageout) { + (path, var) := each(pv) + file := fileName(getField(var, path=path)) + file := basename(file) + label := niceName(var, path = path) + to(graph) { + concat(str:quote(file), " [shape=parallelogram,label=", str:quote(label), "]") + concat(str:quote(tp), " -> ", str:quote(file)) + } + } + } + } + ) + + fileSizes := function(files) { + math:sum( + for(f, files, file:size(f)) + ) + } + + transferStandardFiles := function(rhost, tmpdir, jobid, stdout, stderr) { + concat( + for(f, list(list("stderr.txt", stderr), list("stdout.txt", stdout))) { + (name, file) := each(f) + destfile := concat("{jobid}-", basename(file)) + try { + task:transfer(srchost=rhost, srcdir=tmpdir, srcfile=file, destfile=destfile) + "\n {name}: " + strip(file:read(destfile)) + } + else { + "" + } + maybe(file:remove(destfile)) + } + ) + } + + + transferWrapperLog := function(rhost, wfdir, jobid, jobdir) { + recfile := "{jobid}-info" + srcdir := dircat("{wfdir}/info/", jobdir) + try { + task:transfer(srchost=rhost, srcdir=srcdir, srcfile=recfile, destdir="{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.d/") + } + else catch (exception) { + maybe(file:remove(recfile)) + log(LOG:INFO, "Failed to transfer wrapper log for job {jobid} from {srcdir} on {rhost}", exception) + } + recfile + } + + export(cleanups, + function(cleanup) { + log(LOG:INFO, "START cleanups={cleanup}") + parallelFor(i, cleanup) { + (dir, host) := each(i) + try { + cleanup(dir, host) + } + else catch(exception) { + log(LOG:DEBUG, "EXCEPTION - Exception caught while cleaning up", exception) + to(warnings, exception("Cleanup on {host} failed", exception)) + } + } + log(LOG:INFO, "END cleanups={cleanup}") + } + ) + + export(execute2, + function(progress, tr, stagein, stageout, restartout + replicationGroup, replicationChannel + arguments = [], stdin = null, stdout = null, stderr = null, attributes = null) { + + stagein := list(unique(each(stagein))) + stageout := list(unique(each(stageout))) + + allocateHost(rhost, constraints = jobConstraints(tr, stagein = stagein)) { + + ddir := initDDir(), + (wfdir, sharedDir) := + try { + initSharedDir(progress, rhost) + } + else catch(exception) { + throw(exception("Could not initialize shared directory on {rhost}", exception)) + } + + uid := UID() + jobdir := substring(uid, from=0, to=1) + jobid := "{tr}-{uid}" + + log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread=", currentThread(), " host={rhost} replicationGroup={replicationGroup}") + + statusMode := configProperty("status.mode",host=rhost) + wrapperMode := configProperty("wrapper.parameter.mode",host=rhost) + + wrapfile := "{ddir}/param-{jobid}" + + stdout := if (stdout == null, "stdout.txt", getFieldValue(stdout)) + stderr := if (stderr == null, "stderr.txt", getFieldValue(stderr)) + + fileDirs := fileDirs(stagein, stageout) + os := siteProfile(rhost, "SYSINFO:OS") + + if(wrapperMode == "files") { + sys:file:write(wrapfile) { + "-scratch ", try(siteProfile(rhost, "scratch"), ""), + "\n-e ", executable(tr, rhost), + "\n-out ", stdout, + "\n-err ", stderr, + "\n-i ", if (stdin != null, getFieldValue(stdin)), + "\n-d ", flatten(each(fileDirs)), + "\n-if ", flatten(inFiles(stagein)), + "\n-of ", flatten(outFiles(stageout)), + "\n-cdmfile ", cdm:file(), + "\n-status ", statusMode, + for(a, arguments) { + "\n-a ", a + } + } + } + + setProgress(progress, "Stage in") + tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid) + + try { + createDirSet(jobid, sharedDir, rhost, fileDirs) + doStagein(jobid, stagein, sharedDir, rhost) + if(wrapperMode == "files") { + stageWrapperParams(jobid, jobdir, wrapfile, wfdir, rhost) + } + + log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}") + + setProgress(progress, "Submitting") + + if (wrapperMode == "files") { + swift:execute( + progress, + siteProfile(rhost, "swift:wrapperInterpreter"), + list( + siteProfile(rhost, "swift:wrapperInterpreterOptions"), + dircat("shared", siteProfile(rhost, "swift:wrapperScript"), os=os), + jobid, "-p", jobdir + ) + directory = wfdir + redirect = false + host = rhost + TCProfile(rhost, if (attributes != null, attributes = attributes), tr = tr) + replicationGroup = replicationGroup + replicationChannel = replicationChannel + jobid = jobid + ) + } + else if (wrapperMode == "args") { + swift:execute( + progress, + siteProfile(rhost, "swift:wrapperInterpreter"), + list( + siteProfile(rhost, "swift:wrapperInterpreterOptions"), + dircat("shared", siteProfile(rhost, "swift:wrapperScript"), os=os), + jobid, + "-jobdir", jobdir, + "-scratch", try(siteProfile(rhost, "scratch"), "") + "-e", executable(tr, rhost), + "-out", stdout, + "-err", stderr, + "-i", if (stdin != null, getFieldValue(stdin)), + "-d", flatten(each(fileDirs)), + "-if", flatten(inFiles(stagein)), + "-of", flatten(outFiles(stageout)), + "-cdmfile", cdm:file(), + "-status", statusMode, + "-a", if (arguments != null, each(arguments)) + ) + directory = wfdir + redirect = false + host = rhost + TCProfile(rhost, if (attributes != null, attributes = attributes), tr = tr) + replicationGroup = replicationGroup + replicationChannel = replicationChannel + jobid = jobid + ) + } + + setProgress(progress, "Checking status") + if (statusMode == "files") { + checkJobStatus(rhost, wfdir, jobid, tr, jobdir) + } + + if (wrapperMode == "files") { + file:remove(wrapfile) + } + + log(LOG:DEBUG, "STAGING_OUT jobid={jobid}") + + + /* need to stage the files to upper scratch area in case they are not transfered to another site + before all the files get cleaned out */ + + + setProgress(progress, "Stage out") + doStageout(jobid, stageout, sharedDir, rhost) + doRestartLog(restartout) + + if (configProperty("wrapperlog.always.transfer") == "true") { + discard(transferWrapperLog(rhost, wfdir, jobid, jobdir)) + } + + cacheUnlockFiles(stagein, sharedDir, rhost) { + cleanupFiles(cacheFilesToRemove, rhost) + } + + log(LOG:DEBUG, "JOB_END jobid={jobid}") + } + else catch(prev) { + if (matches(prev, "^Abort$")) { + log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") + cacheUnlockFiles(stagein, sharedDir, rhost, force=false) { + cleanupFiles(cacheFilesToRemove, rhost) + } + throw(prev) + } + else { + setProgress(progress, "Failed but can retry") + exception := try(exception(checkErrorFile(rhost, wfdir, jobid, jobdir)), prev) + + log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) + + if (matches(exception,".*executable bit.*")) { + throw(exception) + } + + cacheUnlockFiles(stagein, sharedDir, rhost, force=false) { + cleanupFiles(cacheFilesToRemove, rhost) + } + + outs := transferStandardFiles(rhost, tmpdir, jobid, stdout, stderr) + + discard(maybe(transferWrapperLog(rhost, wfdir, jobid, jobdir))) + + + throw( + exception( + concat( + "Exception in {tr}:", + maybe("\n Arguments: ", arguments), + "\n Host: {rhost}", + "\n Directory: {tmpdir}", + "{outs}", + ) + exception + ) + ) + } + } + } + } + ) + + export(generateProvenanceGraph, + function(gdata) { + pgraph := configProperty("pgraph") + gname := if(pgraph == "true", "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.dot", pgraph) + file:write(gname) { + "digraph SwiftProvenance \{\n", + " graph [", configProperty("pgraph.graph.options"), "];\n", + " node [", configProperty("pgraph.node.options"), "];\n", + + for(i, gdata) { + " ", i, "\n" + } + "}\n" + } + log(LOG:INFO, "Provenance graph saved in ", gname) + } + ) +} + +// Local variables: +// mode: scheme +// tab-width: 4 +// indent-tabs-mode: t +// End: Added: branches/faster/libexec/swift-lib.k =================================================================== --- branches/faster/libexec/swift-lib.k (rev 0) +++ branches/faster/libexec/swift-lib.k 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,145 @@ +namespace(swiftscript) { + export(extractInt, def("org.griphyn.vdl.karajan.lib.swiftscript.ExtractInt")) + export(readData, def("org.griphyn.vdl.karajan.lib.swiftscript.ReadData")) + export(readData2, def("org.griphyn.vdl.karajan.lib.swiftscript.ReadStructured")) + export(readStructured, def("org.griphyn.vdl.karajan.lib.swiftscript.ReadStructured")) + export(writeData, def("org.griphyn.vdl.karajan.lib.swiftscript.WriteData")) + export(strCat, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$StrCat")) + export(strCut, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$StrCut")) + export(strStr, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$StrStr")) + export(strSplit, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$StrSplit")) + export(strJoin, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$StrJoin")) + export(regexp, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$Regexp")) + export(toInt, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$ToInt")) + export(toFloat, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$ToFloat")) + export(format, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$Format")) + export(pad, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$Pad")) + export(sprintf, def("org.griphyn.vdl.karajan.lib.swiftscript.Sprintf")) + export(toString, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$ToString")) + export(assert, def("org.griphyn.vdl.karajan.lib.swiftscript.Assert")) + export(trace, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$Trace")) + export(tracef, def("org.griphyn.vdl.karajan.lib.swiftscript.Tracef")) + export(fprintf, def("org.griphyn.vdl.karajan.lib.swiftscript.Fprintf")) + + /* included for backwards compatibility */ + export(readdata, def("org.griphyn.vdl.karajan.lib.swiftscript.ReadData")) + export(readdata2, def("org.griphyn.vdl.karajan.lib.swiftscript.ReadStructured")) + export(readstructured, def("org.griphyn.vdl.karajan.lib.swiftscript.ReadStructured")) + export(writedata, def("org.griphyn.vdl.karajan.lib.swiftscript.WriteData")) + export(toint, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$ToInt")) + export(tofloat, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$ToFloat")) + export(tostring, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$ToString")) + + export(java, def("org.griphyn.vdl.karajan.lib.swiftscript.Java")) + export(filename, def("org.griphyn.vdl.karajan.lib.swiftscript.FileName")) + export(filenames, def("org.griphyn.vdl.karajan.lib.swiftscript.FileNames")) + export(dirname, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$Dirname")) + export(length, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$Length")) + export(exists, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$Exists")) + export(arg, def("org.griphyn.vdl.karajan.lib.swiftscript.FnArg")) +} + +namespace(swift) { + import(sys) + export(SWIFT_CONFIG, contextAttribute("config")) + export(SWIFT_TICKER, null) + + export(getDatasetProvenanceID, def("org.griphyn.vdl.karajan.lib.GetDatasetProvenanceID")) + export(startProgressTicker, def("org.griphyn.vdl.karajan.lib.RuntimeStats$StartProgressTicker")) + export(stopProgressTicker, def("org.griphyn.vdl.karajan.lib.RuntimeStats$StopProgressTicker")) + export(initProgressState, def("org.griphyn.vdl.karajan.lib.RuntimeStats$InitProgressState")) + export(setProgress, def("org.griphyn.vdl.karajan.lib.RuntimeStats$SetProgress")) + export(new, def("org.griphyn.vdl.karajan.lib.New")) + export(createArray, def("org.griphyn.vdl.karajan.lib.CreateArray")) + /* used from VDL2 for arguments to apps and returns relative paths */ + export(fileName, def("org.griphyn.vdl.karajan.lib.FileName")) + /* for internal use and does not force relative paths */ + export(absFileName, def("org.griphyn.vdl.karajan.lib.AbsFileName")) + + export(jobConstraints, def("org.griphyn.vdl.karajan.lib.JobConstraints")) + export(getField, def("org.griphyn.vdl.karajan.lib.GetField")) + export(getFieldSubscript, def("org.griphyn.vdl.karajan.lib.GetFieldSubscript")) + export(setFieldValue, def("org.griphyn.vdl.karajan.lib.SetFieldValue")) + export(appendArray, def("org.griphyn.vdl.karajan.lib.AppendArray")) + export(getFieldValue, def("org.griphyn.vdl.karajan.lib.GetFieldValue")) + export(waitFieldValue, def("org.griphyn.vdl.karajan.lib.WaitFieldValue")) + export(getArrayIterator, def("org.griphyn.vdl.karajan.lib.GetArrayIterator")) + export(sliceArray, def("org.griphyn.vdl.karajan.lib.SliceArray")) + export(isFileBound, def("org.griphyn.vdl.karajan.lib.IsFileBound")) + export(isRestartable, def("org.griphyn.vdl.karajan.lib.IsRestartable")) + export(fringePaths, def("org.griphyn.vdl.karajan.lib.FringePaths")) + export(closeDataset, def("org.griphyn.vdl.karajan.lib.CloseDataset")) + export(partialCloseDataset, def("org.griphyn.vdl.karajan.lib.PartialCloseDataset")) + export(setWaitCount, def("org.griphyn.vdl.karajan.lib.SetWaitCount")) + export(cleanDataset, def("org.griphyn.vdl.karajan.lib.CleanDataset")) + export(range, def("org.griphyn.vdl.karajan.lib.Range")) + + export(isLogged, def("org.griphyn.vdl.karajan.lib.IsLogged")) + export(logVar, def("org.griphyn.vdl.karajan.lib.LogVar")) + + export(executable, def("org.griphyn.vdl.karajan.lib.Executable")) + export(TCProfile, def("org.griphyn.vdl.karajan.lib.TCProfile")) + export(siteProfile, def("org.griphyn.vdl.karajan.lib.SiteProfile")) + + export(setFutureFault, def("org.griphyn.vdl.karajan.lib.SetFutureFault")) + + export(configProperty, def("org.griphyn.vdl.karajan.functions.ConfigProperty")) + export(processBulkErrors, def("org.griphyn.vdl.karajan.functions.ProcessBulkErrors")) + + export(niceName, def("org.griphyn.vdl.karajan.lib.NiceName")) + + export(cacheAddAndLockFile, def("org.griphyn.vdl.karajan.lib.CacheAddAndLockFile")) + export(cacheAddFile, def("org.griphyn.vdl.karajan.lib.CacheAddFile")) + export(cacheFileRemoved, def("org.griphyn.vdl.karajan.lib.CacheFileRemoved")) + export(cacheUnlockFiles, def("org.griphyn.vdl.karajan.lib.CacheUnlockFiles")) + + export(LOG:DEBUG, "debug") + export(LOG:INFO, "info") + export(LOG:WARN, "warn") + export(LOG:ERROR, "error") + export(LOG:FATAL, "fatal") + + + export(log, def("org.griphyn.vdl.karajan.lib.Log")) + + export(unitStart, def("org.griphyn.vdl.karajan.lib.UnitStart")) + export(unitEnd, def("org.griphyn.vdl.karajan.lib.UnitEnd")) + export(currentThread, def("org.griphyn.vdl.karajan.lib.CurrentThread")) + + export(dirname, def("org.griphyn.vdl.karajan.lib.PathUtils$DirName")) + export(reldirname, def("org.griphyn.vdl.karajan.lib.PathUtils$RelDirName")) + export(basename, def("org.griphyn.vdl.karajan.lib.PathUtils$BaseName")) + export(provider, def("org.griphyn.vdl.karajan.lib.PathUtils$Provider")) + export(hostname, def("org.griphyn.vdl.karajan.lib.PathUtils$HostName")) + export(dircat, def("org.griphyn.vdl.karajan.lib.PathUtils$DirCat")) + export(pathnames, def("org.griphyn.vdl.karajan.lib.PathUtils$PathNames")) + + export(execute, def("org.griphyn.vdl.karajan.lib.Execute")) + export(expandArguments, def("org.griphyn.vdl.karajan.lib.ExpandArguments")) + + export(tParallelFor, def("org.griphyn.vdl.karajan.lib.ThrottledParallelFor")) + export(throttled, def("org.griphyn.vdl.karajan.lib.Throttled")) + + export(appStageins, def("org.griphyn.vdl.karajan.lib.AppStageins")) + export(appStageouts, def("org.griphyn.vdl.karajan.lib.AppStageouts")) + export(isDone, def("org.griphyn.vdl.karajan.lib.IsDone")) + export(mark, def("org.griphyn.vdl.karajan.lib.Mark")) + export(flatten, def("org.griphyn.vdl.karajan.lib.Flatten")) + export(parameterlog, def("org.griphyn.vdl.karajan.lib.Parameterlog")) + export(inFileDirs, def("org.griphyn.vdl.karajan.lib.InFileDirs")) + export(outFileDirs, def("org.griphyn.vdl.karajan.lib.OutFileDirs")) + export(outFiles, def("org.griphyn.vdl.karajan.lib.OutFiles")) + export(doRestartLog, def("org.griphyn.vdl.karajan.lib.DoRestartLog")) + export(unwrapClosedList, def("org.griphyn.vdl.karajan.lib.UnwrapClosedList")) + + export(siteCatalog, def("org.griphyn.vdl.karajan.lib.SiteCatalog")) +} + +namespace(cdm) { + export(query, def("org.globus.swift.data.Query$Q")) + export(get, def("org.globus.swift.data.Query$Get")) + export(file, def("org.globus.swift.data.Query$File")) + export(broadcast, def("org.globus.swift.data.Action$Broadcast")) + export(external, def("org.globus.swift.data.Action$External")) + export(wait, def("org.globus.swift.data.Action$Wait")) +} Added: branches/faster/libexec/swift-operators.k =================================================================== --- branches/faster/libexec/swift-operators.k (rev 0) +++ branches/faster/libexec/swift-operators.k 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,17 @@ +namespace(swiftop) { + export(sum, def("org.griphyn.vdl.karajan.lib.Operators$Sum")) + export(subtraction, def("org.griphyn.vdl.karajan.lib.Operators$Difference")) + export(product, def("org.griphyn.vdl.karajan.lib.Operators$Product")) + export(fquotient, def("org.griphyn.vdl.karajan.lib.Operators$FQuotient")) + export(iquotient, def("org.griphyn.vdl.karajan.lib.Operators$IQuotient")) + export(remainder, def("org.griphyn.vdl.karajan.lib.Operators$Remainder")) + export(le, def("org.griphyn.vdl.karajan.lib.Operators$LE")) + export(ge, def("org.griphyn.vdl.karajan.lib.Operators$GE")) + export(gt, def("org.griphyn.vdl.karajan.lib.Operators$GT")) + export(lt, def("org.griphyn.vdl.karajan.lib.Operators$LT")) + export(eq, def("org.griphyn.vdl.karajan.lib.Operators$EQ")) + export(ne, def("org.griphyn.vdl.karajan.lib.Operators$NE")) + export(and, def("org.griphyn.vdl.karajan.lib.Operators$And")) + export(or, def("org.griphyn.vdl.karajan.lib.Operators$Or")) + export(not, def("org.griphyn.vdl.karajan.lib.Operators$Not")) +} \ No newline at end of file Added: branches/faster/libexec/swift-xs.k =================================================================== --- branches/faster/libexec/swift-xs.k (rev 0) +++ branches/faster/libexec/swift-xs.k 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,85 @@ +import(sys) + +namespace(xs) { + import(java) + + UnresolvedType := function(name, isArray) { + java:new("org.griphyn.vdl.type.impl.UnresolvedType", types=["String"], name) + } + + typesST := function() { + types = ["String", "org.griphyn.vdl.type.Type"] + } + + newSimpleNode := function(name, type) { + x := java:new("org.griphyn.vdl.type.impl.TypeImpl", types=["String"], name) + invokeMethod("setBaseType", object=x, types=["org.griphyn.vdl.type.Type"] + invokeMethod("getType", classname="org.griphyn.vdl.type.Types", type) + ) + x + } + + newComplexNode := function(name) { + java:new("org.griphyn.vdl.type.impl.TypeImpl", types=["String"], name) + } + + addNode := function(node) { + invokeMethod("addType", classname="org.griphyn.vdl.type.Types" + types=["org.griphyn.vdl.type.Type"] + node + ) + } + + addField := function(node, name, type) { + invokeMethod("addField", object=node, typesST(), name, type) + } + + export(schema, + function() { + invokeMethod("resolveTypes", classname="org.griphyn.vdl.type.Types") + } + ) + + export(simpleType, + function(name, type) { + addNode(newSimpleNode(name, type)) + } + ) + + export(restriction, + function(base) { + type = last(split(base, ":")) + } + ) + + export(complexType, + function(name, ...) { + node := newComplexNode(name) + for(field, ...) { + (name, type) := each(field) + addField(node, name, type) + } + addNode(node) + } + ) + + export(sequence, + function(minOccurs = 0, maxOccurs = 0, ...) { + (name, type) := each(first(...)) + if (maxOccurs == "unbounded") { + list(name, UnresolvedType(type, true)) + } + else { + each(...) + } + } + ) + + //should be noted that we're dealing with type names here + export(element, + function(name, type) { + type := last(split(type, ":")) + list(name, UnresolvedType(type, false)) + } + ) +} Added: branches/faster/libexec/swift.k =================================================================== --- branches/faster/libexec/swift.k (rev 0) +++ branches/faster/libexec/swift.k 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,219 @@ +import(sys) +import(task) +import(rlog) +import('swift-operators', export = true) +import('swift-lib', export = true) +import('swift-xs', export = true) +import('swift-int') + +SWIFT:SCRIPT_NAME := contextAttribute("SWIFT:SCRIPT_NAME") +SWIFT:RUN_ID := contextAttribute("SWIFT:RUN_ID") +SWIFT:HOME := contextAttribute("SWIFT:HOME") + +namespace(swift) { + + pstaging := configProperty("use.provider.staging") + wstaging := configProperty("use.wrapper.staging") + + import(java) + + once("vdl.k-print-version") { + log(LOG:INFO,sys:file:read("{SWIFT:HOME}/libexec/version.txt")) + echo(sys:file:read("{SWIFT:HOME}/libexec/version.txt")) + log(LOG:INFO,"RUNID id=run:{SWIFT:RUN_ID}") + echo("RunID: {SWIFT:RUN_ID}") + } + + export(stagein, def("org.griphyn.vdl.karajan.lib.Stagein")) + export(stageout, def("org.griphyn.vdl.karajan.lib.Stageout")) + + export(parameterLog, + function(direction, variable, id, thread) { + if (configProperty("provenance.log") == "true") { + log("info","PARAM thread={thread} direction={direction} variable={variable} provenanceid={id}") + } + } + ) + + export(split, + function(var) { + each(str:split(getFieldValue(var), " ")) + } + ) + + export(quote, + function(var, path = null) { + str:quote(getFieldValue(var, if (path != null) { path = path })) + } + ) + + export(types, function() {}) + + export(arguments, + function(...) { + arguments = expandArguments(each(...)) + } + ) + + export(mapping, + function(descriptor, ...) { + mapping=map(map:entry("swift#descriptor", descriptor), each(...)) + } + ) + + export(parameter, + function(name, value) { + map:entry(name, value) + } + ) + + export(stdout, + function(file) { + stdout = file + } + ) + + export(stdin, + function(file) { + stdin = file + } + ) + + export(stderr, + function(file) { + stderr = file + } + ) + + export(tr, + function(name) { + tr = name + } + ) + + export(attributes, + function(attrs) { + attributes = attrs + } + ) + + export(mains, + function(channel(cleanup), channel(errors), channel(warnings)) { + anyerrors := processBulkErrors("The following errors have occurred:", errors) + + //this should be reached after everything is done + if (!anyerrors) { + log(LOG:DEBUG, "Starting cleanups") + discard(append(warnings, from(warnings, cleanups(cleanup)))) + log(LOG:DEBUG, "Ending cleanups") + } + else { + log(LOG:INFO, "Errors detected. Cleanup not done.") + } + anywarnings := processBulkErrors("The following warnings have occurred:", warnings, onStdout=true) + if (anyerrors) { + throw("Execution completed with errors") + } + } + ) + + /* + * This would run in parallel with the workflow so that we don't keep + * all the data in memory until the workflow is done + */ + export(mainp, + CBFFunction(channel(graph), channel(cleanup)) { + parallel( + if(configProperty("pgraph") != "false") { + generateProvenanceGraph(graph) + } + to(cleanup, unique(for(c, cleanup, c))) + ) + } + ) + + export(execute, + function( + tr, arguments = null, + stdin = null, stdout = null, stderr = null, + attributes = null, + deperror = false, mdeperror = false, + channel(stagein), channel(stageout), channel(restartout)) { + + progress := initProgressState() + + done := isDone(restartout) + derr := try(deperror, false) + merr := try(mdeperror, false) + + if(derr == false) { + if(!done) { + try { + throttled { + setProgress(progress, "Selecting site") + restartOnError(number(swift:configProperty("execution.retries"))) { + replicationChannel := channel:new() + //trigger the first job + discard(append(replicationChannel, true)) + replicationGroup := UID() + parallelFor(i, replicationChannel) { + try { + execute2( + progress, + tr, maybe(arguments=unwrapClosedList(arguments)), + maybe(stdin=stdin), maybe(stdout=stdout), maybe(stderr=stderr), maybe(attributes=attributes), + stagein, stageout, restartout, replicationGroup, replicationChannel + ) + } + else catch(exception) { + if (matches(exception, "^Abort$")) { + // ignored + } + else { + throw(exception) + } + } + } + } + mark(restartout, err=false, mapping=false) + log(LOG:INFO, "END_SUCCESS thread=", currentThread(), " tr={tr}") + setProgress(progress, "Finished successfully") + } + } + else catch(exception) { + log(LOG:INFO, "END_FAILURE thread=", currentThread(), " tr={tr}") + setProgress(progress, "Failed") + if(swift:configProperty("lazy.errors") == "false") { + throw(exception) + } + else { + to(errors, exception) + log(LOG:INFO, exception) + echo(exception) + mark(restartout, err=true, mapping=false) + graphStuff(tr, stagein, stageout, err=true, maybe(args=arguments)) + } + } + } + else { + setProgress(progress, "Finished in previous run") + } + } + else { //derr == true + if (merr) { + exception := exception(concat("Swift: Application ", str:quote(tr), + " not executed due to errors in mapping dependencies")) + to(errors, exception) + log(LOG:INFO, exception) + } + else if (derr) { + exception := exception("Swift: Application {tr} not executed due to errors in dependencies") + to(errors, exception) + log(LOG:INFO, exception) + } + mark(restartout, err=true, mapping=merr) + graphStuff(tr, stagein, stageout, err=true, maybe(args=arguments)) + } + } + ) +} Modified: branches/faster/libexec/vdl-int-staging.k =================================================================== --- branches/faster/libexec/vdl-int-staging.k 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/libexec/vdl-int-staging.k 2013-01-29 07:31:09 UTC (rev 6170) @@ -19,73 +19,72 @@ global(PIN, if(pinOption == "true", "pinned:", "")) global(PROVENANCE_GRAPH_ENABLED, vdl:configProperty("pgraph") != "false") -namespace("vdl" +namespace("vdl") { export( - element(initDDir, [] + function(initDDir) { ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d" - once(ddir - if(sys:not(file:exists(ddir)) + once(ddir) { + if(sys:not(file:exists(ddir))) { task:dir:make(ddir) - ) - ) + } + } ddir - ) + } - element(inFiles, [stageins] + function(inFiles, stageins) { pathnames(stageins) - ) + } - element(graphStuff, [tr, stagein, stageout, err, optional(args)] - if(PROVENANCE_GRAPH_ENABLED - then( - errprops := if(err ",color=lightsalmon" ",color=lightsteelblue1") - tp := vdl:threadPrefix() - to(graph, - concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]") - ) - for(si, stagein - si := basename(si) - to(graph - concat(str:quote(si), " [shape=parallelogram]") - concat(str:quote(si), " -> ", str:quote(tp)) - ) - ) - for(pv, stageout - [path, var] := each(pv) - file := vdl:fileName(vdl:getfield(var, path=path)) - file := basename(file) - label := vdl:niceName(var, path = path) - to(graph - concat(str:quote(file), " [shape=parallelogram,label=", - str:quote(label), "]") - concat(str:quote(tp), " -> ", str:quote(file)) - ) - ) - ) - ) - ) + function(graphStuff, tr, stagein, stageout, err, optional(args)) { + if(PROVENANCE_GRAPH_ENABLED) { + errprops := if(err ",color=lightsalmon" ",color=lightsteelblue1") + tp := vdl:threadPrefix() + to(graph) { + concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]") + } + for(si, stagein) { + si := basename(si) + to(graph) { + concat(str:quote(si), " [shape=parallelogram]") + concat(str:quote(si), " -> ", str:quote(tp)) + } + } + for(pv, stageout) { + (path, var) := each(pv) + file := vdl:fileName(vdl:getfield(var, path=path)) + file := basename(file) + label := vdl:niceName(var, path = path) + to(graph) { + concat(str:quote(file), " [shape=parallelogram,label=", + str:quote(label), "]") + concat(str:quote(tp), " -> ", str:quote(file)) + } + } + } + } - element(fileSizes, [files] + function(fileSizes, files) { math:sum( for(f, files, file:size(file)) ) - ) + } - element(cleanups, [cleanup] + function(cleanups, cleanup) { log(LOG:INFO, "START cleanups={cleanup}") - ) + } - element(readErrorFile, [dir, jobid] + function(readErrorFile, dir, jobid) { str:strip(file:read("{dir}/{jobid}.error")) file:remove("{dir}/{jobid}.error") - ) + } - element(execute2, [tr, optional(arguments, stdin, stdout, stderr), stagein, stageout, restartout, - replicationGroup, replicationChannel] + function(execute2, tr, optional(arguments, stdin, stdout, stderr), stagein, stageout, restartout, + replicationGroup, replicationChannel) { + stagein := list(unique(each(stagein))) stageout := list(unique(each(stageout))) - allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein) + allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein)) { ddir := initDDir() @@ -93,7 +92,7 @@ jobdir := substring(uid, from=0, to=1) jobid := "{tr}-{uid}" - log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") + log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread=", currentThread(), " host={rhost} replicationGroup={replicationGroup}") wrapper := "_swiftwrap.staging" wrapfile := "{ddir}/param-{jobid}" @@ -105,110 +104,107 @@ tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid) cdmfile := cdm:file() - try( - sequential( - log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}") + try { + log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}") - vdl:setprogress("Submitting") + vdl:setprogress("Submitting") - vdl:execute( - vdl:siteprofile(rhost, "swift:wrapperInterpreter"), - list( - vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"), - wrapper, - "-e", vdl:executable(tr, rhost), - "-out", stdout, - "-err", stderr, - "-i", maybe(getFieldValue(stdin)), - "-d", flatten(unique(outFileDirs(stageout))), - "-if", flatten(infiles(stagein)), - "-of", flatten(outfiles(stageout)), - "-k", - "-cdmfile", cdmfile, - "-status", "provider" - "-a", maybe(each(arguments)) - ) - directory = tmpdir - redirect = false - host = rhost + vdl:execute( + vdl:siteprofile(rhost, "swift:wrapperInterpreter"), + list( + vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"), + wrapper, + "-e", vdl:executable(tr, rhost), + "-out", stdout, + "-err", stderr, + "-i", maybe(getFieldValue(stdin)), + "-d", flatten(unique(outFileDirs(stageout))), + "-if", flatten(infiles(stagein)), + "-of", flatten(outfiles(stageout)), + "-k", + "-cdmfile", cdmfile, + "-status", "provider" + "-a", maybe(each(arguments)) + ) + directory = tmpdir + redirect = false + host = rhost - vdl:tcprofile(rhost, tr = tr) //this gets various app params from the tc, such as environment, walltime, etc - replicationGroup = replicationGroup - replicationChannel = replicationChannel - jobid = jobid + vdl:tcprofile(rhost, tr = tr) //this gets various app params from the tc, such as environment, walltime, etc + replicationGroup = replicationGroup + replicationChannel = replicationChannel + jobid = jobid + + stagingMethod := vdl:siteProfile(rhost, "swift:stagingMethod", default="proxy") - stagingMethod := vdl:siteProfile(rhost, "swift:stagingMethod", default="proxy") + stageIn("{PIN}{stagingMethod}://localhost/{swift.home}/libexec/{wrapper}", wrapper) - stageIn("{PIN}{stagingMethod}://localhost/{swift.home}/libexec/{wrapper}", wrapper) + if (cdmfile != "") { + d := vdl:dirname(cdmfile) + file := basename(cdmfile) + dir := if (d == "", "./", str:concat(d,"/")) + loc := "{PIN}{stagingMethod}://localhost/" + stageIn("{loc}{dir}{file}", cdmfile) + stageIn("{loc}{swift.home}/libexec/cdm.pl", "cdm.pl") + stageIn("{loc}{swift.home}/libexec/cdm_lib.sh", "cdm_lib.sh") + } - if ( cdmfile != "" then( - d := vdl:dirname(cdmfile) - file := basename(cdmfile) - dir := if ( d == "" then("./") else(str:concat(d,"/"))) - loc := "{PIN}{stagingMethod}://localhost/" - stageIn("{loc}{dir}{file}", cdmfile) - stageIn("{loc}{swift.home}/libexec/cdm.pl", "cdm.pl") - stageIn("{loc}{swift.home}/libexec/cdm_lib.sh", "cdm_lib.sh") - )) + appStageins(jobid, stagein, ".", stagingMethod) - appStageins(jobid, stagein, ".", stagingMethod) + stageOut("wrapper.log", "{stagingMethod}://localhost/{ddir}/{jobid}.info", + mode = WRAPPER_TRANSFER_MODE) + //stageOut("{stdout}", "{stagingMethod}://localhost/{ddir}/{stdout}") + //stageOut("{stderr}", "{stagingMethod}://localhost/{ddir}/{stderr}") + stageOut("wrapper.error", "{stagingMethod}://localhost/{ddir}/{jobid}.error", + mode = STAGING_MODE:IF_PRESENT) + appStageouts(jobid, stageout, ".", stagingMethod) - stageOut("wrapper.log", "{stagingMethod}://localhost/{ddir}/{jobid}.info", - mode = WRAPPER_TRANSFER_MODE) - //stageOut("{stdout}", "{stagingMethod}://localhost/{ddir}/{stdout}") - //stageOut("{stderr}", "{stagingMethod}://localhost/{ddir}/{stderr}") - stageOut("wrapper.error", "{stagingMethod}://localhost/{ddir}/{jobid}.error", - mode = STAGING_MODE:IF_PRESENT) - appStageouts(jobid, stageout, ".", stagingMethod) - - task:cleanUp(".") //the whole job directory - ) - doRestartlog(restartout) - log(LOG:DEBUG, "JOB_END jobid={jobid}") - ) - catch("^Abort$" - log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") - throw(exception) - ) - catch("^(?!Abort$).*" - vdl:setprogress("Failed but can retry") - prev := exception - exception := try(exception(readErrorFile(ddir, jobid)), prev) - log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) - - throw( - exception( - concat( - "Exception in {tr}:", nl(), - maybe(" Arguments: {arguments}", nl()), - " Host: {rhost}", nl(), - " Directory: {tmpdir}", nl() - ) - exception + task:cleanUp(".") //the whole job directory + ) // execute + doRestartlog(restartout) + log(LOG:DEBUG, "JOB_END jobid={jobid}") + } + else catch(exception, "^Abort$") { + log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") + throw(exception) + } + else catch(prev, "^(?!Abort$).*") { + vdl:setprogress("Failed but can retry") + exception := try(exception(readErrorFile(ddir, jobid)), prev) + log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) + + throw( + exception( + concat( + "Exception in {tr}:", nl(), + maybe(" Arguments: {arguments}", nl()), + " Host: {rhost}", nl(), + " Directory: {tmpdir}", nl() ) + exception ) ) - ) - ) - ) + } + } + } - element(generateProvenanceGraph, [gdata] + function(generateProvenanceGraph, gdata) { pgraph := vdl:configProperty("pgraph") gname := if(pgraph == "true" "{VDL:SCRIPTNAME}-{VDL:RUNID}.dot" pgraph) - file:write(gname + file:write(gname) { "digraph SwiftProvenance {{", nl() " graph [", vdl:configProperty("pgraph.graph.options"), "];", nl() " node [", vdl:configProperty("pgraph.node.options"), "];", nl() - for(i, gdata + for(i, gdata) { " ", i, nl() - ) + } "}", nl() - ) + } log(LOG:INFO, "Provenance graph saved in ", gname) - ) + } ) -) +} // Local variables: // tab-width: 4 Deleted: branches/faster/libexec/vdl-int.k =================================================================== --- branches/faster/libexec/vdl-int.k 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/libexec/vdl-int.k 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,637 +0,0 @@ -import("sys.k") -import("task.k") -import("vdl-lib.xml") -/* - * Things that are not exposed to the translated file - */ - -global(LOG:DEBUG, "debug") -global(LOG:INFO, "info") -global(LOG:WARN, "warn") -global(LOG:ERROR, "error") -global(LOG:FATAL, "fatal") - -namespace("vdl" - export( - element(rmdir, [dir, host] - parallelFor(entry, file:list(dir, host=host) - epath := "{dir}/{entry}" - if( - file:isDirectory(epath, host=host) rmdir(epath, host) - file:remove(epath, host=host) - ) - ) - dir:remove(dir, host=host) - ) - - element(createdirs, [path, dir, host] - dc := dircat(dir, path) - log(LOG:INFO, "START path={path} dir={dir} - Creating directory structure") - - dir:make(dc, host=host) - ) - - element(checkJobStatus, [rhost, wfdir, jobid, tr, jobdir] - log(LOG:DEBUG, "START jobid={jobid}") - try( - sequential( - /* - * This is a bit of optimization, but I'm not completely - * sure of its correctness. The goal is to both detect - * the presence of the success file and remove it, all - * in one operation. It relies on file:remove() throwing - * an exception if the file is not there. - */ - file:remove("{wfdir}/status/{jobdir}/{jobid}-success", host=rhost) - log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found") - ) - sequential( - try ( - msg = checkErrorFile(rhost, wfdir, jobid, jobdir) - sequential ( - log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing") - throw("No status file was found. Check the shared filesystem on {rhost}") - ) - ) - ) - throw(checkErrorFile(rhost, wfdir, jobid, jobdir)) - ) - ) - - element(checkErrorFile, [rhost, wfdir, jobid, jobdir] - if ( - file:exists("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost) then( - log(LOG:INFO, "FAILURE jobid={jobid} - Failure file found") - task:transfer(srchost=rhost, srcdir="{wfdir}/status/{jobdir}", srcfile="{jobid}-error") - error := parallel( - file:remove("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost) - sequential( - str:strip(file:read("{jobid}-error")) - file:remove("{jobid}-error") - ) - ) - error - ) - else ( - log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Error file missing") - throw("No status file was found. Check the shared filesystem on {rhost}") - ) - ) - ) - - element(initSharedDir, [rhost] - once(list(rhost, "shared") - vdl:setprogress("Initializing site shared directory") - - log(LOG:INFO, "START host={rhost} - Initializing shared directory") - - wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" - sharedDir := dircat(wfdir, "shared") - dir:make(sharedDir, host=rhost) - transfer(srcdir="{swift.home}/libexec/", srcfile=vdl:siteprofile(rhost, "swift:wrapperScript"), destdir=sharedDir, desthost=rhost) - transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftseq", destdir=sharedDir, desthost=rhost) - dir:make(dircat(wfdir, "kickstart"), host=rhost) - - statusMode := configProperty("status.mode",host=rhost) - if(statusMode == "files" - dir:make(dircat(wfdir, "status"), host=rhost) - ) - - wrapperMode := configProperty("wrapper.parameter.mode",host=rhost) - if(wrapperMode == "files" - dir:make(dircat(wfdir, "parameters"), host=rhost) - ) - - dir:make(dircat(wfdir, "info"), host=rhost) - wfdir, sharedDir - //we send the cleanup data to vdl:main() - to(cleanup, list(wfdir, rhost)) - log(LOG:INFO, "END host={rhost} - Done initializing shared directory") - ) - ) - - element(initDDir, [] - ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d" - once(ddir - if(sys:not(file:exists(ddir)) - task:dir:make(ddir) - ) - ) - ddir - ) - - element(inFiles, [stageins] - pathnames(stageins) - ) - - element(fileDirs, [stageins, stageouts] - list( - unique( - inFileDirs(stageins) - outFileDirs(stageouts) - ) - ) - ) - - element(createDirSet, [jobid, destdir, host, dirs] - /* - * Ideally this would be done by creating a tree of the directories - * to be created and (eventually) exploiting the concurrency in that. - */ - log(LOG:INFO, "START jobid={jobid} host={host} - Initializing directory structure") - for(u, dirs - cacheOn(list(u, destdir, host) - createdirs(u, destdir, host) - ) - ) - log(LOG:INFO, "END jobid={jobid} - Done initializing directory structure") - ) - - element(cleanup, [dir, host] - log(LOG:INFO, "START dir={dir} host={host}") - cdmfile := cdm:file() - log(LOG:DEBUG, "cdmfile {cdmfile}") - if(cdmfile != "" & - cdm:get("GATHER_DIR") != "UNSET" then( - log(LOG:INFO, "submitting cdm_cleanup.sh to {dir}") - task:transfer(srcfile="cdm_cleanup.sh", - srcdir="{swift.home}/libexec", - desthost=host, destdir=dir) - task:transfer(srcfile="cdm_lib.sh", - srcdir="{swift.home}/libexec", - desthost=host, destdir=dir) - log(LOG:INFO, "execute: cdm_cleanup.sh") - task:execute( - executable="/bin/bash", - arguments=list("{dir}/cdm_cleanup.sh", - cdm:get("GATHER_DIR"), cdm:get("GATHER_TARGET") - sys:uid() ) - host=host, batch=true, tcprofile(host)) - ) - ) - if(vdl:configProperty("sitedir.keep") == "false" - task:execute( - vdl:siteprofile(host, "swift:cleanupCommand"), - arguments=list( - vdl:siteprofile(host, "swift:cleanupCommandOptions"), - dir - ) - host=host, batch=true, tcprofile(host)) - ) - log(LOG:INFO, "END dir={dir} host={host}") - ) - - element(cleanups, [cleanup] - log(LOG:INFO, "START cleanups={cleanup}") - parallelFor(i, cleanup - [dir, host] := each(i) - try( - vdl:cleanup(dir, host) - catch(".*", - log(LOG:DEBUG, "EXCEPTION - Exception caught while cleaning up", exception) - to(warnings, exception("Cleanup on {host} failed", exception)) - ) - ) - ) - log(LOG:INFO, "END cleanups={cleanup}") - ) - - element(cleanupFiles, [files, host] - uParallelFor(r, files - log(LOG:INFO, "Purging ", r, " on ", host) - file:remove(r, host=host) - vdl:cacheFileRemoved(r, host) - ) - ) - - element(stageWrapperParams, [jobid, jobdir, wrapfile, dir, host] - log(LOG:INFO, "START jobid={jobid} - staging in wrapper params") - provider := provider(wrapfile) - srchost := hostname(wrapfile) - srcdir := vdl:dirname(wrapfile) - destdir := dircat(dir, "/parameters/{jobdir}/") - filename := basename(wrapfile) - - cacheOn(list(destdir, host) - dir:make(destdir, host=host, provider=provider) - ) - - task:transfer(srcprovider=provider, srchost=srchost, srcfile=filename, srcdir=srcdir, desthost=host, destdir=destdir) - log(LOG:INFO, "END jobid={jobid}") - ) - - element(doStagein, [jobid, files, dir, host] - log(LOG:INFO, "START jobid={jobid} - Staging in files") - - cdmfile := cdm:file() - libexec := "{swift.home}/libexec" - - if (cdmfile != "" then( - doStageinFile(provider="file", srchost="localhost", srcfile=basename(cdmfile), - srcdir=vdl:dirname(cdmfile), desthost=host, destdir=dir, - size=file:size(cdmfile), policy="DEFAULT") - doStageinFile(provider="file", srchost="localhost", srcfile="cdm.pl", - srcdir=libexec, desthost=host, destdir=dir, - size=file:size("{libexec}/cdm.pl}"), policy="DEFAULT") - doStageinFile(provider="file", srchost="localhost", srcfile="cdm_lib.sh", - srcdir=libexec, desthost=host, destdir=dir, - size=file:size("{libexec}/cdm_lib.sh}"), policy="DEFAULT") - )) - - uParallelFor(file, files - provider := provider(file) - srchost := hostname(file) - srcdir := vdl:dirname(file) - destdir := dircat(dir, reldirname(file)) - filename := basename(file) - size := file:size("{srcdir}/{filename}", host=srchost, provider=provider) - - policy := cdm:query(query=file) - log(LOG:DEBUG, "CDM: {file} : {policy}") - - doStageinFile(provider=provider, srchost=srchost, srcfile=filename, - srcdir=srcdir, desthost=host, destdir=destdir, size=size, policy=policy) - ) - log(LOG:INFO, "END jobid={jobid} - Staging in finished") - ) - - element(doStageinFile, [provider, srchost, srcfile, srcdir, desthost, destdir, size, policy] - vdl:cacheAddAndLockFile(srcfile, destdir, desthost, size - cleanupFiles(cacheFilesToRemove, desthost) - - log(LOG:DEBUG, "FILE_STAGE_IN_START file={srcfile} ", - "srchost={srchost} srcdir={srcdir} srcname={srcfile} ", - "desthost={desthost} destdir={destdir} provider={provider} ", - "policy={policy}") - if (policy == "DEFAULT" then( - restartOnError(".*", 2 - task:transfer(srcprovider=provider, srchost=srchost, srcfile=srcfile, - srcdir=srcdir, desthost=desthost, destdir=destdir, - transferParams(srchost, desthost, provider)))) - policy == "BROADCAST" then( - log(LOG:DEBUG, "FILE_STAGE_IN_BROADCAST file={srcfile} policy={policy}") - cdm:broadcast(srcfile=srcfile, srcdir=srcdir)) - policy == "EXTERNAL" then( - log(LOG:DEBUG, "FILE_STAGE_IN_EXTERNAL file={srcfile} policy={policy}") - cdm:external(srcfile=srcfile, srcdir=srcdir, - desthost=desthost, destdir=destdir)) - else(log(LOG:DEBUG, "FILE_STAGE_IN_SKIP file={srcfile} policy={policy}"))) - log(LOG:DEBUG, "FILE_STAGE_IN_END file={srcfile} ", - "srchost={srchost} srcdir={srcdir} srcname={srcfile} ", - "desthost={desthost} destdir={destdir} provider={provider}") - ) - cdm:wait() - ) - - element(doStageout, [jobid, stageouts, dir, host] - log(LOG:INFO, "START jobid={jobid} - Staging out files") - log(LOG:DEBUG, "stageouts: {stageouts}") - done := list( - uParallelFor(pv, stageouts - [path, var] := each(pv) - file := vdl:absfilename(vdl:getfield(var, path = path)) - provider := vdl:provider(file) - dhost := vdl:hostname(file) - rdir := dircat(dir, reldirname(file)) - bname := basename(file) - ldir := vdl:dirname(file) - fullLocal := dircat(ldir, bname) - fullRemote := dircat(rdir, bname) - - log(LOG:DEBUG, "FILE_STAGE_OUT_START srcname={bname} srcdir={rdir} srchost={host} ", - "destdir={ldir} desthost={dhost} provider={provider}") - //make sure we do have the directory on the client side - dir:make(ldir, host=dhost, provider=provider) - policy := cdm:query(query=file) - log(LOG:DEBUG, "CDM: {file} : {policy}") - if (sys:or(policy == "DEFAULT", policy == "BROADCAST") - then( - restartOnError(".*", 2 - task:transfer(srchost=host, srcfile=bname,srcdir=rdir, - destdir=ldir, desthost=dhost, destprovider=provider, - transferParams(host, dhost, provider)))) - else(log(LOG:DEBUG, "FILE_STAGE_OUT_SKIP srcname={bname}")) - ) - log(LOG:DEBUG, "FILE_STAGE_OUT_END srcname={bname} srcdir={rdir} srchost={host} ", - "destdir={ldir} desthost={dhost} provider={provider}") - - list(bname, rdir, host, file:size(fullLocal)) - ) - ) - uParallelFor(f, done - [bname, rdir, host, size] := each(f) - vdl:cacheAddFile(bname, rdir, host, size - cleanupFiles(cacheFilesToRemove, host) - ) - ) - log(LOG:INFO, "END jobid={jobid} - Staging out finished") - ) - - element(graphStuff, [tr, stagein, stageout, err, optional(args)] - if( - vdl:configProperty("pgraph") != "false" then( - errprops := if(err ",color=lightsalmon" ",color=lightsteelblue1") - tp := vdl:threadPrefix() - to(graph, - concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]") - ) - for(si, stagein - si := basename(si) - to(graph - concat(str:quote(si), " [shape=parallelogram]") - concat(str:quote(si), " -> ", str:quote(tp)) - ) - ) - for(pv, stageout - [path, var] := each(pv) - file := vdl:fileName(vdl:getfield(var, path=path)) - file := basename(file) - label := vdl:niceName(var, path = path) - to(graph - concat(str:quote(file), " [shape=parallelogram,label=", - str:quote(label), "]") - concat(str:quote(tp), " -> ", str:quote(file)) - ) - ) - ) - ) - ) - - element(fileSizes, [files] - math:sum( - for(f, files, file:size(file)) - ) - ) - - element(transferStandardFiles, [rhost, tmpdir, jobid, stdout, stderr] - concat( - for(f, list(list("stderr.txt", stderr), list("stdout.txt", stdout)) - [name, file] := each(f) - destfile := concat("{jobid}-", basename(file)) - try( - sequential( - task:transfer(srchost=rhost, srcdir=tmpdir, srcfile=file, - destfile=destfile) - nl() - " {name}: " - strip(file:read(destfile)) - ) - "" - ) - maybe(file:remove(destfile)) - ) - ) - ) - - element(transferKickstartRec, [rhost, wfdir, jobid, jobdir] - recfile := "{jobid}-kickstart.xml" - srcdir := dircat("{wfdir}/kickstart/", jobdir) - try( - task:transfer(srchost=rhost, srcdir=srcdir, srcfile=recfile, destdir="{VDL:SCRIPTNAME}-{VDL:RUNID}.d/") - ( - maybe(file:remove(recfile)) - log(LOG:WARN, "Failed to transfer kickstart records from {srcdir} on {rhost}", exception) - ) - ) - recfile - ) - - element(transferWrapperLog, [rhost, wfdir, jobid, jobdir] - recfile := "{jobid}-info" - srcdir := dircat("{wfdir}/info/", jobdir) - try( - task:transfer(srchost=rhost, srcdir=srcdir, srcfile=recfile, destdir="{VDL:SCRIPTNAME}-{VDL:RUNID}.d/") - ( - maybe(file:remove(recfile)) - log(LOG:INFO, "Failed to transfer wrapper log for job {jobid} from {srcdir} on {rhost}", exception) - ) - ) - recfile - ) - - element(execute2, [tr, optional(arguments, stdin, stdout, stderr, attributes), stagein, stageout, restartout, - replicationGroup, replicationChannel] - stagein := list(unique(each(stagein))) - stageout := list(unique(each(stageout))) - - allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein) - - ddir := initDDir() - [wfdir, sharedDir] := try( - initSharedDir(rhost) - throw(exception("Could not initialize shared directory on {rhost}", exception)) - ) - - uid := uid() - jobdir := substring(uid, from=0, to=1) - jobid := "{tr}-{uid}" - - log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") - - statusMode := configProperty("status.mode",host=rhost) - wrapperMode := configProperty("wrapper.parameter.mode",host=rhost) - - wrapfile := "{ddir}/param-{jobid}" - - stdout := try(getFieldValue(stdout), "stdout.txt") - stderr := try(getFieldValue(stderr), "stderr.txt") - kickstart := vdl:kickstart(rhost) - fileDirs := fileDirs(stagein, stageout) - os := vdl:siteprofile(rhost, "SYSINFO:OS") - - if(wrapperMode == "files" - sequential( - sys:file:write(wrapfile, - "-scratch ", try(vdl:siteprofile(rhost, "scratch"), ""), nl(), - "-e ",vdl:executable(tr, rhost), nl(), - "-out ", stdout, nl(), - "-err ", stderr, nl(), - "-i ", maybe(getFieldValue(stdin)), nl(), - "-d ", flatten(each(fileDirs)), nl(), - "-if ", flatten(infiles(stagein)), nl(), - "-of ", flatten(outfiles(stageout)), nl(), - "-k ", kickstart, nl(), - "-cdmfile ", cdm:file(), nl(), - "-status ", statusMode, nl(), - for(a, arguments, "-a ", a, nl()) - ) - ) - ) - - vdl:setprogress("Stage in") - tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid) - - - - try( - sequential( - - createDirSet(jobid, sharedDir, rhost, fileDirs) - doStagein(jobid, stagein, sharedDir, rhost) - if(wrapperMode == "files" - stageWrapperParams(jobid, jobdir, wrapfile, wfdir, rhost) - ) - - - log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}") - - vdl:setprogress("Submitting") - - if(wrapperMode == "files" - vdl:execute( - vdl:siteprofile(rhost, "swift:wrapperInterpreter"), - list( - vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"), - dircat("shared", vdl:siteprofile(rhost, "swift:wrapperScript"), os=os), - jobid, "-p", jobdir - ) - directory=wfdir - redirect=false - host=rhost - vdl:tcprofile(rhost, maybe(attributes=attributes), tr=tr) //this gets various app params from the tc, such as environment, walltime, etc - replicationGroup=replicationGroup - replicationChannel=replicationChannel - jobid=jobid - ) - ) - if(wrapperMode == "args" - vdl:execute( - vdl:siteprofile(rhost, "swift:wrapperInterpreter"), - list( - vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"), - dircat("shared", vdl:siteprofile(rhost, "swift:wrapperScript"), os=os), - jobid, - "-jobdir", jobdir, - "-scratch", try(vdl:siteprofile(rhost, "scratch"), "") - "-e", vdl:executable(tr, rhost), - "-out", stdout, - "-err", stderr, - "-i", maybe(getFieldValue(stdin)), - "-d", flatten(each(fileDirs)), - "-if", flatten(infiles(stagein)), - "-of", flatten(outfiles(stageout)), - "-k", kickstart, - "-cdmfile", cdm:file(), - "-status", statusMode, - "-a", maybe(each(arguments)) - ) - directory=wfdir - redirect=false - host=rhost - vdl:tcprofile(rhost, maybe(attributes=attributes), tr=tr) - replicationGroup=replicationGroup - replicationChannel=replicationChannel - jobid=jobid - ) - ) - - vdl:setprogress("Checking status") - if(statusMode == "files" - checkJobStatus(rhost, wfdir, jobid, tr, jobdir) - ) - - if(wrapperMode == "files" - file:remove(wrapfile) - ) - - log(LOG:DEBUG, "STAGING_OUT jobid={jobid}") - - - /* need to stage the files to upper scratch area in case they are not transfered to another site - before all the files get cleaned out */ - - - vdl:setprogress("Stage out") - doStageout(jobid, stageout, sharedDir, rhost) - doRestartlog(restartout) - if( - kickstart != "" & vdl:configProperty("kickstart.always.transfer") == "true" - discard(transferKickstartRec(rhost, wfdir, jobid, jobdir)) - ) - if( - vdl:configProperty("wrapperlog.always.transfer") == "true" - discard(transferWrapperLog(rhost, wfdir, jobid, jobdir)) - ) - vdl:cacheUnlockFiles(stagein, sharedDir, rhost, cleanupFiles(cacheFilesToRemove, rhost)) - log(LOG:DEBUG, "JOB_END jobid={jobid}") - ) - catch("^Abort$" - log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") - vdl:cacheUnlockFiles(stagein, sharedDir, rhost, force=false - cleanupFiles(cacheFilesToRemove, rhost) - ) - throw(exception) - ) - catch("^(?!Abort$).*" - vdl:setprogress("Failed but can retry") - prev := exception - exception := try(exception(checkErrorFile(rhost, wfdir, jobid, jobdir)), prev) - - log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) - - if(matches(exception,".*executable bit.*") - generateError(exception) - ) - - vdl:cacheUnlockFiles(stagein, sharedDir, rhost, force=false - cleanupFiles(cacheFilesToRemove, rhost) - ) - - outs := transferStandardFiles(rhost, tmpdir, jobid, stdout, stderr) - - discard(maybe(transferWrapperLog(rhost, wfdir, jobid, jobdir))) - - kickstartRec := if( - kickstart == "" "" - else( - try( - ( - recfile := transferKickstartRec(rhost, wfdir, jobid, jobdir) - "KickstartRecord: {recfile}" - ) - "" - ) - ) - ) - - throw( - exception( - concat( - "Exception in {tr}:", nl(), - maybe(" Arguments: ", arguments, nl()), - " Host: {rhost}", nl(), - " Directory: {tmpdir}", - "{outs}", - kickstartRec - ) - exception - ) - ) - ) - ) - ) - ) - - element(generateProvenanceGraph, [gdata] - pgraph := vdl:configProperty("pgraph") - gname := if(pgraph == "true" "{VDL:SCRIPTNAME}-{VDL:RUNID}.dot" pgraph) - file:write(gname - "digraph SwiftProvenance {{", nl() - " graph [", vdl:configProperty("pgraph.graph.options"), "];", nl() - " node [", vdl:configProperty("pgraph.node.options"), "];", nl() - - for(i, gdata - " ", i, nl() - ) - "}", nl() - ) - log(LOG:INFO, "Provenance graph saved in ", gname) - ) - ) -) - -// Local variables: -// mode: scheme -// tab-width: 4 -// indent-tabs-mode: t -// End: Deleted: branches/faster/libexec/vdl-lib.xml =================================================================== --- branches/faster/libexec/vdl-lib.xml 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/libexec/vdl-lib.xml 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,153 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Deleted: branches/faster/libexec/vdl-sc.k =================================================================== --- branches/faster/libexec/vdl-sc.k 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/libexec/vdl-sc.k 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,115 +0,0 @@ -import("sys.k") - -namespace("vdl" - export(siteCatalog - element([file] - - import("task.k") - - element(pool, [handle, optional(gridlaunch), optional(sysinfo), optional(lrc), ..., channel(properties)] - host(name = handle - each(...) - to(properties - each(properties) - maybe(property("gridlaunch", gridlaunch)) - maybe(property("sysinfo", sysinfo)) - ) - ) - ) - - element(gridftp, [url, optional(storage), optional(major), optional(minor), optional(patch)] - if( - url == "local://localhost" - service(type="file", provider="local") - service(type="file", provider="gsiftp", url=url) - ) - ) - - element(jobmanager, [url, major, optional(universe), optional(minor), optional(patch)] - provider := if( - url == "local://localhost" "local" - url == "pbs://localhost" "pbs" - major == "4" "GT4" - major == "2" "GT2" - throw("Unknown job manager version: major = {major}, minor = {minor}, patch = {patch}, url = {url}") - ) - service(type="execution", provider=provider, url=url) - ) - - element(execution, [provider, url, optional(jobManager)] - service(type="execution", provider=provider, maybe(url=url), maybe(jobManager=jobManager)) - ) - - element(filesystem, [provider, url, optional(storage)] - service(type="file", provider=provider, url=url) - ) - - element(profile, [namespace, key, value] - if( - namespace == "karajan" - property("{key}", value) - property("{namespace}:{key}", value) - ) - ) - - element(env, [name, value] - property("env:{name}", value) - ) - - element(workdirectory, [dir] - property("workdir", dir) - ) - - element(scratch, [dir] - property("scratch", dir) - ) - - element(p, [name, value] - print(" {name} = {value}") - property(name, value) - ) - - element(intrepidCoasterParams, [optional(averageJobTime, blockSize)] - print("Setting Intrepid coaster params:") - p("scratch", "/scratch") - p("jobsPerNode", "4") - p("kernelprofile", "zeptoos") - p("alcfbgpnat", "true") - maybe( - discard(averageJobTime) - blockSize := try(blockSize, 512) - parallelism := averageJobTime * 100 - p("slots", math:round(parallelism / (blockSize * 4))) - p("nodeGranularity", blockSize) - p("maxNodes", blockSize) - jt := parallelism * 2 - p("jobThrottle", jt / 100) - print("For best results, please set the foreach.max.threads property in swift.properties to {jt}") - ) - ) - - element(surveyorCoasterParams, [optional(averageJobTime, blockSize)] - print("Setting Surveyor coaster params:") - p("scratch", "/scratch") - p("jobsPerNode", "4") - p("kernelprofile", "zeptoos") - p("alcfbgpnat", "true") - maybe( - discard(averageJobTime) - blockSize := try(blockSize, 64) - parallelism := averageJobTime * 100 - p("slots", 12) - p("nodeGranularity", blockSize) - p("maxTime", 3500) - jt := parallelism * 2 - p("jobThrottle", jt / 100) - print("For best results, please set the foreach.max.threads property in swift.properties to {jt}") - ) - ) - - resources( - executeFile(file) - ) - ) - ) -) Deleted: branches/faster/libexec/vdl-xs.k =================================================================== --- branches/faster/libexec/vdl-xs.k 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/libexec/vdl-xs.k 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,72 +0,0 @@ -import("sys.k") - -namespace("xs" - import("java.k") - - element(UnresolvedType, [name, isArray] - new("org.griphyn.vdl.type.impl.UnresolvedType", types=["String"], name) - ) - - element(typesST, [], types=["String", "org.griphyn.vdl.type.Type"]) - - element(newSimpleNode, [name, type] - x := new("org.griphyn.vdl.type.impl.TypeImpl", types=["String"], name) - invokeMethod("setBaseType", object=x, types=["org.griphyn.vdl.type.Type"] - invokeMethod("getType", classname="org.griphyn.vdl.type.Types", type) - ) - x - ) - - element(newComplexNode, [name] - new("org.griphyn.vdl.type.impl.TypeImpl", types=["String"], name) - ) - - element(addNode, [node] - invokeMethod("addType", classname="org.griphyn.vdl.type.Types" - types=["org.griphyn.vdl.type.Type"] - node - ) - ) - - element(addField, [node, name, type] - invokeMethod("addField", object=node, typesST(), name, type) - ) - - export( - element(schema, [] - invokeMethod("resolveTypes", classname="org.griphyn.vdl.type.Types") - ) - - element(simpleType, [name, type] - addNode(newSimpleNode(name, type)) - ) - - element(restriction, [base] - type=last(split(base, ":")) - ) - - element(complexType, [name, ...] - node := newComplexNode(name) - for(field, ... - [name, type] := each(field) - addField(node, name, type) - ) - addNode(node) - ) - - element(sequence, [optional(minOccurs, maxOccurs), ...] - maxOccurs := choice(maxOccurs, 0) - [name, type] := each(first(...)) - if ( - maxOccurs == "unbounded" list(name, UnresolvedType(type, true)) - else(each(...)) - ) - ) - - //should be noted that we're dealing with type names here - element(xs:element, [name, type] - type := last(split(type, ":")) - list(name, UnresolvedType(type, false)) - ) - ) -) Deleted: branches/faster/libexec/vdl.k =================================================================== --- branches/faster/libexec/vdl.k 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/libexec/vdl.k 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,128 +0,0 @@ -import("sys.k") -import("task.k") -import("rlog.k") -import("operators.xml", export = true) - -import("vdl-xs.k", export = true) - -namespace("vdl" - - import("vdl-sc.k", export = true) - import("vdl-lib.xml", export = true) - - pstaging := configProperty("use.provider.staging") - wstaging := configProperty("use.wrapper.staging") - int := if ( - pstaging == "true", "vdl-int-staging.k", - wstaging == "true", "vdl-int-wrapper-staging.k", - "vdl-int.k" - ) - - import(int) - import("java.k") - - once("vdl.k-print-version" - log("info",sys:file:read("{swift.home}/libexec/version.txt")) - echo(sys:file:read("{swift.home}/libexec/version.txt")) - log("info","RUNID id=run:{VDL:RUNID}") - echo("RunID: {VDL:RUNID}") - ) - - export("stagein", elementDef(classname="org.griphyn.vdl.karajan.lib.Stagein")) - export("stageout", elementDef(classname="org.griphyn.vdl.karajan.lib.Stageout")) - - export( - - element(parameterlog, [direction, variable, id, thread], - if( - vdl:configProperty("provenance.log") == "true" - log("info","PARAM thread={thread} direction={direction} variable={variable} provenanceid={id}") - ) - ) - element(split, [var], each(str:split(vdl:getFieldValue(var), " "))) - - element(quote, [var, optional(path)], - str:quote(vdl:getFieldValue(var, maybe(path = path))) - ) - - element(types, []) - - element(arguments, [...] - arguments=expandArguments(each(...)) - ) - - export(execute - executeFile( - if( - vdl:operation == "dryrun" - "execute-dryrun.k" - vdl:operation == "typecheck" - "execute-typecheck.k" - vdl:operation == "run" - "execute-default.k" - ) - ) - ) - - element(mapping, [descriptor, ...] - mapping=map(map:entry("swift#descriptor", descriptor), each(...)) - ) - - element(parameter, [name, value] - map:entry(name, value) - ) - - element(stdout, [file] - stdout = file - ) - - element(stdin, [file] - stdin = file - ) - - element(stderr, [file] - stderr = file - ) - - element(tr, [name] - tr = name - ) - - element(attributes, [attrs] - attributes = attrs - ) - - element(mains, [channel(cleanup), channel(errors), channel(warnings)] - anyerrors := vdl:processBulkErrors("The following errors have occurred:", errors) - - //this should be reached after everything is done - if( - sys:not(anyerrors) then( - //hmm, you can append to channels! - log(LOG:DEBUG, "Starting cleanups") - append(warnings, from(warnings, cleanups(cleanup))) - log(LOG:DEBUG, "Ending cleanups") - ) - else( - log(LOG:INFO, "Errors detected. Cleanup not done.") - ) - ) - anywarnings := vdl:processBulkErrors("The following warnings have occurred:", warnings, onStdout=true) - if (anyerrors throw("Execution completed with errors")) - ) - - /* - * This would run in parallel with the workflow so that we don't keep - * all the data in memory until the workflow is done - */ - parallelElement(mainp, [channel(graph), channel(cleanup)] - parallel( - if( - vdl:configProperty("pgraph") != "false" - generateProvenanceGraph(graph) - ) - to(cleanup, unique(for(c, cleanup, c))) - ) - ) - ) -) Modified: branches/faster/resources/Karajan.stg =================================================================== --- branches/faster/resources/Karajan.stg 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/resources/Karajan.stg 2013-01-29 07:31:09 UTC (rev 6170) @@ -4,75 +4,74 @@ // are smaller program(types,procedures,declarations,statements,constants,buildversion,cleanups) ::= << - - - - - - $if(types)$ - - - $types;separator="\n"$ - - - $endif$ - $constants;separator="\n"$ - $procedures;separator="\n"$ - $declarations;separator="\n"$ - $if(statements)$ - - - - - $parallel(statements=statements)$ - - - - - $endif$ - $cleanups:vdl_cleandataset();separator="\n"$ - - +// CACHE ID $buildversion$ +import(sys) +import(scheduler) +import(rlog) +import(swift) + +$if(types)$ +types( + xs:schema( + $types;separator="\n"$ + ) +) + +$endif$ +$constants;separator="\n"$ +$procedures;separator="\n"$ +$declarations;separator="\n"$ +$if(statements)$ + +restartLog() { + swift:mains( + swift:startProgressTicker() + swift:mainp( + $parallel(statements=statements)$ + ) + swift:stopProgressTicker() + ) +} +$endif$ +$cleanups:swift_cleandataset();separator="\n"$ +swift:cleanDataset(shutdown=true) >> typeDef(name,type,members,sourcelocation) ::= << - $if(type)$ - - $else$ - $if(!members)$ - - - - $else$ - - - $members;separator="\n"$ - - - $endif$ - $endif$ +$if(type)$ + xs:element("$name$", "$type$") +$else$ + $if(!members)$ + xs:simpleType("$name$", xs:restriction("string")) + $else$ + xs:complexType("$name$", + xs:sequence( + $members;separator="\n"$ + ) + ) + $endif$ +$endif$ >> memberdefinition(type,name,sourcelocation) ::= << - + xs:element("$name$", "$type$") >> procedure(name, outputs, inputs, arguments, optargs, binding, declarations, statements, config, line, initWaitCounts, cleanups, closes) ::= << - - $optargs:default_arg();separator="\n"$ - $inputs:vdl_log_input();separator="\n"$ - $outputs:vdl_log_output();separator="\n"$ + +$name$ := function($if(optargs)$ "$proc_oargs(args=optargs)$"$endif$$if(arguments)$ "$proc_args(args=arguments)$"$endif$) { + $inputs:swift_log_input();separator="\n"$ + $outputs:swift_log_output();separator="\n"$ $if(binding)$ - $vdl_execute(outputs=outputs,inputs=inputs,application=binding.application, name=name,line=line)$ + $swift_execute(outputs=outputs,inputs=inputs,application=binding.application, name=name,line=line)$ $else$ - $compound(outputs=outputs, inputs=inputs, declarations=declarations, statements=statements, name=name, initWaitCounts=initWaitCounts, cleanups=cleanups)$ + $compound(outputs=outputs, inputs=inputs, declarations=declarations, statements=statements, name=name, initWaitCounts=initWaitCounts, cleanups=cleanups)$ $endif$ - - +} >> compound(outputs, inputs, declarations, statements, config, name, initWaitCounts, cleanups) ::= << - +unitStart("COMPOUND", name="$name$", outputs="$outputs:list();separator=","$") $if(initWaitCounts)$ $initWaitCounts;separator="\n"$ $endif$ @@ -81,11 +80,10 @@ $parallel(statements=statements)$ $endif$ -$outputs:vdl_closedataset();separator="\n"$ +$outputs:swift_closedataset();separator="\n"$ -$cleanups:vdl_cleandataset();separator="\n"$ - - +$cleanups:swift_cleandataset();separator="\n"$ +unitEnd("COMPOUND", name="$name$") >> proc_args(args) ::= << @@ -96,116 +94,90 @@ $it.name$ >> -default_arg() ::= << -$if(it.default)$ - -$it.default$ - -$endif$ +proc_oargs(args) ::= << +$args:proc_optarg()$ >> -vdl_execute(outputs,inputs,attributes,application,name,line) ::= << - - - $attributes$ - $application.exec$ - $inputs:vdl_stagein();separator="\n"$ - $outputs:vdl_stageout();separator="\n"$ - $vdl_arguments(attributes=application.attributes,arguments=application.arguments, stdin=application.stdin,stdout=application.stdout,stderr=application.stderr)$ - -$outputs:vdl_closedataset();separator="\n"$ - +proc_optarg() ::= << +$it.name$=$if(it.default)$$it.default$$else$null$endif$ >> -vdl_log_input() ::= << - -input -$it.name$ - -{#thread} - +swift_execute(outputs,inputs,attributes,application,name,line) ::= << +unitStart("PROCEDURE", name="$name$", line="$line$", outputs="$outputs:list();separator=","$") +swift:execute( + $attributes$ + swift:tr("$application.exec$") + $inputs:swift_stagein();separator="\n"$ + $outputs:swift_stageout();separator="\n"$ + $swift_arguments(attributes=application.attributes,arguments=application.arguments, stdin=application.stdin,stdout=application.stdout,stderr=application.stderr)$ +) +$outputs:swift_closedataset();separator="\n"$ +unitEnd("PROCEDURE", name="$name$", line="$line$") >> -vdl_log_output() ::= << - -output -$it.name$ - -{#thread} - +swift_log_input() ::= << +parameterLog("input", "$it.name$", swift:getDatasetProvenanceID($it.name$), currentThread()) >> -vdl_stagein() ::= << - +swift_log_output() ::= << +parameterLog("output", "$it.name$", swift:getDatasetProvenanceID($it.name$), currentThread()) >> -vdl_stageout(outputs) ::= << - +swift_stagein() ::= << +swift:stageIn($it.name$) >> -vdl_closedataset() ::= << - +swift_stageout(outputs) ::= << +swift:stageOut($it.name$) >> -vdl_cleandataset() ::= << - +swift_closedataset() ::= << +swift:closeDataset($it.name$) >> +swift_cleandataset() ::= << +swift:cleanDataset($it$) +>> + list() ::= <<$it.name$>> -vdl_arguments(attributes,arguments,stdin,stdout,stderr) ::= << - $attributes$ - - $arguments;separator="\n"$ - +swift_arguments(attributes,arguments,stdin,stdout,stderr) ::= << +$attributes$ +swift:arguments( + $arguments;separator="\n"$ +) $if (stdin)$ - - $stdin$ - +swift:stdin($stdin$) $endif$ $if (stdout)$ - - $stdout$ - +swift:stdout($stdout$) $endif$ $if (stderr)$ - - $stderr$ - +swift:stderr($stderr$) $endif$ >> -vdl_attributes(entries) ::= << - - - $entries;separator="\n"$ - - - +swift_attributes(entries) ::= << +swift:attributes( + map( + $entries;separator="\n"$ + ) +) >> map_entry(key,value) ::= << - - - $key$ - - - $value$ - - + map:entry(swift:getFieldValue($key$), swift:getFieldValue($value$)) >> -// use unbuffered parallel here -// since calls don't use this macro parallel(statements) ::= << $if(rest(statements))$ - - $statements;separator="\n"$ - +parallel( + $statements;separator="\n"$ +) $else$ - - $statements$ - +sequential( + $statements$ +) $endif$ >> @@ -214,104 +186,76 @@ // that calls a function in the 'swiftscript' namespace. function(name, args, datatype, line) ::= << - - $if(args)$ $args$ $endif$ - +swiftscript:$name$( + $if(args)$ $args$ $endif$ +) >> iterate(declarations,statements,cond,var,cleanups,trace,line) ::= << - - - $sub_comp(declarations=declarations, statements=statements, cleanups=cleanups)$ - - $cond$ - - - - +sys:while($var$, 0) { + if (vdl:getFieldValue($cond$)) { + $sub_comp(declarations=declarations, statements=statements, cleanups=cleanups)$ + next($var$ + 1) + } +} >> foreach(var, in, indexVar, indexVarType, declarations, statements, line, refs, selfClose, cleanups, trace) ::= << - +swift:tParallelFor(\$, _traceline="$line$"$if(indexVar)$, _kvar="$indexVar$"$endif$, _vvar="$var$"$if(selfClose)$, selfClose="true"$endif$$if(refs)$, refs="$refs;separator=" "$"$endif$, $! The iterator !$ - - $in$ - -$! Body !$ - - - + getArrayIterator($in$)) { + (\$\$, $var$) := each(\$) $if(indexVar)$ - - - - + $indexVar$ := swift:new($indexVarType$, \$\$) $endif$ - + unitStart("FOREACH_IT", line="$line$") $declarations;separator="\n"$ $if(statements)$ $parallel(statements=statements)$ - $cleanups:vdl_cleandataset();separator="\n"$ + $cleanups:swift_cleandataset();separator="\n"$ $endif$ - - + unitEnd("FOREACH_IT", line="$line$") +} >> -cs() ::= << - #thread ->> - // need to log inputs and outputs at the calling stage here because // they are not // $outputs:vdl_log_output();separator="\n"$ callInternal(func, outputs, inputs, line, serialize) ::= << - - -$cs()$ -<$func$ _traceline="$line$"> - $if(!serialize)$$endif$ - $outputs:callInternal_log_output();separator="\n"$ - $inputs:callInternal_log_input();separator="\n"$ - $if(!serialize)$$endif$ - - - +sequential( + unitStart("INTERNALPROC", name="$func$", outputs="$outputs:list();separator=","$") + $func$( + $if(!serialize)$parallel($endif$ + $outputs:callInternal_log_output();separator="\n"$ + $inputs:callInternal_log_input();separator="\n"$ + $if(!serialize)$)$endif$ + ) + unitEnd("INTERNALPROC", name="$func$") +) >> callInternal_log_input() ::= << - - $it$ - - input - TODO_name_or_pos - - {swift#cs} - - swift#callInternalValue - +sequential( + parameterLog("input", "TODO_name_or_pos", vdl:getDatasetProvenanceID($it$), currentThread()) + $it$ +) >> callInternal_log_output() ::= << - - $it$ - - output - TODO_name_or_pos - - {swift#cs} - - swift#callInternalValue - +sequential( + parameterLog("output", "TODO_name_or_pos", vdl:getDatasetProvenanceID($it$), currentThread()) + $it$ +) >> callUserDefined(func, outputs, inputs, line, serialize, partialClose) ::= << -<$func$ _traceline="$line$"> - $if(!serialize)$$endif$ +$func$( + $if(!serialize)$parallel($endif$ $outputs;separator="\n"$ $inputs;separator="\n"$ - $if(!serialize)$$endif$ - + $if(!serialize)$)$endif$ +) $if(partialClose)$ $partialClose$ $endif$ @@ -319,159 +263,117 @@ call_arg(bind, expr, datatype) ::= << $if(bind)$ - - $expr$ - +$bind$ = $expr$ $else$ $expr$ $endif$ >> globalConstant(name, expr, datatype) ::= << - - $expr$ - +$name$ := $expr$ >> variable(name, type, expr, mapping, nil, file, waitCount, input, datatype, isGlobal, line) ::= << -$if(isGlobal)$$else$$endif$ - $if(mapping)$ - - $vdl_mapping(mapping=mapping, file=file)$ - - $else$ - $if(file)$ - - $vdl_mapping(mapping=mapping, file=file)$ - - $else$ - - $endif$ - $endif$ -$if(isGlobal)$$else$$endif$ +$name$ := + $if(mapping)$ + swift:new("$type$", dbgname="$name$"$if(waitCount)$, waitCount=$waitCount$$endif$, _defline="$line$"$if(input)$, input=true$endif$ + $swift_mapping(mapping=mapping, file=file)$ + ) + $else$ + $if(file)$ + swift:new("$type$", dbgname="$name$"$if(waitCount)$, waitCount=$waitCount$$endif$, _defline="$line$"$if(input)$, input=true$endif$ + $swift_mapping(mapping=mapping, file=file)$ + ) + $else$ + swift:new("$type$", dbgname="$name$"$if(waitCount)$, waitCount=$waitCount$$endif$, _defline="$line$"$if(input)$, input=true$endif$) + $endif$ + $endif$ $variable_log()$ >> variable_log() ::= << - - intermediate - $name$ - - {#thread} - + parameterLog("intermediate", "$name$", swift:getDatasetProvenanceID($name$), currentThread()) >> -vdl_mapping(mapping, file) ::= << +swift_mapping(mapping, file) ::= << $if(file)$ - - +swift:mapping("single_file_mapper", + swift:parameter("file", "$file.name$") $if(file.params)$$file.params;separator="\n"$$endif$ - +) $else$ - +swift:mapping("$mapping.descriptor$", $mapping.params;separator="\n"$ - +) $endif$ >> -vdl_parameter(name,expr) ::= << -$expr$ +swift_parameter(name,expr) ::= << +swift:parameter("$name$", $expr$) >> assign(var, value, line, partialClose) ::= << - + swift:setFieldValue($if(line)$_traceline="$line$"$else$_traceline="-1"$endif$ $var$ $value$ - + ) $if(partialClose)$ $partialClose$ $endif$ >> append(array, value, partialClose) ::= << - + swift:appendArray( $array$ $value$ - + ) $if(partialClose)$ $partialClose$ $endif$ >> callexpr(call, datatype, prefix) ::= << - - - - - _callintermediate-$prefix$ - - - +sequential( + tmp := swift:new("$datatype$", dbgname="swift#callintermediate", + swift:mapping("concurrent_mapper", + swift:parameter("prefix", "_callintermediate-$prefix$") + ) + ) $call$ - swift#callintermediate - + tmp +) >> array(elements, datatype) ::= << - - - $elements;separator="\n"$ - - +swift:createarray( + list( + $elements;separator="\n"$ + ) +) >> range(from, to, step, datatype) ::= << - - - - $from$ - $to$ -$if(step)$ - $step$ -$endif$ - - - $range_log(from=from, to=to, step=step)$ - swift#rangeout - + sequential( + tmp := swift:range($from$, $to$$if(step)$, step = $step$$endif$) + tmp + $range_log()$ + ) >> -range_log(from, to, step) ::= << - - provenance.logtrue - - - - ARRAYRANGE thread={#thread} array= - - from= $from$ - to= $to$ - $if(step)$ - step= $step$ - $else$ - step=none - $endif$ - - - - +range_log() ::= << >> if(condition,vthen,velse,line,trace) ::= << - - $condition$ - - - $vthen$ - +if (swift:getFieldValue($condition$)) { + unitStart("CONDITION_BLOCK") + $vthen$ +} $if(velse)$ - - - $velse$ - +else { + unitStart("CONDITION_BLOCK") + $velse$ +} $endif$ - - >> sub_comp(declarations, statements, cleanups, preClose) ::= << @@ -481,158 +383,123 @@ $declarations;separator="\n"$ $if(statements)$ $parallel(statements=statements)$ -$cleanups:vdl_cleandataset();separator="\n"$ +$cleanups:swift_cleandataset();separator="\n"$ $endif$ >> switch(condition,cases,sdefault) ::= << - - - $condition$ - - +sequential( + swc := $condition$ $cases:{case | - - - \$_sw - $case.value$ - - - - $sub_comp(declarations=case.declarations, statements=case.statements)$ - + if (swift:getFieldValue(swiftop:eq(swc, $case.value$))) { + $sub_comp(declarations=case.declarations, statements=case.statements)$ + } + else }$ $if(sdefault)$ - + { $sdefault$ - - + } $endif$ - - +) >> // TODO can optimise this like we do with parallel statements so that // the wrapping layer disappers in the (perhaps common?) case of a // single layer. sequential(statements) ::= << - - $statements;separator="\n"$ - + sequential( + $statements;separator="\n"$ + ) >> partialclose(var, count) ::= << - +partialCloseDataset($var$$if(count)$, count=$count$$endif$) >> setWaitCount(name, waitCount) ::= << - +setWaitCount($name$$if(waitCount)$, count=$waitCount$$endif$) >> unitStart(type, outputs) ::= << - + unitStart("$type$", outputs="$outputs$") >> unitEnd(type) ::= << - + unitEnd("$type$") >> operator ::= [ - "+":"vdlop:sum", - "-":"vdlop:subtraction", - "*":"vdlop:product", - "/":"vdlop:fquotient", - "%/":"vdlop:iquotient", - "%%":"vdlop:remainder", - "<=":"vdlop:le", - ">=":"vdlop:ge", - ">":"vdlop:gt", - "<":"vdlop:lt", - "<=":"vdlop:le", - ">=":"vdlop:ge", - ">":"vdlop:gt", - "<":"vdlop:lt", - "==":"vdlop:eq", - "!=":"vdlop:ne", - "&&":"vdlop:and", - "||":"vdlop:or" + "+":"swiftop:sum", + "-":"swiftop:subtraction", + "*":"swiftop:product", + "/":"swiftop:fquotient", + "%/":"swiftop:iquotient", + "%%":"swiftop:remainder", + "<=":"swiftop:le", + ">=":"swiftop:ge", + ">":"swiftop:gt", + "<":"swiftop:lt", + "<=":"swiftop:le", + ">=":"swiftop:ge", + ">":"swiftop:gt", + "<":"swiftop:lt", + "==":"swiftop:eq", + "!=":"swiftop:ne", + "&&":"swiftop:and", + "||":"swiftop:or" ] unaryNegation(exp, datatype) ::= << - - - $exp$ - +swiftop:product(swift:new("int", value = -1), $exp$) >> binaryop(op,left,right,datatype) ::= << -<$operator.(op)$> - $left$ - $right$ - +$operator.(op)$($left$, $right$) >> not(exp, datatype) ::= << - - $exp$ - +swiftop:not($exp$) >> id(var, datatype) ::= << -$var$ +$var$ >> extractarrayelement(parent, arraychild, datatype) ::= << - - $parent$ - $arraychild$ - +swift:getfieldsubscript($parent$, $arraychild$) >> extractstructelement(parent, memberchild, datatype) ::= << - - $parent$ - $memberchild$ - +getfield($parent$, $memberchild$) >> slicearray(parent, memberchild, datatype) ::= << - - $parent$ - - - swift#array - $memberchild$ - $datatype$ - - +sequential( + slice := swift:sliceArray($parent$, $memberchild$, $datatype$) + logSliceArray($memberchild$, $parent$) + slice +) +>> - - SLICEARRAY thread={#thread} slice= - - member=$memberchild$ array= - - - - swift#slice - +logSliceArray(memberchild, array) ::= << >> iConst(value, datatype) ::= << - +swift:new("int", value=$value$) >> fConst(value, datatype) ::= << - +swift:new("float", value=$value$) >> bConst(value, datatype) ::= << - +swift:new("boolean", value=$value$) >> sConst(value,innervalue,datatype) ::= << - +swift:new("string", value="$value$") >> Added: branches/faster/src/org/globus/swift/catalog/site/Parser.java =================================================================== --- branches/faster/src/org/globus/swift/catalog/site/Parser.java (rev 0) +++ branches/faster/src/org/globus/swift/catalog/site/Parser.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,55 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jan 7, 2013 + */ +package org.globus.swift.catalog.site; + +import java.io.File; +import java.io.IOException; +import java.net.URL; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.dom.DOMSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; +import javax.xml.validation.Validator; + +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +public class Parser { + public static final String SCHEMA_RESOURCE = "swift-sites-1.0.xsd"; + + private File src; + + public Parser(String fileName) { + this.src = new File(fileName); + } + + public Document parse() throws ParserConfigurationException, SAXException, IOException { + URL schemaURL = Parser.class.getClassLoader().getResource(SCHEMA_RESOURCE); + + if (schemaURL == null) { + throw new IllegalStateException("Sites schema not found in resources: " + SCHEMA_RESOURCE); + } + + SchemaFactory sfactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema schema = sfactory.newSchema(schemaURL); + + DocumentBuilderFactory dfactory = DocumentBuilderFactory.newInstance(); + dfactory.setSchema(schema); + DocumentBuilder dbuilder = dfactory.newDocumentBuilder(); + Document doc = dbuilder.parse(src); + + return doc; + } + +} Modified: branches/faster/src/org/globus/swift/catalog/transformation/File.java =================================================================== --- branches/faster/src/org/globus/swift/catalog/transformation/File.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/globus/swift/catalog/transformation/File.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1155,6 +1155,8 @@ * @param msg the message to be logged. */ protected void logMessage( String msg ) { - logger.debug(msg); + if (logger.isDebugEnabled()) { + logger.debug(msg); + } } } Modified: branches/faster/src/org/globus/swift/data/Action.java =================================================================== --- branches/faster/src/org/globus/swift/data/Action.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/globus/swift/data/Action.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,92 +17,103 @@ package org.globus.swift.data; +import k.rt.Stack; +import k.thr.LWThread; + import org.apache.log4j.Logger; - -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.globus.cog.karajan.util.BoundContact; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; -import org.globus.swift.data.policy.Broadcast; -import org.globus.swift.data.policy.External; import org.globus.swift.data.policy.Policy; /** * Karajan-accessible CDM functions that change something. * */ -public class Action extends FunctionsCollection { - private static final Logger logger = - Logger.getLogger(Action.class); - - public static final Arg PA_SRCFILE = - new Arg.Positional("srcfile"); - public static final Arg PA_SRCDIR = - new Arg.Positional("srcdir"); - public static final Arg PA_DESTHOST = - new Arg.Positional("desthost"); - public static final Arg PA_DESTDIR = - new Arg.Positional("destdir"); +public class Action { + private static final Logger logger = Logger.getLogger(Action.class); - static { - setArguments("cdm_broadcast", new Arg[]{ PA_SRCFILE, - PA_SRCDIR }); - setArguments("cdm_external", new Arg[]{ PA_SRCFILE, - PA_SRCDIR, - PA_DESTHOST, - PA_DESTDIR }); - setArguments("cdm_wait", new Arg[]{}); - } - /** Register a file for broadcast by CDM. The actual broadcast is triggered by {@link cdm_wait}. */ - public void cdm_broadcast(VariableStack stack) - throws ExecutionException { - String srcfile = (String) PA_SRCFILE.getValue(stack); - String srcdir = (String) PA_SRCDIR.getValue(stack); + public static class Broadcast extends InternalFunction { + private ArgRef srcfile; + private ArgRef srcdir; + + @Override + protected Signature getSignature() { + return new Signature(params("srcfile", "srcdir")); + } - logger.debug("cdm_broadcast()"); + @Override + protected void runBody(LWThread thr) { + Stack stack = thr.getStack(); + String srcfile = this.srcfile.getValue(stack); + String srcdir = this.srcdir.getValue(stack); + + logger.debug("cdm_broadcast()"); - Policy policy = Director.lookup(srcfile); + Policy policy = Director.lookup(srcfile); - if (!(policy instanceof Broadcast)) { - throw new RuntimeException - ("Attempting to BROADCAST the wrong file: " + - srcdir + " " + srcfile + " -> " + policy); + if (!(policy instanceof org.globus.swift.data.policy.Broadcast)) { + throw new RuntimeException("Attempting to BROADCAST the wrong file: " + + srcdir + " " + srcfile + " -> " + policy); + } + + if (srcdir == "") { + srcdir = "."; + } + + Director.addBroadcast(srcdir, srcfile); } + } + + public static class External extends InternalFunction { + private ArgRef srcfile; + private ArgRef srcdir; + private ArgRef desthost; + private ArgRef destdir; - if (srcdir == "") { - srcdir = "."; + @Override + protected Signature getSignature() { + return new Signature(params("srcfile", "srcdir", "desthost", "destdir")); } - Director.addBroadcast(srcdir, srcfile); - } - - public void cdm_external(VariableStack stack) - throws ExecutionException - { - String srcfile = (String) PA_SRCFILE.getValue(stack); - String srcdir = (String) PA_SRCDIR.getValue(stack); - BoundContact bc = (BoundContact) PA_DESTHOST.getValue(stack); - String destdir = (String) PA_DESTDIR.getValue(stack); + @Override + protected void runBody(LWThread thr) { + Stack stack = thr.getStack(); + String srcfile = this.srcfile.getValue(stack); + String srcdir = this.srcdir.getValue(stack); + BoundContact bc = this.desthost.getValue(stack); + String destdir = this.destdir.getValue(stack); - if (srcdir.length() == 0) - srcdir = "."; - String desthost = bc.getHost(); - String workdir = (String) bc.getProperty("workdir"); - - External.doExternal(srcfile, srcdir, - desthost, workdir+"/"+destdir); + if (srcdir.length() == 0) { + srcdir = "."; + } + String desthost = bc.getHost(); + String workdir = (String) bc.getProperty("workdir"); + + org.globus.swift.data.policy.External.doExternal(srcfile, srcdir, + desthost, workdir + "/" + destdir); + } } /** Wait until CDM has ensured that all data has been propagated. */ - public void cdm_wait(VariableStack stack) - throws ExecutionException { - logger.debug("cdm_wait()"); - Director.doBroadcast(); + public static class Wait extends InternalFunction { + + @Override + protected Signature getSignature() { + return new Signature(params()); + } + + @Override + protected void runBody(LWThread thr) { + // TODO busy waiting is not good + logger.debug("cdm_wait()"); + Director.doBroadcast(); + } } } Modified: branches/faster/src/org/globus/swift/data/Query.java =================================================================== --- branches/faster/src/org/globus/swift/data/Query.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/globus/swift/data/Query.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,57 +17,75 @@ package org.globus.swift.data; +import k.rt.Stack; + import org.apache.log4j.Logger; - -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; - +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Param; +import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; +import org.globus.cog.karajan.compiled.nodes.functions.NullaryOp; import org.globus.swift.data.policy.Policy; /** Karajan-accessible read-queries to CDM functionality. */ -public class Query extends FunctionsCollection { +public class Query { private static final Logger logger = Logger.getLogger(Query.class); - public static final Arg PA_QUERY = new Arg.Positional("query"); - public static final Arg PA_NAME = new Arg.Positional("name"); - - static { - setArguments("cdm_query", new Arg[]{ PA_QUERY }); - setArguments("cdm_get", new Arg[]{ PA_NAME }); - setArguments("cdm_file", new Arg[]{}); - } - /** Do CDM policy lookup based on the CDM file. */ - public String cdm_query(VariableStack stack) throws ExecutionException { - String file = (String) PA_QUERY.getValue(stack); - Policy policy = Director.lookup(file); - logger.debug("Director.lookup(): " + file + " -> " + policy); - return policy.toString(); + public static class Q extends AbstractSingleValuedFunction { + private ArgRef query; + + @Override + protected Param[] getParams() { + return params("query"); + } + + @Override + public Object function(Stack stack) { + String file = query.getValue(stack); + Policy policy = Director.lookup(file); + if (logger.isDebugEnabled()) { + logger.debug("Director.lookup(): " + file + " -> " + policy); + } + return policy.toString(); + } } /** Get a CDM property */ - public String cdm_get(VariableStack stack) throws ExecutionException { - String name = (String) PA_NAME.getValue(stack); - String value = Director.property(name); - return value; + public static class Get extends AbstractSingleValuedFunction { + private ArgRef name; + + @Override + protected Param[] getParams() { + return params("name"); + } + + @Override + public Object function(Stack stack) { + String name = this.name.getValue(stack); + String value = Director.property(name); + return value; + } } + /** Obtain the CDM policy file given on the command-line, conventionally "fs.data". If not set, returns an empty String. */ - public String cdm_file(VariableStack stack) throws ExecutionException { - String file = ""; - if (Director.policyFile != null) - file = Director.policyFile.toString(); - return file; + public static class File extends NullaryOp { + @Override + protected String value() { + String file = ""; + if (Director.policyFile != null) { + file = Director.policyFile.toString(); + } + return file; + } } } Modified: branches/faster/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -27,22 +27,19 @@ import java.util.Map; import java.util.Set; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.events.EventBus; -import org.globus.cog.karajan.workflow.events.EventTargetPair; -import org.globus.cog.karajan.workflow.futures.FutureEvaluationException; -import org.globus.cog.karajan.workflow.futures.FutureIterator; -import org.globus.cog.karajan.workflow.futures.FutureList; -import org.globus.cog.karajan.workflow.futures.FutureListener; -import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; -import org.globus.cog.karajan.workflow.futures.ListenerStackPair; +import k.rt.FutureListener; + +import org.globus.cog.karajan.futures.FutureEvaluationException; +import org.globus.cog.karajan.futures.FutureIterator; +import org.globus.cog.karajan.futures.FutureList; +import org.globus.cog.karajan.futures.FutureNotYetAvailable; import org.griphyn.vdl.mapping.ArrayDataNode; import org.griphyn.vdl.mapping.DSHandle; public class ArrayIndexFutureList implements FutureList, FutureWrapper { private ArrayList keys; private Map values; - private List listeners; + private LinkedList listeners; private ArrayDataNode node; private boolean purged; @@ -86,10 +83,6 @@ return new FuturePairIterator(this); } - public FutureIterator futureIterator(VariableStack stack) { - return new FuturePairIterator(this, stack); - } - public void close() { throw new UnsupportedOperationException("Not used here"); } @@ -122,52 +115,38 @@ return node; } - public void addModificationAction(FutureListener target, VariableStack stack) { - synchronized(node) { + @Override + public void addListener(FutureListener l) { + boolean closed; + synchronized(this) { if (listeners == null) { - listeners = new LinkedList(); + listeners = new LinkedList(); } - listeners.add(new ListenerStackPair(target, stack)); - WaitingThreadsMonitor.addThread(stack, node); - if (!node.isClosed()) { - return; - } + listeners.add(l); + closed = isClosed(); } - // closed == true; - notifyListeners(); + if (closed) { + notifyListeners(); + } } - + public void notifyListeners() { - List l; - synchronized(node) { + List ls; + synchronized(this) { if (listeners == null) { return; } - - l = listeners; + ls = listeners; listeners = null; } - - for (final ListenerStackPair lsp : l) { - WaitingThreadsMonitor.removeThread(lsp.stack); - EventBus.post(new Runnable() { - @Override - public void run() { - lsp.listener.futureModified(ArrayIndexFutureList.this, lsp.stack); - } - }); + for (FutureListener l : ls) { + l.futureUpdated(this); } } - - public EventTargetPair[] getListenerEvents() { - synchronized(node) { - if (listeners != null) { - return listeners.toArray(new EventTargetPair[0]); - } - else { - return null; - } - } + + @Override + public synchronized List getListeners() { + return new LinkedList(listeners); } public int size() { Modified: branches/faster/src/org/griphyn/vdl/karajan/AssertFailedException.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/AssertFailedException.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/AssertFailedException.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -18,7 +18,7 @@ package org.griphyn.vdl.karajan; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.ExecutionException; /** * Generated only by SwiftScript @assert(). Modified: branches/faster/src/org/griphyn/vdl/karajan/DSHandleFutureWrapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/DSHandleFutureWrapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/DSHandleFutureWrapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -23,71 +23,20 @@ import java.util.LinkedList; import java.util.List; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.events.EventBus; -import org.globus.cog.karajan.workflow.events.EventTargetPair; -import org.globus.cog.karajan.workflow.futures.FutureEvaluationException; -import org.globus.cog.karajan.workflow.futures.FutureListener; -import org.globus.cog.karajan.workflow.futures.ListenerStackPair; +import k.rt.FutureListener; + +import org.globus.cog.karajan.futures.FutureEvaluationException; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; public class DSHandleFutureWrapper implements FutureWrapper { - private LinkedList listeners; - private final AbstractDataNode node; + private LinkedList listeners; + private AbstractDataNode node; public DSHandleFutureWrapper(AbstractDataNode node) { this.node = node; } - public void addModificationAction(FutureListener target, VariableStack stack) { - /** - * TODO So, the strategy is the following: getValue() or something else - * throws a future exception; then some entity catches that and calls - * this method. There is no way to ensure that the future was not closed - * in the mean time. What has to be done is that this method should - * check if the future was closed or modified at the time of the call of - * this method and call notifyListeners(). - */ - synchronized(node) { - if (listeners == null) { - listeners = new LinkedList(); - } - listeners.add(new ListenerStackPair(target, stack)); - WaitingThreadsMonitor.addThread(stack, node); - if (!node.isClosed()) { - return; - } - } - // closed == true; - notifyListeners(); - } - - public DSHandle getHandle() { - return node; - } - - public void notifyListeners() { - List l; - synchronized(node) { - if (listeners == null) { - return; - } - - l = listeners; - listeners = null; - } - - for (final ListenerStackPair lsp : l) { - WaitingThreadsMonitor.removeThread(lsp.stack); - EventBus.post(new Runnable() { - public void run() { - lsp.listener.futureModified(DSHandleFutureWrapper.this, lsp.stack); - } - }); - } - } - public void close() { node.closeShallow(); } @@ -108,6 +57,37 @@ node.setValue(e); } + @Override + public void addListener(FutureListener l) { + boolean closed; + synchronized(this) { + if (listeners == null) { + listeners = new LinkedList(); + } + WaitingThreadsMonitor.addThread(l, node); + listeners.add(l); + closed = isClosed(); + } + if (closed) { + notifyListeners(); + } + } + + public void notifyListeners() { + List ls; + synchronized(this) { + if (listeners == null) { + return; + } + ls = listeners; + listeners = null; + } + for (FutureListener l : ls) { + WaitingThreadsMonitor.removeThread(l); + l.futureUpdated(this); + } + } + public int listenerCount() { synchronized(node) { if (listeners == null) { @@ -119,20 +99,17 @@ } } - private static final EventTargetPair[] EVENT_ARRAY = new EventTargetPair[0]; + @Override + public synchronized List getListeners() { + return new LinkedList(listeners); + } - public EventTargetPair[] getListenerEvents() { - synchronized(node) { - if (listeners != null) { - return listeners.toArray(EVENT_ARRAY); - } - else { - return null; - } - } - } - - public String toString() { + public String toString() { return "F/" + node; } + + @Override + public DSHandle getHandle() { + return node; + } } Modified: branches/faster/src/org/griphyn/vdl/karajan/FuturePairIterator.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/FuturePairIterator.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/FuturePairIterator.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,13 +20,14 @@ */ package org.griphyn.vdl.karajan; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.futures.FutureEvaluationException; -import org.globus.cog.karajan.workflow.futures.FutureIterator; -import org.globus.cog.karajan.workflow.futures.FutureIteratorIncomplete; -import org.globus.cog.karajan.workflow.futures.FutureListener; -import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; +import k.rt.FutureListener; +import k.thr.LWThread; +import org.globus.cog.karajan.futures.FutureEvaluationException; +import org.globus.cog.karajan.futures.FutureIterator; +import org.globus.cog.karajan.futures.FutureIteratorIncomplete; +import org.globus.cog.karajan.futures.FutureNotYetAvailable; + public class FuturePairIterator implements FutureIterator { private ArrayIndexFutureList array; private int crt; @@ -35,10 +36,6 @@ this.array = array; } - public FuturePairIterator(ArrayIndexFutureList array, VariableStack stack) { - this.array = array; - } - public synchronized boolean hasAvailable() { return crt < array.available(); } @@ -108,11 +105,15 @@ public Object getValue() { return this; } + + - public synchronized void addModificationAction(FutureListener target, VariableStack stack) { - WaitingThreadsMonitor.addThread(stack, array.getHandle()); - array.addModificationAction(target, stack); - } + @Override + public void addListener(FutureListener l) { + WaitingThreadsMonitor.addThread(LWThread.currentThread(), array.getHandle()); + array.addListener(l); + + } private static volatile int cnt = 0; Modified: branches/faster/src/org/griphyn/vdl/karajan/FutureTracker.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/FutureTracker.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/FutureTracker.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,16 +20,12 @@ import java.util.HashMap; import java.util.Map; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.futures.Future; +import k.rt.Future; + import org.griphyn.vdl.mapping.DSHandle; public class FutureTracker { public static final String VAR_NAME = "#swift:futureTracker"; - - public static FutureTracker get(VariableStack stack) { - return (FutureTracker) stack.firstFrame().getVar(VAR_NAME); - } private static final FutureTracker ft = new FutureTracker(); Modified: branches/faster/src/org/griphyn/vdl/karajan/FutureWrapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/FutureWrapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/FutureWrapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,16 +17,19 @@ package org.griphyn.vdl.karajan; -import org.globus.cog.karajan.workflow.events.EventTargetPair; -import org.globus.cog.karajan.workflow.futures.Future; +import java.util.List; + +import k.rt.Future; +import k.rt.FutureListener; + import org.griphyn.vdl.mapping.DSHandle; public interface FutureWrapper extends Future { - void notifyListeners(); - int listenerCount(); - - EventTargetPair[] getListenerEvents(); + List getListeners(); + + void notifyListeners(); + DSHandle getHandle(); } Modified: branches/faster/src/org/griphyn/vdl/karajan/HangChecker.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/HangChecker.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/HangChecker.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -31,27 +31,29 @@ import java.util.Timer; import java.util.TimerTask; +import k.rt.Context; +import k.rt.ExecutionException; +import k.thr.LWThread; +import k.thr.Scheduler; + import org.apache.log4j.Logger; +import org.globus.cog.karajan.analyzer.VariableNotFoundException; +import org.globus.cog.karajan.compiled.nodes.grid.SchedulerNode; import org.globus.cog.karajan.scheduler.WeightedHostScoreScheduler; -import org.globus.cog.karajan.stack.VariableNotFoundException; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.ThreadingContext; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.events.EventBus; -import org.globus.cog.karajan.workflow.nodes.grid.SchedulerNode; import org.griphyn.vdl.mapping.DSHandle; +import com.sun.org.apache.xpath.internal.VariableStack; + public class HangChecker extends TimerTask { public static final Logger logger = Logger.getLogger(HangChecker.class); - public static final int CHECK_INTERVAL = 10000; + public static final int CHECK_INTERVAL = 1000; public static final int MAX_CYCLES = 10; private Timer timer; - private long lastEventCount; - private VariableStack stack; + private Context context; - public HangChecker(VariableStack stack) throws ExecutionException { - this.stack = stack; + public HangChecker(Context context) throws ExecutionException { + this.context = context; } public void start() { @@ -61,11 +63,11 @@ public void run() { try { - WeightedHostScoreScheduler s = (WeightedHostScoreScheduler) stack.getGlobal(SchedulerNode.SCHEDULER); + WeightedHostScoreScheduler s = (WeightedHostScoreScheduler) context.getAttribute(SchedulerNode.CONTEXT_ATTR_NAME); if (s != null) { int running = s.getRunning(); boolean allOverloaded = s.allOverloaded(); - if (running == 0 && EventBus.eventCount == lastEventCount && !allOverloaded) { + if (running == 0 && !Scheduler.getScheduler().isAnythingRunning() && !allOverloaded) { logger.warn("No events in " + (CHECK_INTERVAL / 1000) + "s."); ByteArrayOutputStream os = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(os); @@ -84,7 +86,6 @@ ps.close(); } } - lastEventCount = EventBus.eventCount; } catch (Exception e) { logger.warn("Exception caught during hang check", e); @@ -92,10 +93,10 @@ } private void findThreadsToBlame(PrintStream ps, Graph g) { - Map wt = WaitingThreadsMonitor.getAllThreads(); - Set sl = g.nodeSet(); - Set loners = new HashSet(wt.keySet()); - for (VariableStack s : sl) { + Map wt = WaitingThreadsMonitor.getAllThreads(); + Set sl = g.nodeSet(); + Set loners = new HashSet(wt.keySet()); + for (LWThread s : sl) { for (Graph.Edge e : g.getEdgesFrom(s)) { loners.remove(e.to); } @@ -103,7 +104,7 @@ if (!loners.isEmpty()) { ps.println(); ps.println("The following threads are independently hung:"); - for (VariableStack s : loners) { + for (LWThread s : loners) { Monitor.dumpThread(ps, s, wt.get(s)); ps.println(); } @@ -112,14 +113,14 @@ } private Graph buildGraph() throws VariableNotFoundException { - Map> ot = WaitingThreadsMonitor.getOutputs(); - Map wt = WaitingThreadsMonitor.getAllThreads(); - Map> rwt = new HashMap>(); + Map> ot = WaitingThreadsMonitor.getOutputs(); + Map wt = WaitingThreadsMonitor.getAllThreads(); + Map> rwt = new HashMap>(); - for (Map.Entry e : wt.entrySet()) { - List l = rwt.get(e.getValue()); + for (Map.Entry e : wt.entrySet()) { + List l = rwt.get(e.getValue()); if (l == null) { - l = new LinkedList(); + l = new LinkedList(); rwt.put(e.getValue(), l); } l.add(e.getKey()); @@ -128,21 +129,20 @@ Graph g = new Graph(); // if n1 -> n2, then n1 produces an output that is used by n2 - for (Map.Entry> e : ot.entrySet()) { + for (Map.Entry> e : ot.entrySet()) { for (DSHandle h : e.getValue()) { - List sl = rwt.get(h); + List sl = rwt.get(h); if (sl != null) { - for (VariableStack s : sl) { + for (LWThread s : sl) { g.addEdge(e.getKey(), s, h); } } } - ThreadingContext tc; - tc = ThreadingContext.get(e.getKey()); - for (VariableStack stk : ot.keySet()) { - if (tc.isStrictlySubContext(ThreadingContext.get(stk))) { + LWThread tc = e.getKey(); + for (LWThread stk : ot.keySet()) { + if (isStrictlyChildOf(tc, stk)) { g.addEdge(e.getKey(), stk, null); } } @@ -151,14 +151,28 @@ return g; } + private boolean isStrictlyChildOf(LWThread child, LWThread parent) { + if (child == parent) { + return false; + } + child = child.getParent(); + while (child != null) { + if (child == parent) { + return true; + } + child = child.getParent(); + } + return false; + } + private static boolean findCycles(PrintStream ps, Graph g) { System.out.print("Finding dependency loops..."); System.out.flush(); - Set seen = new HashSet(); + Set seen = new HashSet(); LinkedList cycle = new LinkedList(); List> cycles = new ArrayList>(); - for (VariableStack t : g.nodeSet()) { + for (LWThread t : g.nodeSet()) { seen.clear(); cycle.clear(); findLoop(t, g, seen, cycle, cycles); @@ -195,7 +209,7 @@ else { ps.println("\tthe above must complete before the block below can complete:"); } - for (String t : Monitor.getSwiftTrace((VariableStack) o)) { + for (String t : Monitor.getSwiftTrace((LWThread) o)) { ps.println("\t\t" + t); } } @@ -258,8 +272,8 @@ if (a == null || b == null) { return a == b; } - VariableStack sa = (VariableStack) a; - VariableStack sb = (VariableStack) b; + LWThread sa = (LWThread) a; + LWThread sb = (LWThread) b; List ta = Monitor.getSwiftTraceElements(sa); List tb = Monitor.getSwiftTraceElements(sb); @@ -276,7 +290,7 @@ return true; } - private static void findLoop(VariableStack t, Graph g, Set seen, LinkedList cycle, List> cycles) { + private static void findLoop(LWThread t, Graph g, Set seen, LinkedList cycle, List> cycles) { if (cycles.size() > MAX_CYCLES) { return; } @@ -310,18 +324,18 @@ public static class Graph { public static class Edge { - public final VariableStack to; + public final LWThread to; public final DSHandle contents; - public Edge(VariableStack to, DSHandle contents) { + public Edge(LWThread to, DSHandle contents) { this.to = to; this.contents = contents; } } - private Map> outEdges = new HashMap>(); + private Map> outEdges = new HashMap>(); - public void addEdge(VariableStack from, VariableStack to, DSHandle contents) { + public void addEdge(LWThread from, LWThread to, DSHandle contents) { List l = outEdges.get(from); if (l == null) { l = new ArrayList(); @@ -331,7 +345,7 @@ } public void dump(PrintStream ps) { - for (Map.Entry> e : outEdges.entrySet()) { + for (Map.Entry> e : outEdges.entrySet()) { for (Edge edge : e.getValue()) { String tcf = getThreadingContext(e.getKey()); String tct = getThreadingContext(edge.to); @@ -340,16 +354,16 @@ } } - private String getThreadingContext(VariableStack s) { + private String getThreadingContext(LWThread s) { try { - return String.valueOf(ThreadingContext.get(s)); + return String.valueOf(s); } catch (VariableNotFoundException e) { return "?"; } } - public List getEdgesFrom(VariableStack t) { + public List getEdgesFrom(LWThread t) { List l = outEdges.get(t); if (l == null) { return Collections.emptyList(); @@ -359,7 +373,7 @@ } } - public Set nodeSet() { + public Set nodeSet() { return outEdges.keySet(); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/Loader.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/Loader.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -39,25 +39,25 @@ import java.util.List; import java.util.Map; +import k.rt.Context; + import org.apache.log4j.Appender; +import org.apache.log4j.AsyncAppender; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.FileAppender; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.globus.cog.abstraction.impl.execution.fake.JobSubmissionTaskHandler; +import org.globus.cog.karajan.compiled.nodes.Main; +import org.globus.cog.karajan.compiled.nodes.grid.AbstractGridNode; +import org.globus.cog.karajan.parser.WrapperNode; import org.globus.cog.karajan.scheduler.WeightedHostScoreScheduler; -import org.globus.cog.karajan.stack.LinkedStack; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.Monitor; -import org.globus.cog.karajan.workflow.ElementTree; -import org.globus.cog.karajan.workflow.PrintStreamChannel; -import org.globus.cog.karajan.workflow.nodes.FlowElement; -import org.globus.cog.karajan.workflow.nodes.grid.AbstractGridNode; +import org.globus.cog.karajan.util.KarajanProperties; import org.globus.cog.util.ArgumentParser; import org.globus.cog.util.ArgumentParserException; import org.globus.cog.util.TextFileLoader; import org.globus.swift.data.Director; import org.griphyn.vdl.engine.Karajan; -import org.griphyn.vdl.karajan.functions.ConfigProperty; import org.griphyn.vdl.karajan.lib.Execute; import org.griphyn.vdl.karajan.lib.Log; import org.griphyn.vdl.karajan.lib.New; @@ -75,7 +75,6 @@ public static final String ARG_HELP = "help"; public static final String ARG_VERSION = "version"; - public static final String ARG_MONITOR = "monitor"; public static final String ARG_RESUME = "resume"; public static final String ARG_INSTANCE_CONFIG = "config"; public static final String ARG_TYPECHECK = "typecheck"; @@ -114,9 +113,6 @@ version(); System.exit(0); } - if (ap.isPresent(ARG_MONITOR)) { - new Monitor().start(); - } if (!ap.hasValue(ArgumentParser.DEFAULT)) { error("No SwiftScript program specified"); } @@ -177,7 +173,7 @@ System.exit(3); } } - ElementTree tree = null; + WrapperNode tree = null; if (project != null) { tree = load(project); } @@ -186,50 +182,48 @@ shortUsage(); System.exit(1); } + + tree.setProperty("name", projectName + "-" + runID); + tree.setProperty(WrapperNode.FILENAME, project); - tree.setName(projectName + "-" + runID); - tree.getRoot().setProperty(FlowElement.FILENAME, project); + VDL2Config config = loadConfig(ap); + Context context = new Context(); + context.setArguments(ap.getArguments()); + context.setAttribute("SWIFT:CONFIG", config); + context.setAttribute("projectName", projectName); + context.setAttribute("SWIFT:SCRIPT_NAME", projectName); + context.setAttribute("SWIFT:RUN_ID", runID); + context.setAttribute("SWIFT:DRY_RUN", ap.isPresent(ARG_DRYRUN)); + context.setAttribute("SWIFT:HOME", System.getProperty("swift.home")); + + Main root = compileKarajan(tree, context); + root.setFileName(projectName); - VDL2ExecutionContext ec = new VDL2ExecutionContext(tree, - projectName); - ec.setRunID(runID); - // no order - ec.setStdout(new PrintStreamChannel(System.out, true)); - - VariableStack stack = new LinkedStack(ec); - VDL2Config config = loadConfig(ap, stack); + SwiftExecutor ec = new SwiftExecutor(root); + + List arguments = ap.getArguments(); + if (ap.hasValue(ARG_RESUME)) { + arguments.add("-rlog:resume=" + ap.getStringValue(ARG_RESUME)); + } + addCommandLineProperties(config, ap); if (logger.isDebugEnabled()) { logger.debug(config); } debugSitesText(config); - - if (ap.isPresent(ARG_DRYRUN)) { - stack.setGlobal(CONST_VDL_OPERATION, VDL_OPERATION_DRYRUN); - } - else if (ap.isPresent(ARG_TYPECHECK)) { - stack.setGlobal(CONST_VDL_OPERATION, VDL_OPERATION_TYPECHECK); - } - else { - stack.setGlobal(CONST_VDL_OPERATION, VDL_OPERATION_RUN); - } - stack.setGlobal("swift.home", System.getProperty("swift.home")); - stack.setGlobal("PATH_SEPARATOR", File.separator); - List arguments = ap.getArguments(); - if (ap.hasValue(ARG_RESUME)) { - arguments.add("-rlog:resume=" + ap.getStringValue(ARG_RESUME)); - } - ec.setArguments(arguments); - // long start = System.currentTimeMillis(); - new HangChecker(stack).start(); - ec.start(stack); + new HangChecker(context).start(); + long start = System.currentTimeMillis(); + ec.start(context); ec.waitFor(); + long end = System.currentTimeMillis(); + System.out.println(JobSubmissionTaskHandler.jobsRun + " jobs, " + JobSubmissionTaskHandler.jobsRun * 1000 / (end - start) + " j/s"); if (ec.isFailed()) { runerror = true; } } catch (Exception e) { + e.printStackTrace(); logger.debug("Detailed exception:", e); error("Could not start execution" + getMessages(e)); } @@ -248,17 +242,9 @@ private static String getMessages(Throwable e) { StringBuilder sb = new StringBuilder(); - String lastMessage = null; while (e != null) { - String msg = e.getMessage(); - if (msg == null) { - msg = e.toString(); - } - sb.append("\n\t"); - if (lastMessage == null || !lastMessage.contains(msg)) { - sb.append(msg); - lastMessage = msg; - } + sb.append(":\n\t"); + sb.append(e.toString()); e = e.getCause(); } return sb.toString(); @@ -274,7 +260,7 @@ cdmString = ap.getStringValue(ARG_CDMFILE); File cdmFile = new File(cdmString); debugText("CDM FILE", cdmFile); - Director.load(cdmFile); + Director.load(cdmFile); } catch (IOException e) { logger.debug("Detailed exception:", e); @@ -283,6 +269,12 @@ } } + private static Main compileKarajan(WrapperNode n, Context context) + throws org.globus.cog.karajan.analyzer.CompilationException { + return (Main) n.compile(null, new SwiftRootScope(KarajanProperties.getDefault(), + (String) n.getProperty(WrapperNode.FILENAME), context)); + } + public static String compile(String project) throws FileNotFoundException, ParsingException, IncorrectInvocationException, CompilationException, IOException { @@ -295,7 +287,7 @@ File swiftscript = new File(project); debugText("SWIFTSCRIPT", swiftscript); String projectBase = project.substring(0, project.lastIndexOf('.')); - File xml = new File(projectBase + ".swiftx"); + File xml = new File(projectBase + ".xml"); File kml = new File(projectBase + ".kml"); loadBuildVersion(provenanceEnabled); @@ -344,6 +336,7 @@ if (recompile) { VDLt2VDLx.compile(new FileInputStream(swiftscript), new PrintStream(new FileOutputStream(xml))); + try { FileOutputStream f = new FileOutputStream(kml); Karajan.compile(xml.getAbsolutePath(), new PrintStream(f), provenanceEnabled); @@ -364,12 +357,9 @@ // if we leave a kml file around, then a subsequent // re-run will skip recompiling and cause a different // error message for the user - if (e instanceof NullPointerException) { - e.printStackTrace(); - } kml.delete(); throw new CompilationException( - "Failed to convert .swiftx to .kml for " + project, e); + "Failed to convert .xml to .kml for " + project, e); } } else { @@ -425,18 +415,15 @@ } } - private static VDL2Config loadConfig(ArgumentParser ap, VariableStack stack) - throws IOException { + private static VDL2Config loadConfig(ArgumentParser ap) throws IOException { VDL2Config conf; if (ap.hasValue(ARG_INSTANCE_CONFIG)) { String configFile = ap.getStringValue(ARG_INSTANCE_CONFIG); - stack.setGlobal(ConfigProperty.INSTANCE_CONFIG_FILE, configFile); conf = VDL2Config.getConfig(configFile); } else { conf = (VDL2Config) VDL2Config.getConfig().clone(); } - stack.setGlobal(ConfigProperty.INSTANCE_CONFIG, conf); return conf; } @@ -554,6 +541,10 @@ File f = new File(logfile); FileAppender fa = (FileAppender) getAppender(FileAppender.class); + AsyncAppender aa = new AsyncAppender(); + aa.addAppender(fa); + + replaceAppender(fa, aa); if (fa == null) { logger.warn("Failed to configure log file name"); } @@ -583,12 +574,13 @@ Logger.getLogger(Log.class).setLevel(Level.DEBUG); Logger.getLogger(AbstractGridNode.class).setLevel(Level.DEBUG); Logger.getLogger(Execute.class).setLevel(Level.DEBUG); - Logger.getLogger(VDL2ExecutionContext.class).setLevel(Level.INFO); + Logger.getLogger(SwiftExecutor.class).setLevel(Level.INFO); Logger.getLogger(WeightedHostScoreScheduler.class).setLevel( Level.INFO); ca.setThreshold(Level.FATAL); } else if (ap.isPresent(ARG_MINIMAL_LOGGING)) { + Logger.getLogger("swift").setLevel(Level.WARN); Logger.getRootLogger().setLevel(Level.WARN); } else if (ap.isPresent(ARG_REDUCED_LOGGING)) { @@ -599,6 +591,12 @@ } + private static void replaceAppender(FileAppender fa, AsyncAppender aa) { + Logger root = Logger.getRootLogger(); + root.removeAppender(fa); + root.addAppender(aa); + } + @SuppressWarnings({ "rawtypes", "unchecked" }) protected static Appender getAppender(Class cls) { Logger root = Logger.getRootLogger(); Modified: branches/faster/src/org/griphyn/vdl/karajan/Mergeable.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/Mergeable.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/Mergeable.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,7 +20,7 @@ */ package org.griphyn.vdl.karajan; -import org.globus.cog.karajan.workflow.futures.Future; +import k.rt.Future; public interface Mergeable { void mergeListeners(Future f); Modified: branches/faster/src/org/griphyn/vdl/karajan/Monitor.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/Monitor.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/Monitor.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -50,14 +50,14 @@ import javax.swing.JTable; import javax.swing.table.AbstractTableModel; -import org.globus.cog.karajan.stack.Trace; -import org.globus.cog.karajan.stack.VariableNotFoundException; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.ThreadingContext; -import org.globus.cog.karajan.workflow.events.EventTargetPair; -import org.globus.cog.karajan.workflow.futures.Future; -import org.globus.cog.karajan.workflow.nodes.FlowElement; -import org.globus.cog.karajan.workflow.nodes.FlowNode; +import k.rt.ConditionalYield; +import k.rt.Future; +import k.rt.FutureListener; +import k.rt.FutureValue; +import k.thr.LWThread; + +import org.globus.cog.karajan.analyzer.VariableNotFoundException; +import org.globus.cog.karajan.compiled.nodes.Node; import org.griphyn.vdl.engine.Karajan; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.ArrayDataNode; @@ -66,6 +66,8 @@ import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.Path; +import com.sun.org.apache.xpath.internal.VariableStack; + public class Monitor implements ActionListener, MouseListener { public static final int VARS = 0; public static final int THREADS = 1; @@ -74,7 +76,7 @@ private JTable t; private JButton futures, waiting, tasks; private List wr; - private List wt; + private List wt; private int crtdisp; public Monitor() { @@ -151,7 +153,18 @@ entry.add(handle.getType()); entry.add(h); entry.add(value); - entry.add(f.isClosed() ? "Closed" : "Open"); + if (f instanceof FutureValue) { + try { + ((FutureValue) f).getValue(); + entry.add("Closed"); + } + catch (ConditionalYield y) { + entry.add("Open"); + } + } + else { + entry.add("-"); + } entry.add(sz); String fs; if (f instanceof FutureWrapper) { @@ -188,11 +201,11 @@ } crtdisp = THREADS; ArrayList al = new ArrayList(); - wt = new ArrayList(); - Map c = WaitingThreadsMonitor.getAllThreads(); - for (Map.Entry entry : c.entrySet()) { + wt = new ArrayList(); + Map c = WaitingThreadsMonitor.getAllThreads(); + for (Map.Entry entry : c.entrySet()) { try { - al.add(String.valueOf(ThreadingContext.get(entry.getKey()))); + al.add(entry.getKey().getName()); } catch (VariableNotFoundException e1) { al.add("unknown thread"); @@ -259,20 +272,20 @@ public static void dumpThreads(PrintStream pw) { pw.println("\nWaiting threads:"); - Map c = WaitingThreadsMonitor.getAllThreads(); - for (Map.Entry e : c.entrySet()) { + Map c = WaitingThreadsMonitor.getAllThreads(); + for (Map.Entry e : c.entrySet()) { dumpThread(pw, e.getKey(), e.getValue()); pw.println(); } pw.println("----"); } - public static void dumpThread(PrintStream pw, VariableStack stack, DSHandle handle) { + public static void dumpThread(PrintStream pw, LWThread thr, DSHandle handle) { try { - pw.println("Thread: " + String.valueOf(ThreadingContext.get(stack)) + pw.println("Thread: " + thr.getName() + (handle == null ? "" : ", waiting on " + varWithLine(handle))); - for (String t : getSwiftTrace(stack)) { + for (String t : getSwiftTrace(thr)) { pw.println("\t" + t); } } @@ -289,54 +302,54 @@ (line == null ? "" : " (declared on line " + line + ")"); } - public static String getLastCall(VariableStack stack) { - List trace = Trace.getAsList(stack); - for (Object o : trace) { - if (o instanceof FlowNode) { - FlowNode n = (FlowNode) o; - String traceLine = (String) n.getProperty("_traceline"); - if (traceLine != null) { + public static String getLastCall(LWThread thr) { + List trace = thr.getTrace(); + if (trace != null) { + for (Object o : trace) { + if (o instanceof Node) { + Node n = (Node) o; + int line = n.getLine(); return(Karajan.demangle(n.getTextualName()) + ", " + - fileName(n) + ", line " + traceLine); + fileName(n) + ", line " + line); } } } return "?"; } - public static List getSwiftTrace(VariableStack stack) { + public static List getSwiftTrace(LWThread thr) { List ret = new ArrayList(); - List trace = Trace.getAsList(stack); - for (Object o : trace) { - if (o instanceof FlowNode) { - FlowNode n = (FlowNode) o; - String traceLine = (String) n.getProperty("_traceline"); - if (traceLine != null) { + List trace = thr.getTrace(); + if (trace != null) { + for (Object o : trace) { + if (o instanceof Node) { + Node n = (Node) o; + int line = n.getLine(); ret.add(Karajan.demangle(n.getTextualName()) + ", " + - fileName(n) + ", line " + traceLine); + fileName(n) + ", line " + line); + } } - } + } return ret; } - public static List getSwiftTraceElements(VariableStack stack) { + public static List getSwiftTraceElements(LWThread thr) { List ret = new ArrayList(); - List trace = Trace.getAsList(stack); - for (Object o : trace) { - if (o instanceof FlowNode) { - FlowNode n = (FlowNode) o; - String traceLine = (String) n.getProperty("_traceline"); - if (traceLine != null) { - ret.add(o); + List trace = thr.getTrace(); + if (trace != null) { + for (Object o : trace) { + if (o instanceof Node) { + Node n = (Node) o; + ret.add(n.getLine()); } } } return ret; } - private static String fileName(FlowNode n) { - return new File((String) FlowNode.getTreeProperty(FlowElement.FILENAME, n)).getName().replace(".kml", ".swift"); + private static String fileName(Node n) { + return new File(n.getFileName()).getName().replace(".kml", ".swift"); } public class VariableModel extends AbstractTableModel { @@ -365,14 +378,15 @@ return l.get(rowIndex)[columnIndex]; } else { - EventTargetPair[] l = Monitor.this.getListeners(rowIndex); + List l = Monitor.this.getListeners(rowIndex); if (l != null) { ArrayList a = new ArrayList(); - for (int i = 0; i < l.length; i++) { - try { - a.add(ThreadingContext.get(l[i].getEvent())); + for (int i = 0; i < l.size(); i++) { + FutureListener o = l.get(i); + if (o instanceof LWThread.Listener) { + a.add(((LWThread.Listener) o).getThread().getName()); } - catch (VariableNotFoundException e) { + else { a.add("unknown"); } } @@ -439,12 +453,13 @@ if (e.getClickCount() == 2) { int row = t.rowAtPoint(e.getPoint()); if (crtdisp == VARS) { - EventTargetPair[] l = getListeners(row); - if (l != null) { + List ls = getListeners(row); + if (ls != null) { try { - for (int i = 0; i < l.length; i++) { + for (FutureListener l : ls) { + displayPopup("Stack trace for " + t.getValueAt(row, 1), - Trace.get(l[i].getEvent())); + getTrace(l)); } } catch (NullPointerException ex) { @@ -455,21 +470,30 @@ else if (crtdisp == THREADS) { Object o = wt.get(row); if (o instanceof VariableStack) { - displayPopup("Stack trace for " + t.getValueAt(row, 0), - Trace.get((VariableStack) o)); + displayPopup("Stack trace for " + t.getValueAt(row, 0), " N/A"); } } } } - private void displayPopup(String title, String s) { + private String getTrace(FutureListener l) { + if (l instanceof LWThread.Listener) { + LWThread.Listener lt = (LWThread.Listener) l; + return String.valueOf(lt.getThread().getTrace()); + } + else { + return "unknown"; + } + } + + private void displayPopup(String title, String s) { JOptionPane.showMessageDialog(frame, s, title, JOptionPane.INFORMATION_MESSAGE); } - public EventTargetPair[] getListeners(int wrindex) { + public List getListeners(int wrindex) { Object o = wr.get(wrindex); if (o instanceof FutureWrapper) { - return ((FutureWrapper) o).getListenerEvents(); + return ((FutureWrapper) o).getListeners(); } else { return null; Modified: branches/faster/src/org/griphyn/vdl/karajan/Pair.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/Pair.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/Pair.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -37,5 +37,4 @@ public int size() { return 2; } - } Deleted: branches/faster/src/org/griphyn/vdl/karajan/PairIterator.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/PairIterator.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/PairIterator.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,78 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Jun 9, 2006 - */ -package org.griphyn.vdl.karajan; - -import java.util.Iterator; -import java.util.Map; - -import org.globus.cog.karajan.util.KarajanIterator; - -public class PairIterator implements KarajanIterator { - private Iterator it; - private int crt, count; - private Pair crto; - - public PairIterator(Map map) { - this.it = map.entrySet().iterator(); - this.count = map.size(); - } - - public int current() { - return crt; - } - - public int count() { - return count; - } - - public Object peek() { - if (crto == null) { - crto = convert(it.next()); - } - return crto; - } - - public void remove() { - throw new UnsupportedOperationException("remove"); - } - - public boolean hasNext() { - return it.hasNext() || crto != null; - } - - public Object next() { - crt++; - if (crto != null) { - Object o = crto; - crto = null; - return o; - } - else { - return convert(it.next()); - } - } - - private Pair convert(Object o) { - Map.Entry e = (Map.Entry) o; - return new Pair(e.getKey(), e.getValue()); - } - -} Added: branches/faster/src/org/griphyn/vdl/karajan/PairSet.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/PairSet.java (rev 0) +++ branches/faster/src/org/griphyn/vdl/karajan/PairSet.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,83 @@ +/* + * Copyright 2012 University of Chicago + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/* + * Created on Jun 9, 2006 + */ +package org.griphyn.vdl.karajan; + +import java.util.AbstractCollection; +import java.util.AbstractList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.griphyn.vdl.mapping.DSHandle; + +public class PairSet extends AbstractCollection> { + private Set, DSHandle>> set; + + public PairSet(Map, DSHandle> map) { + this.set = map.entrySet(); + } + + @Override + public Iterator> iterator() { + final Iterator, DSHandle>> it = set.iterator(); + return new Iterator>() { + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public List next() { + final Map.Entry, DSHandle> e = it.next(); + return new AbstractList() { + @Override + public Object get(int index) { + switch (index) { + case 0: + return e.getKey(); + case 1: + return e.getValue(); + default: + throw new IndexOutOfBoundsException(String.valueOf(index)); + } + } + + @Override + public int size() { + return 2; + } + + }; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public int size() { + return set.size(); + } +} Added: branches/faster/src/org/griphyn/vdl/karajan/SwiftExecutor.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/SwiftExecutor.java (rev 0) +++ branches/faster/src/org/griphyn/vdl/karajan/SwiftExecutor.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,72 @@ +/* + * Copyright 2012 University of Chicago + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/* + * Created on Dec 23, 2006 + */ +package org.griphyn.vdl.karajan; + +import k.rt.Context; +import k.rt.ExecutionException; +import k.rt.Executor; +import k.rt.Stack; +import k.thr.Scheduler.RootThread; + +import org.apache.log4j.Logger; +import org.globus.cog.karajan.compiled.nodes.Main; +import org.griphyn.vdl.karajan.functions.ProcessBulkErrors; + +public class SwiftExecutor extends Executor { + public static final Logger logger = Logger.getLogger(SwiftExecutor.class); + + public SwiftExecutor(Main root) { + super(root); + } + + protected void printFailure(ExecutionException e) { + if (logger.isDebugEnabled()) { + logger.debug(e.getMessage(), e); + } + String msg = e.getMessage(); + if (!"Execution completed with errors".equals(msg)) { + if (msg == null) { + msg = getMeaningfulMessage(e); + } + System.err.print("Execution failed:\n\t"); + String translation = VDL2ErrorTranslator.getDefault().translate(msg); + if (translation != null) { + System.err.print(translation); + } + else { + System.err.print(ProcessBulkErrors.getMessageChain(e)); + } + System.err.print("\n"); + } + else { + // lazy errors are on and they have already been printed + } + } + + public void start(Context context) { + if (logger.isDebugEnabled()) { + logger.debug(context); + } + logger.info("swift.home = " + + System.getProperty("swift.home")); + start(new RootThread(getRoot(), new Stack()), context); + } +} Added: branches/faster/src/org/griphyn/vdl/karajan/SwiftRootScope.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/SwiftRootScope.java (rev 0) +++ branches/faster/src/org/griphyn/vdl/karajan/SwiftRootScope.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,33 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jan 6, 2013 + */ +package org.griphyn.vdl.karajan; + +import java.io.File; + +import k.rt.Context; + +import org.globus.cog.karajan.analyzer.RootScope; +import org.globus.cog.karajan.util.KarajanProperties; +import org.griphyn.vdl.karajan.lib.swiftscript.FnArg; +import org.griphyn.vdl.mapping.DuplicateMappingChecker; + +public class SwiftRootScope extends RootScope { + + public SwiftRootScope(KarajanProperties props, String file, Context context) { + super(props, file, context); + context.setAttribute("SWIFT:DM_CHECKER", new DuplicateMappingChecker()); + + addVar("PATH_SEPARATOR", File.separator); + addVar("SWIFT:DRY_RUN", context.getAttribute("SWIFT:DRY_RUN")); + addVar("SWIFT:RUN_ID", context.getAttribute("SWIFT:RUN_ID")); + addVar("SWIFT:SCRIPT_NAME", context.getAttribute("SWIFT:SCRIPT_NAME")); + addVar("SWIFT:PARSED_ARGS", FnArg.parseArgs(context.getArguments())); + } +} Deleted: branches/faster/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,122 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Dec 23, 2006 - */ -package org.griphyn.vdl.karajan; - -import java.util.List; - -import org.apache.log4j.Logger; -import org.globus.cog.karajan.stack.Trace; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ElementTree; -import org.globus.cog.karajan.workflow.ExecutionContext; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.functions.ProcessBulkErrors; -import org.griphyn.vdl.mapping.DuplicateMappingChecker; - -public class VDL2ExecutionContext extends ExecutionContext { - public static final Logger logger = Logger.getLogger(VDL2ExecutionContext.class); - - public static final String RUN_ID = "vdl:runid"; - public static final String SCRIPT_NAME = "vdl:scriptname"; - public static final String DM_CHECKER = "vdl:dpmchecker"; - - private String runID; - private final String scriptName; - - public VDL2ExecutionContext(ElementTree tree, String scriptName) { - super(tree); - this.scriptName = scriptName; - } - - protected void printFailure(ExecutionException e) { - if (logger.isDebugEnabled()) { - logger.debug("Karajan level error: " + getKarajanTrace(e)); - } - String msg = e.getMessage(); - if (!"Execution completed with errors".equals(msg)) { - if (msg == null) { - msg = getMeaningfulMessage(e); - } - getStderr().append("Execution failed:\n\t"); - String translation = VDL2ErrorTranslator.getDefault().translate(msg); - if (translation != null) { - getStderr().append(translation); - } - else { - getStderr().append(ProcessBulkErrors.getMessageChain(e)); - } - if (e.getStack() != null) { - List l = Monitor.getSwiftTrace(e.getStack()); - for (String s : l) { - getStderr().append("\n\t"); - getStderr().append(s); - } - } - getStderr().append("\n"); - } - else { - // lazy errors are on and they have already been printed - } - } - - private String getKarajanTrace(ExecutionException e) { - StringBuilder sb = new StringBuilder(); - while (e != null) { - sb.append(e.getMessage()); - if (e.getStack() != null) { - sb.append(" at\n"); - sb.append(Trace.get(e.getStack())); - } - if (e.getCause() instanceof ExecutionException) { - e = (ExecutionException) e.getCause(); - sb.append("\ncaused by: "); - } - else { - e = null; - } - } - return sb.toString(); - } - - protected void setGlobals(VariableStack stack) { - super.setGlobals(stack); - stack.setGlobal(RUN_ID, runID); - stack.setGlobal(SCRIPT_NAME, scriptName); - stack.setGlobal(DM_CHECKER, new DuplicateMappingChecker()); - } - - public String getRunID() { - return runID; - } - - public void setRunID(String runID) { - this.runID = runID; - } - - public void start(VariableStack stack) { - if (logger.isDebugEnabled()) { - logger.debug(stack); - } - logger.info("swift.home = " + - System.getProperty("swift.home")); - super.start(stack); - } -} Deleted: branches/faster/src/org/griphyn/vdl/karajan/VDL2FutureException.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/VDL2FutureException.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/VDL2FutureException.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,61 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Jan 4, 2007 - */ -package org.griphyn.vdl.karajan; - -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.nodes.FlowElement; -import org.griphyn.vdl.mapping.DSHandle; - -public class VDL2FutureException extends RuntimeException { - private final DSHandle handle; - private FlowElement listener; - private VariableStack stack; - - public VDL2FutureException(DSHandle handle, FlowElement listener, VariableStack stack) { - this.handle = handle; - this.listener = listener; - this.stack = stack; - } - - public VDL2FutureException(DSHandle handle) { - this(handle, null, null); - } - - public DSHandle getHandle() { - return handle; - } - - public VariableStack getStack() { - return stack; - } - - public FlowElement getListener() { - return listener; - } - - public void setStack(VariableStack stack) { - this.stack = stack; - } - - public void setListener(FlowElement listener) { - this.listener = listener; - } -} Modified: branches/faster/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -38,9 +38,9 @@ import org.globus.cog.abstraction.interfaces.JobSpecification; import org.globus.cog.abstraction.interfaces.Service; import org.globus.cog.abstraction.interfaces.Status; +import org.globus.cog.abstraction.interfaces.StatusListener; import org.globus.cog.abstraction.interfaces.Task; import org.globus.cog.karajan.scheduler.AbstractScheduler; -import org.globus.cog.karajan.scheduler.NoSuchResourceException; import org.globus.cog.karajan.scheduler.ResourceConstraintChecker; import org.globus.cog.karajan.scheduler.TaskConstraints; import org.globus.cog.karajan.scheduler.WeightedHostScoreScheduler; @@ -61,10 +61,10 @@ private static Timer timer; private TCCache tc; - private LinkedList dq; + private LinkedList dq; private int clusteringQueueDelay = 1; private int minClusterTime = 60; - private Map> tasks; + private Map> tasks; private boolean clusteringEnabled; private int clusterId; @@ -75,8 +75,8 @@ private Map serviceContactMapping; public VDSAdaptiveScheduler() { - dq = new LinkedList(); - tasks = new HashMap>(); + dq = new LinkedList(); + tasks = new HashMap>(); serviceContactMapping = new HashMap(); } @@ -115,18 +115,19 @@ } } - public void enqueue(Task task, Object constraints) { + @Override + public void enqueue(Task task, Object constraints, StatusListener l) { if (shouldBeClustered(task, constraints)) { startTimer(); if (logger.isDebugEnabled()) { logger.debug("Adding task to clustering queue: " + task.getIdentity()); } synchronized (dq) { - dq.addLast(new Object[] { task, constraints }); + dq.addLast(new Entry(task, constraints, l)); } } else { - super.enqueue(task, constraints); + super.enqueue(task, constraints, l); } } @@ -192,23 +193,23 @@ synchronized (dq) { while (!dq.isEmpty()) { int clusterTime = 0; - LinkedList cluster = new LinkedList(); + LinkedList cluster = new LinkedList(); Map env = new HashMap(); Map attrs = new HashMap(); Object constraints = null; String dir = null; - Iterator dqi = dq.iterator(); + Iterator dqi = dq.iterator(); while (clusterTime < minClusterTime && dqi.hasNext()) { - Object[] h = dqi.next(); - Task task = (Task) h[0]; + Entry e = dqi.next(); + Task task = e.task; JobSpecification js = (JobSpecification) task.getSpecification(); if (constraints == null) { - constraints = ((Object[]) h[1])[0]; + constraints = ((Object[]) e.constraints)[0]; } - else if (!constraints.equals(((Object[]) h[1])[0])) { + else if (!constraints.equals(((Object[]) e.constraints)[0])) { continue; } @@ -230,7 +231,7 @@ merge(js, env, attrs); clusterTime += getMaxWallTime(task); - cluster.addLast(h); + cluster.addLast(e); } if (logger.isDebugEnabled()) { @@ -241,8 +242,8 @@ continue; } else if (cluster.size() == 1) { - Object[] h = cluster.removeFirst(); - super.enqueue((Task) h[0], h[1]); + Entry e = cluster.removeFirst(); + super.enqueue(e.task, e.constraints, e.listener); } else if (cluster.size() > 1) { Task t = new TaskImpl(); @@ -264,8 +265,8 @@ logger.info("Creating cluster " + t.getIdentity() + " with size " + cluster.size()); } - for (Object[] h : cluster) { - Task st = (Task) h[0]; + for (Entry e : cluster) { + Task st = e.task; if (logger.isInfoEnabled()) { logger.info("Task " + st.getIdentity() + " clustered in " + t.getIdentity()); } @@ -292,7 +293,7 @@ synchronized (tasks) { tasks.put(t, cluster); } - super.enqueue(t, new Contact[] { (Contact) constraints }); + super.enqueue(t, new Contact[] { (Contact) constraints }, null); } } } @@ -388,59 +389,58 @@ sb.append(String.valueOf(value)); } - protected void failTask(Task t, String message, Exception e) { - if (e instanceof NoSuchResourceException) { - message = "The application \"" + getTaskConstraints(t).getConstraint("tr") - + "\" is not available for any site/pool in your tc.data catalog "; - } + @Override + protected void failTask(Entry e, String message, Exception ex) { if (logger.isDebugEnabled()) { - logger.debug("Failing task " + t.getIdentity()); + logger.debug("Failing task " + e.task.getIdentity()); } - List cluster = null; + Task t = e.task; + List cluster = null; synchronized (tasks) { cluster = tasks.get(t); } if (cluster != null) { - for (Object[] h : cluster) { - super.failTask((Task) h[0], message, e); + for (Entry e1 : cluster) { + super.failTask(e1, message, ex); } } else { - super.failTask(t, message, e); + super.failTask(e, message, ex); } } - public void statusChanged(StatusEvent e) { - Task t = (Task) e.getSource(); + @Override + public void statusChanged(StatusEvent se, Entry e) { + Task t = e.task; try { if (logger.isDebugEnabled()) { logger.debug("Got task status change for " + t.getIdentity()); } - List cluster = null; + List cluster = null; synchronized (tasks) { cluster = tasks.get(t); } if (cluster == null) { - super.statusChanged(e); + super.statusChanged(se, e); } else { if (logger.isDebugEnabled()) { logger.debug("Got cluster status change for " + t.getIdentity()); } - Status clusterMemberStatus = e.getStatus(); + Status clusterMemberStatus = se.getStatus(); if(clusterMemberStatus.getStatusCode() == Status.FAILED) { clusterMemberStatus = new StatusImpl(Status.COMPLETED); } - for (Object[] h : cluster) { - Task ct = (Task) h[0]; + for (Entry e1 : cluster) { + Task ct = e1.task; StatusEvent nse = new StatusEvent(ct, clusterMemberStatus); ct.setStatus(clusterMemberStatus); - fireJobStatusChangeEvent(nse); + fireJobStatusChangeEvent(nse, e1); } - if (e.getStatus().isTerminal()) { + if (se.getStatus().isTerminal()) { if (logger.isInfoEnabled()) { logger.info("Removing cluster " + t.getIdentity()); } @@ -451,7 +451,7 @@ } } catch (Exception ex) { - failTask(t, ex.getMessage(), ex); + failTask(e, ex.getMessage(), ex); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -35,7 +35,6 @@ import org.globus.cog.karajan.scheduler.TaskTransformer; import org.globus.cog.karajan.util.BoundContact; import org.globus.cog.karajan.util.Contact; -import org.globus.cog.karajan.workflow.KarajanRuntimeException; import org.globus.swift.catalog.TCEntry; import org.globus.swift.catalog.types.TCType; import org.griphyn.vdl.util.FQN; @@ -151,7 +150,7 @@ } } catch(IOException ioe) { - throw new KarajanRuntimeException("Could not determine wrapper invocation mode", ioe); + throw new RuntimeException("Could not determine wrapper invocation mode", ioe); } } @@ -176,7 +175,7 @@ l = tc.getTCEntries(fqn, bc.getHost(), TCType.INSTALLED); } catch (Exception e) { - throw new KarajanRuntimeException(e); + throw new RuntimeException(e); } if (l == null || l.isEmpty()) { return; Modified: branches/faster/src/org/griphyn/vdl/karajan/WaitingThreadsMonitor.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/WaitingThreadsMonitor.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/WaitingThreadsMonitor.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -24,48 +24,62 @@ import java.util.List; import java.util.Map; -import org.globus.cog.karajan.stack.VariableStack; +import k.rt.FutureListener; +import k.thr.LWThread; + import org.griphyn.vdl.mapping.DSHandle; public class WaitingThreadsMonitor { - private static Map threads = new HashMap(); - private static Map> outputs = new HashMap>();; + private static Map threads = new HashMap(); + private static Map> outputs = new HashMap>();; - public static void addThread(VariableStack stack, DSHandle waitingOn) { - if (stack != null) { + public static void addThread(FutureListener fl, DSHandle waitingOn) { + if (fl instanceof LWThread.Listener) { + addThread(((LWThread.Listener) fl).getThread(), waitingOn); + } + } + + public static void removeThread(FutureListener fl) { + if (fl instanceof LWThread.Listener) { + removeThread(((LWThread.Listener) fl).getThread()); + } + } + + public static void addThread(LWThread thr, DSHandle waitingOn) { + if (thr != null) { synchronized(threads) { - threads.put(stack, waitingOn); + threads.put(thr, waitingOn); } } } - public static void removeThread(VariableStack stack) { + public static void removeThread(LWThread thr) { synchronized(threads) { - threads.remove(stack); + threads.remove(thr); } } - public static Map getAllThreads() { + public static Map getAllThreads() { synchronized(threads) { - return new HashMap(threads); + return new HashMap(threads); } } - public static void addOutput(VariableStack stack, List outputs) { + public static void addOutput(LWThread thr, List outputs) { synchronized(WaitingThreadsMonitor.outputs) { - WaitingThreadsMonitor.outputs.put(stack, outputs); + WaitingThreadsMonitor.outputs.put(thr, outputs); } } - public static void removeOutput(VariableStack stack) { + public static void removeOutput(LWThread thr) { synchronized(outputs) { - outputs.remove(stack); + outputs.remove(thr); } } - public static Map> getOutputs() { + public static Map> getOutputs() { synchronized(outputs) { - return new HashMap>(outputs); + return new HashMap>(outputs); } } } Modified: branches/faster/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,55 +22,86 @@ import java.io.IOException; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.functions.AbstractFunction; +import k.rt.Context; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.apache.log4j.Logger; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Param; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; +import org.globus.cog.karajan.parser.WrapperNode; +import org.globus.cog.karajan.util.BoundContact; import org.griphyn.vdl.util.VDL2Config; -import org.globus.cog.karajan.util.BoundContact; -import org.apache.log4j.Logger; -public class ConfigProperty extends AbstractFunction { - public static final Arg NAME = new Arg.Positional("name"); - public static final Arg INSTANCE = new Arg.Optional("instance", Boolean.TRUE); - public static final Arg HOST = new Arg.Optional("host",null); - - static { - setArguments(ConfigProperty.class, new Arg[] { NAME, INSTANCE, HOST }); +public class ConfigProperty extends AbstractSingleValuedFunction { + private ArgRef name; + private ArgRef instance; + private ArgRef host; + + private VDL2Config instanceConfig; + private VarRef context; + + + @Override + protected Param[] getParams() { + return params("name", optional("instance", Boolean.TRUE), optional("host", null)); } - public static final String INSTANCE_CONFIG_FILE = "vdl:instanceconfigfile"; - public static final String INSTANCE_CONFIG = "vdl:instanceconfig"; + public static final String INSTANCE_CONFIG = "SWIFT_CONFIG"; public static final Logger logger = Logger.getLogger(ConfigProperty.class); + + + @Override + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) + throws CompilationException { + context = scope.getVarRef("#context"); + return super.compileBody(w, argScope, scope); + } - public Object function(VariableStack stack) throws ExecutionException { - String name = TypeUtil.toString(NAME.getValue(stack)); - boolean instance = TypeUtil.toBoolean(INSTANCE.getValue(stack)); - Object host = HOST.getValue(stack); - if(logger.isDebugEnabled()) { - logger.debug("Getting property "+name+" with host "+host); + @Override + public Object function(Stack stack) { + String name = this.name.getValue(stack); + boolean instance = this.instance.getValue(stack); + BoundContact host = this.host.getValue(stack); + if (logger.isDebugEnabled()) { + logger.debug("Getting property " + name + " with host " + host); } - if(host!= null) { + if (host != null) { // see if the host has this property defined, and if so // get its value - BoundContact h = (BoundContact)host; - String prop = (String) h.getProperty(name); - if(prop != null) { - logger.debug("Found property "+name+" in BoundContact"); + String prop = (String) host.getProperty(name); + if (prop != null) { + if (logger.isDebugEnabled()) { + logger.debug("Found property " + name + " in BoundContact"); + } return prop; } - logger.debug("Could not find property "+name+" in BoundContact"); + if (logger.isDebugEnabled()) { + logger.debug("Could not find property " + name + " in BoundContact"); + } } - return getProperty(name, instance, stack); + return getProperty(name, instance, getInstanceConfig(stack)); } - public static String getProperty(String name, VariableStack stack) throws ExecutionException { - return getProperty(name, true, stack); + private synchronized VDL2Config getInstanceConfig(Stack stack) { + if (instanceConfig == null) { + Context ctx = this.context.getValue(stack); + instanceConfig = (VDL2Config) ctx.getAttribute("SWIFT:CONFIG"); + } + return instanceConfig; } - public static String getProperty(String name, boolean instance, VariableStack stack) throws ExecutionException { + public static String getProperty(String name, VDL2Config instanceConfig) { + return getProperty(name, true, instanceConfig); + } + + public static String getProperty(String name, boolean instance, VDL2Config instanceConfig) { try { VDL2Config conf; String prop; @@ -79,20 +110,8 @@ prop = conf.getProperty(name); } else { - synchronized (stack.firstFrame()) { - conf = (VDL2Config) stack.getGlobal(INSTANCE_CONFIG); - if (conf == null) { - String confFile = (String) stack.getGlobal(INSTANCE_CONFIG_FILE); - if (confFile == null) { - conf = VDL2Config.getConfig(); - } - else { - conf = VDL2Config.getConfig(confFile); - } - stack.setGlobal(INSTANCE_CONFIG, conf); - } - prop = conf.getProperty(name); - } + conf = instanceConfig; + prop = conf.getProperty(name); } if (prop == null) { throw new ExecutionException("Swift config property \"" + name + "\" not found in " Modified: branches/faster/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,42 +22,61 @@ import java.io.CharArrayWriter; import java.io.PrintWriter; +import java.util.Collection; import java.util.ConcurrentModificationException; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; +import k.rt.Channel; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.functions.AbstractFunction; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.compiled.nodes.functions.AbstractFunction; +import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.karajan.VDL2ErrorTranslator; public class ProcessBulkErrors extends AbstractFunction { public static final Logger logger = Logger.getLogger(ProcessBulkErrors.class); + + private ArgRef message; + private ArgRef> errors; + private ArgRef onStdout; + + private ChannelRef cr_stdout, cr_stderr; - public static final Arg MESSAGE = new Arg.Positional("message"); - public static final Arg ERRORS = new Arg.Positional("errors"); - public static final Arg ONSTDOUT = new Arg.Optional("onstdout", Boolean.FALSE); + @Override + protected Signature getSignature() { + return new Signature(params("message", "errors", optional("onStdout", false)), + returns(channel("stdout", DYNAMIC), channel("stderr", DYNAMIC))); + } - static { - setArguments(ProcessBulkErrors.class, new Arg[] { MESSAGE, ERRORS, ONSTDOUT }); - } + @Override + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) + throws CompilationException { + returnDynamic(scope); + return super.compileBody(w, argScope, scope); + } - public Object function(VariableStack stack) throws ExecutionException { - String message = TypeUtil.toString(MESSAGE.getValue(stack)); - boolean onStdout = TypeUtil.toBoolean(ONSTDOUT.getValue(stack)); - List l = TypeUtil.toList(ERRORS.getValue(stack)); + + @Override + public Object function(Stack stack) { + String message = this.message.getValue(stack); + boolean onStdout = this.onStdout.getValue(stack); + Collection l = this.errors.getValue(stack); + VDL2ErrorTranslator translator = VDL2ErrorTranslator.getDefault(); - Map count = new HashMap(); - Iterator i = l.iterator(); - while (i.hasNext()) { - ExecutionException ex = (ExecutionException) i.next(); + Map count = new HashMap(); + for (ExecutionException ex : l) { if (ex.getCause() instanceof ConcurrentModificationException) { ex.printStackTrace(); } @@ -76,26 +95,25 @@ } tmsg = tmsg.trim(); if (count.containsKey(tmsg)) { - Integer j = (Integer) count.get(tmsg); + Integer j = count.get(tmsg); count.put(tmsg, new Integer(j.intValue() + 1)); } else { count.put(tmsg, new Integer(1)); } } - Arg.Channel channel = onStdout ? STDOUT : STDERR; + ChannelRef channel = onStdout ? cr_stdout : cr_stderr; + Channel r = channel.get(stack); if (count.size() != 0) { - channel.ret(stack, message + "\n"); - i = count.entrySet().iterator(); + r.add(message + "\n"); int k = 1; - while (i.hasNext()) { - Map.Entry e = (Map.Entry) i.next(); - Integer j = (Integer) e.getValue(); + for (Map.Entry e : count.entrySet()) { + Integer j = e.getValue(); if (j.intValue() == 1) { - channel.ret(stack, k + ". " + e.getKey() + "\n"); + r.add(k + ". " + e.getKey() + "\n"); } else { - channel.ret(stack, k + ". " + e.getKey() + " (" + j.intValue() + " times)\n"); + r.add(k + ". " + e.getKey() + " (" + j.intValue() + " times)\n"); } k++; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/AbsFileName.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/AbsFileName.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/AbsFileName.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,16 +20,22 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.Stack; -public class AbsFileName extends VDLFunction { - static { - setArguments(AbsFileName.class, new Arg[] { PA_VAR }); - } +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.mapping.DSHandle; - public Object function(VariableStack stack) throws ExecutionException { - return filename(stack); - } +public class AbsFileName extends SwiftFunction { + private ArgRef var; + + @Override + protected Signature getSignature() { + return new Signature(params("var")); + } + + @Override + public Object function(Stack stack) { + return filename(var.getValue(stack)); + } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/AppStageins.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/AppStageins.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/AppStageins.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -23,34 +23,41 @@ import java.util.LinkedList; import java.util.List; +import k.rt.Stack; +import k.thr.LWThread; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.ArgUtil; -import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; import org.globus.swift.data.Director; import org.globus.swift.data.policy.Policy; import org.griphyn.vdl.mapping.AbsFile; -public class AppStageins extends AbstractSequentialWithArguments { +public class AppStageins extends InternalFunction { + private ArgRef jobid; + private ArgRef> files; + private ArgRef dir; + private ArgRef stagingMethod; + + private ChannelRef> cr_stagein; static Logger logger = Logger.getLogger(AppStageins.class); - public static final Arg JOBID = new Arg.Positional("jobid"); - public static final Arg FILES = new Arg.Positional("files"); - public static final Arg DIR = new Arg.Positional("dir"); - public static final Arg STAGING_METHOD = new Arg.Positional("stagingMethod"); - public static final Arg.Channel STAGEIN = new Arg.Channel("stagein"); - - static { - setArguments(AppStageins.class, new Arg[] { JOBID, FILES, DIR, - STAGING_METHOD }); + + @Override + protected Signature getSignature() { + return new Signature(params("jobid", "files", "dir", "stagingMethod"), returns(channel("stagein"))); } - protected void post(VariableStack stack) throws ExecutionException { - List files = TypeUtil.toList(FILES.getValue(stack)); + + protected void runBody(LWThread thr) { + Stack stack = thr.getStack(); + List files = this.files.getValue(stack); + String stagingMethod = this.stagingMethod.getValue(stack); + String dir = this.dir.getValue(stack); for (Object f : files) { AbsFile file = new AbsFile(TypeUtil.toString(f)); Policy policy = Director.lookup(file.toString()); @@ -61,7 +68,7 @@ String protocol = file.getProtocol(); if (protocol.equals("file")) { - protocol = TypeUtil.toString(STAGING_METHOD.getValue(stack)); + protocol = stagingMethod; } String path = file.getDir().equals("") ? file.getName() : file .getDir() @@ -70,11 +77,10 @@ if (logger.isDebugEnabled()) { logger.debug("will stage in: " + relpath + " via: " + protocol); } - ArgUtil.getChannelReturn(stack, STAGEIN).append( + cr_stagein.append(stack, makeList(protocol + "://" + file.getHost() + "/" + path, - TypeUtil.toString(DIR.getValue(stack)) + "/" + relpath)); + dir + "/" + relpath)); } - super.post(stack); } private List makeList(String s1, String s2) { Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/AppStageouts.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/AppStageouts.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/AppStageouts.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -23,54 +23,55 @@ import java.util.LinkedList; import java.util.List; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.ArgUtil; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import k.rt.ExecutionException; +import k.rt.Stack; +import k.thr.LWThread; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.Path; -public class AppStageouts extends AbstractSequentialWithArguments { - - public static final Arg JOBID = new Arg.Positional("jobid"); - public static final Arg FILES = new Arg.Positional("files"); - public static final Arg DIR = new Arg.Positional("dir"); - public static final Arg STAGING_METHOD = new Arg.Positional("stagingMethod"); - public static final Arg VAR = new Arg.Optional("var", null); - public static final Arg.Channel STAGEOUT = new Arg.Channel("stageout"); - - static { - setArguments(AppStageouts.class, new Arg[] { JOBID, FILES, DIR, - STAGING_METHOD, VAR }); +public class AppStageouts extends InternalFunction { + private ArgRef jobid; + private ArgRef>> files; + private ArgRef dir; + private ArgRef stagingMethod; + + private ChannelRef> cr_stageout; + + @Override + protected Signature getSignature() { + return new Signature(params("jobid", "files", "dir", "stagingMethod"), returns(channel("stageout"))); } - protected void post(VariableStack stack) throws ExecutionException { + protected void runBody(LWThread thr) { try { - List files = TypeUtil.toList(FILES.getValue(stack)); - for (Object f : files) { - List pv = TypeUtil.toList(f); + Stack stack = thr.getStack(); + List> files = this.files.getValue(stack); + String stagingMethod = this.stagingMethod.getValue(stack); + String dir = this.dir.getValue(stack); + for (List pv : files) { Path p = (Path) pv.get(0); DSHandle handle = (DSHandle) pv.get(1); - ArgUtil.getNamedArguments(stack).add("var", handle.getField(p)); - AbsFile file = new AbsFile(VDLFunction.filename(stack)[0]); + AbsFile file = new AbsFile(SwiftFunction.filename(handle.getField(p))[0]); String protocol = file.getProtocol(); if (protocol.equals("file")) { - protocol = TypeUtil.toString(STAGING_METHOD.getValue(stack)); + protocol = stagingMethod; } String path = file.getDir().equals("") ? file.getName() : file.getDir() + "/" + file.getName(); String relpath = path.startsWith("/") ? path.substring(1) : path; - ArgUtil.getChannelReturn(stack, STAGEOUT).append( - makeList(TypeUtil.toString(DIR.getValue(stack)) + "/" + relpath, + cr_stageout.append(stack, + makeList(dir + "/" + relpath, protocol + "://" + file.getHost() + "/" + path)); } - super.post(stack); } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/AppendArray.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/AppendArray.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/AppendArray.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,10 +17,10 @@ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.InvalidPathException; @@ -28,25 +28,24 @@ public class AppendArray extends SetFieldValue { - public static final Arg PA_ID = new Arg.Positional("id"); - - static { - setArguments(AppendArray.class, new Arg[] { PA_VAR, PA_VALUE }); + @Override + protected Signature getSignature() { + return new Signature(params("var", "value")); } - + @Override - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); - AbstractDataNode value = (AbstractDataNode) PA_VALUE.getValue(stack); + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); + AbstractDataNode value = this.value.getValue(stack); // while there isn't a way to avoid conflicts between auto generated indices // and a user manually using the same index, adding a "#" may reduce // the incidence of problems - Path path = Path.EMPTY_PATH.addFirst(getThreadPrefix(stack), true); + Path path = Path.EMPTY_PATH.addFirst(getThreadPrefix(), true); try { - deepCopy(var.getField(path), value, stack, 0); + deepCopy(var.getField(path), value, stack); } catch (InvalidPathException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } return null; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/CacheAddAndLockFile.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/CacheAddAndLockFile.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/CacheAddAndLockFile.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,35 +20,54 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; +import java.util.List; + +import k.rt.ExecutionException; +import k.rt.Stack; +import k.thr.LWThread; +import k.thr.Yield; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.futures.FutureNotYetAvailable; +import org.globus.cog.karajan.util.BoundContact; import org.griphyn.vdl.karajan.lib.cache.CacheReturn; import org.griphyn.vdl.karajan.lib.cache.File; import org.griphyn.vdl.karajan.lib.cache.VDLFileCache; public class CacheAddAndLockFile extends CacheFunction { - public static final String PFILE = "#pfile"; + private ArgRef file; + private ArgRef dir; + private ArgRef host; + private ArgRef size; + private Node body; + + private VarRef pfile; + private VarRef> cacheFilesToRemove; + + @Override + protected Signature getSignature() { + return new Signature(params("file", "dir", "host", "size", block("body"))); + } + + @Override + protected void addLocals(Scope scope) { + super.addLocals(scope); + pfile = scope.getVarRef(scope.addVar("#pfile")); + cacheFilesToRemove = scope.getVarRef(scope.addVar(CACHE_FILES_TO_REMOVE)); + } - public static final Arg FILE = new Arg.Positional("file"); - public static final Arg DIR = new Arg.Positional("dir"); - public static final Arg HOST = new Arg.Positional("host"); - public static final Arg SIZE = new Arg.Positional("size"); - - static { - setArguments(CacheAddAndLockFile.class, new Arg[] { FILE, DIR, HOST, SIZE }); - } - - protected void partialArgumentsEvaluated(VariableStack stack) throws ExecutionException { - String file = TypeUtil.toString(FILE.getValue(stack)); - String dir = TypeUtil.toString(DIR.getValue(stack)); - Object host = HOST.getValue(stack); - long size = TypeUtil.toLong(SIZE.getValue(stack)); - VDLFileCache cache = CacheFunction.getCache(stack); + protected boolean lock(Stack stack) { + String file = this.file.getValue(stack); + String dir = this.dir.getValue(stack); + BoundContact host = this.host.getValue(stack); + long size = this.size.getValue(stack).longValue(); + VDLFileCache cache = getCache(stack); File f = new File(file, dir, host, size); - stack.setVar(PFILE, f); + pfile.setValue(stack, f); CacheReturn cr = cache.addAndLockEntry(f); if (cr.alreadyCached) { if (cr.cached.isLockedForProcessing()) { @@ -56,32 +75,60 @@ throw new FutureNotYetAvailable(cr.cached); } else { - complete(stack); + return false; } } else { - super.partialArgumentsEvaluated(stack); - stack.setVar(CACHE_FILES_TO_REMOVE, cr.remove); - startRest(stack); + cacheFilesToRemove.setValue(stack, cr.remove); + return true; } } + + @Override + protected void runBody(LWThread thr) { + int i = thr.checkSliceAndPopState(); + int fc = thr.popIntState(); + Stack stack = thr.getStack(); + try { + switch (i) { + case 0: + fc = stack.frameCount(); + i++; + case 1: + if (!lock(stack)) { + break; + } + i++; + case 2: + body.run(thr); + unlock(stack); + } + } + catch (ExecutionException e) { + stack.dropToFrame(fc); + removeEntry(stack); + throw e; + } + catch (Yield y) { + y.getState().push(fc); + y.getState().push(i); + throw y; + } + } - protected void post(VariableStack stack) throws ExecutionException { - File f = (File) stack.currentFrame().getVar(PFILE); + protected void unlock(Stack stack) { + File f = pfile.getValue(stack); if (f == null) { throw new ExecutionException("Weird inconsistency in " + this + ". The file was not found on the current frame."); } - VDLFileCache cache = CacheFunction.getCache(stack); + VDLFileCache cache = getCache(stack); cache.unlockFromProcessing(f); - super.post(stack); } - public void failed(VariableStack stack, ExecutionException e) - throws ExecutionException { - VDLFileCache cache = CacheFunction.getCache(stack); - cache.entryRemoved((File) stack.currentFrame().getVar(PFILE)); - super.failed(stack, e); + public void removeEntry(Stack stack) { + VDLFileCache cache = getCache(stack); + cache.entryRemoved(pfile.getValue(stack)); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/CacheAddFile.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/CacheAddFile.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/CacheAddFile.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,37 +20,72 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import java.util.List; + +import k.rt.ExecutionException; +import k.rt.Stack; +import k.thr.LWThread; +import k.thr.Yield; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.util.BoundContact; import org.griphyn.vdl.karajan.lib.cache.CacheReturn; import org.griphyn.vdl.karajan.lib.cache.File; import org.griphyn.vdl.karajan.lib.cache.VDLFileCache; public class CacheAddFile extends CacheFunction { - public static final Arg FILE = new Arg.Positional("file"); - public static final Arg DIR = new Arg.Positional("dir"); - public static final Arg HOST = new Arg.Positional("host"); - public static final Arg SIZE = new Arg.Positional("size"); + private ArgRef file; + private ArgRef dir; + private ArgRef host; + private ArgRef size; + private Node body; + + private VarRef> cacheFilesToRemove; - static { - setArguments(CacheAddFile.class, new Arg[] { FILE, DIR, HOST, SIZE }); - } + @Override + protected Signature getSignature() { + return new Signature(params("file", "dir", "host", "size", block("body"))); + } + + @Override + protected void addLocals(Scope scope) { + super.addLocals(scope); + cacheFilesToRemove = scope.getVarRef(scope.addVar(CACHE_FILES_TO_REMOVE)); + } - public void partialArgumentsEvaluated(VariableStack stack) throws ExecutionException { - String file = TypeUtil.toString(FILE.getValue(stack)); - String dir = TypeUtil.toString(DIR.getValue(stack)); - Object host = HOST.getValue(stack); - long size = TypeUtil.toLong(SIZE.getValue(stack)); - VDLFileCache cache = getCache(stack); + @Override + protected void runBody(LWThread thr) { + int i = thr.checkSliceAndPopState(); + try { + switch (i) { + case 0: + add(thr.getStack()); + i++; + case 1: + body.run(thr); + } + } + catch (Yield y) { + y.getState().push(i); + throw y; + } + } + + public void add(Stack stack) { + String file = this.file.getValue(stack); + String dir = this.dir.getValue(stack); + BoundContact host = this.host.getValue(stack); + long size = this.size.getValue(stack).longValue(); + VDLFileCache cache = getCache(stack); File f = new File(file, dir, host, size); CacheReturn cr = cache.addEntry(f); if (cr.alreadyCached) { - throw new ExecutionException("The cache already contains " + f + "."); + throw new ExecutionException(this, "The cache already contains " + f + "."); } - super.partialArgumentsEvaluated(stack); - stack.setVar(CacheFunction.CACHE_FILES_TO_REMOVE, cr.remove); - startNext(stack); + cacheFilesToRemove.setValue(stack, cr.remove); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/CacheFileRemoved.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/CacheFileRemoved.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/CacheFileRemoved.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,28 +20,31 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.Stack; +import k.thr.LWThread; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.lib.cache.File; import org.griphyn.vdl.karajan.lib.cache.VDLFileCache; public class CacheFileRemoved extends CacheFunction { - public static final Arg PATH = new Arg.Positional("path"); - public static final Arg HOST = new Arg.Positional("host"); + private ArgRef path; + private ArgRef host; + + @Override + protected Signature getSignature() { + return new Signature(params("path", "host")); + } - static { - setArguments(CacheFileRemoved.class, new Arg[] { PATH, HOST }); - } - public void partialArgumentsEvaluated(VariableStack stack) throws ExecutionException { - String path = TypeUtil.toString(PATH.getValue(stack)); - Object host = HOST.getValue(stack); - VDLFileCache cache = getCache(stack); - File f = new File(path, host, 0); - cache.entryRemoved(f); - complete(stack); - } - + @Override + protected void runBody(LWThread thr) { + Stack stack = thr.getStack(); + String path = this.path.getValue(stack); + Object host = this.host.getValue(stack); + VDLFileCache cache = getCache(stack); + File f = new File(path, host, 0); + cache.entryRemoved(f); + } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/CacheFunction.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/CacheFunction.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/CacheFunction.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,29 +20,53 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.PartialArgumentsContainer; +import k.rt.Context; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; +import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.karajan.functions.ConfigProperty; import org.griphyn.vdl.karajan.lib.cache.VDLFileCache; import org.griphyn.vdl.karajan.lib.cache.VDLFileCacheFactory; +import org.griphyn.vdl.util.VDL2Config; import org.griphyn.vdl.util.VDL2ConfigProperties; -public abstract class CacheFunction extends PartialArgumentsContainer { - public static final String CACHE_FILES_TO_REMOVE = "cachefilestoremove"; +public abstract class CacheFunction extends InternalFunction { + public static final String CACHE_FILES_TO_REMOVE = "cacheFilesToRemove"; - public static final String VDL_FILE_CACHE = "vdl:filecache"; + private VarRef context; + private VDLFileCache cache; + + @Override + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) + throws CompilationException { + context = scope.getVarRef("#context"); + return super.compileBody(w, argScope, scope); + } - protected static VDLFileCache getCache(VariableStack stack) throws ExecutionException { - VDLFileCache cache; - synchronized (stack.getExecutionContext()) { - cache = (VDLFileCache) stack.getGlobal(VDL_FILE_CACHE); - if (cache == null) { - cache = VDLFileCacheFactory.newInstance(ConfigProperty.getProperty( - VDL2ConfigProperties.CACHING_ALGORITHM, stack)); - stack.setGlobal(VDL_FILE_CACHE, cache); - } - } - return cache; + protected VDLFileCache getCache(Stack stack) { + synchronized(this) { + if (cache == null) { + cache = getOrCreateCache(stack); + } + return cache; + } } + + private VDLFileCache getOrCreateCache(Stack stack) { + Context ctx = context.getValue(stack); + synchronized(ctx) { + VDLFileCache cache = (VDLFileCache) ctx.getAttribute("SWIFT:FILE_CACHE"); + if (cache == null) { + cache = VDLFileCacheFactory.newInstance(ConfigProperty.getProperty( + VDL2ConfigProperties.CACHING_ALGORITHM, (VDL2Config) ctx.getAttribute("SWIFT:CONFIG"))); + ctx.setAttribute("SWIFT:FILE_CACHE", cache); + } + return cache; + } + } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -21,44 +21,77 @@ package org.griphyn.vdl.karajan.lib; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.Stack; +import k.thr.LWThread; +import k.thr.Yield; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.util.BoundContact; import org.griphyn.vdl.karajan.lib.cache.CacheReturn; import org.griphyn.vdl.karajan.lib.cache.File; import org.griphyn.vdl.karajan.lib.cache.VDLFileCache; import org.griphyn.vdl.mapping.AbsFile; public class CacheUnlockFiles extends CacheFunction { - public static final Arg FILE = new Arg.Positional("files"); - public static final Arg DIR = new Arg.Positional("dir"); - public static final Arg HOST = new Arg.Positional("host"); - public static final Arg FORCE = new Arg.Optional("force", Boolean.TRUE); + private ArgRef> file; + private ArgRef dir; + private ArgRef host; + private ArgRef force; + private Node body; + + private VarRef> cacheFilesToRemove; - static { - setArguments(CacheUnlockFiles.class, new Arg[] { FILE, DIR, HOST, FORCE }); - } + @Override + protected Signature getSignature() { + return new Signature(params("file", "dir", "host", optional("force", Boolean.TRUE), block("body"))); + } + + @Override + protected void addLocals(Scope scope) { + super.addLocals(scope); + cacheFilesToRemove = scope.getVarRef(scope.addVar(CACHE_FILES_TO_REMOVE)); + } - public void partialArgumentsEvaluated(VariableStack stack) throws ExecutionException { - List pairs = TypeUtil.toList(FILE.getValue(stack)); - String dir = TypeUtil.toString(DIR.getValue(stack)); - Object host = HOST.getValue(stack); - VDLFileCache cache = getCache(stack); - List rem = new ArrayList(); - - Iterator i = pairs.iterator(); - while (i.hasNext()) { - String file = (String) i.next(); - File f = new File(new AbsFile(file).getPath(), dir, host, 0); - CacheReturn cr = cache.unlockEntry(f, TypeUtil.toBoolean(FORCE.getValue(stack))); - rem.addAll(cr.remove); - } - super.partialArgumentsEvaluated(stack); - stack.setVar(CACHE_FILES_TO_REMOVE, rem); - startRest(stack); - } + @Override + protected void runBody(LWThread thr) { + int i = thr.checkSliceAndPopState(); + try { + switch (i) { + case 0: + remove(thr.getStack()); + i++; + case 1: + body.run(thr); + } + } + catch (Yield y) { + y.getState().push(i); + throw y; + } + } + + public void remove(Stack stack) { + List pairs = this.file.getValue(stack); + String dir = this.dir.getValue(stack); + Object host = this.host.getValue(stack); + VDLFileCache cache = getCache(stack); + List rem = new ArrayList(); + + boolean force = this.force.getValue(stack); + + for (Object o : pairs) { + String file = (String) o; + File f = new File(new AbsFile(file).getPath(), dir, host, 0); + CacheReturn cr = cache.unlockEntry(f, force); + rem.addAll(cr.remove); + } + + cacheFilesToRemove.setValue(stack, rem); + } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/CleanDataset.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/CleanDataset.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/CleanDataset.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,24 +20,25 @@ */ package org.griphyn.vdl.karajan.lib; -import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.file.FileGarbageCollector; -public class CleanDataset extends VDLFunction { - public static final Logger logger = Logger.getLogger(CleanDataset.class); - - public static final Arg.Optional OA_SHUTDOWN = new Arg.Optional("shutdown"); - - static { - setArguments(CleanDataset.class, new Arg[] { PA_VAR, OA_SHUTDOWN }); - } +public class CleanDataset extends SwiftFunction { + private ArgRef var; + private ArgRef shutdown; - public Object function(VariableStack stack) throws ExecutionException { - if (OA_SHUTDOWN.isPresent(stack)) { + @Override + protected Signature getSignature() { + return new Signature(params("var", optional("shutdown", Boolean.FALSE))); + } + + public Object function(Stack stack) { + boolean shutdown = this.shutdown.getValue(stack); + if (shutdown) { // signals that everything is done and the main program should wait for the // garbage collector to finish everything try { @@ -48,7 +49,7 @@ } } else { - AbstractDataNode var = (AbstractDataNode) PA_VAR.getValue(stack); + AbstractDataNode var = this.var.getValue(stack); if (logger.isInfoEnabled()) { logger.info("Cleaning " + var); } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/CloseDataset.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/CloseDataset.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/CloseDataset.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,43 +20,47 @@ */ package org.griphyn.vdl.karajan.lib; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; -public class CloseDataset extends VDLFunction { +public class CloseDataset extends SwiftFunction { public static final Logger logger = Logger.getLogger(CloseDataset.class); - public static final Arg OA_CHILDREN_ONLY = new Arg.Optional("childrenOnly", Boolean.FALSE); + private ArgRef var; + private ArgRef path; + private ArgRef childrenOnly; + + @Override + protected Signature getSignature() { + return new Signature(params("var", optional("path", Path.EMPTY_PATH), optional("childrenOnly", Boolean.FALSE))); + } - static { - setArguments(CloseDataset.class, new Arg[] { PA_VAR, OA_PATH, OA_CHILDREN_ONLY }); - } - - public Object function(VariableStack stack) throws ExecutionException { - Path path = parsePath(OA_PATH.getValue(stack), stack); - DSHandle var = (DSHandle) PA_VAR.getValue(stack); + public Object function(Stack stack) { + Path path = parsePath(this.path.getValue(stack)); + DSHandle var = this.var.getValue(stack); try { if (logger.isDebugEnabled()) { logger.debug("Closing " + var); } var = var.getField(path); - if (TypeUtil.toBoolean(OA_CHILDREN_ONLY.getValue(stack))) { - closeChildren(stack, (AbstractDataNode) var); + if (childrenOnly.getValue(stack)) { + closeChildren((AbstractDataNode) var); } else { var.closeDeep(); } } catch (InvalidPathException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } return null; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/CreateArray.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/CreateArray.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/CreateArray.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -19,37 +19,39 @@ import java.util.List; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.futures.FutureFault; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.futures.FutureFault; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.MappingParamSet; +import org.griphyn.vdl.mapping.OOBYield; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.RootArrayDataNode; import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Type; -public class CreateArray extends VDLFunction { - +public class CreateArray extends SetFieldValue { public static final Logger logger = Logger.getLogger(CreateArray.class); - - public static final Arg PA_VALUE = new Arg.Positional("value"); - static { - setArguments(CreateArray.class, new Arg[] { PA_VALUE }); - } + private ArgRef value; - public Object function(VariableStack stack) throws ExecutionException { - Object value = PA_VALUE.getValue(stack); + @Override + protected Signature getSignature() { + return new Signature(params("value")); + } + + public Object function(Stack stack) { + Object value = this.value.getValue(stack); try { if (!(value instanceof List)) { - throw new RuntimeException( - "An array variable can only be initialized with a list of values"); + throw new RuntimeException("An array variable can only be initialized with a list of values"); } Type type = checkTypes((List) value); @@ -59,12 +61,7 @@ setMapper(handle); } - /* - * The reason this is disabled without provenance is that the identifier - * is essentially a random number plus a counter. It does not help - * in debugging problems. - */ - if (AbstractDataNode.provenance && logger.isInfoEnabled()) { + if (logger.isInfoEnabled()) { logger.info("CREATEARRAY START array=" + handle.getIdentifier()); } @@ -79,9 +76,9 @@ DSHandle dst = handle.getField(p); - SetFieldValue.deepCopy(dst, n, stack, 1); + deepCopy(dst, n, stack); - if (AbstractDataNode.provenance && logger.isInfoEnabled()) { + if (logger.isInfoEnabled()) { logger.info("CREATEARRAY MEMBER array=" + handle.getIdentifier() + " index=" + index + " member=" + n.getIdentifier()); } @@ -90,7 +87,7 @@ handle.closeShallow(); - if (AbstractDataNode.provenance && logger.isInfoEnabled()) { + if (logger.isInfoEnabled()) { logger.info("CREATEARRAY COMPLETED array=" + handle.getIdentifier()); } @@ -100,7 +97,7 @@ throw e; } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } @@ -109,7 +106,15 @@ MappingParamSet params = new MappingParamSet(); params.set(MappingParam.SWIFT_DESCRIPTOR, "concurrent_mapper"); params.set(MappingParam.SWIFT_DBGNAME, "arrayexpr"); - handle.init(params); + try { + handle.init(params); + } + catch (OOBYield y) { + throw y.wrapped(this); + } + catch (HandleOpenException e) { + throw new ExecutionException(this, "Plain HandleOpenException caught", e); + } } private boolean hasMappableFields(Type type) { Added: branches/faster/src/org/griphyn/vdl/karajan/lib/CurrentThread.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/CurrentThread.java (rev 0) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/CurrentThread.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,30 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 13, 2012 + */ +package org.griphyn.vdl.karajan.lib; + +import k.thr.LWThread; + +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; + +public class CurrentThread extends InternalFunction { + private ChannelRef cr_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params(), returns(channel("...", 1))); + } + + @Override + public void run(LWThread thr) { + cr_vargs.append(thr.getStack(), thr.getName()); + } +} Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,40 +20,52 @@ */ package org.griphyn.vdl.karajan.lib; +import java.util.Collection; import java.util.List; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.ArgUtil; -import org.globus.cog.karajan.arguments.VariableArguments; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import k.rt.Channel; +import k.rt.ExecutionException; +import k.rt.Stack; +import k.thr.LWThread; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.Path; -public class DoRestartLog extends AbstractSequentialWithArguments { - public static final Arg RESTARTOUTS = new Arg.Positional("restartouts"); - - static { - setArguments(DoRestartLog.class, new Arg[] { RESTARTOUTS }); +public class DoRestartLog extends InternalFunction { + + private ArgRef>> restartouts; + private ChannelRef cr_vargs; + private ChannelRef cr_restartLog; + + @Override + protected Signature getSignature() { + return new Signature(params("restartouts"), returns(channel("...", DYNAMIC), channel("restartLog", DYNAMIC))); } @Override - protected void post(VariableStack stack) throws ExecutionException { - List files = TypeUtil.toList(RESTARTOUTS.getValue(stack)); - VariableArguments ret = ArgUtil.getVariableReturn(stack); + protected void runBody(LWThread thr) { + Stack stack = thr.getStack(); + Collection> files = restartouts.getValue(stack); + Channel ret = cr_vargs.get(stack); + Channel log = cr_restartLog.get(stack); try { - for (Object f : files) { - List pv = TypeUtil.toList(f); + for (List pv : files) { Path p = (Path) pv.get(0); DSHandle handle = (DSHandle) pv.get(1); - LogVar.logVar(stack, handle, p); + LogVar.logVar(log, handle, p); } } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } - super.post(stack); } + + @Override + public String toString() { + return super.toString(); + } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Executable.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Executable.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Executable.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,27 +20,31 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.util.BoundContact; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; import org.globus.swift.catalog.TCEntry; import org.griphyn.vdl.karajan.TCCache; import org.griphyn.vdl.util.FQN; -public class Executable extends VDLFunction { - public static final Arg PA_TR = new Arg.Positional("tr"); - public static final Arg PA_HOST = new Arg.Positional("host"); +public class Executable extends SwiftFunction { + + private ArgRef tr; + private ArgRef host; + + + @Override + protected Signature getSignature() { + return new Signature(params("tr", "host")); + } - static { - setArguments(Executable.class, new Arg[] { PA_TR, PA_HOST }); - } - public Object function(VariableStack stack) throws ExecutionException { + public Object function(Stack stack) { TCCache tc = getTC(stack); - String tr = TypeUtil.toString(PA_TR.getValue(stack)); - BoundContact bc = (BoundContact) PA_HOST.getValue(stack); + String tr = this.tr.getValue(stack); + BoundContact bc = this.host.getValue(stack); TCEntry tce = getTCE(tc, new FQN(tr), bc); if (tce == null) { return tr; Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Execute.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Execute.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Execute.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,5 +1,5 @@ /* - * Copyright 2012 University of Chicago + * Copyright 2012 University of Chicagou * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,129 +20,145 @@ */ package org.griphyn.vdl.karajan.lib; +import k.rt.Abort; +import k.rt.Channel; +import k.rt.ConditionalYield; +import k.rt.Context; +import k.rt.ExecutionException; +import k.rt.Stack; +import k.thr.LWThread; + import org.apache.log4j.Logger; import org.globus.cog.abstraction.impl.common.StatusEvent; import org.globus.cog.abstraction.interfaces.Status; import org.globus.cog.abstraction.interfaces.Task; -import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Param; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.grid.GridExec; +import org.globus.cog.karajan.compiled.nodes.grid.TaskStateFuture; import org.globus.cog.karajan.scheduler.Scheduler; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.KarajanRuntimeException; -import org.globus.cog.karajan.workflow.futures.FutureVariableArguments; -import org.globus.cog.karajan.workflow.nodes.grid.GridExec; +import org.griphyn.vdl.karajan.lib.RuntimeStats.ProgressState; import org.griphyn.vdl.karajan.lib.replication.CanceledReplicaException; import org.griphyn.vdl.karajan.lib.replication.ReplicationManager; public class Execute extends GridExec { public static final Logger logger = Logger.getLogger(Execute.class); + + private ArgRef replicationGroup; + private ArgRef> replicationChannel; + private ArgRef jobid; + private ArgRef progress; + + private VarRef context; + + @Override + protected Signature getSignature() { + Signature sig = super.getSignature(); + sig.getParams().add(0, new Param("progress", Param.Type.POSITIONAL)); + sig.getParams().add(optional("replicationGroup", null)); + sig.getParams().add(optional("replicationChannel", null)); + sig.getParams().add(optional("jobid", null)); + return sig; + } + + @Override + protected void addLocals(Scope scope) { + super.addLocals(scope); + context = scope.getVarRef("#context"); + } - public static final String REPLICATION_MANAGER = "execute:replication-manager"; - - public static final Arg A_REPLICATION_GROUP = new Arg.Optional("replicationGroup"); - public static final Arg A_REPLICATION_CHANNEL = new Arg.Optional("replicationChannel"); - public static final Arg A_JOBID = new Arg.Optional("jobid"); - - static { - setArguments(Execute.class, new Arg[] { A_EXECUTABLE, A_ARGS, A_ARGUMENTS, A_HOST, - A_STDOUT, A_STDERR, A_STDOUTLOCATION, A_STDERRLOCATION, A_STDIN, A_PROVIDER, - A_COUNT, A_HOST_COUNT, A_JOBTYPE, A_MAXTIME, A_MAXWALLTIME, A_MAXCPUTIME, - A_ENVIRONMENT, A_QUEUE, A_PROJECT, A_MINMEMORY, A_MAXMEMORY, A_REDIRECT, - A_SECURITY_CONTEXT, A_DIRECTORY, A_NATIVESPEC, A_DELEGATION, A_ATTRIBUTES, - C_ENVIRONMENT, A_FAIL_ON_JOB_ERROR, A_BATCH, A_REPLICATION_GROUP, - A_REPLICATION_CHANNEL, A_JOBID, C_STAGEIN, C_STAGEOUT, C_CLEANUP }); - } - - public Execute() { - } - - public void submitScheduled(Scheduler scheduler, Task task, VariableStack stack, - Object constraints) throws ExecutionException { + @Override + public void submitScheduled(Scheduler scheduler, Task task, Stack stack, Object constraints) { try { registerReplica(stack, task); log(task, stack); - scheduler.addJobStatusListener(this, task); - setStack(task, stack); - scheduler.enqueue(task, constraints); + + TaskStateFuture tsf = new SwiftTaskStateFuture(stack, task, false); + scheduler.enqueue(task, constraints, tsf); + throw new ConditionalYield(1, tsf); } catch (CanceledReplicaException e) { if (logger.isDebugEnabled()) { logger.debug("Early abort on replicated task " + task); } - abort(stack); + throw new Abort(); } } - void log(Task task, VariableStack stack) - throws ExecutionException - { + void log(Task task, Stack stack) throws ExecutionException { if (logger.isDebugEnabled()) { logger.debug(task); logger.debug("Submitting task " + task); } - String jobid = (String) A_JOBID.getValue(stack,null); + String jobid = this.jobid.getValue(stack); if (logger.isDebugEnabled()) { - logger.debug("jobid="+jobid+" task=" + task); + logger.debug("jobid=" + jobid + " task=" + task); } } - protected void registerReplica(VariableStack stack, Task task) throws CanceledReplicaException { + protected void registerReplica(Stack stack, Task task) throws CanceledReplicaException { setTaskIdentity(stack, task); - try { - String rg = TypeUtil.toString(A_REPLICATION_GROUP.getValue(stack, null)); - if (rg != null) { - getReplicationManager(stack).register(rg, task); - } + + String rg = this.replicationGroup.getValue(stack); + if (rg != null) { + getReplicationManager(stack).register(rg, task); } - catch (ExecutionException e) { - throw new KarajanRuntimeException(e); - } } + + protected class SwiftTaskStateFuture extends CustomTaskStateFuture { - public void statusChanged(StatusEvent e) { - Task task = (Task) e.getSource(); - VariableStack stack = getStack(task); - try { - if (stack != null) { - int c = e.getStatus().getStatusCode(); - if (c == Status.SUBMITTED) { - RuntimeStats.setProgress(stack, "Submitted"); - getReplicationManager(stack).submitted(task, e.getStatus().getTime()); - } - else if (c == Status.STAGE_IN) { - RuntimeStats.setProgress(stack, "Stage in"); - } - else if (c == Status.STAGE_OUT) { - RuntimeStats.setProgress(stack, "Stage out"); - } - else if (c == Status.ACTIVE) { - RuntimeStats.setProgress(stack, "Active"); - getReplicationManager(stack).active(task, e.getStatus().getTime()); - ((FutureVariableArguments) A_REPLICATION_CHANNEL.getValue(stack)).close(); - } - else if (e.getStatus().isTerminal()) { - getReplicationManager(stack).terminated(task); - } - else if (c == ReplicationManager.STATUS_NEEDS_REPLICATION) { - RuntimeStats.setProgress(stack, "Replicating"); - ((FutureVariableArguments) A_REPLICATION_CHANNEL.getValue(stack)).append(Boolean.TRUE); - } - } - } - catch (ExecutionException ex) { - logger.warn(ex); - } - super.statusChanged(e); + public SwiftTaskStateFuture(Stack stack, Task task, boolean taskHasListener) { + super(stack, task, taskHasListener); + } + + public void statusChanged(StatusEvent e) { + Task task = (Task) e.getSource(); + Stack stack = getStack(); + try { + if (stack != null) { + int c = e.getStatus().getStatusCode(); + ProgressState ps = progress.getValue(stack); + if (c == Status.SUBMITTED) { + ps.setState("Submitted"); + getReplicationManager(stack).submitted(task, e.getStatus().getTime()); + } + else if (c == Status.STAGE_IN) { + ps.setState("Stage in"); + } + else if (c == Status.STAGE_OUT) { + ps.setState("Stage out"); + } + else if (c == Status.ACTIVE) { + ps.setState("Active"); + getReplicationManager(stack).active(task, e.getStatus().getTime()); + Execute.this.replicationChannel.getValue(stack).close(); + } + else if (e.getStatus().isTerminal()) { + getReplicationManager(stack).terminated(task); + } + else if (c == ReplicationManager.STATUS_NEEDS_REPLICATION) { + ps.setState("Replicating"); + Execute.this.replicationChannel.getValue(stack).add(Boolean.TRUE); + } + } + } + catch (ExecutionException ex) { + logger.warn(ex); + } + super.statusChanged(e); + } } - protected ReplicationManager getReplicationManager(VariableStack stack) throws ExecutionException { - synchronized (stack.firstFrame()) { - ReplicationManager rm = (ReplicationManager) stack.firstFrame().getVar( - REPLICATION_MANAGER); + protected ReplicationManager getReplicationManager(Stack stack) throws ExecutionException { + Context ctx = this.context.getValue(stack); + synchronized (ctx) { + ReplicationManager rm = (ReplicationManager) ctx.getAttribute("#replicationManager"); if (rm == null) { rm = new ReplicationManager(getScheduler(stack)); - stack.firstFrame().setVar(REPLICATION_MANAGER, rm); + ctx.setAttribute("#replicationManager", rm); } return rm; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -19,48 +19,47 @@ import java.util.ArrayList; import java.util.Comparator; -import java.util.Iterator; import java.util.Map; -import java.util.Set; -import java.util.TreeSet; +import java.util.SortedMap; +import java.util.TreeMap; +import k.rt.Channel; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.ArrayDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.PathElementComparator; -public class ExpandArguments extends VDLFunction { +public class ExpandArguments extends SwiftFunction { public static final Logger logger = Logger.getLogger(ExpandArguments.class); + + private ChannelRef c_vargs; - static { - setArguments(ExpandArguments.class, new Arg[] { Arg.VARGS }); - } + @Override + protected Signature getSignature() { + return new Signature(params("...")); + } - public Object function(VariableStack stack) throws ExecutionException { - ArrayList l = new ArrayList(); - Object[] items = Arg.VARGS.asArray(stack); - for (int i = 0; i < items.length; i++) { - Object item = items[i]; + public Object function(Stack stack) { + ArrayList l = new ArrayList(); + Channel items = c_vargs.get(stack); + for (Object item : items) { if(!(item instanceof DSHandle)) { - throw new RuntimeException("Cannot handle argument implemented by "+item.getClass()); + throw new RuntimeException("Cannot handle argument implemented by " + item.getClass()); } - if(item instanceof ArrayDataNode) { + if (item instanceof ArrayDataNode) { ArrayDataNode array = (ArrayDataNode) item; - Map m=array.getArrayValue(); - Set keySet = m.keySet(); - TreeSet> sortedKeySet = new TreeSet>(new PathElementComparator()); - sortedKeySet.addAll(keySet); - Iterator it = sortedKeySet.iterator(); - while(it.hasNext()) { - Object key = it.next(); - l.add(m.get(key)); - } - } else { - l.add(item); + Map, DSHandle> m = array.getArrayValue(); + SortedMap, DSHandle> sorted = new TreeMap, DSHandle>(new PathElementComparator()); + sorted.putAll(m); + l.addAll(m.values()); + } + else { + l.add((DSHandle) item); } // TODO this does not correctly handle structs or // externals - at the moment, probably neither of @@ -68,11 +67,11 @@ // does not handle nested arrays. However, none of // those should get here in normal operation due // to static type-checking - } + } return l; } - class StringsAsIntegersComparator implements Comparator { + class StringsAsIntegersComparator implements Comparator { public int compare(Object l, Object r) { Integer lnum = new Integer((String)l); Integer rnum = new Integer((String)r); Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/FileCopier.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/FileCopier.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/FileCopier.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,8 +17,8 @@ package org.griphyn.vdl.karajan.lib; -import java.util.LinkedList; -import java.util.List; +import k.rt.AbstractFuture; +import k.rt.Future; import org.globus.cog.abstraction.impl.common.StatusEvent; import org.globus.cog.abstraction.impl.common.task.FileTransferSpecificationImpl; @@ -35,21 +35,14 @@ import org.globus.cog.abstraction.interfaces.Status; import org.globus.cog.abstraction.interfaces.StatusListener; import org.globus.cog.abstraction.interfaces.TaskHandler; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.futures.Future; -import org.globus.cog.karajan.workflow.futures.FutureEvaluationException; -import org.globus.cog.karajan.workflow.futures.FutureListener; -import org.globus.cog.karajan.workflow.futures.FuturesMonitor; -import org.globus.cog.karajan.workflow.futures.ListenerStackPair; +import org.globus.cog.karajan.futures.FutureEvaluationException; import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.PhysicalFormat; -public class FileCopier implements Future, StatusListener { +public class FileCopier extends AbstractFuture implements Future, StatusListener { private static final TaskHandler fth = new FileTransferTaskHandler(); private FileTransferTask task; - private List actions; private Exception exception; private boolean closed; @@ -72,46 +65,9 @@ task.addStatusListener(this); } - public synchronized void addModificationAction(FutureListener target, - VariableStack stack) { - if (actions == null) { - actions = new LinkedList(); - } - ListenerStackPair etp = new ListenerStackPair(target, stack); - if (FuturesMonitor.debug) { - FuturesMonitor.monitor.add(etp, this); - } - synchronized (actions) { - actions.add(etp); - } - if (closed) { - actions(); - } - } - - public List getModificationActions() { - return actions; - } - - private void actions() { - if (actions != null) { - synchronized (actions) { - java.util.Iterator i = actions.iterator(); - while (i.hasNext()) { - ListenerStackPair etp = i.next(); - if (FuturesMonitor.debug) { - FuturesMonitor.monitor.remove(etp); - } - i.remove(); - etp.listener.futureModified(this, etp.stack); - } - } - } - } - public void fail(FutureEvaluationException e) { this.exception = e; - actions(); + notifyListeners(); } public Object getValue() { @@ -130,7 +86,7 @@ public void close() { closed = true; - actions(); + notifyListeners(); } public void statusChanged(StatusEvent event) { Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/FileName.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/FileName.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/FileName.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,16 +20,24 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.Stack; -public class FileName extends VDLFunction { - static { - setArguments(FileName.class, new Arg[] { PA_VAR }); - } +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.mapping.DSHandle; - public Object function(VariableStack stack) throws ExecutionException { - return argList(filename(stack), true); +public class FileName extends SwiftFunction { + private ArgRef var; + private ChannelRef cr_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("var")); + } + + @Override + public Object function(Stack stack) { + return argList(filename(var.getValue(stack)), true); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Flatten.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Flatten.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Flatten.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,21 +22,24 @@ import java.util.List; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.VariableArguments; -import org.globus.cog.karajan.stack.VariableStack; +import k.rt.Channel; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -public class Flatten extends VDLFunction { +public class Flatten extends SwiftFunction { + private ChannelRef c_vargs; - static { - setArguments(Flatten.class, new Arg[] { Arg.VARGS }); + @Override + protected Signature getSignature() { + return new Signature(params("...")); } @Override - protected Object function(VariableStack stack) throws ExecutionException { - VariableArguments v = Arg.VARGS.get(stack); + public Object function(Stack stack) { + Channel v = c_vargs.get(stack); if (v.isEmpty()) { return ""; } @@ -48,10 +51,10 @@ } } - private void flatten(StringBuilder sb, List l) { + private void flatten(StringBuilder sb, List l) { for (Object o : l) { if (o instanceof List) { - flatten(sb, (List) o); + flatten(sb, (List) o); } else { sb.append(TypeUtil.toString(o)); Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/FringePaths.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/FringePaths.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/FringePaths.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,34 +22,41 @@ import java.util.Collection; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.futures.Future; -import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; +import k.rt.ExecutionException; +import k.rt.Future; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.futures.FutureNotYetAvailable; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidPathException; +import org.griphyn.vdl.mapping.Path; -public class FringePaths extends VDLFunction { +public class FringePaths extends SwiftFunction { + private ArgRef var; + private ArgRef path; + + @Override + protected Signature getSignature() { + return new Signature(params("var", optional("path", Path.EMPTY_PATH))); + } - static { - setArguments(FringePaths.class, new Arg[] { PA_VAR, OA_PATH }); - } - - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); DSHandle root = var.getRoot(); try { - var = var.getField(parsePath(OA_PATH.getValue(stack), stack)); - Collection c; + var = var.getField(parsePath(path.getValue(stack))); + Collection c; synchronized(root) { c = var.getFringePaths(); } return c; } catch (InvalidPathException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } catch (HandleOpenException e) { throw new FutureNotYetAvailable((Future) e.getSource()); Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,34 +17,37 @@ package org.griphyn.vdl.karajan.lib; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.PairIterator; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.PairSet; import org.griphyn.vdl.mapping.ArrayDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; -public class GetArrayIterator extends VDLFunction { +public class GetArrayIterator extends SwiftFunction { public static final Logger logger = Logger.getLogger(GetArrayIterator.class); - static { - setArguments(GetArrayIterator.class, new Arg[] { PA_VAR, OA_PATH }); - } + private ArgRef var; + private ArgRef path; + @Override + protected Signature getSignature() { + return new Signature(params("var", optional("path", Path.EMPTY_PATH))); + } + /** * Takes a supplied variable and path, and returns an array iterator. */ - public Object function(VariableStack stack) throws ExecutionException { - Object var1 = PA_VAR.getValue(stack); - if (!(var1 instanceof DSHandle)) { - return var1; - } - DSHandle var = (DSHandle) var1; + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); try { - Path path = parsePath(OA_PATH.getValue(stack), stack); + Path path = parsePath(this.path.getValue(stack)); if (path.hasWildcards()) { throw new RuntimeException("Wildcards not supported"); } @@ -58,7 +61,7 @@ if (logger.isDebugEnabled()) { logger.debug("Using closed iterator for " + var); } - return new PairIterator(var.getArrayValue()); + return new PairSet(var.getArrayValue()); } else { if (logger.isDebugEnabled()) { @@ -70,8 +73,7 @@ } } catch (InvalidPathException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } - } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/GetDatasetProvenanceID.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/GetDatasetProvenanceID.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/GetDatasetProvenanceID.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,19 +17,23 @@ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.DSHandle; -public class GetDatasetProvenanceID extends VDLFunction { +public class GetDatasetProvenanceID extends SwiftFunction { + private ArgRef var; - static { - setArguments(GetDatasetProvenanceID.class, new Arg[] { PA_VAR }); - } + @Override + protected Signature getSignature() { + return new Signature(params("var")); + } - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); return var.getIdentifier(); } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/GetField.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/GetField.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/GetField.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,9 +20,11 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; @@ -30,28 +32,26 @@ /** * Obtain the DSHandle from within another DSHandle via the given PATH * */ -public class GetField extends VDLFunction { - static { - setArguments(GetField.class, new Arg[] { OA_PATH, PA_VAR }); - } +public class GetField extends SwiftFunction { + private ArgRef var; + private ArgRef path; + + @Override + protected Signature getSignature() { + return new Signature(params("var", optional("path", Path.EMPTY_PATH))); + } - public Object function(VariableStack stack) throws ExecutionException { - Object var1 = PA_VAR.getValue(stack); + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); - if(var1 instanceof DSHandle) { - - try { - DSHandle var = (DSHandle) var1; - - Path path = parsePath(OA_PATH.getValue(stack), stack); - DSHandle field = var.getField(path); - return field; - } - catch (InvalidPathException e) { - throw new ExecutionException(e); - } - } else { - throw new ExecutionException("was expecting a DSHandle, got: "+var1.getClass()); + try { + Path path = parsePath(this.path.getValue(stack)); + DSHandle field = var.getField(path); + return field; } + catch (InvalidPathException e) { + throw new ExecutionException(this, e); + } } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/GetFieldSubscript.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/GetFieldSubscript.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/GetFieldSubscript.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -19,31 +19,31 @@ import java.util.Collection; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; -public class GetFieldSubscript extends VDLFunction { +public class GetFieldSubscript extends SwiftFunction { + private ArgRef var; + private ArgRef subscript; - public static final SwiftArg PA_SUBSCRIPT = new SwiftArg.Positional("subscript"); + @Override + protected Signature getSignature() { + return new Signature(params("var", "subscript")); + } - static { - setArguments(GetFieldSubscript.class, new Arg[] { PA_VAR, PA_SUBSCRIPT }); - } + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); - public Object function(VariableStack stack) throws ExecutionException { - Object var1 = PA_VAR.getValue(stack); - if(!(var1 instanceof DSHandle)) { - throw new ExecutionException("was expecting a dshandle, got: "+var1.getClass()); - } - DSHandle var = (DSHandle) var1; + Object index = this.subscript.getValue(stack); - Object index = PA_SUBSCRIPT.getValue(stack); - try { Path path; if ("*".equals(index)) { @@ -61,10 +61,10 @@ } } catch (InvalidPathException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } catch (HandleOpenException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,35 +20,37 @@ */ package org.griphyn.vdl.karajan.lib; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.futures.FutureFault; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.futures.FutureFault; import org.griphyn.vdl.mapping.AbstractDataNode; -import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.Path; -public class GetFieldValue extends VDLFunction { +public class GetFieldValue extends SwiftFunction { public static final Logger logger = Logger.getLogger(GetFieldValue.class); + + private ArgRef var; + private ArgRef path; + + @Override + protected Signature getSignature() { + return new Signature(params("var", optional("path", Path.EMPTY_PATH))); + } - static { - setArguments(GetFieldValue.class, new Arg[] { PA_VAR, OA_PATH }); - } - /** * Takes a supplied variable and path, and returns the unique value at that * path. Path can contain wildcards, in which case an array is returned. */ - public Object function(VariableStack stack) throws ExecutionException { - Object var1 = PA_VAR.getValue(stack); - if (!(var1 instanceof DSHandle)) { - return var1; - } - AbstractDataNode var = (AbstractDataNode) var1; + @Override + public Object function(Stack stack) { + AbstractDataNode var = this.var.getValue(stack); try { - Path path = parsePath(OA_PATH.getValue(stack), stack); + Path path = parsePath(this.path.getValue(stack)); if (path.hasWildcards()) { return var.getFields(path).toArray(); } @@ -57,7 +59,7 @@ if (var.getType().isArray()) { throw new RuntimeException("Getting value for array " + var + " which is not permitted."); } - var.waitFor(); + var.waitFor(this); return var.getValue(); } } @@ -65,7 +67,7 @@ throw f; } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,30 +20,43 @@ */ package org.griphyn.vdl.karajan.lib; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.List; +import k.rt.Context; +import k.rt.Stack; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import org.globus.cog.karajan.analyzer.Param; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; import org.griphyn.vdl.karajan.functions.ConfigProperty; +import org.griphyn.vdl.util.VDL2Config; -public class GetURLPrefix extends AbstractSequentialWithArguments { +public class GetURLPrefix extends AbstractSingleValuedFunction { + private VarRef context; + private VarRef cwd; @Override - protected void post(VariableStack stack) throws ExecutionException { + protected Param[] getParams() { + return params(); + } + + @Override + protected void addLocals(Scope scope) { + super.addLocals(scope); + context = scope.getVarRef("#context"); + cwd = scope.getVarRef("CWD"); + } + + @Override + public Object function(Stack stack) { + Context ctx = this.context.getValue(stack); + String localServerBase = ConfigProperty.getProperty("wrapper.staging.local.server", + (VDL2Config) ctx.getAttribute("SWIFT:CONFIG")); + String cwd = this.cwd.getValue(stack); - String localServerBase = ConfigProperty.getProperty("wrapper.staging.local.server", stack); - - String cwd = stack.getExecutionContext().getCwd(); if (cwd.endsWith("/.")) { cwd = cwd.substring(0, cwd.length() - 2); } - ret(stack, localServerBase + cwd); - - super.post(stack); + return localServerBase + cwd; } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/InFileDirs.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,28 +22,33 @@ import java.util.List; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.ArgUtil; -import org.globus.cog.karajan.arguments.VariableArguments; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import k.rt.Channel; +import k.rt.Stack; +import k.thr.LWThread; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.griphyn.vdl.mapping.AbsFile; -public class InFileDirs extends AbstractSequentialWithArguments { - public static final Arg STAGEINS = new Arg.Positional("stageins"); +public class InFileDirs extends InternalFunction { - static { - setArguments(InFileDirs.class, new Arg[] { STAGEINS }); + private ArgRef> stageins; + private ChannelRef cr_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("stageins"), returns(channel("...", DYNAMIC))); } + @Override - protected void post(VariableStack stack) throws ExecutionException { - List files = TypeUtil.toList(STAGEINS.getValue(stack)); - VariableArguments ret = ArgUtil.getVariableReturn(stack); - for (Object f : files) { - String path = (String) f; + protected void runBody(LWThread thr) { + Stack stack = thr.getStack(); + List files = stageins.getValue(stack); + Channel ret = cr_vargs.get(stack); + for (String path : files) { AbsFile af = new AbsFile(path); if ("file".equals(af.getProtocol())) { String dir = af.getDir(); @@ -52,17 +57,16 @@ // as "a/b/c.txt". Perhaps absolute paths // should have a unique prefix. if (dir.startsWith("/") && dir.length() != 1) { - ret.append(dir.substring(1)); + ret.add(dir.substring(1)); } else if (dir.length() != 0) { - ret.append(dir); + ret.add(dir); } } else { // also prepend host name to the path - ret.append(af.getHost() + "/" + af.getDir()); + ret.add(af.getHost() + "/" + af.getDir()); } } - super.post(stack); } } Deleted: branches/faster/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,125 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.griphyn.vdl.karajan.lib; - -import java.util.Collections; -import java.util.List; - -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.ThreadingContext; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.FlowElement; -import org.globus.cog.karajan.workflow.nodes.Sequential; -import org.globus.cog.karajan.workflow.nodes.While; -import org.griphyn.vdl.mapping.RootDataNode; -import org.griphyn.vdl.type.Types; - -public class InfiniteCountingWhile extends Sequential { - - public static final String COUNTER_NAME = "$"; - public static final Arg.Positional VAR = new Arg.Positional("var"); - - private Tracer tracer; - - public InfiniteCountingWhile() { - setOptimize(false); - } - - @Override - protected void initializeStatic() { - super.initializeStatic(); - tracer = Tracer.getTracer(this); - } - - public void pre(VariableStack stack) throws ExecutionException { - ThreadingContext tc = (ThreadingContext)stack.getVar("#thread"); - stack.setVar("#iteratethread", tc); - stack.setVar("#thread", tc.split(0)); - stack.setVar(COUNTER_NAME, Collections.singletonList(0)); - String var = (String) VAR.getStatic(this); - if (tracer.isEnabled()) { - tracer.trace(tc.toString(), var + " = 0"); - } - stack.setVar(var, new RootDataNode(Types.INT, 0)); - super.pre(stack); - } - - protected void startNext(VariableStack stack) throws ExecutionException { - if (stack.isDefined("#abort")) { - abort(stack); - return; - } - int index = getIndex(stack); - if (elementCount() == 0) { - post(stack); - return; - } - FlowElement fn = null; - - if (index == elementCount() - 1) { - // the condition is always compiled as the last thing in the loop - // but the increment needs to happen before the condition is - // evaluated - @SuppressWarnings("unchecked") - List c = (List) stack.getVar(COUNTER_NAME); - int i = c.get(0).intValue(); - i++; - ThreadingContext tc = (ThreadingContext)stack.getVar("#iteratethread"); - ThreadingContext ntc = tc.split(i); - stack.setVar("#thread", ntc); - stack.setVar(COUNTER_NAME, Collections.singletonList(i)); - String var = (String) VAR.getStatic(this); - if (tracer.isEnabled()) { - tracer.trace(ntc.toString(), var + " = " + i); - } - stack.setVar(var, new RootDataNode(Types.INT, i)); - } - if (index >= elementCount()) { - // starting new iteration - setIndex(stack, 1); - fn = getElement(0); - } - else { - fn = getElement(index++); - setIndex(stack, index); - } - startElement(fn, stack); - } - - public void failed(VariableStack stack, ExecutionException e) - throws ExecutionException { - if (e instanceof While.Break) { - complete(stack); - return; - } - if (e instanceof While.Continue) { - setIndex(e.getStack(), 0); - startNext(e.getStack()); - return; - } - super.failed(stack, e); - } - - @Override - public String getTextualName() { - return "iterate"; - } - - -} Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/IsDone.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/IsDone.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/IsDone.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,33 +22,36 @@ import java.util.List; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.restartLog.LogChannelOperator; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.Path; -public class IsDone extends VDLFunction { - public static final Arg STAGEOUT = new Arg.Positional("stageout"); +public class IsDone extends SwiftFunction { + private ArgRef>> stageout; - static { - setArguments(IsDone.class, new Arg[] { STAGEOUT }); - } + private ChannelRef cr_restartLog; @Override - protected Object function(VariableStack stack) throws ExecutionException { - List files = TypeUtil.toList(STAGEOUT.getValue(stack)); - for (Object f : files) { - List pv = TypeUtil.toList(f); + protected Signature getSignature() { + return new Signature(params("stageout"), returns(channel("...", 1), channel("restartLog"))); + } + + @Override + public Object function(Stack stack) { + Iterable> files = stageout.getValue(stack); + for (List pv : files) { Path p = (Path) pv.get(0); DSHandle handle = (DSHandle) pv.get(1); - if (!IsLogged.isLogged(stack, handle, p)) { + if (!IsLogged.isLogged((LogChannelOperator) cr_restartLog.get(stack), handle, p)) { return Boolean.FALSE; } } - if (files.isEmpty()) { + if (!files.iterator().hasNext()) { return Boolean.FALSE; } return Boolean.TRUE; Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/IsFileBound.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/IsFileBound.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/IsFileBound.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,19 +20,24 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; -public class IsFileBound extends VDLFunction { - static { - setArguments(IsFileBound.class, new Arg[] { PA_VAR }); - } +public class IsFileBound extends SwiftFunction { + private ArgRef var; + + @Override + protected Signature getSignature() { + return new Signature(params("var")); + } - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); if (var instanceof AbstractDataNode) { return Boolean.valueOf(!((AbstractDataNode) var).isPrimitive()); } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/IsLogged.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/IsLogged.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/IsLogged.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -23,40 +23,51 @@ import java.util.List; import java.util.Map; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.restartLog.LogChannelOperator; +import org.globus.cog.karajan.compiled.nodes.restartLog.LogEntry; import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.restartLog.LogEntry; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.Path; -public class IsLogged extends VDLFunction { - static { - setArguments(IsLogged.class, new Arg[] { PA_VAR, PA_PATH }); - } +public class IsLogged extends SwiftFunction { + private ArgRef var; + private ArgRef path; + + private ChannelRef cr_restartLog; + + @Override + protected Signature getSignature() { + return new Signature(params("var", "path"), returns(channel("restartLog"))); + } - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); Path path; - Object p = PA_PATH.getValue(stack); + Object p = this.path.getValue(stack); if (p instanceof Path) { path = (Path) p; } else { path = Path.parse(TypeUtil.toString(p)); } - return Boolean.valueOf(isLogged(stack, var, path)); + return Boolean.valueOf(isLogged((LogChannelOperator) cr_restartLog.get(stack), var, path)); } - public static boolean isLogged(VariableStack stack, DSHandle var, Path path) throws ExecutionException { + public static boolean isLogged(LogChannelOperator log, DSHandle var, Path path) throws ExecutionException { + Map logData = log.getLogData(); path = var.getPathFromRoot().append(path); LogEntry entry = LogEntry.build(var.getRoot().getParam(MappingParam.SWIFT_RESTARTID) + "." + path.stringForm()); - Map map = getLogData(stack); boolean found = false; - synchronized (map) { - List files = (List) map.get(entry); + synchronized (logData) { + List files = (List) logData.get(entry); if (files != null && !files.isEmpty()) { found = true; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/IsRestartable.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/IsRestartable.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/IsRestartable.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,9 +17,10 @@ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.DSHandle; /** Determines if a variable is 'restartable'; that is, if we restart the @@ -28,14 +29,18 @@ -public class IsRestartable extends VDLFunction { - static { - setArguments(IsRestartable.class, new Arg[] { PA_VAR }); - } +public class IsRestartable extends SwiftFunction { + private ArgRef var; + + @Override + protected Signature getSignature() { + return new Signature(params("var")); + } - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); - return Boolean.valueOf(var.isRestartable()); - } + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); + return Boolean.valueOf(var.isRestartable()); + } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/JobConstraints.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/JobConstraints.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/JobConstraints.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,38 +20,46 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; +import java.util.Collection; + +import k.rt.Stack; +import k.thr.LWThread; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.scheduler.TaskConstraints; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import org.griphyn.vdl.karajan.lib.cache.CacheMapAdapter; import org.griphyn.vdl.util.FQN; -import java.util.Collection; -import org.griphyn.vdl.karajan.lib.cache.CacheMapAdapter; -public class JobConstraints extends VDLFunction { - public static final Arg A_TR = new Arg.Positional("tr"); - public static final Arg STAGE_IN = new Arg.Optional("stagein"); +public class JobConstraints extends CacheFunction { + private ArgRef tr; + private ArgRef> stagein; + private ChannelRef cr_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("tr", optional("stagein", null)), returns(channel("...", 1))); + } - static { - setArguments(JobConstraints.class, new Arg[] { A_TR, STAGE_IN }); - } - private static final String[] STRING_ARRAY = new String[0]; - public Object function(VariableStack stack) throws ExecutionException { - String tr = TypeUtil.toString(A_TR.getValue(stack)); + @Override + public void runBody(LWThread thr) { + Stack stack = thr.getStack(); + String tr = this.tr.getValue(stack); String[] filenames = null; - if (STAGE_IN.isPresent(stack)) { - filenames = (String[]) ((Collection) STAGE_IN.getValue(stack)).toArray(STRING_ARRAY); + Collection c = this.stagein.getValue(stack); + if (c != null) { + filenames = c.toArray(STRING_ARRAY); } TaskConstraints tc = new TaskConstraints(); tc.addConstraint("tr", tr); tc.addConstraint("trfqn", new FQN(tr)); if (filenames != null) { tc.addConstraint("filenames", filenames); - tc.addConstraint("filecache", new CacheMapAdapter(CacheFunction.getCache(stack))); + tc.addConstraint("filecache", new CacheMapAdapter(getCache(stack))); } - return tc; + cr_vargs.append(stack, tc); } } Deleted: branches/faster/src/org/griphyn/vdl/karajan/lib/Kickstart.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Kickstart.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Kickstart.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,66 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Dec 26, 2006 - */ -package org.griphyn.vdl.karajan.lib; - -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.BoundContact; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.functions.ConfigProperty; -import org.griphyn.vdl.util.TriStateBoolean; -import org.griphyn.vdl.util.VDL2ConfigProperties; - -public class Kickstart extends VDLFunction { - public static final String PROPERTY_GRIDLAUNCH = "gridlaunch"; - - public static final Arg A_HOST = new Arg.Positional("host"); - - static { - setArguments(Kickstart.class, new Arg[] { A_HOST }); - } - - public static final String NOTHING = ""; - - public Object function(VariableStack stack) throws ExecutionException { - String enabled = ConfigProperty.getProperty(VDL2ConfigProperties.KICKSTART_ENABLED, stack); - TriStateBoolean tbs = TriStateBoolean.valueOf(enabled); - if (tbs.equals(TriStateBoolean.FALSE)) { - return NOTHING; - } - else { - BoundContact host = (BoundContact) A_HOST.getValue(stack); - String kickstart = (String) host.getProperty(PROPERTY_GRIDLAUNCH); - if (kickstart == null) { - if (tbs.equals(TriStateBoolean.MAYBE)) { - return NOTHING; - } - else { - throw new ExecutionException( - "The \"kickstart.enable\" option is set to \"true\" but Kickstart is not installed on " - + host); - } - } - else { - return kickstart; - } - } - } -} Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Log.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Log.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Log.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -23,22 +23,26 @@ import java.util.HashMap; import java.util.Map; +import k.rt.Channel; +import k.rt.Stack; +import k.thr.LWThread; + import org.apache.log4j.Level; import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; -public class Log extends AbstractSequentialWithArguments { - public static final Arg LEVEL = new Arg.Positional("level"); - public static final Arg MESSAGE = new Arg.Optional("message", null); +public class Log extends InternalFunction { + private ArgRef level; + private ChannelRef c_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("level", "...")); + } - static { - setArguments(Log.class, new Arg[] { LEVEL, MESSAGE, Arg.VARGS }); - } - public static final Logger logger = Logger.getLogger("swift"); private static final Map priorities = new HashMap(); @@ -49,27 +53,21 @@ priorities.put("error", Level.ERROR); priorities.put("fatal", Level.FATAL); } - - public static Level getLevel(String lvl) { + + public static Level getLevel(String lvl) { return priorities.get(lvl); } - protected void post(VariableStack stack) throws ExecutionException { - Level lvl = getLevel((String) LEVEL.getValue(stack)); + protected void runBody(LWThread thr) { + Stack stack = thr.getStack(); + Level lvl = getLevel(this.level.getValue(stack)); if (logger.isEnabledFor(lvl)) { - Object smsg = MESSAGE.getValue(stack); - if (smsg != null) { - logger.log(lvl, smsg); + Channel l = c_vargs.get(stack); + StringBuilder sb = new StringBuilder(); + for (Object o : l) { + sb.append(o); } - else { - Object[] msg = Arg.VARGS.asArray(stack); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < msg.length; i++) { - sb.append(TypeUtil.toString(msg[i])); - } - logger.log(lvl, sb.toString()); - } + logger.log(lvl, sb.toString()); } - super.post(stack); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/LogVar.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/LogVar.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/LogVar.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,37 +20,44 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableNotFoundException; -import org.globus.cog.karajan.stack.VariableStack; +import k.rt.Channel; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VariableNotFoundException; import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.restartLog.RestartLog; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.Path; -public class LogVar extends VDLFunction { +public class LogVar extends SwiftFunction { + private ArgRef var; + private ArgRef path; + private ChannelRef cr_restartlog; + + @Override + protected Signature getSignature() { + return new Signature(params("var", "path"), returns(channel("restartlog", 1))); + } - static { - setArguments(LogVar.class, new Arg[] { PA_VAR, PA_PATH }); - } - - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); Path path; - Object p = PA_PATH.getValue(stack); + Object p = this.path.getValue(stack); if (p instanceof Path) { path = (Path) p; } else { path = Path.parse(TypeUtil.toString(p)); } - logVar(stack, var, path); + logVar(cr_restartlog.get(stack), var, path); return null; } - public static void logVar(VariableStack stack, DSHandle var, Path path) throws VariableNotFoundException { + public static void logVar(Channel log, DSHandle var, Path path) throws VariableNotFoundException { path = var.getPathFromRoot().append(path); String annotation; if(var.getMapper() != null) { @@ -58,7 +65,7 @@ } else { annotation = "unmapped"; } - RestartLog.LOG_CHANNEL.ret(stack, var.getRoot().getParam(MappingParam.SWIFT_RESTARTID) + log.add(var.getRoot().getParam(MappingParam.SWIFT_RESTARTID) + "." + path.stringForm() + "!" + annotation); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Mark.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Mark.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Mark.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,34 +22,34 @@ import java.util.List; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.DataDependentException; -import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.MappingDependentException; import org.griphyn.vdl.mapping.Path; -public class Mark extends VDLFunction { - public static final Arg RESTARTS = new Arg.Positional("restarts"); - public static final Arg ERR = new Arg.Positional("err"); - public static final Arg MAPPING = new Arg.Optional("mapping", Boolean.FALSE); - - static { - setArguments(Mark.class, new Arg[] { RESTARTS, ERR, MAPPING }); +public class Mark extends SwiftFunction { + private ArgRef>> restarts; + private ArgRef err; + private ArgRef mapping; + + @Override + protected Signature getSignature() { + return new Signature(params("restarts", "err", optional("mapping", Boolean.FALSE))); } @Override - protected Object function(VariableStack stack) throws ExecutionException { + public Object function(Stack stack) { try { - if (TypeUtil.toBoolean(ERR.getValue(stack))) { - boolean mapping = TypeUtil.toBoolean(MAPPING.getValue(stack)); - List files = TypeUtil.toList(RESTARTS.getValue(stack)); - for (Object f : files) { - List pv = TypeUtil.toList(f); - Path p = parsePath(pv.get(0), stack); + if (err.getValue(stack)) { + boolean mapping = this.mapping.getValue(stack); + List> files = this.restarts.getValue(stack); + for (List pv : files) { + Path p = parsePath(pv.get(0)); DSHandle handle = (DSHandle) pv.get(1); DSHandle leaf = handle.getField(p); synchronized (leaf) { @@ -65,7 +65,7 @@ } } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } return null; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/New.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/New.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/New.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -24,18 +24,25 @@ import java.util.List; import java.util.Map; +import k.rt.Context; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.VDL2ExecutionContext; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.DuplicateMappingChecker; import org.griphyn.vdl.mapping.ExternalDataNode; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.MappingParamSet; +import org.griphyn.vdl.mapping.OOBYield; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.RootArrayDataNode; import org.griphyn.vdl.mapping.RootDataNode; @@ -43,40 +50,55 @@ import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; -public class New extends VDLFunction { - +public class New extends SwiftFunction { public static final Logger logger = Logger.getLogger(New.class); - - public static final Arg OA_TYPE = new Arg.Optional("type", null); - public static final Arg OA_MAPPING = new Arg.Optional("mapping", null); - public static final Arg OA_VALUE = new Arg.Optional("value", null); - public static final Arg OA_DBGNAME = new Arg.Optional("dbgname", null); - public static final Arg OA_WAITCOUNT = new Arg.Optional("waitcount", null); - public static final Arg OA_INPUT = new Arg.Optional("input", Boolean.FALSE); - - static { - setArguments(New.class, - new Arg[] { OA_TYPE, OA_MAPPING, OA_VALUE, OA_DBGNAME, OA_WAITCOUNT, OA_INPUT}); + + private ArgRef type; + private ArgRef> mapping; + private ArgRef value; + private ArgRef dbgname; + private ArgRef waitCount; + private ArgRef input; + private ArgRef _defline; + + private VarRef context; + private VarRef cwd; + + @Override + protected Signature getSignature() { + return new Signature(params("type", optional("mapping", null), optional("value", null), + optional("dbgname", null), optional("waitCount", null), optional("input", Boolean.FALSE), optional("_defline", null))); } private Tracer tracer; - @Override - protected void initializeStatic() { - super.initializeStatic(); + @Override + protected void addLocals(Scope scope) { + super.addLocals(scope); + context = scope.getVarRef("#context"); + cwd = scope.getVarRef("CWD"); + } + + @Override + public Node compile(WrapperNode w, Scope scope) + throws CompilationException { + Node fn = super.compile(w, scope); + if (_defline.getValue() != null) { + setLine(Integer.parseInt(_defline.getValue())); + } tracer = Tracer.getTracer(this); + return fn; } - public Object function(VariableStack stack) throws ExecutionException { - String typename = TypeUtil.toString(OA_TYPE.getValue(stack)); - Object value = OA_VALUE.getValue(stack); - @SuppressWarnings("unchecked") - Map mapping = - (Map) OA_MAPPING.getValue(stack); - String dbgname = TypeUtil.toString(OA_DBGNAME.getValue(stack)); - String waitCount = (String) OA_WAITCOUNT.getValue(stack); - boolean input = TypeUtil.toBoolean(OA_INPUT.getValue(stack)); - String line = (String) getProperty("_defline"); + @Override + public Object function(Stack stack) { + String typename = this.type.getValue(stack); + Object value = this.value.getValue(stack); + Map mapping = this.mapping.getValue(stack); + String dbgname = this.dbgname.getValue(stack); + Number waitCount = this.waitCount.getValue(stack); + boolean input = this.input.getValue(stack); + String line = this._defline.getValue(stack); MappingParamSet mps = new MappingParamSet(); mps.setAll(mapping); @@ -93,7 +115,7 @@ mps.set(MappingParam.SWIFT_LINE, line); } - String threadPrefix = getThreadPrefix(stack); + String threadPrefix = getThreadPrefix(); mps.set(MappingParam.SWIFT_RESTARTID, threadPrefix + ":" + dbgname); @@ -101,7 +123,7 @@ int initialWriteRefCount; boolean noWriters = input; if (waitCount != null) { - initialWriteRefCount = Integer.parseInt(waitCount); + initialWriteRefCount = waitCount.intValue(); } else { initialWriteRefCount = 0; @@ -115,7 +137,7 @@ if ("concurrent_mapper".equals(mapper)) { mps.set(ConcurrentMapper.PARAM_THREAD_PREFIX, threadPrefix); } - mps.set(MappingParam.SWIFT_BASEDIR, stack.getExecutionContext().getBasedir()); + mps.set(MappingParam.SWIFT_BASEDIR, cwd.getValue(stack)); try { Type type; @@ -134,8 +156,7 @@ } else if (type.isArray()) { // dealing with array variable - handle = new RootArrayDataNode(type, - (DuplicateMappingChecker) stack.getGlobal(VDL2ExecutionContext.DM_CHECKER)); + handle = new RootArrayDataNode(type, getDMChecker(stack)); if (value != null) { if (value instanceof RootArrayDataNode) { if (tracer.isEnabled()) { @@ -184,8 +205,7 @@ handle = (DSHandle) value; } else { - handle = new RootDataNode(type, - (DuplicateMappingChecker) stack.getGlobal(VDL2ExecutionContext.DM_CHECKER)); + handle = new RootDataNode(type, getDMChecker(stack)); handle.init(mps); if (value != null) { if (tracer.isEnabled()) { @@ -206,11 +226,19 @@ handle.setWriteRefCount(initialWriteRefCount); return handle; } + catch (OOBYield y) { + throw y.wrapped(this); + } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } + private DuplicateMappingChecker getDMChecker(Stack stack) { + Context ctx = this.context.getValue(stack); + return (DuplicateMappingChecker) ctx.getAttribute("SWIFT:DM_CHECKER"); + } + private String formatList(List value) { StringBuilder sb = new StringBuilder(); sb.append('['); Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/NiceName.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/NiceName.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/NiceName.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,23 +20,31 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.Path; -public class NiceName extends VDLFunction { - static { - setArguments(NiceName.class, new Arg[] { OA_PATH, PA_VAR }); - } +public class NiceName extends SwiftFunction { + private ArgRef var; + private ArgRef path; - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); + @Override + protected Signature getSignature() { + return new Signature(params("var", optional("path", Path.EMPTY_PATH))); + } + + + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); try { - Path path = parsePath(OA_PATH.getValue(stack), stack); + Path path = parsePath(this.path.getValue(stack)); DSHandle field = var.getField(path); Path p = field.getPathFromRoot(); if (p.equals(Path.EMPTY_PATH)) { @@ -53,7 +61,7 @@ } } catch (InvalidPathException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Operators.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Operators.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Operators.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,82 +17,44 @@ package org.griphyn.vdl.karajan.lib; -import java.io.IOException; +import k.rt.ExecutionException; +import k.thr.LWThread; import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.compiled.nodes.functions.BinaryOp; +import org.globus.cog.karajan.compiled.nodes.functions.UnaryOp; +import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; import org.griphyn.vdl.util.VDL2Config; -public class Operators extends FunctionsCollection { - - public static final SwiftArg L = new SwiftArg.Positional("left"); - public static final SwiftArg R = new SwiftArg.Positional("right"); - public static final SwiftArg U = new SwiftArg.Positional("unaryArg"); - - public static final Logger provenanceLogger = Logger.getLogger("org.globus.swift.provenance.operators"); - - private DSHandle newNum(Type type, double value) throws ExecutionException { - try { - DSHandle handle = new RootDataNode(type); - if (type == Types.INT) { - handle.setValue(Integer.valueOf((int) value)); - } - else { - handle.setValue(new Double(value)); - } - handle.closeShallow(); - return handle; - } - catch (Exception e) { - throw new ExecutionException("Internal error", e); - } - } +public class Operators { - private DSHandle newNum(Type type, int value) throws ExecutionException { - try { - DSHandle handle = new RootDataNode(type); - handle.setValue(new Integer(value)); - handle.closeShallow(); - return handle; - } - catch (Exception e) { - throw new ExecutionException("Internal error", e); - } - } + public static final boolean PROVENANCE_ENABLED; - private DSHandle newString(String value) throws ExecutionException { + static { + boolean v; try { - DSHandle handle = new RootDataNode(Types.STRING); - handle.setValue(value); - handle.closeShallow(); - return handle; + v = VDL2Config.getConfig().getProvenanceLog(); } catch (Exception e) { - throw new ExecutionException("Internal error", e); + v = false; } + PROVENANCE_ENABLED = v; } - private DSHandle newBool(boolean value) throws ExecutionException { - try { - DSHandle handle = new RootDataNode(Types.BOOLEAN); - handle.setValue(new Boolean(value)); - handle.closeShallow(); - return handle; - } - catch (Exception e) { - throw new ExecutionException("Internal error", e); - } - } + public static final Logger provenanceLogger = Logger.getLogger("org.globus.swift.provenance.operators"); - private Type type(VariableStack stack) throws ExecutionException { - if (Types.FLOAT.equals(L.getType(stack)) || Types.FLOAT.equals(R.getType(stack))) { + private static Type type(DSHandle l, DSHandle r) throws ExecutionException { + Type tl = l.getType(); + Type tr = r.getType(); + if (Types.STRING.equals(tl) || Types.STRING.equals(tr)) { + return Types.STRING; + } + else if (Types.FLOAT.equals(tl) || Types.FLOAT.equals(tr)) { return Types.FLOAT; } else { @@ -100,194 +62,231 @@ } } - private static final String[] BINARY_OPERATORS = new String[] { "vdlop_sum", "vdlop_subtraction", - "vdlop_product", "vdlop_quotient", "vdlop_fquotient", "vdlop_iquotient", - "vdlop_remainder", "vdlop_le", "vdlop_ge", "vdlop_lt", "vdlop_gt", "vdlop_eq", "vdlop_ne", "vdlop_and", "vdlop_or" }; - private static final String[] UNARY_OPERATORS = new String[] { "vdlop_not" }; - - private static final Arg[] BINARY_ARGS = new Arg[] { L, R }; - private static final Arg[] UNARY_ARGS = new Arg[] { U }; - - static { - for (int i = 0; i < BINARY_OPERATORS.length; i++) { - setArguments(BINARY_OPERATORS[i], BINARY_ARGS); + private static int getInt(Node n, DSHandle h) { + waitFor(n, h); + Object v = h.getValue(); + if (v instanceof Integer) { + return ((Integer) v).intValue(); } - for (int i = 0; i < UNARY_OPERATORS.length; i++) { - setArguments(UNARY_OPERATORS[i], UNARY_ARGS); - } - } - - public Object vdlop_sum(VariableStack stack) throws ExecutionException { - Object l = L.getValue(stack); - Object r = R.getValue(stack); - Type t = type(stack); - DSHandle ret; - if (l instanceof String || r instanceof String) { - ret = newString(((String) l) + ((String) r)); - } - else if (t == Types.INT) { - ret = newNum(t, SwiftArg.checkInt(l) + SwiftArg.checkInt(r)); - } else { - ret = newNum(t, SwiftArg.checkDouble(l) + SwiftArg.checkDouble(r)); + throw new ExecutionException(n, "Internal error. Expected an int: " + h); } - logBinaryProvenance(stack, "sum", ret); - return ret; } + + private static void waitFor(Node n, DSHandle h) { + ((AbstractDataNode) h).waitFor(n); + } - public Object vdlop_subtraction(VariableStack stack) throws ExecutionException { - double l = L.getDoubleValue(stack); - double r = R.getDoubleValue(stack); - DSHandle ret = newNum(type(stack), l - r); - logBinaryProvenance(stack, "subtraction", ret); - return ret; + private static double getFloat(Node n, DSHandle h) { + waitFor(n, h); + Object v = h.getValue(); + if (v instanceof Number) { + return ((Number) v).doubleValue(); + } + else { + throw new ExecutionException(n, "Internal error. Expected float: " + h); + } + } + + private static boolean getBool(Node n, DSHandle h) { + waitFor(n, h); + Object v = h.getValue(); + if (v instanceof Boolean) { + return ((Boolean) v).booleanValue(); + } + else { + throw new ExecutionException(n, "Internal error. Expected float: " + h); + } + } + + public static class Sum extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + Type t = type(v1, v2); + DSHandle r; + if (t == Types.STRING) { + r = new RootDataNode(Types.STRING, (String.valueOf(v1.getValue()) + String.valueOf(v2.getValue()))); + } + else if (t == Types.INT) { + r = new RootDataNode(Types.INT, getInt(this, v1) + getInt(this, v2)); + } + else { + r = new RootDataNode(Types.FLOAT, getFloat(this, v1) + getFloat(this, v2)); + } + logBinaryProvenance("sum", v1, v2, r); + return r; + } } - - public Object vdlop_product(VariableStack stack) throws ExecutionException { - double l = L.getDoubleValue(stack); - double r = R.getDoubleValue(stack); - DSHandle ret = newNum(type(stack), l * r); - logBinaryProvenance(stack, "product", ret); - return ret; + + public static class Difference extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + Type t = type(v1, v2); + DSHandle r; + if (t == Types.INT) { + r = new RootDataNode(Types.INT, getInt(this, v1) - getInt(this, v2)); + } + else { + r = new RootDataNode(Types.FLOAT, getFloat(this, v1) - getFloat(this, v2)); + } + logBinaryProvenance("difference", v1, v2, r); + return r; + } + } + + public static class Product extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + Type t = type(v1, v2); + DSHandle r; + if (t == Types.INT) { + r = new RootDataNode(Types.INT, getInt(this, v1) * getInt(this, v2)); + } + else { + r = new RootDataNode(Types.FLOAT, getFloat(this, v1) * getFloat(this, v2)); + } + logBinaryProvenance("product", v1, v2, r); + return r; + } + } + + public static class FQuotient extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + DSHandle r = new RootDataNode(Types.FLOAT, getFloat(this, v1) / getFloat(this, v2)); + logBinaryProvenance("fquotient", v1, v2, r); + return r; + } + } + + public static class Quotient extends FQuotient { } - public Object vdlop_quotient(VariableStack stack) throws ExecutionException { - // for now we map to this one - return vdlop_fquotient(stack); - } + public static class IQuotient extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + DSHandle r = new RootDataNode(Types.INT, getInt(this, v1) / getInt(this, v2)); + logBinaryProvenance("iquotient", v1, v2, r); + return r; + } + } - public Object vdlop_fquotient(VariableStack stack) throws ExecutionException { - double l = L.getDoubleValue(stack); - double r = R.getDoubleValue(stack); - DSHandle ret = newNum(Types.FLOAT, l / r); - logBinaryProvenance(stack, "fquotient", ret); - return ret; - } + public static class Remainder extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + Type t = type(v1, v2); + DSHandle r; + if (t == Types.INT) { + r = new RootDataNode(Types.INT, getInt(this, v1) % getInt(this, v2)); + } + else { + r = new RootDataNode(Types.FLOAT, getFloat(this, v1) % getFloat(this, v2)); + } + logBinaryProvenance("remainder", v1, v2, r); + return r; + } + } + + public static class LE extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + DSHandle r = new RootDataNode(Types.BOOLEAN, getFloat(this, v1) <= getFloat(this, v2)); + logBinaryProvenance("le", v1, v2, r); + return r; + } + } + + public static class GE extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + DSHandle r = new RootDataNode(Types.BOOLEAN, getFloat(this, v1) >= getFloat(this, v2)); + logBinaryProvenance("ge", v1, v2, r); + return r; + } + } + + public static class LT extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + DSHandle r = new RootDataNode(Types.BOOLEAN, getFloat(this, v1) < getFloat(this, v2)); + logBinaryProvenance("lt", v1, v2, r); + return r; + } + } + + public static class GT extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + DSHandle r = new RootDataNode(Types.BOOLEAN, getFloat(this, v1) > getFloat(this, v2)); + logBinaryProvenance("gt", v1, v2, r); + return r; + } + } + + public static class EQ extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + DSHandle r = new RootDataNode(Types.BOOLEAN, v1.getValue().equals(v2.getValue())); + logBinaryProvenance("eq", v1, v2, r); + return r; + } + } + + public static class NE extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + DSHandle r = new RootDataNode(Types.BOOLEAN, !v1.getValue().equals(v2.getValue())); + logBinaryProvenance("ne", v1, v2, r); + return r; + } + } + + public static class And extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + DSHandle r = new RootDataNode(Types.BOOLEAN, getBool(this, v1) && getBool(this, v2)); + logBinaryProvenance("and", v1, v2, r); + return r; + } + } - public Object vdlop_iquotient(VariableStack stack) throws ExecutionException { - double l = L.getDoubleValue(stack); - double r = R.getDoubleValue(stack); - DSHandle ret = newNum(Types.INT, l / r); - logBinaryProvenance(stack, "iquotient", ret); - return ret; - } + public static class Or extends BinaryOp { + @Override + protected DSHandle value(DSHandle v1, DSHandle v2) { + DSHandle r = new RootDataNode(Types.BOOLEAN, getBool(this, v1) || getBool(this, v2)); + logBinaryProvenance("or", v1, v2, r); + return r; + } + } + + public static class Not extends UnaryOp { + @Override + protected DSHandle value(DSHandle v) { + DSHandle r = new RootDataNode(Types.BOOLEAN, !getBool(this, v)); + logUnaryProvenance("not", v, r); + return r; + } + } - public Object vdlop_remainder(VariableStack stack) throws ExecutionException { - double l = L.getDoubleValue(stack); - double r = R.getDoubleValue(stack); - DSHandle ret = newNum(type(stack), l % r); - logBinaryProvenance(stack, "remainder", ret); - return ret; - } - - public Object vdlop_le(VariableStack stack) throws ExecutionException { - double l = L.getDoubleValue(stack); - double r = R.getDoubleValue(stack); - DSHandle ret = newBool(l <= r); - logBinaryProvenance(stack, "le", ret); - return ret; - } - - public Object vdlop_ge(VariableStack stack) throws ExecutionException { - double l = L.getDoubleValue(stack); - double r = R.getDoubleValue(stack); - DSHandle ret = newBool(l >= r); - logBinaryProvenance(stack, "ge", ret); - return ret; - } - - public Object vdlop_gt(VariableStack stack) throws ExecutionException { - double l = L.getDoubleValue(stack); - double r = R.getDoubleValue(stack); - DSHandle ret = newBool(l > r); - logBinaryProvenance(stack, "gt", ret); - return ret; - } - - public Object vdlop_lt(VariableStack stack) throws ExecutionException { - double l = L.getDoubleValue(stack); - double r = R.getDoubleValue(stack); - DSHandle ret = newBool(l < r); - logBinaryProvenance(stack, "lt", ret); - return ret; - } - - public Object vdlop_eq(VariableStack stack) throws ExecutionException { - Object l = L.getValue(stack); - Object r = R.getValue(stack); - if (l == null) { - throw new ExecutionException("First operand is null"); + private static void logBinaryProvenance(String name, DSHandle v1, DSHandle v2, DSHandle result) throws ExecutionException { + if (PROVENANCE_ENABLED) { + String thread = LWThread.currentThread().getName(); + String lhsid = v1.getIdentifier(); + String rhsid = v2.getIdentifier(); + String rid = result.getIdentifier(); + provenanceLogger.info("OPERATOR thread=" + thread + " operator=" + name + + " lhs=" + lhsid + " rhs=" + rhsid + " result=" + rid); } - if (r == null) { - throw new ExecutionException("Second operand is null"); - } - DSHandle ret = newBool(l.equals(r)); - logBinaryProvenance(stack, "eq", ret); - return ret; } - public Object vdlop_ne(VariableStack stack) throws ExecutionException { - Object l = L.getValue(stack); - Object r = R.getValue(stack); - if (l == null) { - throw new ExecutionException("First operand is null"); + private static void logUnaryProvenance(String name, DSHandle v, DSHandle r) throws ExecutionException { + if (PROVENANCE_ENABLED) { + String thread = LWThread.currentThread().getName(); + String vid = v.getIdentifier(); + String rid = r.getIdentifier(); + provenanceLogger.info("UNARYOPERATOR thread=" + thread + " operator=" + name + + " operand=" + vid + " result=" + rid); } - if (r == null) { - throw new ExecutionException("Second operand is null"); - } - DSHandle ret = newBool(!(l.equals(r))); - logBinaryProvenance(stack, "ne", ret); - return ret; } - - public Object vdlop_and(VariableStack stack) throws ExecutionException { - boolean l = ((Boolean)L.getValue(stack)).booleanValue(); - boolean r = ((Boolean)R.getValue(stack)).booleanValue(); - DSHandle ret = newBool(l && r); - logBinaryProvenance(stack, "and", ret); - return ret; - } - - public Object vdlop_or(VariableStack stack) throws ExecutionException { - boolean l = ((Boolean)L.getValue(stack)).booleanValue(); - boolean r = ((Boolean)R.getValue(stack)).booleanValue(); - DSHandle ret = newBool(l || r); - logBinaryProvenance(stack, "or", ret); - return ret; - } - - public Object vdlop_not(VariableStack stack) throws ExecutionException { - boolean u = ((Boolean)U.getValue(stack)).booleanValue(); - DSHandle ret = newBool(!u); - logUnaryProvenance(stack, "not", ret); - return ret; - } - - private void logBinaryProvenance(VariableStack stack, String name, DSHandle resultDataset) throws ExecutionException { - try { - if(VDL2Config.getConfig().getProvenanceLog()) { - String thread = stack.getVar("#thread").toString(); - String lhs = L.getRawValue(stack).getIdentifier(); - String rhs = R.getRawValue(stack).getIdentifier(); - String result = resultDataset.getIdentifier(); - provenanceLogger.info("OPERATOR thread="+thread+" operator="+name+" lhs="+lhs+" rhs="+rhs+" result="+result); - } - } catch(IOException ioe) { - throw new ExecutionException("Exception when logging provenance for binary operator", ioe); - } - } - - private void logUnaryProvenance(VariableStack stack, String name, DSHandle resultDataset) throws ExecutionException { - try { - if(VDL2Config.getConfig().getProvenanceLog()) { - String thread = stack.getVar("#thread").toString(); - String lhs = U.getRawValue(stack).getIdentifier(); - String result = resultDataset.getIdentifier(); - provenanceLogger.info("UNARYOPERATOR thread="+thread+" operator="+name+" operand="+lhs+" result="+result); - } - } catch(IOException ioe) { - throw new ExecutionException("Exception when logging provenance for unary operator", ioe); - } - } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,51 +22,53 @@ import java.util.List; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.ArgUtil; -import org.globus.cog.karajan.arguments.VariableArguments; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.Path; -public class OutFileDirs extends VDLFunction { - public static final Arg STAGEOUTS = new Arg.Positional("stageouts"); - - static { - setArguments(OutFileDirs.class, new Arg[] { STAGEOUTS }); +public class OutFileDirs extends SwiftFunction { + + private ArgRef>> stageouts; + private ChannelRef cr_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("stageouts"), returns(channel("...", DYNAMIC))); } + @Override - public Object function(VariableStack stack) throws ExecutionException { - List files = TypeUtil.toList(STAGEOUTS.getValue(stack)); - VariableArguments ret = ArgUtil.getVariableReturn(stack); + public Object function(Stack stack) { + List> files = stageouts.getValue(stack); try { - for (Object f : files) { - List pv = TypeUtil.toList(f); - Path p = parsePath(pv.get(0), stack); + for (List pv : files) { + Path p = parsePath(pv.get(0)); DSHandle handle = (DSHandle) pv.get(1); DSHandle leaf = handle.getField(p); - String fname = VDLFunction.filename(leaf)[0]; + String fname = SwiftFunction.filename(leaf)[0]; AbsFile af = new AbsFile(fname); if ("file".equals(af.getProtocol())) { String dir = af.getDir(); if (dir.startsWith("/") && dir.length() != 1) { - ret.append(dir.substring(1)); + cr_vargs.append(stack, dir.substring(1)); } else if (dir.length() != 0) { - ret.append(dir); + cr_vargs.append(stack, dir); } } else { - ret.append(af.getHost() + "/" + af.getDir()); + cr_vargs.append(stack, af.getHost() + "/" + af.getDir()); } } } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } return null; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/OutFiles.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/OutFiles.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/OutFiles.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,40 +22,40 @@ import java.util.List; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.ArgUtil; -import org.globus.cog.karajan.arguments.VariableArguments; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; -import org.griphyn.vdl.mapping.AbsFile; +import k.rt.Channel; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.Path; -public class OutFiles extends VDLFunction { - public static final Arg STAGEOUTS = new Arg.Positional("stageouts"); - - static { - setArguments(OutFiles.class, new Arg[] { STAGEOUTS }); +public class OutFiles extends SwiftFunction { + private ArgRef>> stageouts; + private ChannelRef cr_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("stageouts"), returns(channel("...", DYNAMIC))); } @Override - protected Object function(VariableStack stack) throws ExecutionException { - List files = TypeUtil.toList(STAGEOUTS.getValue(stack)); - VariableArguments ret = ArgUtil.getVariableReturn(stack); + public Object function(Stack stack) { + List> files = this.stageouts.getValue(stack); + Channel ret = cr_vargs.get(stack); try { - for (Object f : files) { - List pv = TypeUtil.toList(f); - Path p = parsePath(pv.get(0), stack); + for (List pv : files) { + Path p = parsePath(pv.get(0)); DSHandle handle = (DSHandle) pv.get(1); DSHandle leaf = handle.getField(p); - String fname = argList(VDLFunction.filename(leaf), true); - ret.append(fname); + String fname = argList(SwiftFunction.filename(leaf), true); + ret.add(fname); } } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } return null; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Parameterlog.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Parameterlog.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Parameterlog.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,52 +20,60 @@ */ package org.griphyn.vdl.karajan.lib; +import k.rt.Stack; +import k.thr.LWThread; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; +import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.karajan.functions.ConfigProperty; +import org.griphyn.vdl.util.VDL2Config; -public class Parameterlog extends AbstractSequentialWithArguments { +public class Parameterlog extends InternalFunction { public static final Logger logger = Logger.getLogger(Parameterlog.class); - - public static final Arg DIRECTION = new Arg.Positional("direction"); - public static final Arg VAR = new Arg.Positional("variable"); - public static final Arg ID = new Arg.Positional("id"); - public static final Arg THREAD = new Arg.Positional("thread"); - - static { - setArguments(Parameterlog.class, new Arg[] { DIRECTION, VAR, ID, THREAD }); + + private ArgRef direction; + private ArgRef variable; + private ArgRef id; + + @Override + protected Signature getSignature() { + return new Signature(params("direction", "variable", "id")); } private Boolean enabled; + private VarRef config; @Override - public void pre(VariableStack stack) throws ExecutionException { - if (enabled == null) { - enabled = "true".equals(ConfigProperty.getProperty("provenance.log", true, stack)); - } + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) + throws CompilationException { + config = scope.getVarRef("SWIFT_CONFIG"); + return super.compileBody(w, argScope, scope); } - - - @Override - protected void executeChildren(VariableStack stack) throws ExecutionException { - if (enabled) { - super.executeChildren(stack); - } - else { - complete(stack); - } - } + @Override - protected void post(VariableStack stack) throws ExecutionException { - if (enabled) { - logger.info("PARAM thread=" + THREAD.getValue(stack) + " direction=" - + DIRECTION.getValue(stack) + " variable=" + VAR.getValue(stack) - + " provenanceid=" + ID.getValue(stack)); + protected void runBody(LWThread thr) { + Stack stack = thr.getStack(); + boolean run; + synchronized(this) { + if (enabled == null) { + enabled = "true".equals(ConfigProperty.getProperty("provenance.log", true, config.getValue(stack))); + } + run = enabled; } + if (run) { + super.run(thr); + logger.info("PARAM thread=" + thr.getName() + " direction=" + + direction.getValue(stack) + " variable=" + variable.getValue(stack) + + " provenanceid=" + id.getValue(stack)); + } } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/PartialCloseDataset.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/PartialCloseDataset.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/PartialCloseDataset.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,24 +17,27 @@ package org.griphyn.vdl.karajan.lib; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.DSHandle; -public class PartialCloseDataset extends VDLFunction { +public class PartialCloseDataset extends SwiftFunction { public static final Logger logger = Logger.getLogger(CloseDataset.class); + + private ArgRef var; + private ArgRef count; - public static final Arg OA_COUNT = new Arg.Optional("count", 1); + @Override + protected Signature getSignature() { + return new Signature(params("var", optional("count", 1))); + } - static { - setArguments(PartialCloseDataset.class, new Arg[] { PA_VAR, OA_COUNT }); - } - - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); if (logger.isDebugEnabled()) { logger.debug("Partially closing " + var); } @@ -44,7 +47,7 @@ return null; } - int count = TypeUtil.toInt(OA_COUNT.getValue(stack)); + int count = this.count.getValue(stack).intValue(); var.updateWriteRefCount(-count); return null; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/PathUtils.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -18,99 +18,122 @@ package org.griphyn.vdl.karajan.lib; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Param; +import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; import org.globus.swift.catalog.types.Os; import org.griphyn.vdl.mapping.AbsFile; -public class PathUtils extends FunctionsCollection { +public class PathUtils { - public static final Arg PATH = new Arg.Positional("path"); + public static class DirName extends AbstractSingleValuedFunction { + private ArgRef path; - static { - setArguments("vdl_dirname", new Arg[] { PATH }); - } + @Override + protected Param[] getParams() { + return params("path"); + } - public String vdl_dirname(VariableStack stack) throws ExecutionException { - String path = TypeUtil.toString(PATH.getValue(stack)); - return new AbsFile(path).getDir(); - } + @Override + public Object function(Stack stack) { + return new AbsFile(path.getValue(stack)).getDir(); + } + } - static { - setArguments("vdl_reldirname", new Arg[] { PATH }); - } + public static class RelDirName extends AbstractSingleValuedFunction { + private ArgRef path; - public String vdl_reldirname(VariableStack stack) throws ExecutionException { - String path = TypeUtil.toString(PATH.getValue(stack)); - String dir = new AbsFile(path).getDir(); - if (dir.startsWith("/")) { - return dir.substring(1); + @Override + protected Param[] getParams() { + return params("path"); } - else { - return dir; + + @Override + public Object function(Stack stack) { + String dir = new AbsFile(path.getValue(stack)).getDir(); + if (dir.startsWith("/")) { + return dir.substring(1); + } + else { + return dir; + } } } - - static { - setArguments("vdl_basename", new Arg[] { PATH }); - } - public String vdl_basename(VariableStack stack) throws ExecutionException { - String path = TypeUtil.toString(PATH.getValue(stack)); - return new AbsFile(path).getName(); + public static class BaseName extends AbstractSingleValuedFunction { + private ArgRef path; + + @Override + protected Param[] getParams() { + return params("path"); + } + + @Override + public Object function(Stack stack) { + return new AbsFile(path.getValue(stack)).getName(); + } } - static { - setArguments("vdl_provider", new Arg[] { PATH }); - } + public static class Provider extends AbstractSingleValuedFunction { + private ArgRef path; - public String vdl_provider(VariableStack stack) throws ExecutionException { - String path = TypeUtil.toString(PATH.getValue(stack)); - return new AbsFile(path).getProtocol(); + @Override + protected Param[] getParams() { + return params("path"); + } + + @Override + public Object function(Stack stack) { + return new AbsFile(path.getValue(stack)).getProtocol(); + } } - - static { - setArguments("vdl_hostname", new Arg[] { PATH }); - } - public String vdl_hostname(VariableStack stack) throws ExecutionException { - String path = TypeUtil.toString(PATH.getValue(stack)); - return new AbsFile(path).getHost(); + public static class HostName extends AbstractSingleValuedFunction { + private ArgRef path; + + @Override + protected Param[] getParams() { + return params("path"); + } + + @Override + public Object function(Stack stack) { + return new AbsFile(path.getValue(stack)).getHost(); + } } - public static final Arg DIR = new Arg.Positional("dir"); - public static final Arg OS = new Arg.Optional("os"); - - static { - setArguments("vdl_dircat", new Arg[] { DIR, PATH, OS }); - } + public static class DirCat extends AbstractSingleValuedFunction { + private ArgRef dir; + private ArgRef path; + private ArgRef os; - public String vdl_dircat(VariableStack stack) throws ExecutionException { - String dir = TypeUtil.toString(DIR.getValue(stack)); - String path = TypeUtil.toString(PATH.getValue(stack)); - boolean windows = false; - if (OS.isPresent(stack)) { - Os os = (Os) OS.getValue(stack); - windows = Os.WINDOWS.equals(os); - } - if (dir.equals("")) { - return windowsify(path, windows); + @Override + protected Param[] getParams() { + return params("dir", "path", optional("os", Os.LINUX)); } - else if (dir.endsWith("/")) { - return windowsify(dir + path, windows); + + @Override + public Object function(Stack stack) { + String dir = this.dir.getValue(stack); + String path = this.path.getValue(stack); + boolean windows = this.os.getValue(stack).equals(Os.WINDOWS); + if (dir.equals("")) { + return windowsify(path, windows); + } + else if (dir.endsWith("/")) { + return windowsify(dir + path, windows); + } + else { + return windowsify(dir + '/' + path, windows); + } } - else { - return windowsify(dir + '/' + path, windows); - } } - private String windowsify(String path, boolean windows) { + private static String windowsify(String path, boolean windows) { if (windows) { return path.replace('/', '\\'); } @@ -118,18 +141,22 @@ return path; } } + + public static class PathNames extends AbstractSingleValuedFunction { + private ArgRef> files; - public static final Arg FILES = new Arg.Positional("files"); - static { - setArguments("vdl_pathnames", new Arg[] { FILES }); - } + @Override + protected Param[] getParams() { + return params("files"); + } - public Object[] vdl_pathnames(VariableStack stack) throws ExecutionException { - List l = new ArrayList(); - Iterator i = TypeUtil.toIterator(FILES.getValue(stack)); - while (i.hasNext()) { - l.add(new AbsFile((String) i.next()).getPath()); + @Override + public String[] function(Stack stack) { + List l = new ArrayList(); + for (String f : files.getValue(stack)) { + l.add(new AbsFile(f).getPath()); + } + return l.toArray(new String[0]); } - return l.toArray(new String[0]); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Range.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Range.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Range.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -28,12 +28,13 @@ import java.util.Map; import java.util.Set; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.RootArrayDataNode; @@ -41,29 +42,33 @@ import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; -public class Range extends VDLFunction { - public static final SwiftArg PA_FROM = new SwiftArg.Positional("from"); - public static final SwiftArg PA_TO = new SwiftArg.Positional("to"); - public static final SwiftArg OA_STEP = new SwiftArg.Optional("step", new Double(1), Types.FLOAT); +public class Range extends SwiftFunction { + private ArgRef from; + private ArgRef to; + private ArgRef step; + + @Override + protected Signature getSignature() { + return new Signature(params("from", "to", optional("step", new RootDataNode(Types.FLOAT, 1)))); + } - static { - setArguments(Range.class, new Arg[] { PA_FROM, PA_TO, OA_STEP }); - } - - public Object function(final VariableStack stack) throws ExecutionException { + @Override + public Object function(Stack stack) { // TODO: deal with expression - final Type type = PA_FROM.getType(stack); - final double start = PA_FROM.getDoubleValue(stack); - final double stop = PA_TO.getDoubleValue(stack); - final double incr = OA_STEP.getDoubleValue(stack); + DSHandle from = this.from.getValue(stack); + DSHandle to = this.to.getValue(stack); + DSHandle step = this.step.getValue(stack); + final Type type = from.getType(); + final double start = ((Number) from.getValue()).doubleValue(); + final double stop = ((Number) to.getValue()).doubleValue(); + final double incr = ((Number) step.getValue()).doubleValue(); // only deal with int and float try { final AbstractDataNode handle; handle = new RootArrayDataNode(type.arrayType()) { - final DSHandle h = this; - + final DSHandle h = this; { closeShallow(); } @@ -78,10 +83,7 @@ } else { int index = (Integer) path.getFirst(); - DSHandle value = new RootDataNode(type); - value.init(null); - value.setValue(new Double(start + incr * index)); - value.closeShallow(); + DSHandle value = new RootDataNode(type, new Double(start + incr * index)); return Collections.singletonList(value); } } @@ -106,15 +108,12 @@ private int key; { - value = new RootDataNode(type); - value.init(null); if (type == Types.INT) { - value.setValue(new Integer((int) crt)); + value = new RootDataNode(Types.INT, new Integer((int) crt)); } else { - value.setValue(new Double(crt)); + value = new RootDataNode(Types.FLOAT, new Double(crt)); } - value.closeShallow(); key = index; } @@ -159,7 +158,7 @@ return handle; } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -19,32 +19,56 @@ import java.io.IOException; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; +import k.rt.Context; +import k.rt.ExecutionException; +import k.rt.Stack; +import k.thr.LWThread; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableNotFoundException; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.Var; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.util.VDL2Config; /** this is an icky class that does too much with globals, but is for proof of concept. */ -public class RuntimeStats extends FunctionsCollection { +public class RuntimeStats { - public static final String TICKER = "#swift-runtime-progress-ticker"; - public static final String PROGRESS = "#swift-runtime-progress"; + public static final boolean TICKER_DISABLED; + + static{ + boolean disabled; + try{ + disabled = "true".equalsIgnoreCase(VDL2Config.getConfig().getProperty("ticker.disable")); + } + catch (Exception e) { + disabled = false; + } + TICKER_DISABLED = disabled; + } + + public static final String TICKER = "SWIFT_TICKER"; - public static final Arg PA_STATE = new Arg.Positional("state"); //formatter for timestamp against std.err lines public static SimpleDateFormat formatter = new SimpleDateFormat("E, dd MMM yyyy HH:mm:ss Z"); public static final int MIN_PERIOD_MS=1000; - public static final int MAX_PERIOD_MS=30000; + public static final int MAX_PERIOD_MS=1000; public static final String[] preferredOutputOrder = { "uninitialized", @@ -62,83 +86,110 @@ "Finished in previous run", "Finished successfully" }; + + public static class StartProgressTicker extends Node { + private VarRef context; - static { - setArguments("vdl_startprogressticker", new Arg[0]); - setArguments("vdl_stopprogressticker", new Arg[0]); - setArguments("vdl_setprogress", new Arg[] { PA_STATE } ); - setArguments("vdl_initprogressstate", new Arg[] { PA_STATE }); + @Override + public Node compile(WrapperNode w, Scope scope) + throws CompilationException { + super.compile(w, scope); + context = scope.getVarRef("#context"); + return TICKER_DISABLED ? null : this; + } + + @Override + public void run(LWThread thr) { + ProgressTicker t = new ProgressTicker(); + t.setDaemon(true); + t.start(); + context.getValue(thr.getStack()).setAttribute(TICKER, t); + // Allow user to reformat output date + String format; + try { + format = VDL2Config.getDefaultConfig().getTickerDateFormat(); + } + catch (IOException e) { + throw new ExecutionException(this, e); + } + if (format != null && format.length() > 0) { + formatter = new SimpleDateFormat(format); + } + } } - public static void setTicker(VariableStack stack, ProgressTicker ticker) { - stack.setGlobal(TICKER, ticker); + public static class InitProgressState extends Node { + private VarRef context; + private ChannelRef cr_vargs; + + @Override + public Node compile(WrapperNode w, Scope scope) + throws CompilationException { + super.compile(w, scope); + context = scope.getVarRef("#context"); + Var.Channel r = scope.lookupChannel("..."); + r.appendDynamic(); + cr_vargs = scope.getChannelRef(r); + return TICKER_DISABLED ? null : this; + } + + @Override + public void run(LWThread thr) { + ProgressState ps = new ProgressState(); + ps.crt = "Initializing"; + Stack stack = thr.getStack(); + ((ProgressTicker) context.getValue(stack).getAttribute(TICKER)).addState(ps); + cr_vargs.get(stack).add(ps); + } } - public static ProgressTicker getTicker(VariableStack stack) { - return (ProgressTicker) stack.getGlobal(TICKER); - } - - public static void setProgress(VariableStack stack, RuntimeProgress p) { - stack.parentFrame().setVar(PROGRESS, p); - } + public static class SetProgress extends InternalFunction { + private ArgRef ps; + private ArgRef state; + + @Override + protected Signature getSignature() { + return new Signature(params("ps", "state")); + } + + @Override + protected void runBody(LWThread thr) { + Stack stack = thr.getStack(); + ps.getValue(stack).crt = state.getValue(stack); + } + } - public static RuntimeProgress getProgress(VariableStack stack) throws VariableNotFoundException { - return (RuntimeProgress) stack.getDeepVar(PROGRESS); - } - public Object vdl_startprogressticker(VariableStack stack) throws ExecutionException { - ProgressTicker t = new ProgressTicker(); - t.setDaemon(true); - t.start(); - setTicker(stack, t); - - // Allow user to reformat output date - String format; - try { - format = VDL2Config.getDefaultConfig().getTickerDateFormat(); - } - catch (IOException e) { - throw new ExecutionException(e); - } - if (format != null && format.length() > 0) - formatter = - new SimpleDateFormat(format); - return null; - } + public static class StopProgressTicker extends Node { + private VarRef context; - public Object vdl_setprogress(VariableStack stack) throws ExecutionException { - setProgress(stack, TypeUtil.toString(PA_STATE.getValue(stack))); - return null; - } + @Override + public Node compile(WrapperNode w, Scope scope) + throws CompilationException { + super.compile(w, scope); + context = scope.getVarRef("#context"); + return TICKER_DISABLED ? null : this; + } - static public void setProgress(VariableStack stack, String newState) throws ExecutionException { - RuntimeProgress p = getProgress(stack); - ProgressTicker t = getTicker(stack); - synchronized(t) { - t.dec(p.status); - t.inc(newState); - } - p.status = newState; - t.dumpState(); - } + @Override + public void run(LWThread thr) { + ProgressTicker t = (ProgressTicker) context.getValue(thr.getStack()).getAttribute(TICKER); + t.finalDumpState(); + t.shutdown(); + } + } - public Object vdl_initprogressstate(VariableStack stack) throws ExecutionException { - RuntimeProgress rp = new RuntimeProgress(); - ProgressTicker p = getTicker(stack); - setProgress(stack, rp); - rp.status = "Initializing"; - synchronized(p) { - p.inc(rp.status); - } - p.dumpState(); - return null; - } + + public static class ProgressState { + private String crt; - public synchronized Object vdl_stopprogressticker(VariableStack stack) throws ExecutionException { - ProgressTicker p = getTicker(stack); - p.finalDumpState(); - p.shutdown(); - return null; + public void setState(String state) { + this.crt = state; + } + + public String toString() { + return "ProgressState: " + crt; + } } @@ -146,7 +197,7 @@ public static final Logger logger = Logger.getLogger(ProgressTicker.class); - private Map counts; + private Set states; long start; long lastDumpTime = 0; @@ -157,7 +208,7 @@ public ProgressTicker() { super("Progress ticker"); - counts = new HashMap(); + states = new HashSet(); try { if ("true".equalsIgnoreCase(VDL2Config.getConfig().getProperty("ticker.disable"))) { logger.info("Ticker disabled in configuration file"); @@ -171,6 +222,12 @@ } start = System.currentTimeMillis(); } + + public void addState(ProgressState ps) { + synchronized(states) { + states.add(ps); + } + } public void run() { if (disabled) { @@ -186,29 +243,7 @@ } } } - - public void inc(String state) { - Integer crt = counts.get(state); - if (crt == null) { - counts.put(state, 1); - } - else { - counts.put(state, crt + 1); - } - } - - public void dec(String state) { - Integer crt = counts.get(state); - if (crt != null) { - if (crt == 1) { - counts.remove(state); - } - else { - counts.put(state, crt - 1); - } - } - } - + void shutdown() { shutdown = true; } @@ -230,11 +265,33 @@ printStates("Final status:"); } - public synchronized Map getSummary() { - return new HashMap(counts); + private Map getSummary() { + List states; + + synchronized(this.states) { + states = new ArrayList(this.states); + } + Map m = new HashMap(); + + for (ProgressState s : states) { + inc(m, s); + } + return m; + } + + private void inc(Map m, ProgressState s) { + String v = s.crt; + Integer i = m.get(v); + if (i == null) { + i = 1; + } + else { + i = i + 1; + } + m.put(v, i); } - - void printStates(String prefix) { + + synchronized void printStates(String prefix) { Map summary = getSummary(); StringBuilder sb = new StringBuilder(); @@ -274,8 +331,4 @@ } } - - class RuntimeProgress { - String status = "uninitialized"; - } } Deleted: branches/faster/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,39 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.griphyn.vdl.karajan.lib; - -import org.globus.cog.karajan.workflow.nodes.Sequential; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.ThreadingContext; -import org.globus.cog.karajan.workflow.ExecutionException; - - -/** launch a single child, giving it a new thread ID, but without starting - an entire karajan thread -*/ - -public class SequentialWithID extends Sequential { - - protected void executeChildren(VariableStack stack) throws ExecutionException { - ThreadingContext tc = (ThreadingContext) stack.getVar("#thread"); - stack.setVar("#thread", tc.split(1)); - super.executeChildren(stack); - } - -} - Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,19 +20,28 @@ */ package org.griphyn.vdl.karajan.lib; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; +import java.util.List; import java.util.Map; +import k.rt.ExecutionException; +import k.rt.Stack; +import k.thr.LWThread; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableNotFoundException; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.futures.FutureFault; -import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; -import org.griphyn.vdl.karajan.Pair; -import org.griphyn.vdl.karajan.PairIterator; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.analyzer.VariableNotFoundException; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.futures.FutureFault; +import org.globus.cog.karajan.futures.FutureNotYetAvailable; +import org.globus.cog.karajan.parser.WrapperNode; +import org.griphyn.vdl.karajan.PairSet; import org.griphyn.vdl.karajan.WaitingThreadsMonitor; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; @@ -41,30 +50,76 @@ import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.type.Type; -public class SetFieldValue extends VDLFunction { +public class SetFieldValue extends SwiftFunction { public static final Logger logger = Logger.getLogger(SetFieldValue.class); - public static final Arg PA_VALUE = new Arg.Positional("value"); + protected ArgRef var; + protected ArgRef path; + protected ArgRef value; + + protected ArgRef _traceline; - static { - setArguments(SetFieldValue.class, new Arg[] { OA_PATH, PA_VAR, PA_VALUE }); + @Override + protected Signature getSignature() { + return new Signature(params("var", "value", optional("path", Path.EMPTY_PATH), optional("_traceline", null))); + } + + private String src, dest; + private Tracer tracer; + + protected VarRef state; + + private static class State { + public final List l = new ArrayList(); } - private String src, dest; - private Tracer tracer; + private static class StateEntry { + private Object value; + private Object it; + + @SuppressWarnings("unchecked") + public T it() { + return (T) it; + } + + public void it(Object it) { + this.it= it; + } + + @SuppressWarnings("unchecked") + public T value() { + return (T) value; + } + + public void value(Object value) { + this.value= value; + } + } + + @Override + protected void addLocals(Scope scope) { + super.addLocals(scope); + state = scope.getVarRef(scope.addVar("#state")); + } - @Override - protected void initializeStatic() { - super.initializeStatic(); + @Override + public Node compile(WrapperNode w, Scope scope) + throws CompilationException { + Node fn = super.compile(w, scope); + if (_traceline.getValue() != null) { + setLine(Integer.parseInt(_traceline.getValue())); + } tracer = Tracer.getTracer(this); + return fn; } - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); try { - Path path = parsePath(OA_PATH.getValue(stack), stack); + Path path = parsePath(this.path.getValue(stack)); DSHandle leaf = var.getField(path); - AbstractDataNode value = (AbstractDataNode) PA_VALUE.getValue(stack); + AbstractDataNode value = this.value.getValue(stack); if (src == null) { dest = Tracer.getVarName(var); @@ -72,7 +127,7 @@ } if (tracer.isEnabled()) { - log(leaf, value, stack); + log(leaf, value, LWThread.currentThread()); } // TODO want to do a type check here, for runtime type checking @@ -81,23 +136,25 @@ // for type conversion here; but would be useful to have // type checking. - deepCopy(leaf, value, stack, 0); + deepCopy(leaf, value, stack); + return null; } catch (FutureFault f) { + LWThread thr = LWThread.currentThread(); if (tracer.isEnabled()) { - tracer.trace(stack, var + " waiting for " + Tracer.getFutureName(f.getFuture())); + tracer.trace(thr, var + " waiting for " + Tracer.getFutureName(f.getFuture())); } - WaitingThreadsMonitor.addOutput(stack, Collections.singletonList(var)); + WaitingThreadsMonitor.addOutput(thr, Collections.singletonList(var)); throw f; } catch (Exception e) { // TODO tighten this - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } - private void log(DSHandle leaf, DSHandle value, VariableStack stack) throws VariableNotFoundException { - tracer.trace(stack, dest + " = " + Tracer.unwrapHandle(value)); + private void log(DSHandle leaf, DSHandle value, LWThread thr) throws VariableNotFoundException { + tracer.trace(thr, dest + " = " + Tracer.unwrapHandle(value)); } String unpackHandles(DSHandle handle, Map, DSHandle> handles) { @@ -119,21 +176,33 @@ return sb.toString(); } + protected void deepCopy(DSHandle dest, DSHandle source, Stack stack) { + State state = this.state.getValue(stack); + if (state == null) { + state = new State(); + this.state.setValue(stack, state); + } + + deepCopy(dest, source, state, 0); + + this.state.setValue(stack, null); + } + /** make dest look like source - if its a simple value, copy that and if its an array then recursively copy */ - public static void deepCopy(DSHandle dest, DSHandle source, VariableStack stack, int level) throws ExecutionException { - ((AbstractDataNode) source).waitFor(); + public void deepCopy(DSHandle dest, DSHandle source, State state, int level) { + ((AbstractDataNode) source).waitFor(this); if (source.getType().isPrimitive()) { dest.setValue(source.getValue()); } else if (source.getType().isArray()) { - copyArray(dest, source, stack, level); + copyArray(dest, source, state, level); } else if (source.getType().isComposite()) { - copyStructure(dest, source, stack, level); + copyStructure(dest, source, state, level); } else { - copyNonComposite(dest, source, stack, level); + copyNonComposite(dest, source, state, level); } } @@ -147,27 +216,26 @@ } } - @SuppressWarnings("unchecked") - private static void copyStructure(DSHandle dest, DSHandle source, - VariableStack stack, int level) throws ExecutionException { + private void copyStructure(DSHandle dest, DSHandle source, State state, int level) { Type type = dest.getType(); - Iterator fni = (Iterator) stack.currentFrame().getVar("it" + level); + StateEntry se = getStateEntry(state, level); + Iterator fni = se.it(); if (fni == null) { fni = type.getFieldNames().iterator(); - stack.currentFrame().setVar("it" + level, fni); + se.it = fni; } - String fname = (String) stack.currentFrame().getVar("f" + level); + String fname = se.value(); while (fni.hasNext() || fname != null) { if (fname == null) { fname = fni.next(); - stack.currentFrame().setVar("f" + level, fname); + se.value(fname); } Path fpath = Path.EMPTY_PATH.addFirst(fname); try { DSHandle dstf = dest.getField(fpath); try { DSHandle srcf = source.getField(fpath); - deepCopy(dstf, srcf, stack, level + 1); + deepCopy(dstf, srcf, state, level + 1); } catch (InvalidPathException e) { // do nothing. It's an unused field in the source. @@ -177,15 +245,29 @@ throw new ExecutionException("Internal type inconsistency detected. " + dest + " claims not to have a " + fname + " field"); } - stack.currentFrame().deleteVar("f" + level); + se.value(null); fname = null; } - stack.currentFrame().deleteVar("it" + level); + popStateEntry(state); dest.closeShallow(); } - private static void copyNonComposite(DSHandle dest, DSHandle source, - VariableStack stack, int level) throws ExecutionException { + private static StateEntry getStateEntry(State state, int level) { + if (state.l.size() == level) { + StateEntry e = new StateEntry(); + state.l.add(e); + return e; + } + else { + return state.l.get(level); + } + } + + private static void popStateEntry(State state) { + state.l.remove(state.l.size() - 1); + } + + private static void copyNonComposite(DSHandle dest, DSHandle source, State state, int level) { Path dpath = dest.getPathFromRoot(); Mapper dmapper = dest.getRoot().getMapper(); if (dmapper.canBeRemapped(dpath)) { @@ -196,8 +278,9 @@ dest.closeShallow(); } else { - if (stack.currentFrame().isDefined("fc")) { - FileCopier fc = (FileCopier) stack.currentFrame().getVarAndDelete("fc"); + StateEntry se = getStateEntry(state, level); + FileCopier fc = se.value(); + if (fc != null) { if (!fc.isClosed()) { throw new FutureNotYetAvailable(fc); } @@ -209,9 +292,9 @@ dest.closeShallow(); } else { - FileCopier fc = new FileCopier(source.getMapper().map(source.getPathFromRoot()), + fc = new FileCopier(source.getMapper().map(source.getPathFromRoot()), dest.getMapper().map(dpath)); - stack.setVar("fc", fc); + se.value(fc); try { fc.start(); } @@ -220,24 +303,22 @@ } throw new FutureNotYetAvailable(fc); } + popStateEntry(state); } } - private static void copyArray(DSHandle dest, DSHandle source, - VariableStack stack, int level) throws ExecutionException { - PairIterator it; - if (stack.isDefined("it" + level)) { - it = (PairIterator) stack.getVar("it" + level); + private void copyArray(DSHandle dest, DSHandle source, State state, int level) { + StateEntry se = getStateEntry(state, level); + Iterator> it = se.it(); + if (it == null) { + it = new PairSet(source.getArrayValue()).iterator(); + se.it(it); } - else { - it = new PairIterator(source.getArrayValue()); - stack.setVar("it" + level, it); - } - Pair pair = (Pair) stack.currentFrame().getVar("p" + level); + List pair = se.value(); while (it.hasNext() || pair != null) { if (pair == null) { - pair = (Pair) it.next(); - stack.currentFrame().setVar("p" + level, pair); + pair = it.next(); + se.value(pair); } Comparable lhs = (Comparable) pair.get(0); DSHandle rhs = (DSHandle) pair.get(1); @@ -250,11 +331,11 @@ catch (InvalidPathException ipe) { throw new ExecutionException("Could not get destination field",ipe); } - deepCopy(field, rhs, stack, level + 1); - stack.currentFrame().deleteVar("p" + level); + deepCopy(field, rhs, state, level + 1); + se.value(null); pair = null; } - stack.currentFrame().deleteVar("it" + level); + popStateEntry(state); dest.closeShallow(); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/SetFutureFault.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/SetFutureFault.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/SetFutureFault.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,47 +20,53 @@ */ package org.griphyn.vdl.karajan.lib; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.DataDependentException; import org.griphyn.vdl.mapping.MappingDependentException; import org.griphyn.vdl.mapping.Path; -public class SetFutureFault extends VDLFunction { +public class SetFutureFault extends SwiftFunction { public static final Logger logger = Logger.getLogger(SetFutureFault.class); + + private ArgRef var; + private ArgRef path; + private ArgRef exception; + private ArgRef mapping; - public static final Arg OA_EXCEPTION = new Arg.Optional("exception", null); - public static final Arg OA_MAPPING = new Arg.Optional("mapping", Boolean.FALSE); + @Override + protected Signature getSignature() { + return new Signature(params("var", optional("path", Path.EMPTY_PATH), + optional("exception", null), optional("mapping", false))); + } - static { - setArguments(SetFutureFault.class, new Arg[] { OA_PATH, PA_VAR, OA_EXCEPTION, OA_MAPPING }); - } - - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); - boolean mapping = TypeUtil.toBoolean(OA_MAPPING.getValue(stack)); + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); + boolean mapping = this.mapping.getValue(stack); try { - Path path = parsePath(OA_PATH.getValue(stack), stack); + Path path = parsePath(this.path.getValue(stack)); DSHandle leaf = var.getField(path); if (logger.isInfoEnabled()) { logger.info("Failing " + leaf + " (mapping=" + mapping + ")"); } synchronized (leaf) { - Object value = OA_EXCEPTION.getValue(stack); + Exception e = this.exception.getValue(stack); if (mapping) { - leaf.setValue(new MappingDependentException(leaf, (Exception) value)); + leaf.setValue(new MappingDependentException(leaf, e)); } else { - leaf.setValue(new DataDependentException(leaf, (Exception) value)); + leaf.setValue(new DataDependentException(leaf, e)); } } } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } return null; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/SetWaitCount.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/SetWaitCount.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/SetWaitCount.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,30 +17,34 @@ package org.griphyn.vdl.karajan.lib; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.DSHandle; -public class SetWaitCount extends VDLFunction { +public class SetWaitCount extends SwiftFunction { public static final Logger logger = Logger.getLogger(CloseDataset.class); - public static final Arg OA_COUNT = new Arg.Optional("count", 1); + private ArgRef var; + private ArgRef count; - static { - setArguments(SetWaitCount.class, new Arg[] { PA_VAR, OA_COUNT }); - } + @Override + protected Signature getSignature() { + return new Signature(params("var", optional("count", 1))); + } - public Object function(VariableStack stack) throws ExecutionException { - DSHandle var = (DSHandle) PA_VAR.getValue(stack); + @Override + public Object function(Stack stack) { + DSHandle var = this.var.getValue(stack); if (var.isClosed()) { throw new ExecutionException("Attempted to set a wait count for a closed variable " + var); } - int count = TypeUtil.toInt(OA_COUNT.getValue(stack)); + int count = this.count.getValue(stack).intValue(); var.setWriteRefCount(count); return null; } Added: branches/faster/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java (rev 0) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,223 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jan 7, 2013 + */ +package org.griphyn.vdl.karajan.lib; + +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.abstraction.impl.common.AbstractionFactory; +import org.globus.cog.abstraction.impl.common.ProviderMethodException; +import org.globus.cog.abstraction.impl.common.task.ExecutionServiceImpl; +import org.globus.cog.abstraction.impl.common.task.InvalidProviderException; +import org.globus.cog.abstraction.impl.common.task.ServiceContactImpl; +import org.globus.cog.abstraction.impl.common.task.ServiceImpl; +import org.globus.cog.abstraction.interfaces.Service; +import org.globus.cog.abstraction.interfaces.ServiceContact; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Param; +import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; +import org.globus.cog.karajan.util.BoundContact; +import org.globus.cog.karajan.util.ContactSet; +import org.globus.swift.catalog.site.Parser; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +public class SiteCatalog extends AbstractSingleValuedFunction { + private ArgRef fileName; + + @Override + protected Param[] getParams() { + return params("fileName"); + } + + @Override + public Object function(Stack stack) { + String fn = fileName.getValue(stack); + Parser p = new Parser(fn); + try { + Document doc = p.parse(); + return buildResources(doc); + } + catch (Exception e) { + throw new ExecutionException(this, "Failed to parse site catalog", e); + } + } + + private Object buildResources(Document doc) { + ContactSet cs = new ContactSet(); + NodeList pools = doc.getElementsByTagName("config").item(0).getChildNodes(); + for (int i = 0; i < pools.getLength(); i++) { + try { + BoundContact bc = pool(pools.item(i)); + if (bc != null) { + cs.addContact(bc); + } + } + catch (Exception e) { + System.err.println("Invalid pool entry '" + attr(pools.item(i), "name") + "': " + e.getMessage()); + } + } + return cs; + } + + private BoundContact pool(Node n) throws InvalidProviderException, ProviderMethodException { + if (n.getNodeType() != Node.ELEMENT_NODE) { + return null; + } + String name = attr(n, "handle"); + BoundContact bc = new BoundContact(name); + + String sysinfo = attr(n, "sysinfo"); + if (sysinfo != null) { + bc.addProperty("sysinfo", sysinfo); + } + + NodeList cs = n.getChildNodes(); + + for (int i = 0; i < cs.getLength(); i++) { + Node c = cs.item(i); + if (c.getNodeType() != Node.ELEMENT_NODE) { + continue; + } + String ctype = c.getNodeName(); + + if (ctype.equals("gridftp")) { + bc.addService(gridftp(c)); + } + else if (ctype.equals("jobmanager")) { + bc.addService(jobmanager(c)); + } + else if (ctype.equals("execution")) { + bc.addService(execution(c)); + } + else if (ctype.equals("filesystem")) { + bc.addService(filesystem(c)); + } + else if (ctype.equals("workdirectory")) { + bc.addProperty("workdir", text(c)); + } + else if (ctype.equals("scratch")) { + bc.addProperty("scratch", text(c)); + } + else if (ctype.equals("env")) { + env(bc, c); + } + else if (ctype.equals("profile")) { + profile(bc, c); + } + else { + System.err.println("Unknown node type: " + ctype); + } + } + return bc; + } + + private Service jobmanager(Node n) throws InvalidProviderException, ProviderMethodException { + String provider; + String url = attr(n, "url"); + String major = attr(n, "major"); + if (url.equals("local://localhost")) { + provider = "local"; + } + else if (url.equals("pbs://localhost")) { + provider = "pbs"; + } + else if ("2".equals(major)) { + provider = "gt2"; + } + else if ("4".equals(major)) { + provider = "gt4"; + } + else { + throw new IllegalArgumentException("Unknown job manager version: " + major + ", url = '" + url + "'"); + } + + ServiceContact contact = new ServiceContactImpl(url); + return new ServiceImpl(provider, Service.EXECUTION, + contact, AbstractionFactory.newSecurityContext(provider, contact)); + } + + private Service gridftp(Node n) throws InvalidProviderException, ProviderMethodException { + String url = attr(n, "url"); + if (url.equals("local://localhost")) { + return new ServiceImpl("local", Service.FILE_OPERATION, new ServiceContactImpl("localhost"), null); + } + else { + ServiceContact contact = new ServiceContactImpl(url); + return new ServiceImpl("gsiftp", Service.FILE_OPERATION, + contact, AbstractionFactory.newSecurityContext("gsiftp", contact)); + } + } + + private Service execution(Node n) throws InvalidProviderException, ProviderMethodException { + String provider = attr(n, "provider"); + String url = attr(n, "url"); + String jobManager = attr(n, "jobManager"); + ServiceContact contact = null; + if (url != null) { + contact = new ServiceContactImpl(url); + } + return new ExecutionServiceImpl(provider, contact, + AbstractionFactory.newSecurityContext(provider, contact), jobManager); + } + + private Service filesystem(Node n) throws InvalidProviderException, ProviderMethodException { + String provider = attr(n, "provider"); + String url = attr(n, "url"); + ServiceContact contact = null; + if (url != null) { + contact = new ServiceContactImpl(url); + } + return new ServiceImpl(provider, Service.FILE_OPERATION, + contact, AbstractionFactory.newSecurityContext(provider, contact)); + } + + private void env(BoundContact bc, Node n) { + String key = attr(n, "key"); + String value = text(n); + + bc.addProperty("env:" + key, value); + } + + private void profile(BoundContact bc, Node n) { + String ns = attr(n, "namespace"); + String key = attr(n, "key"); + String value = text(n); + + if (ns.equals("karajan")) { + bc.addProperty(key, value); + } + else { + bc.addProperty(ns + ":" + key, value); + } + } + + private String text(Node n) { + return n.getFirstChild().getNodeValue(); + } + + private String attr(Node n, String name) { + NamedNodeMap attrs = n.getAttributes(); + if (attrs != null) { + Node attr = attrs.getNamedItem(name); + if (attr == null) { + return null; + } + else { + return attr.getNodeValue(); + } + } + else { + return null; + } + } +} Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/SiteProfile.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/SiteProfile.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/SiteProfile.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -25,30 +25,34 @@ import java.util.Map; import java.util.Set; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.util.BoundContact; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; import org.globus.swift.catalog.types.Os; import org.griphyn.vdl.util.FQN; -public class SiteProfile extends VDLFunction { +public class SiteProfile extends SwiftFunction { public static final Logger logger = Logger.getLogger(SiteProfile.class); - public static final Arg PA_HOST = new Arg.Positional("host"); - public static final Arg PA_FQN = new Arg.Positional("fqn"); - public static final Arg OA_DEFAULT = new Arg.Optional("default", null); + private ArgRef host; + private ArgRef fqn; + private ArgRef _default; + private ChannelRef cr_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("host", "fqn", optional("default", null)), returns(channel("...", 1))); + } - static { - setArguments(SiteProfile.class, new Arg[] { PA_HOST, PA_FQN, OA_DEFAULT }); + public Object function(Stack stack) throws ExecutionException { + BoundContact bc = host.getValue(stack); + return getSingle(bc, new FQN(fqn.getValue(stack)), _default.getValue(stack)); } - - public Object function(VariableStack stack) throws ExecutionException { - BoundContact bc = (BoundContact) PA_HOST.getValue(stack); - return getSingle(stack, bc, new FQN(TypeUtil.toString(PA_FQN.getValue(stack))), OA_DEFAULT.getValue(stack)); - } public static final FQN SWIFT_WRAPPER_INTERPRETER = new FQN("swift:wrapperInterpreter"); public static final FQN SWIFT_WRAPPER_INTERPRETER_OPTIONS = new FQN("swift:wrapperInterpreterOptions"); @@ -70,7 +74,6 @@ osm.put(fqn, value); } - @SuppressWarnings("unused") private static boolean hasDefault(Os os, FQN fqn) { Map osm = DEFAULTS.get(os); if (osm == null) { @@ -104,7 +107,7 @@ addDefault(null, SWIFT_CLEANUP_COMMAND_OPTIONS, new String[] {"-rf"}); } - private Object getSingle(VariableStack stack, BoundContact bc, FQN fqn, Object defval) + private Object getSingle(BoundContact bc, FQN fqn, Object defval) throws ExecutionException { String value = getProfile(bc, fqn); if (value == null) { @@ -119,7 +122,7 @@ return defval; } else { - throw new ExecutionException(stack, "Missing profile: " + fqn); + throw new ExecutionException(this, "Missing profile: " + fqn); } } else { @@ -145,7 +148,7 @@ else { String[] p = o.toString().split("::"); if (p.length < 2) { - throw new IllegalArgumentException("Invalid sysinfo for " + bc + ": " + o); + throw new ExecutionException("Invalid sysinfo for " + bc + ": " + o); } return Os.fromString(p[1]); } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/SliceArray.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/SliceArray.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/SliceArray.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,11 +20,14 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.Pair; -import org.griphyn.vdl.karajan.PairIterator; +import java.util.List; + +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.PairSet; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.InvalidPathException; @@ -36,99 +39,93 @@ -public class SliceArray extends VDLFunction { +public class SliceArray extends SwiftFunction { + private ArgRef var; + private ArgRef path; + private ArgRef type; - public static final Arg PA_TYPE = new Arg.Positional("type"); + @Override + protected Signature getSignature() { + return new Signature(params("var", "path", "type")); + } - static { - setArguments(SliceArray.class, new Arg[] { PA_VAR, PA_PATH, PA_TYPE }); - } - public Object function(VariableStack stack) throws ExecutionException { - Object var1 = PA_VAR.getValue(stack); - -// TODO for now, this insists the the array be closed entirely before we -// execute. This may cause overserialisation; and worse, will break when -// we are trying to use the cut as an output parameter, not an input -// parameter (likely resulting in a hang). -// Need to think hard about how to handle this. Static assignment -// analysis is going to fail, I think - its like pointer aliasing in C, -// a bit. If I get a ref to an array element using this, then I can -// assign to it, but the compiler isn't going to be aware that I am -// assigning to it so can't construct partialCloseDatasets correctly... -// perhaps thats another argument for map? (as in, moving away from -// [] based assignments... - - - if(var1 instanceof DSHandle) { - - try { - AbstractDataNode sourceArray = (AbstractDataNode) var1; - sourceArray.waitFor(); - - Type sourceType = sourceArray.getType(); - - if(!sourceType.isArray()) { - throw new RuntimeException("SliceArray can only slice arrays."); - } - - String destinationTypeName = (String) PA_TYPE.getValue(stack); - Type destinationType = Types.getType(destinationTypeName); - RootArrayDataNode destinationArray = new RootArrayDataNode(destinationType); - - - Path cutPath = Path.EMPTY_PATH.addLast((String) PA_PATH.getValue(stack), false); - - PairIterator it = new PairIterator(sourceArray.getArrayValue()); - - while(it.hasNext()) { - Pair pair = (Pair) it.next(); - Object index = pair.get(0); - DSHandle sourceElement = (DSHandle) pair.get(1); - - Path p = Path.EMPTY_PATH.addLast((Comparable) index, true); - - DSHandle n = sourceElement.getField(cutPath); - - destinationArray.getField(p).set(n); - } - - // all of the inputs should be closed, so - // we only need shallow close - destinationArray.closeShallow(); - - return destinationArray; - -/* code from setfieldvalue to look at: - } else if(source.getType().isArray()) { - PairIterator it = new PairIterator(source.getArrayValue()); - while(it.hasNext()) { - Pair pair = (Pair) it.next(); - Object lhs = pair.get(0); - DSHandle rhs = (DSHandle) pair.get(1); - Path memberPath = Path.EMPTY_PATH.addLast(String.valueOf(lhs),true); - DSHandle field; - try { - field = dest.getField(memberPath); - } catch(InvalidPathException ipe) { - throw new ExecutionException("Could not get destination field",ipe); - } - deepCopy(field,rhs,stack); - } - closeShallow(stack, dest); - -*/ - } - catch(NoSuchTypeException nste) { - throw new ExecutionException("No such type",nste); - } - catch (InvalidPathException e) { - throw new ExecutionException(e); - } - } else { - throw new ExecutionException("was expecting a DSHandle or collection of DSHandles, got: "+var1.getClass()); + @Override + public Object function(Stack stack) { + // TODO for now, this insists the the array be closed entirely before we + // execute. This may cause overserialisation; and worse, will break when + // we are trying to use the cut as an output parameter, not an input + // parameter (likely resulting in a hang). + // Need to think hard about how to handle this. Static assignment + // analysis is going to fail, I think - its like pointer aliasing in C, + // a bit. If I get a ref to an array element using this, then I can + // assign to it, but the compiler isn't going to be aware that I am + // assigning to it so can't construct partialCloseDatasets correctly... + // perhaps thats another argument for map? (as in, moving away from + // [] based assignments... + + + try { + AbstractDataNode sourceArray = this.var.getValue(stack); + sourceArray.waitFor(this); + + Type sourceType = sourceArray.getType(); + + if(!sourceType.isArray()) { + throw new RuntimeException("SliceArray can only slice arrays."); + } + + String destinationTypeName = this.type.getValue(stack); + Type destinationType = Types.getType(destinationTypeName); + RootArrayDataNode destinationArray = new RootArrayDataNode(destinationType); + + + Path cutPath = Path.EMPTY_PATH.addLast(this.path.getValue(stack), false); + + PairSet s = new PairSet(sourceArray.getArrayValue()); + + for (List pair : s) { + Object index = pair.get(0); + DSHandle sourceElement = (DSHandle) pair.get(1); + + Path p = Path.EMPTY_PATH.addLast((Comparable) index, true); + + DSHandle n = sourceElement.getField(cutPath); + + destinationArray.getField(p).set(n); + } + + // all of the inputs should be closed, so + // we only need shallow close + destinationArray.closeShallow(); + + return destinationArray; + + /* code from setfieldvalue to look at: + } else if(source.getType().isArray()) { + PairIterator it = new PairIterator(source.getArrayValue()); + while(it.hasNext()) { + Pair pair = (Pair) it.next(); + Object lhs = pair.get(0); + DSHandle rhs = (DSHandle) pair.get(1); + Path memberPath = Path.EMPTY_PATH.addLast(String.valueOf(lhs),true); + DSHandle field; + try { + field = dest.getField(memberPath); + } catch(InvalidPathException ipe) { + throw new ExecutionException("Could not get destination field",ipe); + } + deepCopy(field,rhs,stack); + } + closeShallow(stack, dest); + + */ } + catch(NoSuchTypeException nste) { + throw new ExecutionException("No such type",nste); + } + catch (InvalidPathException e) { + throw new ExecutionException(this, e); + } } - - } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Stagein.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Stagein.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Stagein.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,52 +22,55 @@ import java.util.Collection; +import k.rt.ExecutionException; +import k.rt.Stack; +import k.thr.LWThread; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.Arg.Channel; -import org.globus.cog.karajan.arguments.ArgUtil; -import org.globus.cog.karajan.arguments.NamedArguments; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.futures.FutureFault; -import org.globus.cog.karajan.workflow.nodes.FlowNode; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.futures.FutureFault; +import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.engine.Karajan; import org.griphyn.vdl.mapping.AbstractDataNode; -import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.DependentException; import org.griphyn.vdl.mapping.MappingDependentException; import org.griphyn.vdl.mapping.Path; -public class Stagein extends VDLFunction { +public class Stagein extends SwiftFunction { public static final Logger logger = Logger.getLogger(Stagein.class); - public static final Arg VAR = new Arg.Positional("var"); + private ArgRef var; + private ChannelRef cr_stagein; + private VarRef r_deperror; + private VarRef r_mdeperror; - public static final Channel STAGEIN = new Channel("stagein"); - private Tracer tracer; private String procName; - - static { - setArguments(Stagein.class, new Arg[] { VAR }); - } @Override - protected void initializeStatic() { - super.initializeStatic(); - FlowNode def = (FlowNode) getParent().getParent(); + protected Signature getSignature() { + return new Signature(params("var"), returns("deperror", "mdeperror", channel("stagein", DYNAMIC))); + } + + @Override + public Node compile(WrapperNode w, Scope scope) + throws CompilationException { + Node def = getParent().getParent(); procName = Karajan.demangle(def.getTextualName()); tracer = Tracer.getTracer(def, "APPCALL"); + return super.compile(w, scope); } - private boolean isPrimitive(DSHandle var) { - return (var instanceof AbstractDataNode && ((AbstractDataNode) var) - .isPrimitive()); - } - - protected Object function(VariableStack stack) throws ExecutionException { - AbstractDataNode var = (AbstractDataNode) VAR.getValue(stack); - if (!isPrimitive(var)) { + @Override + public Object function(Stack stack) { + AbstractDataNode var = this.var.getValue(stack); + if (var.isPrimitive()) { boolean deperr = false; boolean mdeperr = false; try { @@ -75,19 +78,21 @@ try { for (Path p : fp) { AbstractDataNode n = (AbstractDataNode) var.getField(p); - n.waitFor(); + n.waitFor(this); } } catch (DependentException e) { deperr = true; } + + k.rt.Channel stagein = cr_stagein.get(stack); for (Path p : fp) { - STAGEIN.ret(stack, filename(stack, var.getField(p))[0]); + stagein.add(filename(var.getField(p))[0]); } } catch (FutureFault f) { if (tracer.isEnabled()) { - tracer.trace(stack, procName + " WAIT " + Tracer.getFutureName(f.getFuture())); + tracer.trace(LWThread.currentThread(), procName + " WAIT " + Tracer.getFutureName(f.getFuture())); } throw f; } @@ -97,27 +102,28 @@ mdeperr = true; } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } - if (deperr || mdeperr) { - NamedArguments named = ArgUtil.getNamedReturn(stack); - named.add("deperror", deperr); - named.add("mdeperror", mdeperr); + if (deperr) { + this.r_deperror.setValue(stack, true); } + if (mdeperr) { + this.r_mdeperror.setValue(stack, true); + } } else { // we still wait until the primitive value is there if (tracer.isEnabled()) { try { - var.waitFor(); + var.waitFor(this); } catch (FutureFault f) { - tracer.trace(stack, procName + " WAIT " + Tracer.getFutureName(f.getFuture())); + tracer.trace(LWThread.currentThread(), procName + " WAIT " + Tracer.getFutureName(f.getFuture())); throw f; } } else { - var.waitFor(); + var.waitFor(this); } } return null; Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Stageout.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Stageout.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Stageout.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -24,35 +24,33 @@ import java.util.Collection; import java.util.List; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.Arg.Channel; -import org.globus.cog.karajan.arguments.ArgUtil; -import org.globus.cog.karajan.arguments.NamedArguments; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.DataNode; import org.griphyn.vdl.mapping.MappingDependentException; import org.griphyn.vdl.mapping.Path; -public class Stageout extends VDLFunction { +public class Stageout extends SwiftFunction { public static final Logger logger = Logger.getLogger(Stageout.class); - public static final Arg VAR = new Arg.Positional("var"); - - public static final Channel STAGEOUT = new Channel("stageout"); - public static final Channel RESTARTOUT = new Channel("restartout"); - - static { - setArguments(Stageout.class, new Arg[] { VAR }); + private ArgRef var; + private ChannelRef cr_stageout; + private ChannelRef cr_restartout; + private VarRef r_deperror; + private VarRef r_mdeperror; + + @Override + protected Signature getSignature() { + return new Signature(params("var"), returns("deperror", "mdeperror", + channel("stageout", DYNAMIC), channel("restartout", DYNAMIC))); } - - private boolean isPrimitive(DSHandle var) { - return (var instanceof AbstractDataNode && ((AbstractDataNode) var) - .isPrimitive()); - } private List list(Path p, DSHandle var) { ArrayList l = new ArrayList(2); @@ -61,8 +59,9 @@ return l; } - protected Object function(VariableStack stack) throws ExecutionException { - AbstractDataNode var = (AbstractDataNode) VAR.getValue(stack); + @Override + public Object function(Stack stack) { + AbstractDataNode var = this.var.getValue(stack); boolean deperr = false; boolean mdeperr = false; // currently only static arrays are supported as app returns @@ -71,14 +70,14 @@ // race conditions (e.g. if this array's mapper had some parameter // dependencies that weren't closed at the time the app was started). if (var.getType().isArray()) { - var.waitFor(); + var.waitFor(this); } try { - if (!isPrimitive(var)) { - retPaths(STAGEOUT, stack, var); + if (var.isPrimitive()) { + retPaths(cr_stageout.get(stack), var); } if (var.isRestartable()) { - retPaths(RESTARTOUT, stack, var); + retPaths(cr_restartout.get(stack), var); } } catch (MappingDependentException e) { @@ -86,23 +85,24 @@ deperr = true; mdeperr = true; } - if (deperr || mdeperr) { - NamedArguments named = ArgUtil.getNamedReturn(stack); - named.add("deperror", deperr); - named.add("mdeperror", mdeperr); + if (deperr) { + this.r_deperror.setValue(stack, true); } + if (mdeperr) { + this.r_mdeperror.setValue(stack, true); + } return null; } - private void retPaths(Channel channel, VariableStack stack, DSHandle var) throws ExecutionException { + private void retPaths(k.rt.Channel channel, DSHandle var) throws ExecutionException { try { Collection fp = var.getFringePaths(); for (Path p : fp) { - channel.ret(stack, list(p, var)); + channel.add(list(p, var)); } } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } } Deleted: branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftArg.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftArg.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftArg.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,244 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Jul 31, 2007 - */ -package org.griphyn.vdl.karajan.lib; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.ArgUtil; -import org.globus.cog.karajan.arguments.VariableArguments; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.mapping.AbstractDataNode; -import org.griphyn.vdl.mapping.ArrayDataNode; -import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.DataNode; -import org.griphyn.vdl.type.Type; -import org.griphyn.vdl.type.Types; - -public abstract class SwiftArg extends Arg { - public static final Logger logger = Logger.getLogger(SwiftArg.class); - - public SwiftArg(String name, int index) { - super(name, index); - } - - public SwiftArg(String name) { - super(name, IMPLICIT); - } - - protected Object unwrap(VariableStack stack, Object val) throws ExecutionException { - if (val instanceof DSHandle) { - AbstractDataNode handle = (AbstractDataNode) val; - if (logger.isDebugEnabled()) { - logger.debug("SwiftArg.getValue(" + handle + ")"); - } - if (handle.getType().isArray()) { - return handle; - } - else { - handle.waitFor(); - return handle.getValue(); - } - } - else { - throw new ExecutionException("Expected Swift data, but got some primitive type (" + val - + ")"); - } - } - - public Object getValue(VariableStack stack) throws ExecutionException { - Object v = super.getValue(stack); - if (v == null) { - return v; - } - else { - return unwrap(stack, v); - } - } - - public double getDoubleValue(VariableStack stack) throws ExecutionException { - return checkDouble(getValue(stack)); - } - - public static double checkDouble(Object dbl) throws ExecutionException { - if (dbl instanceof Number) { - return ((Number) dbl).doubleValue(); - } - else { - throw new ExecutionException("Internal type error. Expected a Double. Got " + classOf(dbl)); - } - } - - public static int checkInt(Object i) throws ExecutionException { - if (i instanceof Integer) { - return ((Integer) i).intValue(); - } - else { - throw new ExecutionException("Internal type error. Expected a Integer. Got " + classOf(i)); - } - } - - public DSHandle getRawValue(VariableStack stack) throws ExecutionException { - Object v = super.getValue(stack); - if(v instanceof DSHandle) { - return (DSHandle)v; - } else if(v == null) { - return null; - } else { - throw new ExecutionException("Expected Swift data, but got some primitive type (" + v + ")"); - } - } - - private static Class classOf(Object object) { - if (object == null) { - return null; - } - else { - return object.getClass(); - } - } - - protected Type getType0(Object o) throws ExecutionException { - if (o instanceof DSHandle) { - DSHandle handle = (DSHandle) o; - return handle.getType(); - } - else { - throw new ExecutionException("Expected Swift data, but got some primitive type (" + o - + ")"); - } - } - - public Type getType(VariableStack stack) throws ExecutionException { - return getType0(super.getValue(stack)); - } - - public static class Positional extends SwiftArg { - public Positional(String name, int index) { - super(name, index); - } - - public Positional(String name) { - super(name); - } - } - - public static class Optional extends SwiftArg { - private final Object defaultValue; - private final Type defaultType; - - public Optional(String name, Object defaultValue, Type defaultType) { - super(name, NOINDEX); - this.defaultValue = defaultValue; - this.defaultType = defaultType; - } - - public Optional(String name) { - this(name, null, Types.ANY); - } - - public Object getValue(VariableStack stack) throws ExecutionException { - Object o = super.getValue(stack); - if (o == null) { - return defaultValue; - } - else { - return o; - } - } - - public Type getType(VariableStack stack) throws ExecutionException { - Object o = super.getValue(stack, defaultValue); - if (o == null) { - return defaultType; - } - else { - return getType0(o); - } - } - } - - public static final class Vargs extends SwiftArg { - public Vargs() { - super("..."); - } - - public Object[] asArray(VariableStack stack) throws ExecutionException { - VariableArguments args = get(stack); - Object[] ret = new Object[args.size()]; - for (int i = 0; i < ret.length; i++) { - ret[i] = unwrap(stack, args.get(i)); - } - return ret; - } - - public DSHandle[] asDSHandleArray(VariableStack stack) throws ExecutionException { - VariableArguments args = get(stack); - DSHandle[] ret = new DSHandle[args.size()]; - for (int i = 0; i < ret.length; i++) { - ret[i] = (DSHandle) args.get(i); - } - return ret; - } - - public AbstractDataNode[] asDataNodeArray(VariableStack stack) throws ExecutionException { - VariableArguments args = get(stack); - AbstractDataNode[] ret = new AbstractDataNode[args.size()]; - for (int i = 0; i < ret.length; i++) { - ret[i] = (AbstractDataNode) args.get(i); - } - return ret; - } - - public List asList(VariableStack stack) throws ExecutionException { - VariableArguments args = get(stack); - List ret = new ArrayList(); - Iterator i = args.iterator(); - while (i.hasNext()) { - ret.add(unwrap(stack, i.next())); - } - return get(stack).getAll(); - } - - public VariableArguments get(VariableStack stack) throws ExecutionException { - VariableArguments args = ArgUtil.getVariableArguments(stack); - if (args == null) { - throw new ExecutionException("No default channel found on stack"); - } - return args; - } - - public boolean isPresent(VariableStack stack) throws ExecutionException { - return ArgUtil.getVariableArguments(stack) != null; - } - - public String getVariableName() { - return "#vargs"; - } - - } - - public static final Vargs VARGS = new Vargs(); -} Added: branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java (rev 0) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,442 @@ +/* + * Copyright 2012 University of Chicago + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.griphyn.vdl.karajan.lib; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import k.rt.Channel; +import k.rt.Context; +import k.rt.ExecutionException; +import k.rt.MemoryChannel; +import k.rt.Stack; +import k.thr.LWThread; + +import org.apache.log4j.Logger; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.compiled.nodes.functions.AbstractFunction; +import org.globus.cog.karajan.parser.WrapperNode; +import org.globus.cog.karajan.util.BoundContact; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.swift.catalog.TCEntry; +import org.globus.swift.catalog.transformation.File; +import org.globus.swift.catalog.types.TCType; +import org.griphyn.vdl.karajan.AssertFailedException; +import org.griphyn.vdl.karajan.Loader; +import org.griphyn.vdl.karajan.TCCache; +import org.griphyn.vdl.karajan.functions.ConfigProperty; +import org.griphyn.vdl.mapping.AbsFile; +import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.GeneralizedFileFormat; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.InvalidPathException; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.PathComparator; +import org.griphyn.vdl.mapping.PhysicalFormat; +import org.griphyn.vdl.type.Type; +import org.griphyn.vdl.type.Types; +import org.griphyn.vdl.util.FQN; +import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.VDL2ConfigProperties; + +public abstract class SwiftFunction extends AbstractFunction { + public static final Logger logger = Logger.getLogger(SwiftFunction.class); + + public static final boolean PROVENANCE_ENABLED; + + static { + boolean v; + try { + v = VDL2Config.getConfig().getProvenanceLog(); + } + catch (IOException e) { + v = false; + } + PROVENANCE_ENABLED = v; + } + + + private VarRef context; + + @Override + protected void addLocals(Scope scope) { + super.addLocals(scope); + context = scope.getVarRef("#context"); + } + + @Override + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) + throws CompilationException { + returnDynamic(scope); + return super.compileBody(w, argScope, scope); + } + + + + @Override + public void runBody(LWThread thr) { + try { + Stack stack = thr.getStack(); + ret(stack, function(stack)); + } + catch (AssertFailedException e) { + logger.fatal("swift: assert failed: " + e.getMessage()); + throw e; + } + catch (DependentException e) { + // This would not be the primal fault so in non-lazy errors mode it + // should not matter + throw new ExecutionException("Wrapping a dependent exception in VDLFunction.post() - errors in data dependencies",e); + } + } + + /* + * This will likely break if the engine changes in fundamental ways. It also + * depends on the fact that iteration variable is named '$' in this + * particular implementation. + */ + public static String getThreadPrefix() throws ExecutionException { + return LWThread.currentThread().getName(); + } + + // TODO - is this needed any more? its doing some type inferencing and + // object creation and dequoting of strings, but the necessary behaviour + // here has possibly moved elsewhere, into a more strongly typed + // intermediate + // XML form that removes the need for this inference. + + // we might need to do some casting here for the numerical stuff - eg when + // asking for a float but we're given an int? not sure? might be the case + // that we already have value in the Double form already, in which case + // deference the internal value? + + // this is only used by VDL new (and really should only be used by + // VDL new, and should perhaps move to the VDL new source?) + + protected Object internalValue(Type type, Object value) { + if (Types.FLOAT.equals(type)) { + return new Double(TypeUtil.toDouble(value)); + } + else if (Types.INT.equals(type)) { + return new Integer(TypeUtil.toInt(value)); + } + else if (Types.BOOLEAN.equals(type)) { + return new Boolean(TypeUtil.toBoolean(value)); + } + else { + return value; + } + } + + public static final String[] EMPTY_STRING_ARRAY = new String[0]; + + + public static String[] filename(DSHandle var) throws ExecutionException { + try { + if (var.getType().isArray()) { + return leavesFileNames(var); + } + else if(var.getType().getFields().size() > 0) { + return leavesFileNames(var); + } + else { + return new String[] { leafFileName(var) }; + } + } + catch (DependentException e) { + return new String[0]; + } + catch (HandleOpenException e) { + throw new ExecutionException("The current implementation should not throw this exception", e); + } + } + + private static String[] leavesFileNames(DSHandle var) throws ExecutionException, HandleOpenException { + Mapper mapper; + + synchronized (var.getRoot()) { + mapper = var.getMapper(); + } + + if (mapper == null) { + throw new ExecutionException(var.getType() + " is not a mapped type"); + } + + List l = new ArrayList(); + try { + Collection fp = var.getFringePaths(); + List src; + if (fp instanceof List) { + src = (List) fp; + } + else { + src = new ArrayList(fp); + } + Collections.sort(src, new PathComparator()); + + for (Path p : src) { + l.add(leafFileName(var.getField(p), mapper)); + } + } + catch (InvalidPathException e) { + throw new ExecutionException("DSHandle is lying about its fringe paths"); + } + return l.toArray(EMPTY_STRING_ARRAY); + } + + private static String leafFileName(DSHandle var) throws ExecutionException { + return leafFileName(var, var.getMapper()); + } + + private static String leafFileName(DSHandle var, Mapper mapper) throws ExecutionException { + if (Types.STRING.equals(var.getType())) { + return relativize(String.valueOf(var.getValue())); + } + else { + if (var.getMapper() == null) { + throw new ExecutionException("Cannot invoke filename() on data without a mapper: " + var); + } + PhysicalFormat f = var.getMapper().map(var.getPathFromRoot()); + if (f instanceof GeneralizedFileFormat) { + String filename = ((GeneralizedFileFormat) f).getURIAsString(); + if (filename == null) { + throw new ExecutionException("Mapper did not provide a file name"); + } + else { + return filename; + } + } + else if (f == null) { + throw new ExecutionException("Mapper failed to map " + var); + } + else { + throw new ExecutionException("Only file formats are supported for now"); + } + } + } + + protected Object pathOnly(Object f) { + if (f instanceof String[]) { + return pathOnly((String[]) f); + } + else { + return pathOnly((String) f); + } + } + + protected static String pathOnly(String file) { + AbsFile af = new AbsFile(file); + if ("file".equals(af.getProtocol())) { + return af.getPath(); + } + else { + return af.getHost() + "/" + af.getPath(); + } + } + + protected String[] pathOnly(String[] files) { + String[] p = new String[files.length]; + for (int i = 0; i < files.length; i++) { + p[i] = pathOnly(files[i]); + } + return p; + } + + /** + * Given an input of an array of strings, returns a single string with the + * input strings separated by a space. If the 'relative' flag is set to + * true, then each input string will be passed through the relativize + * function. + */ + public String argList(String[] s, boolean relative) { + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < s.length; i++) { + if (relative) { + s[i] = relativize(s[i]); + } + sb.append(s[i]); + if (i < s.length - 1) { + sb.append(' '); + } + } + return sb.toString(); + } + + /** + * removes leading / character from a supplied filename if present, so that + * the path can be used as a relative path. + */ + public static String relativize(String name) { + name = pathOnly(name); + if (name != null && name.length() > 0 && name.charAt(0) == '/') { + return name.substring(1); + } + else { + return name; + } + } + + protected boolean compatible(Type expectedType, Type actualType) { + if (expectedType.equals(Types.FLOAT)) { + if (actualType.equals(Types.FLOAT) || actualType.equals(Types.INT)) { + return true; + } + else { + return false; + } + } + else if (expectedType.equals(Types.FLOAT.arrayType())) { + if (actualType.equals(Types.FLOAT.arrayType()) + || actualType.equals(Types.INT.arrayType())) { + return true; + } + else { + return false; + } + } + else if (expectedType.equals(Types.ANY)) { + return true; + } + else { + return actualType.equals(expectedType); + } + } + + protected void closeChildren(AbstractDataNode handle) throws InvalidPathException { + // Close the future + handle.closeShallow(); + // Mark all leaves + for (DSHandle child : handle.getFields(Path.CHILDREN)) { + child.closeShallow(); + } + } + + public static void waitForAll(Node who, Channel vargs) throws ExecutionException { + for (AbstractDataNode n : vargs) { + n.waitFor(who); + } + } + + public static Map, DSHandle> waitForArray(Node who, AbstractDataNode n) throws ExecutionException { + n.waitFor(who); + Map, DSHandle> v = n.getArrayValue(); + for (DSHandle h : v.values()) { + ((AbstractDataNode) h).waitFor(who); + } + return v; + } + + public static Channel unwrapAll(Node who, Channel vargs) throws ExecutionException { + waitForAll(who, vargs); + MemoryChannel mc = new MemoryChannel(); + for (AbstractDataNode n : vargs) { + mc.add(n.getValue()); + } + return mc; + } + + @SuppressWarnings("unchecked") + public static T unwrap(Node who, AbstractDataNode n) throws ExecutionException { + n.waitFor(who); + return (T) n.getValue(); + } + + public static Path parsePath(Object o) { + if (o instanceof Path) { + return (Path) o; + } + else { + return Path.parse((String) o); + } + } + + private static Set> warnset = new HashSet>(); + + protected TCEntry getTCE(TCCache tc, FQN fqn, BoundContact bc) { + List l; + try { + l = tc.getTCEntries(fqn, bc.getHost(), TCType.INSTALLED); + } + catch (Exception e) { + throw new ExecutionException(this, e); + } + if (l == null || l.isEmpty()) { + return null; + } + if (l.size() > 1) { + synchronized (warnset) { + LinkedList wl = new LinkedList(); + wl.add(fqn); + wl.add(bc); + if (!warnset.contains(wl)) { + logger.warn("Multiple entries found for " + fqn + " on " + bc + + ". Using the first one"); + warnset.add(wl); + } + } + } + return l.get(0); + } + + public static final String TC = "vdl:TC"; + + public TCCache getTC(Stack stack) throws ExecutionException { + Context c = this.context.getValue(stack); + synchronized (c) { + TCCache tc = (TCCache) c.getAttribute(TC); + if (tc == null) { + String prop = ConfigProperty.getProperty(VDL2ConfigProperties.TC_FILE, (VDL2Config) c.getAttribute("SWIFT:CONFIG")); + Loader.debugText("TC", new java.io.File(prop)); + tc = new TCCache(File.getNonSingletonInstance(prop)); + c.setAttribute(TC, tc); + } + return tc; + } + } + + private static int provenanceIDCount = 451000; + + public static synchronized int nextProvenanceID() { + return provenanceIDCount++; + } + + public static void logProvenanceResult(int id, DSHandle result, String name) { + if (logger.isDebugEnabled()) + logger.debug("FUNCTION id="+id+" name="+name+" result="+result.getIdentifier()); + else if (logger.isInfoEnabled()) + logger.info("FUNCTION: " + name + "()"); + } + + public static void logProvenanceParameter(int id, DSHandle parameter, String paramName) { + if (logger.isDebugEnabled()) + logger.debug("FUNCTIONPARAMETER id="+id+" input="+parameter.getIdentifier()+" name="+paramName); + } +} Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/TCProfile.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -27,85 +27,84 @@ import java.util.Map; import java.util.Set; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; import org.globus.cog.abstraction.impl.common.execution.WallTime; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.ArgUtil; -import org.globus.cog.karajan.arguments.NamedArguments; -import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.functions.Map.Entry; import org.globus.cog.karajan.util.BoundContact; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.grid.GridExec; import org.globus.swift.catalog.TCEntry; import org.globus.swift.catalog.util.Profile; import org.griphyn.vdl.karajan.TCCache; import org.griphyn.vdl.util.FQN; -public class TCProfile extends VDLFunction { +public class TCProfile extends SwiftFunction { public static final Logger logger = Logger.getLogger(TCProfile.class); - public static final Arg OA_TR = new Arg.Optional("tr"); - - /** - Allows for dynamic attributes from the SwiftScript - profile statements. - These override any other attributes. - */ - public static final Arg OA_ATTRS = new Arg.Positional("attributes"); - - public static final Arg PA_HOST = new Arg.Positional("host"); - - static { - setArguments(TCProfile.class, new Arg[] { PA_HOST, OA_ATTRS, OA_TR }); - } + private ArgRef host; + /** + Allows for dynamic attributes from the SwiftScript + profile statements. + These override any other attributes. + */ + private ArgRef> attributes; + private ArgRef tr; + + private VarRef r_count; + private VarRef r_jobType; + private VarRef r_attributes; + private ChannelRef> cr_environment; + + private enum Attr { + COUNT, JOB_TYPE; + } + + private static final Map ATTR_TYPES; + + static { + ATTR_TYPES = new HashMap(); + ATTR_TYPES.put("count", Attr.COUNT); + ATTR_TYPES.put("jobType", Attr.JOB_TYPE); + } - private static Map PROFILE_T; + @Override + protected Signature getSignature() { + return new Signature( + params("host", optional("attributes", null), optional("tr", null)), + returns("count", "jobType", + "attributes", channel("environment", DYNAMIC)) + ); + } - static { - PROFILE_T = new HashMap(); - PROFILE_T.put("count", GridExec.A_COUNT); - PROFILE_T.put("jobtype", GridExec.A_JOBTYPE); - PROFILE_T.put("maxcputime", GridExec.A_MAXCPUTIME); - PROFILE_T.put("maxmemory", GridExec.A_MAXMEMORY); - PROFILE_T.put("maxtime", GridExec.A_MAXTIME); - PROFILE_T.put("maxwalltime", GridExec.A_MAXWALLTIME); - PROFILE_T.put("minmemory", GridExec.A_MINMEMORY); - PROFILE_T.put("project", GridExec.A_PROJECT); - PROFILE_T.put("queue", GridExec.A_QUEUE); - } - - public Object function(VariableStack stack) throws ExecutionException { + public Object function(Stack stack) { TCCache tc = getTC(stack); - String tr = null; + String tr = this.tr.getValue(stack); - Map dynamicAttributes = - readDynamicAttributes(stack); + Map dynamicAttributes = readDynamicAttributes(stack); - if (OA_TR.isPresent(stack)) { - tr = TypeUtil.toString(OA_TR.getValue(stack)); - } - BoundContact bc = (BoundContact) PA_HOST.getValue(stack); + BoundContact bc = this.host.getValue(stack); - NamedArguments named = ArgUtil.getNamedReturn(stack); Map attrs = null; - attrs = attributesFromHost(bc, attrs, named); + attrs = attributesFromHost(bc, attrs, stack); TCEntry tce = null; if (tr != null) { tce = getTCE(tc, new FQN(tr), bc); } - Map env = new HashMap(); if (tce != null) { - addEnvironment(env, tce); - addEnvironment(env, bc); - attrs = attributesFromTC(tce, attrs, named); + addEnvironment(stack, tce); + addEnvironment(stack, bc); + attrs = attributesFromTC(tce, attrs, stack); } - named.add(GridExec.A_ENVIRONMENT, env); - checkWalltime(tr, named); attrs = addDynamicAttributes(attrs, dynamicAttributes); - addAttributes(named, attrs); + checkWalltime(attrs, tr, stack); + addAttributes(attrs, stack); return null; } @@ -113,14 +112,8 @@ Bring in the dynamic attributes from the Karajan stack @return Map, may be null */ - @SuppressWarnings("unchecked") - private Map - readDynamicAttributes(VariableStack stack) - throws ExecutionException { - Map result = null; - if (OA_ATTRS.isPresent(stack)) - result = (Map) OA_ATTRS.getValue(stack); - return result; + private Map readDynamicAttributes(Stack stack) { + return this.attributes.getValue(stack); } /** @@ -143,13 +136,11 @@ return result; } - private void checkWalltime(String tr, NamedArguments attrs) { - Object walltime = null; - if (attrs != null) { - if (attrs.hasArgument("maxwalltime")) { - walltime = attrs.getArgument("maxwalltime"); - } - } + private void checkWalltime(Map attrs, String tr, Stack stack) { + if (attrs == null) { + return; + } + Object walltime = attrs.get("maxwalltime"); if (walltime == null) { return; } @@ -157,7 +148,7 @@ //validate walltime WallTime.timeToSeconds(walltime.toString()); } - catch (IllegalArgumentException e) { + catch (ExecutionException e) { warn(tr, "Warning: invalid walltime specification for \"" + tr + "\" (" + walltime + ")."); } @@ -174,33 +165,30 @@ } } - private void addEnvironment(Map m, - TCEntry tce) { + private void addEnvironment(Stack stack, TCEntry tce) { List list = tce.getProfiles(Profile.ENV); if (list != null) { for (Profile p : list) { - m.put(p.getProfileKey(), p.getProfileValue()); + cr_environment.append(stack, new Entry(p.getProfileKey(), p.getProfileValue())); } } } - public static final String PROFILE_GLOBUS_PREFIX = - (Profile.GLOBUS + "::").toLowerCase(); + public static final String PROFILE_GLOBUS_PREFIX = (Profile.GLOBUS + "::").toLowerCase(); - private void addEnvironment(Map m, - BoundContact bc) { + private void addEnvironment(Stack stack, BoundContact bc) { Map props = bc.getProperties(); for (Map.Entry e : props.entrySet()) { String name = e.getKey(); FQN fqn = new FQN(name); String value = (String) e.getValue(); if (Profile.ENV.equalsIgnoreCase(fqn.getNamespace())) { - m.put(fqn.getName(), value); + cr_environment.append(stack, new Entry(fqn.getName(), value)); } } } - private void addAttributes(NamedArguments named, Map attrs) { + private void addAttributes(Map attrs, Stack stack) { if (logger.isDebugEnabled()) { logger.debug("Attributes: " + attrs); } @@ -210,26 +198,23 @@ Iterator> i = attrs.entrySet().iterator(); while (i.hasNext()) { Map.Entry e = i.next(); - Arg a = PROFILE_T.get(e.getKey()); + Attr a = ATTR_TYPES.get(e.getKey()); if (a != null) { - named.add(a, e.getValue()); + setAttr(a, stack, e.getValue()); i.remove(); } } if (attrs.size() == 0) { return; } - named.add(GridExec.A_ATTRIBUTES, attrs); + this.r_attributes.setValue(stack, attrs); } - private Map - attributesFromTC(TCEntry tce, - Map attrs, - NamedArguments named) { + private Map attributesFromTC(TCEntry tce, Map attrs, Stack stack) { List list = tce.getProfiles(Profile.GLOBUS); if (list != null) { for (Profile p : list) { - Arg a = PROFILE_T.get(p.getProfileKey()); + Attr a = ATTR_TYPES.get(p.getProfileKey()); if (a == null) { if (attrs == null) { attrs = new HashMap(); @@ -237,7 +222,7 @@ attrs.put(p.getProfileKey(), p.getProfileValue()); } else { - named.add(a, p.getProfileValue()); + setAttr(a, stack, p.getProfileValue()); } } } @@ -248,16 +233,13 @@ Inserts namespace=globus attributes from BoundContact bc into given attrs */ - private Map - attributesFromHost(BoundContact bc, - Map attrs, - NamedArguments named) { + private Map attributesFromHost(BoundContact bc, Map attrs, Stack stack) { Map props = bc.getProperties(); if (props != null) { for (Map.Entry e : props.entrySet()) { FQN fqn = new FQN(e.getKey()); if (Profile.GLOBUS.equalsIgnoreCase(fqn.getNamespace())) { - Arg a = PROFILE_T.get(fqn.getName()); + Attr a = ATTR_TYPES.get(fqn.getName()); if (a == null) { if (attrs == null) { attrs = new HashMap(); @@ -265,11 +247,22 @@ attrs.put(fqn.getName(), e.getValue()); } else { - named.add(a, e.getValue()); + setAttr(a, stack, e.getValue()); } } } } return attrs; } + + private void setAttr(Attr a, Stack stack, Object value) { + switch (a) { + case COUNT: + r_count.setValue(stack, value); + break; + case JOB_TYPE: + r_jobType.setValue(stack, value); + break; + } + } } Deleted: branches/faster/src/org/griphyn/vdl/karajan/lib/ThreadPrefix.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/ThreadPrefix.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/ThreadPrefix.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,32 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Dec 26, 2006 - */ -package org.griphyn.vdl.karajan.lib; - -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; - -public class ThreadPrefix extends VDLFunction { - - public Object function(VariableStack stack) throws ExecutionException { - return getThreadPrefix(stack); - } - -} Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Throttled.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Throttled.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Throttled.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -23,55 +23,69 @@ import java.io.IOException; import java.util.LinkedList; -import org.globus.cog.karajan.stack.VariableStack; +import k.rt.ConditionalYield; +import k.rt.FutureObject; +import k.thr.LWThread; +import k.thr.Yield; + +import org.globus.cog.karajan.compiled.nodes.Sequential; import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.Sequential; import org.griphyn.vdl.util.VDL2Config; public class Throttled extends Sequential { public static final int DEFAULT_MAX_THREADS = 1000000; - private LinkedList waiting; + private LinkedList waiting; private int maxThreadCount, current; public Throttled() { try { maxThreadCount = TypeUtil.toInt(VDL2Config.getConfig() - .getProperty("exec.throttle", String.valueOf(DEFAULT_MAX_THREADS))); + .getProperty("max.threads", String.valueOf(DEFAULT_MAX_THREADS))); } catch (IOException e) { maxThreadCount = DEFAULT_MAX_THREADS; } current = 0; - waiting = new LinkedList(); + waiting = new LinkedList(); } - + @Override - protected void executeChildren(VariableStack stack) - throws ExecutionException { - synchronized(this) { - if (current == maxThreadCount) { - waiting.addLast(stack); - return; + public void run(LWThread thr) { + int i = thr.checkSliceAndPopState(); + try { + switch (i) { + case 0: + tryIncCurrent(); + i++; + default: + super.run(thr); + decCurrent(); } - else { - current++; - } } - super.executeChildren(stack); + catch (Yield y) { + y.getState().push(i); + throw y; + } + catch (RuntimeException e) { + decCurrent(); + throw e; + } } - - @Override - protected void post(VariableStack stack) throws ExecutionException { - synchronized(this) { - if (!waiting.isEmpty()) { - super.executeChildren(waiting.removeFirst()); - } - else { - current--; - } + + private synchronized void decCurrent() { + current--; + if (!waiting.isEmpty()) { + waiting.removeFirst().setValue(Boolean.TRUE); } - super.post(stack); - } + } + + private synchronized void tryIncCurrent() { + if (current == maxThreadCount) { + FutureObject fo = new FutureObject(); + waiting.addLast(fo); + throw new ConditionalYield(fo); + } + current++; + } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -23,87 +23,99 @@ import java.util.List; import java.util.StringTokenizer; +import k.rt.ExecutionException; +import k.rt.KRunnable; +import k.rt.Stack; +import k.thr.LWThread; +import k.thr.ThreadSet; +import k.thr.ThrottledThreadSet; +import k.thr.Yield; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableNotFoundException; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.Identifier; -import org.globus.cog.karajan.util.KarajanIterator; -import org.globus.cog.karajan.util.ThreadingContext; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.CompilerSettings; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.compiled.nodes.UParallelFor; +import org.globus.cog.karajan.parser.WrapperNode; import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.futures.FutureEvaluationException; -import org.globus.cog.karajan.workflow.futures.FutureFault; -import org.globus.cog.karajan.workflow.futures.FutureIterator; -import org.globus.cog.karajan.workflow.futures.FutureIteratorIncomplete; -import org.globus.cog.karajan.workflow.futures.FutureListener; -import org.globus.cog.karajan.workflow.futures.ListenerStackPair; -import org.globus.cog.karajan.workflow.nodes.AbstractParallelIterator; import org.griphyn.vdl.karajan.Pair; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.util.VDL2Config; -public class ThrottledParallelFor extends AbstractParallelIterator { - public static final Logger logger = Logger.getLogger(ThrottledParallelFor.class); +public class ThrottledParallelFor extends UParallelFor { + public static final Logger logger = Logger + .getLogger(ThrottledParallelFor.class); - public static final int DEFAULT_MAX_THREADS = 10000000; + public static final int DEFAULT_MAX_THREADS = 1024; - public static final Arg A_NAME = new Arg.Positional("name"); - public static final Arg A_IN = new Arg.Positional("in"); - public static final Arg O_SELFCLOSE = new Arg.Optional("selfclose", Boolean.FALSE); - public static final Arg O_REFS = new Arg.Optional("refs", null); + private ArgRef selfClose; + private ArgRef refs; + private ArgRef _kvar; + private ArgRef _vvar; + private ArgRef _traceline; + + @Override + protected Signature getSignature() { + return new Signature( + params( + identifier("name"), "in", + optional("selfClose", Boolean.FALSE), optional("refs", null), + optional("_kvar", null), optional("_vvar", null), optional("_traceline", null), + block("body") + ) + ); + } - static { - setArguments(ThrottledParallelFor.class, new Arg[] { A_NAME, A_IN, O_SELFCLOSE, O_REFS }); - } - - public static final String THREAD_COUNT = "#threadcount"; - private int maxThreadCount = -1; private Tracer forTracer, iterationTracer; - private String kvar, vvar; - private List srefs; - - private static class StaticRefCount { - public final String name; + private List srefs; + + private static class StaticRefCount { + public final VarRef ref; public final int count; - - public StaticRefCount(String name, int count) { - this.name = name; + + public StaticRefCount(VarRef ref, int count) { + this.ref = ref; this.count = count; } - } - - private static class RefCount { - public final DSHandle var; - public final int count; - - public RefCount(DSHandle var, int count) { - this.var = var; - this.count = count; - } - - public void inc() { - - } - - public void dec() { - - } - } - + } + + private static class RefCount { + public final DSHandle var; + public final int count; + + public RefCount(DSHandle var, int count) { + this.var = var; + this.count = count; + } + + public void inc() { + + } + + public void dec() { + + } + } + @Override - protected void initializeStatic() { - super.initializeStatic(); + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) + throws CompilationException { + srefs = buildStaticRefs(scope); + if (_traceline.getValue() != null) { + setLine(Integer.parseInt(_traceline.getValue())); + } forTracer = Tracer.getTracer(this, "FOREACH"); iterationTracer = Tracer.getTracer(this, "ITERATION"); - kvar = (String) getProperty("_kvar"); - vvar = (String) getProperty("_vvar"); - srefs = buildStaticRefs(); + return super.compileBody(w, argScope, scope); } - - private List buildStaticRefs() { - String refs = (String) O_REFS.getStatic(this); + + private List buildStaticRefs(Scope scope) { + String refs = this.refs.getValue(); if (refs == null) { return null; } @@ -117,115 +129,141 @@ } else { int count = Integer.parseInt(st.nextToken()); - l.add(new StaticRefCount(name.toLowerCase(), count)); + l.add(new StaticRefCount(scope.getVarRef(name), count)); } flip = !flip; } return l; } - private List buildRefs(VariableStack stack) throws VariableNotFoundException { + private List buildRefs(Stack stack) { if (srefs == null) { return null; } List l = new ArrayList(srefs.size()); for (StaticRefCount s : srefs) { - l.add(new RefCount((DSHandle) stack.getVar(s.name), s.count)); + l.add(new RefCount((DSHandle) s.ref.getValue(stack), s.count)); } return l; } - protected void partialArgumentsEvaluated(VariableStack stack) - throws ExecutionException { - if (forTracer.isEnabled()) { - forTracer.trace(ThreadingContext.get(stack).toString()); + @SuppressWarnings("unchecked") + @Override + protected void runBody(final LWThread thr) { + int i = thr.checkSliceAndPopState(); + Iterator it = (Iterator) thr.popState(); + ThrottledThreadSet ts = (ThrottledThreadSet) thr.popState(); + int fc = thr.popIntState(); + List drefs = (List) thr.popState(); + Stack stack = thr.getStack(); + try { + switch(i) { + case 0: + it = in.getValue(stack).iterator(); + ts = new ThrottledThreadSet(getMaxThreads()); + drefs = buildRefs(stack); + ts.lock(); + fc = stack.frameCount() + 1; + + if (forTracer.isEnabled()) { + forTracer.trace(thr); + } + + i++; + case 1: + final ThreadSet tsf = ts; + + ts.checkFailed(); + + startBulk(thr, ts, it, fc, drefs); + startRest(thr, ts, it, fc, drefs); + + ts.unlock(); + decRefs(drefs); + ts.waitFor(); + } } - super.partialArgumentsEvaluated(stack); + catch (Yield y) { + y.getState().push(drefs); + y.getState().push(fc); + y.getState().push(ts); + y.getState().push(it); + y.getState().push(i); + throw y; + } } - public void iterate(VariableStack stack, Identifier var, KarajanIterator i) - throws ExecutionException { - if (elementCount() > 0) { - if (logger.isDebugEnabled()) { - logger.debug("iterateParallel: " + stack.parentFrame()); - } - stack.setVar(VAR, var); - setChildFailed(stack, false); - stack.setCaller(this); - initThreadCount(stack, TypeUtil.toBoolean(O_SELFCLOSE.getStatic(this)), i); - citerate(stack, var, i); - } - else { - complete(stack); - } - } - - protected void citerate(VariableStack stack, Identifier var, - KarajanIterator i) throws ExecutionException { - ThreadCount tc = getThreadCount(stack); - - // we can bulk operations at the start to avoid contention - // on the counter since at least as many - // threads as reported by available() are available - int available = tc.available(); - try { - int j = 0; - try { - for (; j < available && i.hasNext(); j++) { - startIteration(tc, var, i.current(), i.next(), stack); - } - } - finally { - tc.add(j); - } - while (i.hasNext()) { - startIteration(tc, var, i.current(), tc.tryIncrement(), stack); - } - - decRefs(tc.rc); - - int left; - synchronized(tc) { - // can only have closed and running = 0 in one place - tc.close(); - left = tc.current(); - } - if (left == 0) { - complete(stack); - } - } - catch (FutureIteratorIncomplete fii) { - synchronized (stack.currentFrame()) { - stack.setVar(ITERATOR, i); + private boolean startBulk(LWThread thr, ThrottledThreadSet ts, Iterator it, int fcf, List refs) { + int available = ts.freeSlots(); + int j = 0; + Stack stack = thr.getStack(); + for (; j < available && it.hasNext(); j++) { + if (startOne(thr, ts, it.next(), fcf, refs)) { + // aborted + return true; } - fii.getFutureIterator().addModificationAction(this, stack); - } - } + } + return false; + } - private void startIteration(ThreadCount tc, Identifier var, int id, Object value, - VariableStack stack) throws ExecutionException { - incRefs(tc.rc); - VariableStack copy = stack.copy(); - copy.enter(); - ThreadingContext ntc = ThreadingContext.get(copy).split(id); - ThreadingContext.set(copy, ntc); - setIndex(copy, 2); - if (iterationTracer.isEnabled()) { - iterationTracer.trace(ntc.toString(), unwrap(value)); + private boolean startRest(LWThread thr, ThrottledThreadSet ts, Iterator it, int fcf, List refs) { + Stack stack = thr.getStack(); + while (it.hasNext()) { + ts.waitForSlot(); + if (startOne(thr, ts, it.next(), fcf, refs)) { + return true; + } } - copy.setVar(var.getName(), value); - startElement(1, copy); + return false; } + private boolean startOne(final LWThread thr, final ThreadSet ts, final Object value, final int fcf, List refs) { + incRefs(refs); + LWThread ct = thr.fork(new KRunnable() { + @Override + public void run(LWThread thr2) { + try { + if (iterationTracer.isEnabled()) { + iterationTracer.trace(thr2, unwrap(value)); + } + + if (CompilerSettings.PERFORMANCE_COUNTERS) { + startCount++; + } + body.run(thr2); + ts.threadDone(thr2, null); + } + catch (ExecutionException e) { + throw e; + } + catch (Exception e) { + thr2.getStack().dropToFrame(fcf); + ts.threadDone(thr2, new ExecutionException(ThrottledParallelFor.this, e)); + ts.abortAll(); + thr.awake(); + } + } + }); + if(ts.add(ct)) { + return true; + } + + Stack cs = ct.getStack(); + cs.enter(this, frameSize); + this.var.setValue(cs, value); + ct.start(); + return false; + } + private void decRefs(List rcs) throws ExecutionException { - if (rcs != null) { - for (RefCount rc : rcs) { - rc.var.updateWriteRefCount(-rc.count); - } - } - } - - private void incRefs(List rcs) throws ExecutionException { + if (rcs != null) { + for (RefCount rc : rcs) { + rc.var.updateWriteRefCount(-rc.count); + } + } + } + + private void incRefs(List rcs) throws ExecutionException { if (rcs != null) { for (RefCount rc : rcs) { rc.var.updateWriteRefCount(rc.count); @@ -233,218 +271,38 @@ } } - private Object unwrap(Object value) { + + private int getMaxThreads() { + if (maxThreadCount < 0) { + try { + maxThreadCount = TypeUtil.toInt(VDL2Config.getConfig() + .getProperty("foreach.max.threads", String.valueOf(DEFAULT_MAX_THREADS))); + } + catch (IOException e) { + maxThreadCount = DEFAULT_MAX_THREADS; + } + } + return maxThreadCount; + } + + protected Object unwrap(Object value) { if (value instanceof Pair) { Pair p = (Pair) value; - if (kvar != null) { - return kvar + "=" + p.get(0) + ", " + vvar + "=" + Tracer.unwrapHandle(p.get(1)); + if (_kvar.getValue() != null) { + return _kvar.getValue() + "=" + p.get(0) + ", " + _vvar.getValue() + "=" + Tracer.unwrapHandle(p.get(1)); } else { - return vvar + "=" + Tracer.unwrapHandle(p.get(1)); + return _vvar.getValue() + "=" + Tracer.unwrapHandle(p.get(1)); } } else { return "!"; } } - - @Override - public void completed(VariableStack stack) throws ExecutionException { - int index = preIncIndex(stack) - 1; - if (index == 1) { - // iterator - stack.currentFrame().deleteVar(QUOTED); - processArguments(stack); - try { - partialArgumentsEvaluated(stack); - } - catch (FutureFault e) { - e.getFuture().addModificationAction(new PartialResume(), stack); - } - } - else if (index == elementCount()) { - iterationCompleted(stack); - } - else { - startElement(index, stack); - } - } - public void failed(VariableStack stack, ExecutionException e) throws ExecutionException { - if (!testAndSetChildFailed(stack)) { - if (stack.parentFrame().isDefined(VAR)) { - stack.leave(); - } - failImmediately(stack, e); - } - } - protected void iterationCompleted(VariableStack stack) - throws ExecutionException { - stack.leave(); - ThreadCount tc = getThreadCount(stack); - int running; - boolean closed; - boolean iteratorHasValues; - synchronized(tc) { - closed = tc.isClosed(); - running = tc.decrement(); - iteratorHasValues = tc.iteratorHasValues(); - } - boolean done = false; - if (running == 0) { - if (closed) { - complete(stack); - } - if (tc.selfClose && !iteratorHasValues) { - decRefs(tc.rc); - complete(stack); - } - } - } - - private void initThreadCount(VariableStack stack, boolean selfClose, KarajanIterator i) throws VariableNotFoundException { - if (maxThreadCount < 0) { - try { - maxThreadCount = TypeUtil.toInt(VDL2Config.getConfig() - .getProperty("foreach.max.threads", String.valueOf(DEFAULT_MAX_THREADS))); - } - catch (IOException e) { - maxThreadCount = DEFAULT_MAX_THREADS; - } - } - stack.setVar(THREAD_COUNT, new ThreadCount(maxThreadCount, selfClose, i, buildRefs(stack))); - } - - private ThreadCount getThreadCount(VariableStack stack) - throws VariableNotFoundException { - return (ThreadCount) stack.getVar(THREAD_COUNT); - } - @Override public String getTextualName() { return "foreach"; } - - private static class ThreadCount implements FutureIterator { - public boolean selfClose; - private int maxThreadCount; - private int crt; - private boolean closed; - private List listeners; - private KarajanIterator i; - private final List rc; - - public ThreadCount(int maxThreadCount, boolean selfClose, KarajanIterator i, List rc) { - this.maxThreadCount = maxThreadCount; - this.i = i; - crt = 0; - this.selfClose = selfClose; - this.rc = rc; - } - - public boolean raiseWaiting() { - return false; - } - - public boolean iteratorHasValues() { - try { - return i.hasNext(); - } - catch (FutureFault e) { - return false; - } - } - - public synchronized int available() { - return maxThreadCount - crt; - } - - public synchronized void add(int count) { - crt += count; - } - - public synchronized Object tryIncrement() { - // there is no way that both crt == 0 and i has no values outside this critical section - if (crt < maxThreadCount) { - Object o = i.next(); - crt++; - return o; - } - else { - throw new FutureIteratorIncomplete(this, this); - } - } - - public synchronized int decrement() { - crt--; - notifyListeners(); - return crt; - } - - private void notifyListeners() { - if (listeners != null) { - Iterator i = listeners.iterator(); - listeners = null; - while (i.hasNext()) { - ListenerStackPair etp = i.next(); - i.remove(); - etp.listener.futureModified(this, etp.stack); - } - } - } - - public boolean hasAvailable() { - return false; - } - - public int count() { - return 0; - } - - public synchronized int current() { - return crt; - } - - public Object peek() { - return null; - } - - public boolean hasNext() { - return false; - } - - public Object next() { - return null; - } - - public void remove() { - } - - public synchronized void addModificationAction(FutureListener target, - VariableStack stack) { - if (listeners == null) { - listeners = new ArrayList(); - } - listeners.add(new ListenerStackPair(target, stack)); - if (crt < maxThreadCount) { - notifyListeners(); - } - } - - public synchronized void close() { - this.closed = true; - } - - public void fail(FutureEvaluationException e) { - } - - public Object getValue() { - return null; - } - - public synchronized boolean isClosed() { - return closed; - } - } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/Tracer.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Tracer.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Tracer.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -13,13 +13,12 @@ import java.util.HashMap; import java.util.Map; +import k.rt.Future; +import k.thr.LWThread; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.stack.VariableNotFoundException; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.ThreadingContext; -import org.globus.cog.karajan.workflow.futures.Future; -import org.globus.cog.karajan.workflow.nodes.FlowElement; -import org.globus.cog.karajan.workflow.nodes.FlowNode; +import org.globus.cog.karajan.analyzer.VariableNotFoundException; +import org.globus.cog.karajan.compiled.nodes.Node; import org.griphyn.vdl.engine.Karajan; import org.griphyn.vdl.karajan.FutureWrapper; import org.griphyn.vdl.mapping.AbstractDataNode; @@ -56,7 +55,7 @@ this.enabled = enabled; } - private Tracer(FlowNode fe, String name) { + private Tracer(Node fe, String name) { source = buildSource(fe, name); if (source == null) { enabled = false; @@ -76,11 +75,11 @@ enabled = true; } - private Tracer(FlowNode fe) { + private Tracer(Node fe) { this(fe, null); } - private String buildSource(FlowNode fe, String name) { + private String buildSource(Node fe, String name) { String line = findLine(fe); if (line == null) { return null; @@ -99,7 +98,7 @@ return sb.toString(); } - private String getType(FlowNode fe) { + private String getType(Node fe) { String t = Karajan.demangle(fe.getTextualName()); String nt = NAME_MAPPINGS.get(t); if (nt == null) { @@ -110,31 +109,16 @@ } } - private String findLine(FlowElement fe) { - String line; - if (fe.hasProperty("_traceline")) { - line = (String) fe.getProperty("_traceline"); - } - else if (fe.hasProperty("_defline")) { - line = (String) fe.getProperty("_defline"); - } - else { - line = null; - } - if (line == null || line.equals("-1") || line.equals("")) { - return null; - } - else { - return line; - } + private String findLine(Node fe) { + return String.valueOf(fe.getLine()); } public boolean isEnabled() { return enabled; } - public void trace(VariableStack stack, Object msg) throws VariableNotFoundException { - trace(threadName(stack), msg); + public void trace(LWThread thr, Object msg) throws VariableNotFoundException { + trace(threadName(thr), msg); } public void trace(String thread, Object msg) { @@ -158,7 +142,7 @@ logger.info(str); } - public void trace(String thread) { + public void trace(LWThread thread) { logger.info(source + ", thread " + threadName(thread)); } @@ -171,17 +155,17 @@ } } - private String threadName(VariableStack stack) throws VariableNotFoundException { - return threadName(ThreadingContext.get(stack).toString()); + private String threadName(LWThread thr) throws VariableNotFoundException { + return thr.getName(); } private static Tracer disabledTracer, enabledTracer; - public static Tracer getTracer(FlowNode fe) { + public static Tracer getTracer(Node fe) { return getTracer(fe, null); } - public static Tracer getTracer(FlowNode fe, String name) { + public static Tracer getTracer(Node fe, String name) { if (globalTracingEnabled) { return new Tracer(fe, name); } Deleted: branches/faster/src/org/griphyn/vdl/karajan/lib/Typecheck.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/Typecheck.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/Typecheck.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,69 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Dec 26, 2006 - */ -package org.griphyn.vdl.karajan.lib; - -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.type.NoSuchTypeException; -import org.griphyn.vdl.type.Type; -import org.griphyn.vdl.type.Types; - -public class Typecheck extends VDLFunction { - public static final Arg PA_TYPE = new Arg.Positional("type"); - public static final Arg OA_ARGNAME = new Arg.Optional("argname"); - - static { - setArguments(Typecheck.class, new Arg[] { PA_VAR, PA_TYPE, OA_ARGNAME }); - } - - public Object function(VariableStack stack) throws ExecutionException { - String type = TypeUtil.toString(PA_TYPE.getValue(stack)); - Object ovar = PA_VAR.getValue(stack); - if (!(ovar instanceof DSHandle)) { - throw new ExecutionException("Wrong java type for argument. " - + "Expected DSHandle containing " + type - + "; got java object of class " + ovar.getClass() + " with value " + ovar); - } - DSHandle var = (DSHandle) ovar; - String argname = TypeUtil.toString(OA_ARGNAME.getValue(stack, null)); - - try { - Type t = Types.getType(type); - if (!compatible(t, var.getType())) { - if (argname != null) { - throw new ExecutionException("Wrong type for argument '" + argname + "'. Expected " - + type + "; got " + var.getType() + ". Actual argument: " + var); - } - else { - throw new ExecutionException("Wrong type for argument. Expected " + type + "; got " - + var.getType() + ". Actual argument: " + var); - } - } - } - catch (NoSuchTypeException e) { - throw new ExecutionException(e); - } - return null; - } -} Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/UnitEnd.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/UnitEnd.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/UnitEnd.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -9,37 +9,32 @@ */ package org.griphyn.vdl.karajan.lib; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.ThreadingContext; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.FlowNode; +import k.rt.ExecutionException; +import k.thr.LWThread; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.griphyn.vdl.karajan.WaitingThreadsMonitor; -public class UnitEnd extends FlowNode { - public static final Arg.Positional TYPE = new Arg.Positional("type"); - public static final Arg.Optional NAME = new Arg.Optional("name", null); - public static final Arg.Optional LINE = new Arg.Optional("line", null); +public class UnitEnd extends InternalFunction { - @Override - public void execute(VariableStack stack) throws ExecutionException { - executeSimple(stack); - complete(stack); - } + private ArgRef type; + private ArgRef name; + private ArgRef line; @Override - public boolean isSimple() { - return super.isSimple(); + protected Signature getSignature() { + return new Signature(params("type", optional("name", null), optional("line", null))); } @Override - public void executeSimple(VariableStack stack) throws ExecutionException { - String type = (String) TYPE.getStatic(this); - ThreadingContext thread = ThreadingContext.get(stack); - String name = (String) NAME.getStatic(this); - String line = (String) LINE.getStatic(this); + public void run(LWThread thr) throws ExecutionException { + String type = this.type.getValue(); + String name = this.name.getValue(); + String line = this.line.getValue(); - UnitStart.log(false, type, thread, name, line); - WaitingThreadsMonitor.removeOutput(stack); + UnitStart.log(false, type, thr, name, line); + WaitingThreadsMonitor.removeOutput(thr); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/UnitStart.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/UnitStart.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/UnitStart.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -10,163 +10,182 @@ package org.griphyn.vdl.karajan.lib; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Set; +import k.rt.Stack; +import k.thr.LWThread; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableNotFoundException; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.ThreadingContext; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.FlowNode; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; +import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.engine.Karajan; import org.griphyn.vdl.karajan.WaitingThreadsMonitor; import org.griphyn.vdl.mapping.DSHandle; -public class UnitStart extends FlowNode { +public class UnitStart extends InternalFunction { public static final Logger uslogger = Logger.getLogger(UnitStart.class); // keep compatibility with log() public static final Logger logger = Logger.getLogger("swift"); - public static final Arg.Positional TYPE = new Arg.Positional("type"); - public static final Arg.Optional NAME = new Arg.Optional("name", null); - public static final Arg.Optional LINE = new Arg.Optional("line", null); - public static final Arg.Optional OUTPUTS = new Arg.Optional("outputs", null); + private ArgRef type; + private ArgRef name; + private ArgRef line; + private ArgRef arguments; + private ArgRef outputs; + @Override + protected Signature getSignature() { + return new Signature(params("type", optional("name", null), optional("line", null), + optional("outputs", null), optional("arguments", null))); + } + + private static class NamedRef { + public final String name; + public final VarRef ref; + + public NamedRef(String name, VarRef ref) { + this.name = name; + this.ref = ref; + } + } + private Tracer tracer; - private List inputArgs, outputArgs; - + private List inputArgs, outputArgs; + @Override - protected void initializeStatic() { - super.initializeStatic(); - String type = (String) TYPE.getStatic(this); + public Node compile(WrapperNode w, Scope scope) throws CompilationException { + Node fn = super.compile(w, scope); + String type = this.type.getValue(); if (type.equals("PROCEDURE")) { - tracer = Tracer.getTracer((FlowNode) getParent(), "APPCALL"); + tracer = Tracer.getTracer(line.getValue(), "APPCALL"); } else if (type.equals("COMPOUND")) { - tracer = Tracer.getTracer((FlowNode) getParent(), "CALL"); + tracer = Tracer.getTracer(line.getValue(), "CALL"); } if (tracer != null && tracer.isEnabled()) { - populateArgNames(); + populateArgNames(scope); } + return fn; } - private void populateArgNames() { - String outs = (String) getStaticArguments().get("outputs"); + private void populateArgNames(Scope scope) { + String outs = this.outputs.getValue(); + Set outNames = new HashSet(); if (outs != null && outs.length() > 0) { - outputArgs = Arrays.asList(outs.split(",")); + outputArgs = new ArrayList(); + for (String name : outs.split(",")) { + VarRef ref = scope.getVarRef(name); + outputArgs.add(new NamedRef(name, ref)); + outNames.add(name); + } } else { - outputArgs = Collections.emptyList(); + outputArgs = null; } - String args = (String) getParent().getStaticArguments().get("arguments"); + String args = this.arguments.getValue(); if (args != null && args.length() > 0) { - inputArgs = new ArrayList(Arrays.asList(args.split(","))); - inputArgs.removeAll(outputArgs); + inputArgs = new ArrayList(); + for (String name : args.split(",")) { + if (outNames.contains(name)) { + continue; + } + VarRef ref = scope.getVarRef(name); + inputArgs.add(new NamedRef(name, ref)); + } } else { - inputArgs = Collections.emptyList(); + inputArgs = null; } - } - - @Override - public void execute(VariableStack stack) throws ExecutionException { - executeSimple(stack); - complete(stack); - } + } - @Override - public boolean isSimple() { - return super.isSimple(); - } @Override - public void executeSimple(VariableStack stack) throws ExecutionException { - String type = (String) TYPE.getStatic(this); - ThreadingContext thread = ThreadingContext.get(stack); - String name = (String) NAME.getStatic(this); - String line = (String) LINE.getStatic(this); + protected void runBody(LWThread thr) { + String type = this.type.getValue(); + String name = this.name.getValue(); + String line = this.line.getValue(); if (tracer != null && tracer.isEnabled()) { - tracer.trace(thread.toString(), Karajan.demangle(name) + "(" + formatArguments(stack) + ")"); + tracer.trace(thr, Karajan.demangle(name) + "(" + formatArguments(thr.getStack()) + ")"); } - log(true, type, thread, name, line); + log(true, type, thr, name, line); - String outputs = (String) OUTPUTS.getStatic(this); - if (outputs != null) { - trackOutputs(stack, outputs, "SCOPE".equals(type)); + if (outputArgs != null) { + trackOutputs(thr); } } - private String formatArguments(VariableStack stack) { - StringBuilder sb = new StringBuilder(); - boolean first = true; - for (String name : inputArgs) { - if (first) { - first = false; + private String formatArguments(Stack stack) { + if (inputArgs != null) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (NamedRef nr : inputArgs) { + if (first) { + first = false; + } + else { + sb.append(", "); + } + sb.append(Karajan.demangle(nr.name)); + sb.append(" = "); + sb.append(Tracer.unwrapHandle(nr.ref.getValue(stack))); } - else { - sb.append(", "); - } - sb.append(Karajan.demangle(name)); - sb.append(" = "); - sb.append(Tracer.unwrapHandle(stack.parentFrame().getVar(name))); + return sb.toString(); } - return sb.toString(); + else { + return ""; + } } private static final List EMPTY_OUTPUTS = Collections.emptyList(); - private void trackOutputs(VariableStack stack, String outputs, boolean deep) { - if (outputs.length() != 0) { - String[] names = outputs.split(","); + private void trackOutputs(LWThread thr) { + Stack stack = thr.getStack(); + if (!outputArgs.isEmpty()) { List l = new LinkedList(); - for (String name : names) { - if (deep) { - try { - l.add((DSHandle) stack.getVar(name.toLowerCase())); - } - catch (VariableNotFoundException e) { - logger.info("Could not find variable " + name, e); - } - } - else { - l.add((DSHandle) stack.parentFrame().getVar(name)); - } + for (NamedRef nr : outputArgs) { + l.add(nr.ref.getValue(stack)); } - WaitingThreadsMonitor.addOutput(stack, l); + WaitingThreadsMonitor.addOutput(thr, l); } } - protected static void log(boolean start, String type, ThreadingContext thread, String name, String line) { + protected static void log(boolean start, String type, LWThread thread, String name, String line) { if (type.equals("COMPOUND")) { - logger.info((start ? "START" : "END") + type + " thread=" + thread + " name=" + name); + logger.info((start ? "START" : "END") + type + " thread=" + thread.getName() + " name=" + name); } else if (type.equals("PROCEDURE")) { if (start) { - logger.debug("PROCEDURE line=" + line + " thread=" + thread + " name=" + name); + logger.debug("PROCEDURE line=" + line + " thread=" + thread.getName() + " name=" + name); } else { - logger.debug("PROCEDURE_END line=" + line + " thread=" + thread + " name=" + name); + logger.debug("PROCEDURE_END line=" + line + " thread=" + thread.getName() + " name=" + name); } } else if (type.equals("FOREACH_IT")) { - logger.debug("FOREACH_IT_" + (start ? "START" : "END") + " line=" + line + " thread=" + thread); + logger.debug("FOREACH_IT_" + (start ? "START" : "END") + " line=" + line + " thread=" + thread.getName()); if (start) { - logger.debug("SCOPE thread=" + thread); + logger.debug("SCOPE thread=" + thread.getName()); } } else if (type.equals("INTERNALPROC")) { - logger.debug("INTERNALPROC_" + (start ? "START" : "END") + " thread=" + thread + " name=" + name); + logger.debug("INTERNALPROC_" + (start ? "START" : "END") + " thread=" + thread.getName() + " name=" + name); } else if (type.equals("CONDITION_BLOCK")) { if (start) { - logger.debug("SCOPE thread=" + thread); + logger.debug("SCOPE thread=" + thread.getName()); } } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -24,24 +24,26 @@ import java.util.List; import java.util.Map; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.DSHandle; -public class UnwrapClosedList extends VDLFunction { +public class UnwrapClosedList extends SwiftFunction { public static final Logger logger = Logger.getLogger(UnwrapClosedList.class); - public static final Arg.Positional PA_LIST = new Arg.Positional("list"); + private ArgRef> list; + + @Override + protected Signature getSignature() { + return new Signature(params("list")); + } - static { - setArguments(UnwrapClosedList.class, new Arg[] { PA_LIST }); - } - - public Object function(VariableStack stack) throws ExecutionException { - @SuppressWarnings("unchecked") - List l = (List) PA_LIST.getValue(stack); + @Override + public Object function(Stack stack) { + List l = this.list.getValue(stack); List r = new ArrayList(l.size()); Deleted: branches/faster/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -1,475 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.griphyn.vdl.karajan.lib; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Stack; - -import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.arguments.ArgUtil; -import org.globus.cog.karajan.arguments.VariableArguments; -import org.globus.cog.karajan.stack.StackFrame; -import org.globus.cog.karajan.stack.VariableNotFoundException; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.BoundContact; -import org.globus.cog.karajan.util.ThreadingContext; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.KarajanRuntimeException; -import org.globus.cog.karajan.workflow.futures.Future; -import org.globus.cog.karajan.workflow.nodes.SequentialWithArguments; -import org.globus.cog.karajan.workflow.nodes.restartLog.RestartLog; -import org.globus.swift.catalog.TCEntry; -import org.globus.swift.catalog.transformation.File; -import org.globus.swift.catalog.types.TCType; -import org.griphyn.vdl.karajan.AssertFailedException; -import org.griphyn.vdl.karajan.FutureWrapper; -import org.griphyn.vdl.karajan.Loader; -import org.griphyn.vdl.karajan.TCCache; -import org.griphyn.vdl.karajan.functions.ConfigProperty; -import org.griphyn.vdl.mapping.AbsFile; -import org.griphyn.vdl.mapping.AbstractDataNode; -import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.DependentException; -import org.griphyn.vdl.mapping.GeneralizedFileFormat; -import org.griphyn.vdl.mapping.HandleOpenException; -import org.griphyn.vdl.mapping.InvalidPathException; -import org.griphyn.vdl.mapping.Mapper; -import org.griphyn.vdl.mapping.Path; -import org.griphyn.vdl.mapping.PathComparator; -import org.griphyn.vdl.mapping.PhysicalFormat; -import org.griphyn.vdl.type.Type; -import org.griphyn.vdl.type.Types; -import org.griphyn.vdl.util.FQN; -import org.griphyn.vdl.util.VDL2ConfigProperties; - -public abstract class VDLFunction extends SequentialWithArguments { - public static final Logger logger = Logger.getLogger(VDLFunction.class); - - public static final Arg.Channel ERRORS = new Arg.Channel("errors"); - - public static final Arg OA_PATH = new Arg.Optional("path", ""); - public static final Arg PA_PATH = new Arg.Positional("path"); - public static final Arg PA_VAR = new Arg.Positional("var"); - public static final Arg OA_ISARRAY = new Arg.Optional("isArray", Boolean.FALSE); - - public final void post(VariableStack stack) throws ExecutionException { - try { - Object o = function(stack); - if (o != null) { - ret(stack, o); - } - super.post(stack); - } - catch (AssertFailedException e) { - logger.fatal("swift: assert failed: " + e.getMessage()); - stack.getExecutionContext().failedQuietly(stack, e); - } - catch (ExecutionException e) { - if (e.getStack() == null) { - e.setStack(stack); - } - throw e; - } - catch (DependentException e) { - // This would not be the primal fault so in non-lazy errors mode it - // should not matter - throw new ExecutionException(stack, e); - } - } - - protected void ret(VariableStack stack, final Object value) throws ExecutionException { - if (value != null) { - final VariableArguments vret = ArgUtil.getVariableReturn(stack); - if (value.getClass().isArray()) { - if (value.getClass().getComponentType().isPrimitive()) { - vret.append(value); - } - else { - Object[] array = (Object[]) value; - for (int i = 0; i < array.length; i++) { - vret.append(array[i]); - } - } - } - else { - vret.append(value); - } - } - } - - protected abstract Object function(VariableStack stack) throws ExecutionException; - - /* - * This will likely break if the engine changes in fundamental ways. It also - * depends on the fact that iteration variable is named '$' in this - * particular implementation. - */ - public static String getThreadPrefix(VariableStack stack) throws ExecutionException { - stack = stack.copy(); - ThreadingContext last = ThreadingContext.get(stack); - Stack s = new Stack(); - while (stack.frameCount() > 1) { - StackFrame frame = stack.currentFrame(); - if (frame.isDefined("$")) { - List itv = (List) frame.getVar("$"); - s.push(itv.get(0)); - stack.leave(); - last = ThreadingContext.get(stack); - } - else { - ThreadingContext tc = ThreadingContext.get(stack); - if (!last.equals(tc)) { - s.push(String.valueOf(last.getLastID())); - last = tc; - } - stack.leave(); - } - } - - StringBuffer sb = new StringBuffer(); - while (!s.isEmpty()) { - sb.append(s.pop()); - if (!s.isEmpty()) { - sb.append('-'); - } - } - return sb.toString(); - } - - // TODO - is this needed any more? its doing some type inferencing and - // object creation and dequoting of strings, but the necessary behaviour - // here has possibly moved elsewhere, into a more strongly typed - // intermediate - // XML form that removes the need for this inference. - - // we might need to do some casting here for the numerical stuff - eg when - // asking for a float but we're given an int? not sure? might be the case - // that we already have value in the Double form already, in which case - // deference the internal value? - - // this is only used by VDL new (and really should only be used by - // VDL new, and should perhaps move to the VDL new source?) - - protected Object internalValue(Type type, Object value) { - if (Types.FLOAT.equals(type)) { - return new Double(TypeUtil.toDouble(value)); - } - else if (Types.INT.equals(type)) { - return new Integer(TypeUtil.toInt(value)); - } - else if (Types.BOOLEAN.equals(type)) { - return new Boolean(TypeUtil.toBoolean(value)); - } - else { - return value; - } - } - - public static final String[] EMPTY_STRING_ARRAY = new String[0]; - - public static String[] filename(VariableStack stack) throws ExecutionException { - DSHandle handle = (DSHandle)PA_VAR.getValue(stack); - return filename(stack, handle); - } - - public static String[] filename(VariableStack stack, DSHandle handle) throws ExecutionException { - return filename(handle); - } - - public static String[] filename(DSHandle var) throws ExecutionException { - try { - if (var.getType().isArray()) { - return leavesFileNames(var); - } - else if(var.getType().getFields().size() > 0) { - return leavesFileNames(var); - } - else { - return new String[] { leafFileName(var) }; - } - } - catch (DependentException e) { - return new String[0]; - } - catch (HandleOpenException e) { - throw new ExecutionException("The current implementation should not throw this exception", e); - } - } - - private static String[] leavesFileNames(DSHandle var) throws ExecutionException, HandleOpenException { - Mapper mapper; - - synchronized (var.getRoot()) { - mapper = var.getMapper(); - } - - if (mapper == null) { - throw new ExecutionException(var.getType() + " is not a mapped type"); - } - - List l = new ArrayList(); - try { - Collection fp = var.getFringePaths(); - List src; - if (fp instanceof List) { - src = (List) fp; - } - else { - src = new ArrayList(fp); - } - Collections.sort(src, new PathComparator()); - - for (Path p : src) { - l.add(leafFileName(var.getField(p), mapper)); - } - } - catch (InvalidPathException e) { - throw new ExecutionException("DSHandle is lying about its fringe paths"); - } - return l.toArray(EMPTY_STRING_ARRAY); - } - - private static String leafFileName(DSHandle var) throws ExecutionException { - return leafFileName(var, var.getMapper()); - } - - private static String leafFileName(DSHandle var, Mapper mapper) throws ExecutionException { - if (Types.STRING.equals(var.getType())) { - return relativize(String.valueOf(var.getValue())); - } - else { - if (var.getMapper() == null) { - throw new ExecutionException("Cannot invoke filename() on data without a mapper: " + var); - } - PhysicalFormat f = var.getMapper().map(var.getPathFromRoot()); - if (f instanceof GeneralizedFileFormat) { - String filename = ((GeneralizedFileFormat) f).getURIAsString(); - if (filename == null) { - throw new ExecutionException("Mapper did not provide a file name"); - } - else { - return filename; - } - } - else if (f == null) { - throw new ExecutionException("Mapper failed to map " + var); - } - else { - throw new ExecutionException("Only file formats are supported for now"); - } - } - } - - protected Object pathOnly(Object f) { - if (f instanceof String[]) { - return pathOnly((String[]) f); - } - else { - return pathOnly((String) f); - } - } - - protected static String pathOnly(String file) { - AbsFile af = new AbsFile(file); - if ("file".equals(af.getProtocol())) { - return af.getPath(); - } - else { - return af.getHost() + "/" + af.getPath(); - } - } - - protected String[] pathOnly(String[] files) { - String[] p = new String[files.length]; - for (int i = 0; i < files.length; i++) { - p[i] = pathOnly(files[i]); - } - return p; - } - - /** - * Given an input of an array of strings, returns a single string with the - * input strings separated by a space. If the 'relative' flag is set to - * true, then each input string will be passed through the relativize - * function. - */ - public String argList(String[] s, boolean relative) { - StringBuffer sb = new StringBuffer(); - for (int i = 0; i < s.length; i++) { - if (relative) { - s[i] = relativize(s[i]); - } - sb.append(s[i]); - if (i < s.length - 1) { - sb.append(' '); - } - } - return sb.toString(); - } - - /** - * removes leading / character from a supplied filename if present, so that - * the path can be used as a relative path. - */ - public static String relativize(String name) { - name = pathOnly(name); - if (name != null && name.length() > 0 && name.charAt(0) == '/') { - return name.substring(1); - } - else { - return name; - } - } - - protected static Map getLogData(VariableStack stack) throws ExecutionException { - try { - return (Map) stack.getDeepVar(RestartLog.LOG_DATA); - } - catch (VariableNotFoundException e) { - throw new ExecutionException("No log data found. Missing restartLog()?"); - } - } - - protected boolean compatible(Type expectedType, Type actualType) { - if (expectedType.equals(Types.FLOAT)) { - if (actualType.equals(Types.FLOAT) || actualType.equals(Types.INT)) { - return true; - } - else { - return false; - } - } - else if (expectedType.equals(Types.FLOAT.arrayType())) { - if (actualType.equals(Types.FLOAT.arrayType()) - || actualType.equals(Types.INT.arrayType())) { - return true; - } - else { - return false; - } - } - else if (expectedType.equals(Types.ANY)) { - return true; - } - else { - return actualType.equals(expectedType); - } - } - - protected void closeChildren(VariableStack stack, AbstractDataNode handle) throws ExecutionException, - InvalidPathException { - // Close the future - handle.closeShallow(); - // Mark all leaves - for (DSHandle child : handle.getFields(Path.CHILDREN)) { - child.closeShallow(); - } - } - - public static AbstractDataNode[] waitForAllVargs(VariableStack stack) throws ExecutionException { - AbstractDataNode[] args = SwiftArg.VARGS.asDataNodeArray(stack); - - for (int i = 0; i < args.length; i++) { - args[i].waitFor(); - } - - return args; - } - - public static Path parsePath(Object o, VariableStack stack) throws ExecutionException { - if (o instanceof Path) { - return (Path) o; - } - else { - return Path.parse(TypeUtil.toString(o)); - } - } - - private static Set warnset = new HashSet(); - - protected TCEntry getTCE(TCCache tc, FQN fqn, BoundContact bc) { - List l; - try { - l = tc.getTCEntries(fqn, bc.getHost(), TCType.INSTALLED); - } - catch (Exception e) { - throw new KarajanRuntimeException(e); - } - if (l == null || l.isEmpty()) { - return null; - } - if (l.size() > 1) { - synchronized (warnset) { - LinkedList wl = new LinkedList(); - wl.add(fqn); - wl.add(bc); - if (!warnset.contains(wl)) { - logger.warn("Multiple entries found for " + fqn + " on " + bc - + ". Using the first one"); - warnset.add(wl); - } - } - } - return (TCEntry) l.get(0); - } - - public static final String TC = "vdl:TC"; - - public static TCCache getTC(VariableStack stack) throws ExecutionException { - synchronized (stack.firstFrame()) { - TCCache tc = (TCCache) stack.firstFrame().getVar(TC); - if (tc == null) { - String prop = ConfigProperty.getProperty(VDL2ConfigProperties.TC_FILE, stack); - Loader.debugText("TC", new java.io.File(prop)); - tc = new TCCache(File.getNonSingletonInstance(prop)); - stack.firstFrame().setVar(TC, tc); - } - return tc; - } - } - - - private static int provenanceIDCount = 451000; - - public static synchronized int nextProvenanceID() { - return provenanceIDCount++; - } - - public static void logProvenanceResult(int id, DSHandle result, - String name) - throws ExecutionException { - if (logger.isDebugEnabled()) - logger.debug("FUNCTION id="+id+" name="+name+" result="+result.getIdentifier()); - else if (logger.isInfoEnabled()) - logger.info("FUNCTION: " + name + "()"); - } - - public static void logProvenanceParameter(int id, DSHandle parameter, String paramName) throws ExecutionException { - if (logger.isDebugEnabled()) - logger.debug("FUNCTIONPARAMETER id="+id+" input="+parameter.getIdentifier()+" name="+paramName); - } -} Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/WaitFieldValue.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/WaitFieldValue.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/WaitFieldValue.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,35 +17,40 @@ package org.griphyn.vdl.karajan.lib; -import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; -public class WaitFieldValue extends VDLFunction { - public static final Logger logger = Logger.getLogger(WaitFieldValue.class); +public class WaitFieldValue extends SwiftFunction { + private ArgRef var; + private ArgRef path; + + @Override + protected Signature getSignature() { + return new Signature(params("var", optional("path", Path.EMPTY_PATH))); + } - static { - setArguments(WaitFieldValue.class, new Arg[] { PA_VAR, OA_PATH }); - } /** * Takes a supplied variable and path, and returns the unique value at that * path. Path can contain wildcards, in which case an array is returned. */ - public Object function(VariableStack stack) throws ExecutionException { - AbstractDataNode var = (AbstractDataNode) PA_VAR.getValue(stack); + @Override + public Object function(Stack stack) { + AbstractDataNode var = this.var.getValue(stack); try { - Path path = parsePath(OA_PATH.getValue(stack), stack); + Path path = parsePath(this.path.getValue(stack)); var = (AbstractDataNode) var.getField(path); - var.waitFor(); + var.waitFor(this); return null; } catch (InvalidPathException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/cache/CacheReturn.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/cache/CacheReturn.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/cache/CacheReturn.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -24,10 +24,10 @@ public class CacheReturn { public final boolean alreadyCached; - public final List remove; + public final List remove; public final File cached; - public CacheReturn(boolean alreadyCached, List remove, File cached) { + public CacheReturn(boolean alreadyCached, List remove, File cached) { this.alreadyCached = alreadyCached; this.remove = remove; this.cached = cached; Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/cache/File.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/cache/File.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/cache/File.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,25 +20,17 @@ */ package org.griphyn.vdl.karajan.lib.cache; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; +import k.rt.AbstractFuture; +import k.rt.Future; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.events.EventBus; -import org.globus.cog.karajan.workflow.futures.Future; -import org.globus.cog.karajan.workflow.futures.FutureEvaluationException; -import org.globus.cog.karajan.workflow.futures.FutureListener; -import org.globus.cog.karajan.workflow.futures.ListenerStackPair; +import org.globus.cog.karajan.futures.FutureEvaluationException; -public class File implements Future { +public class File extends AbstractFuture implements Future { private String path; private Object host; private long size, lastAccess; private int locked; private boolean processingLock; - private List listeners; public File(String file, String dir, Object host, long size) { if (dir.endsWith("/")) { @@ -166,31 +158,6 @@ return processingLock; } - public void notifyListeners() { - if (listeners != null) { - Iterator i = listeners.iterator(); - while (i.hasNext()) { - final ListenerStackPair etp = i.next(); - i.remove(); - EventBus.post(new Runnable() { - public void run() { - etp.listener.futureModified(File.this, etp.stack); - } - }); - } - } - } - - public synchronized void addModificationAction(FutureListener target, VariableStack stack) { - if (listeners == null) { - listeners = new LinkedList(); - } - listeners.add(new ListenerStackPair(target, stack)); - if (isClosed()) { - notifyListeners(); - } - } - public void close() { } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -18,11 +18,13 @@ package org.griphyn.vdl.karajan.lib.swiftscript; // import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.AssertFailedException; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.type.Types; @@ -31,36 +33,29 @@ Throw AssertionException if input is false or 0. Optional second argument is string message printed on failure. */ -public class Assert extends VDLFunction { - - // private static final Logger logger = - // Logger.getLogger(Assert.class); +public class Assert extends SwiftFunction { + private ArgRef value; + private ArgRef message; - static { - setArguments(Assert.class, new Arg[] { Arg.VARGS }); + @Override + protected Signature getSignature() { + return new Signature(params("value", "message")); } - + @Override - protected Object function(VariableStack stack) - throws ExecutionException { - AbstractDataNode[] args = waitForAllVargs(stack); - String message = ""; + public Object function(Stack stack) { + AbstractDataNode hmessage = this.message.getValue(stack); + hmessage.waitFor(this); + String message = (String) hmessage.getValue(); + AbstractDataNode hvalue = this.value.getValue(stack); + hvalue.waitFor(this); + + checkAssert(hvalue, message); - if (args.length == 2) - if (args[1].getType() == Types.STRING) - message = (String) args[1].getValue(); - else - throw new ExecutionException - ("Second argument to assert() must be a String!"); - - checkAssert(args[0], message); - return null; } - private void checkAssert(DSHandle value, String message) - throws ExecutionException - { + private void checkAssert(DSHandle value, String message) { boolean success = true; if (value.getType() == Types.BOOLEAN) { if (! (Boolean) value.getValue()) @@ -71,10 +66,11 @@ if (d == 0) success = false; } - else - throw new ExecutionException - ("First argument to assert() must be boolean or int!"); - if (! success) + else { + throw new ExecutionException(this, "First argument to assert() must be boolean or int!"); + } + if (!success) { throw new AssertFailedException(message); + } } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,26 +22,31 @@ import java.io.IOException; import java.io.Reader; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.type.Types; -public class ExtractInt extends VDLFunction { - static { - setArguments(ExtractInt.class, new Arg[] { PA_VAR }); - } +public class ExtractInt extends SwiftFunction { + private ArgRef var; - public Object function(VariableStack stack) throws ExecutionException { - AbstractDataNode handle = null; + @Override + protected Signature getSignature() { + return new Signature(params("var")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode handle = this.var.getValue(stack); try { - handle = (AbstractDataNode) PA_VAR.getValue(stack); - handle.waitFor(); + handle.waitFor(this); String fn = argList(filename(handle), true); Reader freader = new FileReader(fn); @@ -49,9 +54,9 @@ String str = breader.readLine(); freader.close(); DSHandle result = new RootDataNode(Types.INT, Integer.parseInt(str)); - int provid = VDLFunction.nextProvenanceID(); - VDLFunction.logProvenanceResult(provid, result, "extractint"); - VDLFunction.logProvenanceParameter(provid, handle, "filename"); + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceResult(provid, result, "extractint"); + SwiftFunction.logProvenanceParameter(provid, handle, "filename"); return result; } catch (IOException ioe) { Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,25 +17,35 @@ package org.griphyn.vdl.karajan.lib.swiftscript; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.lib.SwiftFunction; +import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.type.Types; -public class FileName extends VDLFunction { - static { - setArguments(FileName.class, new Arg[] { PA_VAR }); - } +public class FileName extends SwiftFunction { + private ArgRef var; - public Object function(VariableStack stack) throws ExecutionException { - String s = argList(filename(stack), true); + @Override + protected Signature getSignature() { + return new Signature(params("var")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode var = this.var.getValue(stack); + String s = argList(filename(var), true); DSHandle result = new RootDataNode(Types.STRING, s); - int provid = VDLFunction.nextProvenanceID(); - //VDLFunction.logProvenanceParameter(provid, (DSHandle) PA_VAR.getValue(stack), "input"); - //VDLFunction.logProvenanceResult(provid, result, "filename"); + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceParameter(provid, var, "input"); + SwiftFunction.logProvenanceResult(provid, result, "filename"); + } + return result; } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,28 +20,35 @@ */ package org.griphyn.vdl.karajan.lib.swiftscript; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.lib.SwiftFunction; +import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; -import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.mapping.RootArrayDataNode; import org.griphyn.vdl.type.Types; -public class FileNames extends VDLFunction { - static { - setArguments(FileNames.class, new Arg[] { PA_VAR }); - } +public class FileNames extends SwiftFunction { + private ArgRef var; - public Object function(VariableStack stack) throws ExecutionException { - String[] f = filename(stack); + @Override + protected Signature getSignature() { + return new Signature(params("var")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode var = this.var.getValue(stack); + String[] f = filename(var); DSHandle returnArray = new RootArrayDataNode(Types.STRING.arrayType()); try { for (int i = 0; i < f.length; i++) { - Path p = parsePath("["+i+"]", stack); + Path p = parsePath("["+i+"]"); DSHandle h = returnArray.getField(p); h.setValue(relativize(f[i])); } @@ -50,9 +57,11 @@ } returnArray.closeShallow(); - int provid = VDLFunction.nextProvenanceID(); - logProvenanceParameter(provid, (DSHandle) PA_VAR.getValue(stack), "input"); - logProvenanceResult(provid, returnArray, "filenames"); + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + logProvenanceParameter(provid, var, "input"); + logProvenanceResult(provid, returnArray, "filenames"); + } return returnArray; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -21,78 +21,90 @@ package org.griphyn.vdl.karajan.lib.swiftscript; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionContext; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.functions.AbstractFunction; -import org.griphyn.vdl.karajan.lib.SwiftArg; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.VarRef; +import org.griphyn.vdl.karajan.lib.SwiftFunction; +import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.type.Types; -public class FnArg extends AbstractFunction { - public static final String PARSED_ARGS = "cmdline:named"; +public class FnArg extends SwiftFunction { + private ArgRef name; + private ArgRef value; + + private VarRef> parsedArgs; + + @Override + protected Signature getSignature() { + return new Signature(params("name", optional("value", null))); + } - public static final SwiftArg P_NAME = new SwiftArg.Positional("name"); - public static final SwiftArg P_VALUE = new SwiftArg.Positional("value"); + @Override + protected void addLocals(Scope scope) { + super.addLocals(scope); + parsedArgs = scope.getVarRef("SWIFT:PARSED_ARGS"); + } - static { - setArguments(FnArg.class, new Arg[] { P_NAME, P_VALUE }); - } - - public Object function(VariableStack stack) throws ExecutionException { - synchronized (stack.firstFrame()) { - if (!stack.firstFrame().isDefined(PARSED_ARGS)) { - List argv = (List) stack.firstFrame().getVar(ExecutionContext.CMDLINE_ARGS); - Map named = new HashMap(); - Iterator i = argv.iterator(); - while (i.hasNext()) { - String arg = (String) i.next(); - if (!arg.startsWith("-")) { - continue; - } - int index = arg.indexOf('='); - if (index == -1 || (arg.charAt(0) != '-')) { - throw new ExecutionException("Invalid command line argument: " + arg); - } - else { - String name = arg.substring(1, index); - named.put(name, arg.substring(index + 1)); - } - } - stack.firstFrame().setVar(PARSED_ARGS, named); - } + @Override + public Object function(Stack stack) { + Map args = this.parsedArgs.getValue(stack); + AbstractDataNode hname = this.name.getValue(stack); + hname.waitFor(this); + AbstractDataNode hvalue = this.value.getValue(stack); + if (hvalue != null) { + hvalue.waitFor(this); } - Map args = (Map) stack.firstFrame().getVar(PARSED_ARGS); - String name = TypeUtil.toString(P_NAME.getValue(stack)); + String name = (String) hname.getValue(); name = name.trim(); if (name.startsWith("\"") && name.endsWith("\"")) { name = name.substring(1, name.length() - 1); - } + } Object value = args.get(name); - if (value == null && P_VALUE.isPresent(stack)) { - value = P_VALUE.getValue(stack); + if (value == null && hvalue != null) { + value = hvalue.getValue(); } if (value == null) { - throw new ExecutionException(stack, "Missing command line argument: " + name); + throw new ExecutionException("Missing command line argument: " + name); } else { - DSHandle result = new RootDataNode(Types.STRING, value); - int provid=VDLFunction.nextProvenanceID(); - VDLFunction.logProvenanceResult(provid, result, "arg"); - VDLFunction.logProvenanceParameter(provid, P_NAME.getRawValue(stack), "name"); - if(P_VALUE.getRawValue(stack) != null) { - VDLFunction.logProvenanceParameter(provid, P_VALUE.getRawValue(stack), "value"); + DSHandle result = new RootDataNode(Types.STRING, value); + if (PROVENANCE_ENABLED) { + int provid = nextProvenanceID(); + logProvenanceResult(provid, result, "arg"); + logProvenanceParameter(provid, hname, "name"); + if (hvalue != null) { + logProvenanceParameter(provid, hvalue, "value"); + } } return result; } } + + public static Map parseArgs(List argv) { + Map named = new HashMap(); + for (String arg : argv) { + if (!arg.startsWith("-")) { + continue; + } + int index = arg.indexOf('='); + if (index == -1 || (arg.charAt(0) != '-')) { + throw new ExecutionException("Invalid command line argument: " + arg); + } + else { + String name = arg.substring(1, index); + named.put(name, arg.substring(index + 1)); + } + } + return named; + } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -21,14 +21,16 @@ import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; +import k.rt.Channel; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbstractDataNode; -import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.type.Types; /** Formatted file output.
@@ -37,31 +39,38 @@ @see Tracef, Sprintf @author wozniak */ -public class Fprintf extends VDLFunction { - - private static final Logger logger = - Logger.getLogger(Fprintf.class); +public class Fprintf extends SwiftFunction { + private static final Logger logger = Logger.getLogger(Fprintf.class); - static { - setArguments(Fprintf.class, new Arg[] { Arg.VARGS }); + private ArgRef filename; + private ArgRef spec; + private ChannelRef c_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("filename", "spec", "...")); } - static ConcurrentHashMap openFiles = - new ConcurrentHashMap(); + static ConcurrentHashMap openFiles = new ConcurrentHashMap(); @Override - protected Object function(VariableStack stack) - throws ExecutionException { - AbstractDataNode[] args = waitForAllVargs(stack); + public Object function(Stack stack) { + AbstractDataNode hfilename = this.filename.getValue(stack); + AbstractDataNode hspec = this.spec.getValue(stack); + hfilename.waitFor(this); + hspec.waitFor(this); + Channel args = c_vargs.get(stack); + waitForAll(this, args); + String filename = (String) hfilename.getValue(); + String spec = (String) hspec.getValue(); - check(args); - - String filename = (String) args[0].getValue(); - String spec = (String) args[1].getValue(); - DSHandle[] vars = Sprintf.copyArray(args, 2, args.length-2); - StringBuilder output = new StringBuilder(); - Sprintf.format(spec, vars, output); + try { + Sprintf.format(spec, args, output); + } + catch (RuntimeException e) { + throw new ExecutionException(this, e.getMessage()); + } String msg = output.toString(); logger.debug("file: " + filename + " msg: " + msg); @@ -69,19 +78,6 @@ return null; } - private static void check(DSHandle[] args) - throws ExecutionException { - if (args.length < 2) - throw new ExecutionException - ("fprintf(): requires at least 2 arguments!"); - if (! args[0].getType().equals(Types.STRING)) - throw new ExecutionException - ("fprintf(): first argument must be a string filename!"); - if (! args[0].getType().equals(Types.STRING)) - throw new ExecutionException - ("fprintf(): second argument must be a string specifier!"); - } - private static void write(String filename, String msg) throws ExecutionException { acquire(filename); Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,29 +20,40 @@ import java.lang.reflect.Method; import java.util.Arrays; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import k.rt.Channel; +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; -public class Java extends VDLFunction -{ - - static - { - setArguments(Java.class, new Arg[] { Arg.VARGS }); +public class Java extends SwiftFunction { + private ArgRef lib; + private ArgRef name; + private ChannelRef c_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("lib", "name", "...")); } - protected Object function(VariableStack stack) throws ExecutionException - { - AbstractDataNode[] args = waitForAllVargs(stack); + @Override + public Object function(Stack stack) { + AbstractDataNode hlib = this.lib.getValue(stack); + AbstractDataNode hname = this.name.getValue(stack); + Channel args = this.c_vargs.get(stack); + hlib.waitFor(this); + hname.waitFor(this); + waitForAll(this, args); - Method method = getMethod(args); + Method method = getMethod((String) hlib.getValue(), (String) hname.getValue(), args); Object[] p = convertInputs(method, args); Type type = returnType(method); Object value = invoke(method, p); @@ -55,30 +66,18 @@ Given the user args, locate the Java Method. */ - Method getMethod(DSHandle[] args) - { + private Method getMethod(String lib, String name, Channel args) { Method result; Class clazz; - String lib = "unset"; - String name = "unset"; + Class[] parameterTypes = new Class[args.size()]; - Class[] parameterTypes = new Class[args.length-2]; - - if (args.length < 2) - throw new RuntimeException - ("@java() requires at least two arguments"); - - try - { - lib = (String) args[0].getValue(); - name = (String) args[1].getValue(); + try { clazz = Class.forName(lib); - for (int i = 2; i < args.length; i++) - { - Class p = null; - Type t = args[i].getType(); + for (int i = 0; i < args.size(); i++) { + Class p = null; + Type t = args.get(i).getType(); if (t.equals(Types.FLOAT)) p = double.class; else if (t.equals(Types.INT)) p = int.class; @@ -86,20 +85,19 @@ else if (t.equals(Types.STRING)) p = String.class; else throw new RuntimeException("Cannot use @java with non-primitive types"); - parameterTypes[i-2] = p; + parameterTypes[i] = p; } result = clazz.getMethod(name, parameterTypes); } - catch (Exception e) - { + catch (Exception e) { e.printStackTrace(); - throw new RuntimeException - ("@java(): Error attempting to use: " + args[0].getValue()); + throw new ExecutionException(this, "@java(): Error attempting to use: " + lib); } if (result == null) - throw new RuntimeException - ("No method: " + name + " in " + lib + "with parameter types" + Arrays.toString(parameterTypes)); + throw new ExecutionException(this, "No method: " + + name + " in " + lib + "with parameter types" + + Arrays.toString(parameterTypes)); return result; } @@ -107,34 +105,12 @@ /** Convert the user args to a Java Object array. */ - Object[] convertInputs(Method method, DSHandle[] args) - { - Object[] result = new Object[args.length-2]; + private Object[] convertInputs(Method method, Channel args) { + Object[] result = new Object[args.size()]; Object a = null; - try - { - for (int i = 2; i < args.length; i++) - { - Type t = args[i].getType(); - Object v = args[i].getValue(); - if (t.equals(Types.FLOAT)) - a = (Double) v; - else if (t.equals(Types.INT)) - a = (Integer) v; - else if (t.equals(Types.BOOLEAN)) - a = (Boolean) v; - else if (t.equals(Types.STRING)) - a = (String) v; - result[i-2] = a; - } + for (int i = 0; i < args.size(); i++) { + result[i] = args.get(i).getValue(); } - catch (Exception e) { - e.printStackTrace(); - throw new RuntimeException - ("Error converting input arguments: \n" + - " to: " + method.getDeclaringClass() + - "." + method + " \n argument: " + a); - } return result; } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,14 +22,19 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import k.rt.Channel; +import k.rt.ExecutionException; +import k.rt.Stack; +import k.thr.LWThread; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; +import org.globus.cog.karajan.compiled.nodes.functions.AbstractFunction; import org.globus.cog.karajan.util.TypeUtil; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; -import org.griphyn.vdl.karajan.lib.SwiftArg; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; @@ -40,424 +45,571 @@ import org.griphyn.vdl.type.Types; import org.griphyn.vdl.util.VDL2Config; -public class Misc extends FunctionsCollection { +public class Misc { private static final Logger logger = Logger.getLogger(Misc.class); - - public static final SwiftArg PA_INPUT = new SwiftArg.Positional("input"); - public static final SwiftArg PA_PATTERN = new SwiftArg.Positional("regexp"); - public static final SwiftArg PA_TRANSFORM = new SwiftArg.Positional("transform"); - public static final SwiftArg PA_FILE = new SwiftArg.Positional("file"); - public static final SwiftArg PA_ARRAY = new SwiftArg.Positional("array"); - + + public static final boolean PROVENANCE_ENABLED; + static { - setArguments("swiftscript_dirname", new Arg[] { PA_FILE }); - setArguments("swiftscript_exists", new Arg[] { Arg.VARGS }); - setArguments("swiftscript_existsfile", new Arg[] { PA_FILE }); - setArguments("swiftscript_format", new Arg[] { PA_INPUT, PA_TRANSFORM }); - setArguments("swiftscript_length", new Arg[] { PA_ARRAY }); - setArguments("swiftscript_pad", new Arg[] { PA_INPUT, PA_TRANSFORM }); - setArguments("swiftscript_regexp", new Arg[] { PA_INPUT, PA_PATTERN, PA_TRANSFORM }); - setArguments("swiftscript_strcat", new Arg[] { Arg.VARGS }); - setArguments("swiftscript_strcut", new Arg[] { PA_INPUT, PA_PATTERN }); - setArguments("swiftscript_strsplit", new Arg[] { PA_INPUT, PA_PATTERN }); - setArguments("swiftscript_strjoin", new Arg[] { PA_ARRAY, PA_INPUT }); - setArguments("swiftscript_strstr", new Arg[] { PA_INPUT, PA_PATTERN }); - setArguments("swiftscript_trace", new Arg[] { Arg.VARGS }); - setArguments("swiftscript_to_int", new Arg[] { PA_INPUT }); - setArguments("swiftscript_toint", new Arg[] { PA_INPUT }); - setArguments("swiftscript_to_float", new Arg[] { PA_INPUT }); - setArguments("swiftscript_tofloat", new Arg[] { PA_INPUT }); - setArguments("swiftscript_to_string", new Arg[] { PA_INPUT }); - setArguments("swiftscript_tostring", new Arg[] { PA_INPUT }); + boolean v; + try { + v = VDL2Config.getConfig().getProvenanceLog(); + } + catch (IOException e) { + v = false; + } + PROVENANCE_ENABLED = v; } - private static final Logger traceLogger = - Logger.getLogger("org.globus.swift.trace"); - public DSHandle swiftscript_trace(VariableStack stack) - throws ExecutionException { + private static final Logger traceLogger = Logger.getLogger("org.globus.swift.trace"); + + public static class Trace extends InternalFunction { + private ChannelRef c_vargs; - AbstractDataNode[] args = VDLFunction.waitForAllVargs(stack); + @Override + protected Signature getSignature() { + return new Signature(params("...")); + } - StringBuilder buf = new StringBuilder(); - buf.append("SwiftScript trace: "); - for (int i = 0; i < args.length; i++) { - DSHandle handle = args[i]; - if (i != 0) { - buf.append(", "); - } - Object v = args[i].getValue(); - //buf.append(v == null ? args[i] : v); - prettyPrint(buf, args[i]); - } - traceLogger.warn(buf); - return null; + @Override + protected void runBody(LWThread thr) { + Channel vargs = c_vargs.get(thr.getStack()); + SwiftFunction.waitForAll(this, vargs); + + StringBuilder buf = new StringBuilder(); + buf.append("SwiftScript trace: "); + boolean first = true; + for (AbstractDataNode n : vargs) { + if (!first) { + buf.append(", "); + } + else { + first = false; + } + //buf.append(v == null ? args[i] : v); + prettyPrint(buf, n); + } + traceLogger.warn(buf); + } } - - private void prettyPrint(StringBuilder buf, DSHandle h) { - Object o = h.getValue(); - if (o == null) { - buf.append(h); - } - else { - if (h.getType().isPrimitive()) { - if (h.getType().equals(Types.INT)) { - buf.append(((Number) o).intValue()); - } - else { - buf.append(o); - } - } - else if (h.getType().isArray()) { - buf.append('{'); - boolean first = true; - for (Map.Entry, DSHandle> e : h.getArrayValue().entrySet()) { - if (first) { - first = false; - } - else { - buf.append(", "); - } - buf.append(e.getKey()); - buf.append(" = "); - prettyPrint(buf, e.getValue()); - } - buf.append('}'); - } - else { - buf.append(h); - } - } + + private static void prettyPrint(StringBuilder buf, DSHandle h) { + Object o = h.getValue(); + if (o == null) { + buf.append(h); + } + else { + if (h.getType().isPrimitive()) { + if (h.getType().equals(Types.INT)) { + buf.append(((Number) o).intValue()); + } + else { + buf.append(o); + } + } + else if (h.getType().isArray()) { + buf.append('{'); + boolean first = true; + for (Map.Entry, DSHandle> e : h.getArrayValue().entrySet()) { + if (first) { + first = false; + } + else { + buf.append(", "); + } + buf.append(e.getKey()); + buf.append(" = "); + prettyPrint(buf, e.getValue()); + } + buf.append('}'); + } + else { + buf.append(h); + } + } } + + public static class StrCat extends AbstractFunction { + private ChannelRef c_vargs; - public DSHandle swiftscript_strcat(VariableStack stack) throws ExecutionException { - Object[] args = SwiftArg.VARGS.asArray(stack); - int provid = VDLFunction.nextProvenanceID(); - StringBuffer buf = new StringBuffer(); - - for (int i = 0; i < args.length; i++) { - buf.append(TypeUtil.toString(args[i])); - } - - DSHandle handle = new RootDataNode(Types.STRING, buf.toString()); - - try { - if(VDL2Config.getConfig().getProvenanceLog()) { - DSHandle[] provArgs = SwiftArg.VARGS.asDSHandleArray(stack); - for (int i = 0; i < provArgs.length; i++) { - VDLFunction.logProvenanceParameter(provid, provArgs[i], ""+i); - } - VDLFunction.logProvenanceResult(provid, handle, "strcat"); - } - } catch(IOException ioe) { - throw new ExecutionException("When logging provenance for strcat", ioe); - } - return handle; + @Override + protected Signature getSignature() { + return new Signature(params("...")); + } + + @Override + public Object function(Stack stack) { + Channel vargs = c_vargs.get(stack); + Channel args = SwiftFunction.unwrapAll(this, vargs); + + StringBuffer buf = new StringBuffer(); + + for (Object o : args) { + buf.append(TypeUtil.toString(o)); + } + + DSHandle handle = new RootDataNode(Types.STRING, buf.toString()); + + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + int index = 0; + for (AbstractDataNode dn : vargs) { + SwiftFunction.logProvenanceParameter(provid, dn, String.valueOf(index++)); + } + SwiftFunction.logProvenanceResult(provid, handle, "strcat"); + } + return handle; + } } + + public static class Exists extends AbstractFunction { + private ArgRef file; - public DSHandle swiftscript_exists(VariableStack stack) - throws ExecutionException { - logger.debug(stack); - Object[] args = SwiftArg.VARGS.asArray(stack); - int provid = VDLFunction.nextProvenanceID(); + @Override + protected Signature getSignature() { + return new Signature(params("file")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode dn = file.getValue(stack); + String filename = SwiftFunction.unwrap(this, dn); - if (args.length != 1) - throw new ExecutionException - ("Wrong number of arguments to @exists()"); + AbsFile file = new AbsFile(filename); + if (logger.isDebugEnabled()) { + logger.debug("exists: " + file); + } + DSHandle handle = new RootDataNode(Types.BOOLEAN, file.exists()); + + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceParameter(provid, dn, "file"); + SwiftFunction.logProvenanceResult(provid, handle, "exists"); + } + + return handle; + } + } + + public static class StrCut extends AbstractFunction { + private ArgRef input; + private ArgRef pattern; - String filename = TypeUtil.toString(args[0]); + @Override + protected Signature getSignature() { + return new Signature(params("input", "pattern")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode hinput = this.input.getValue(stack); + String input = SwiftFunction.unwrap(this, hinput); + AbstractDataNode hpattern = this.pattern.getValue(stack); + String pattern = SwiftFunction.unwrap(this, hpattern); - AbsFile file = new AbsFile(filename); - logger.debug("exists: " + file); - DSHandle handle = new RootDataNode(Types.BOOLEAN, file.exists()); + if (logger.isDebugEnabled()) { + logger.debug("strcut will match '" + input + "' with pattern '" + pattern + "'"); + } - try { - if(VDL2Config.getConfig().getProvenanceLog()) { - DSHandle[] provArgs = - SwiftArg.VARGS.asDSHandleArray(stack); - for (int i = 0; i < provArgs.length; i++) { - VDLFunction.logProvenanceParameter - (provid, provArgs[i], ""+i); - } - VDLFunction.logProvenanceResult - (provid, handle, "exists"); - } - } catch (IOException ioe) { - throw new ExecutionException - ("When logging provenance for exists", - ioe); - } - - return handle; + String group; + try { + Pattern p = Pattern.compile(pattern); + // TODO probably should memoize this? + + Matcher m = p.matcher(input); + m.find(); + group = m.group(1); + } + catch (IllegalStateException e) { + throw new ExecutionException("@strcut could not match pattern " + pattern + + " against string " + input, e); + } + if (logger.isDebugEnabled()) { + logger.debug("strcut matched '" + group + "'"); + } + DSHandle handle = new RootDataNode(Types.STRING, group); + + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceResult(provid, handle, "strcut"); + SwiftFunction.logProvenanceParameter(provid, hinput, "input"); + SwiftFunction.logProvenanceParameter(provid, hpattern, "pattern"); + } + return handle; + } } + + public static class StrStr extends AbstractFunction { + private ArgRef input; + private ArgRef pattern; - public DSHandle swiftscript_strcut(VariableStack stack) - throws ExecutionException { - int provid = VDLFunction.nextProvenanceID(); - String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); - String pattern = TypeUtil.toString(PA_PATTERN.getValue(stack)); - if (logger.isDebugEnabled()) { - logger.debug("strcut will match '" + inputString + "' with pattern '" + pattern + "'"); - } + @Override + protected Signature getSignature() { + return new Signature(params("input", "pattern")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode hinput = this.input.getValue(stack); + String input = SwiftFunction.unwrap(this, hinput); + AbstractDataNode hpattern = this.pattern.getValue(stack); + String pattern = SwiftFunction.unwrap(this, hpattern); - String group; - try { - Pattern p = Pattern.compile(pattern); - // TODO probably should memoize this? + if (logger.isDebugEnabled()) { + logger.debug("strstr will search '" + input + "' for pattern '" + pattern + "'"); + } + + DSHandle result = new RootDataNode(Types.INT, input.indexOf(pattern)); - Matcher m = p.matcher(inputString); - m.find(); - group = m.group(1); - } - catch (IllegalStateException e) { - throw new ExecutionException("@strcut could not match pattern " + pattern - + " against string " + inputString, e); - } - if (logger.isDebugEnabled()) { - logger.debug("strcut matched '" + group + "'"); - } - DSHandle handle = new RootDataNode(Types.STRING, group); - - VDLFunction.logProvenanceResult(provid, handle, "strcut"); - VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "input"); - VDLFunction.logProvenanceParameter(provid, PA_PATTERN.getRawValue(stack), "pattern"); - return handle; - } - - public DSHandle swiftscript_strstr(VariableStack stack) - throws ExecutionException { - String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); - String pattern = TypeUtil.toString(PA_PATTERN.getValue(stack)); - if (logger.isDebugEnabled()) { - logger.debug("strstr will search '" + inputString + - "' for pattern '" + pattern + "'"); + + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceResult(provid, result, "strstr"); + SwiftFunction.logProvenanceParameter(provid, hinput, "input"); + SwiftFunction.logProvenanceParameter(provid, hpattern, "pattern"); + } + return result; } - int result = inputString.indexOf(pattern); - return new RootDataNode(Types.INT, result); } + + public static class StrSplit extends AbstractFunction { + private ArgRef input; + private ArgRef pattern; - public DSHandle swiftscript_strsplit(VariableStack stack) - throws ExecutionException, InvalidPathException { + @Override + protected Signature getSignature() { + return new Signature(params("input", "pattern")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode hinput = this.input.getValue(stack); + String input = SwiftFunction.unwrap(this, hinput); + AbstractDataNode hpattern = this.pattern.getValue(stack); + String pattern = SwiftFunction.unwrap(this, hpattern); - String str = TypeUtil.toString(PA_INPUT.getValue(stack)); - String pattern = TypeUtil.toString(PA_PATTERN.getValue(stack)); + String[] split = input.split(pattern); - String[] split = str.split(pattern); - - DSHandle handle = new RootArrayDataNode(Types.STRING.arrayType()); - for (int i = 0; i < split.length; i++) { - DSHandle el = handle.getField(Path.EMPTY_PATH.addFirst(i, true)); - el.setValue(split[i]); - } - handle.closeDeep(); - int provid=VDLFunction.nextProvenanceID(); - VDLFunction.logProvenanceResult(provid, handle, "strsplit"); - VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "input"); - VDLFunction.logProvenanceParameter(provid, PA_PATTERN.getRawValue(stack), "pattern"); - return handle; - } + DSHandle handle = new RootArrayDataNode(Types.STRING.arrayType()); + for (int i = 0; i < split.length; i++) { + DSHandle el; + try { + el = handle.getField(Path.EMPTY_PATH.addFirst(i, true)); + el.setValue(split[i]); + } + catch (InvalidPathException e) { + throw new ExecutionException(this, e); + } + } + handle.closeDeep(); + + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceResult(provid, handle, "strsplit"); + SwiftFunction.logProvenanceParameter(provid, hinput, "input"); + SwiftFunction.logProvenanceParameter(provid, hpattern, "pattern"); + } + return handle; + } + } /** - * swiftscript_strjoin (@strjoin) - Combine elements of an array into a single string with a specified delimiter + * StrJoin (@strjoin) - Combine elements of an array into a single string with a specified delimiter * @param stack * @return DSHandle representing the resulting string * @throws ExecutionException */ - public DSHandle swiftscript_strjoin(VariableStack stack) throws ExecutionException - { - AbstractDataNode array = (AbstractDataNode) PA_ARRAY.getRawValue(stack); - String delim = TypeUtil.toString(PA_INPUT.getValue(stack)); - String result = ""; + public static class StrJoin extends AbstractFunction { + private ArgRef array; + private ArgRef delim; - array.waitFor(); + @Override + protected Signature getSignature() { + return new Signature(params("array", optional("delim", new RootDataNode(Types.STRING, ", ")))); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode harray = this.array.getValue(stack); + Map, DSHandle> arrayValues = SwiftFunction.waitForArray(this, harray); + AbstractDataNode hdelim = this.delim.getValue(stack); + String delim = SwiftFunction.unwrap(this, hdelim); - Map arrayValues = array.getArrayValue(); - for (Object value : arrayValues.values()) { - if (result == "") { result += ((DSHandle) value).getValue(); } - else { result += delim + ((DSHandle) value).getValue(); } + StringBuilder result = new StringBuilder(); + + boolean first = true; + for (DSHandle h : arrayValues.values()) { + if (first) { + first = false; + } + else { + result.append(delim); + } + result.append(h.getValue()); + } + + DSHandle handle = new RootDataNode(Types.STRING, result.toString()); + + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceResult(provid, handle, "strjoin"); + SwiftFunction.logProvenanceParameter(provid, harray, "array"); + SwiftFunction.logProvenanceParameter(provid, hdelim, "delim"); + } + return handle; } - - DSHandle handle = new RootDataNode(Types.STRING, result); - return handle; } + + public static class Regexp extends AbstractFunction { + private ArgRef input; + private ArgRef pattern; + private ArgRef transform; - public DSHandle swiftscript_regexp(VariableStack stack) - throws ExecutionException { - String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); - String pattern = TypeUtil.toString(PA_PATTERN.getValue(stack)); - String transform = TypeUtil.toString(PA_TRANSFORM.getValue(stack)); - if (logger.isDebugEnabled()) { - logger.debug("regexp will match '" + inputString + "' with pattern '" + pattern + "'"); - } + @Override + protected Signature getSignature() { + return new Signature(params("input", "pattern", "transform")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode hinput = this.input.getValue(stack); + String input = SwiftFunction.unwrap(this, hinput); + AbstractDataNode hpattern = this.pattern.getValue(stack); + String pattern = SwiftFunction.unwrap(this, hpattern); + AbstractDataNode htransform = this.transform.getValue(stack); + String transform = SwiftFunction.unwrap(this, htransform); - String group; - try { - Pattern p = Pattern.compile(pattern); - // TODO probably should memoize this? + if (logger.isDebugEnabled()) { + logger.debug("regexp will match '" + input + "' with pattern '" + pattern + "'"); + } + + String group; + try { + Pattern p = Pattern.compile(pattern); + + Matcher m = p.matcher(input); + m.find(); + group = m.replaceFirst(transform); + } + catch (IllegalStateException e) { + throw new ExecutionException("@regexp could not match pattern " + pattern + + " against string " + input, e); + } + if (logger.isDebugEnabled()) { + logger.debug("regexp replacement produced '" + group + "'"); + } + DSHandle handle = new RootDataNode(Types.STRING); + handle.setValue(group); + handle.closeShallow(); - Matcher m = p.matcher(inputString); - m.find(); - group = m.replaceFirst(transform); - } - catch (IllegalStateException e) { - throw new ExecutionException("@regexp could not match pattern " + pattern - + " against string " + inputString, e); - } - if (logger.isDebugEnabled()) { - logger.debug("regexp replacement produced '" + group + "'"); - } - DSHandle handle = new RootDataNode(Types.STRING); - handle.setValue(group); - handle.closeShallow(); + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceResult(provid, handle, "regexp"); + SwiftFunction.logProvenanceParameter(provid, hinput, "input"); + SwiftFunction.logProvenanceParameter(provid, hpattern, "pattern"); + SwiftFunction.logProvenanceParameter(provid, htransform, "transform"); + } + return handle; + } + } + + public static class ToInt extends AbstractFunction { + private ArgRef str; - int provid=VDLFunction.nextProvenanceID(); - VDLFunction.logProvenanceResult(provid, handle, "regexp"); - VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "input"); - VDLFunction.logProvenanceParameter(provid, PA_PATTERN.getRawValue(stack), "pattern"); - VDLFunction.logProvenanceParameter(provid, PA_TRANSFORM.getRawValue(stack), "transform"); - return handle; - } + @Override + protected Signature getSignature() { + return new Signature(params("str")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode hstr = str.getValue(stack); + String str = SwiftFunction.unwrap(this, hstr); + + DSHandle handle = new RootDataNode(Types.INT, Integer.valueOf(str)); - public DSHandle swiftscript_toint(VariableStack stack) - throws ExecutionException { - return swiftscript_to_int(stack); - } + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceParameter(provid, hstr, "str"); + SwiftFunction.logProvenanceResult(provid, handle, "toint"); + } + + return handle; + } + } - public DSHandle swiftscript_to_int(VariableStack stack) - throws ExecutionException { - String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); - - DSHandle handle = new RootDataNode(Types.INT, new Double(inputString).intValue()); + public static class ToFloat extends AbstractFunction { + private ArgRef str; - int provid=VDLFunction.nextProvenanceID(); - VDLFunction.logProvenanceResult(provid, handle, "toint"); - VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "string"); - return handle; - } + @Override + protected Signature getSignature() { + return new Signature(params("str")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode hstr = str.getValue(stack); + String str = SwiftFunction.unwrap(this, hstr); + + DSHandle handle = new RootDataNode(Types.FLOAT, Double.valueOf(str)); - public DSHandle swiftscript_tofloat(VariableStack stack) - throws ExecutionException { - return swiftscript_to_float(stack); - } + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceParameter(provid, hstr, "str"); + SwiftFunction.logProvenanceResult(provid, handle, "tofloat"); + } + + return handle; + } + } - public DSHandle swiftscript_to_float(VariableStack stack) - throws ExecutionException { - String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); - DSHandle handle = new RootDataNode(Types.FLOAT); - - try - { - handle.setValue(new Double(inputString)); - } - catch(NumberFormatException e) - { - throw new ExecutionException(stack, "Could not convert value \""+inputString+"\" to type float"); - } - handle.closeShallow(); - int provid=VDLFunction.nextProvenanceID(); - VDLFunction.logProvenanceResult(provid, handle, "tofloat"); - VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "string"); - return handle; - } - /* * Takes in a float and formats to desired precision and returns a string */ - public DSHandle swiftscript_format(VariableStack stack) - throws ExecutionException { - String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); - String inputFormat = TypeUtil.toString(PA_TRANSFORM.getValue(stack)); - DSHandle handle = new RootDataNode(Types.STRING); + public static class Format extends AbstractFunction { + private ArgRef format; + private ArgRef value; - String output = String.format("%."+inputFormat+"f", Double.parseDouble(inputString)); - handle.setValue(output); - handle.closeShallow(); + @Override + protected Signature getSignature() { + return new Signature(params("format", "value")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode hformat = this.format.getValue(stack); + String format = SwiftFunction.unwrap(this, hformat); + AbstractDataNode hvalue = this.value.getValue(stack); + Double value = SwiftFunction.unwrap(this, hvalue); + + DSHandle handle = new RootDataNode(Types.STRING, + String.format("%." + format + "f", value)); - int provid=VDLFunction.nextProvenanceID(); - VDLFunction.logProvenanceResult(provid, handle, "format"); - VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "float"); - VDLFunction.logProvenanceParameter(provid, PA_TRANSFORM.getRawValue(stack), "float"); - return handle; - } - + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceResult(provid, handle, "format"); + SwiftFunction.logProvenanceParameter(provid, hformat, "format"); + SwiftFunction.logProvenanceParameter(provid, hvalue, "value"); + } + return handle; + } + } + /* * Takes in an int and pads zeros to the left and returns a string */ - public DSHandle swiftscript_pad(VariableStack stack) - throws ExecutionException { - String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); - String inputFormat = TypeUtil.toString(PA_TRANSFORM.getValue(stack)); - DSHandle handle = new RootDataNode(Types.STRING); + public static class Pad extends AbstractFunction { + private ArgRef size; + private ArgRef value; - int num_length = inputString.length(); - int zeros_to_pad = Integer.parseInt(inputFormat); - zeros_to_pad += num_length; + @Override + protected Signature getSignature() { + return new Signature(params("size", "value")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode hsize = this.size.getValue(stack); + Integer size = SwiftFunction.unwrap(this, hsize); + AbstractDataNode hvalue = this.value.getValue(stack); + Integer value = SwiftFunction.unwrap(this, hvalue); + + DSHandle handle = new RootDataNode(Types.STRING, + String.format("%0" + size + "d", value)); - String output = String.format("%0"+zeros_to_pad+"d", - Integer.parseInt(inputString)); - handle.setValue(output); - handle.closeShallow(); + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceResult(provid, handle, "pad"); + SwiftFunction.logProvenanceParameter(provid, hsize, "size"); + SwiftFunction.logProvenanceParameter(provid, hvalue, "value"); + } + return handle; + } + } + + public static class ToString extends AbstractFunction { + private ArgRef value; - int provid=VDLFunction.nextProvenanceID(); - VDLFunction.logProvenanceResult(provid, handle, "pad"); - VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "int"); - VDLFunction.logProvenanceParameter(provid, PA_TRANSFORM.getRawValue(stack), "int"); - return handle; - } + @Override + protected Signature getSignature() { + return new Signature(params("value")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode hvalue = this.value.getValue(stack); + + StringBuilder sb = new StringBuilder(); + prettyPrint(sb, hvalue); + DSHandle handle = new RootDataNode(Types.STRING, sb.toString()); - public DSHandle swiftscript_tostring(VariableStack stack) - throws ExecutionException { - return swiftscript_to_string(stack); - } + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceParameter(provid, hvalue, "value"); + SwiftFunction.logProvenanceResult(provid, handle, "tostring"); + } + + return handle; + } + } - public DSHandle swiftscript_to_string(VariableStack stack) - throws ExecutionException { - Object input = PA_INPUT.getValue(stack); - DSHandle handle = new RootDataNode(Types.STRING); - if (input instanceof DSHandle) { - StringBuilder sb = new StringBuilder(); - prettyPrint(sb, (DSHandle) input); - handle.setValue(sb.toString()); - } - else { - handle.setValue(String.valueOf(input)); - } - handle.closeShallow(); - return handle; - } + /* + * This is copied from swiftscript_dirname. + * Both the functions could be changed to be more readable. + * Returns length of an array. + * Good for debugging because array needs to be closed + * before the length is determined + */ + public static class Dirname extends AbstractFunction { + private ArgRef file; - public DSHandle swiftscript_dirname(VariableStack stack) - throws ExecutionException { - AbstractDataNode n = (AbstractDataNode) PA_FILE.getRawValue(stack); - n.waitFor(); - String name = VDLFunction.filename(n)[0]; - String result = new AbsFile(name).getDir(); - return new RootDataNode(Types.STRING, result); - } + @Override + protected Signature getSignature() { + return new Signature(params("file")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode dn = file.getValue(stack); + String name = SwiftFunction.filename(dn)[0]; - /* - * This is copied from swiftscript_dirname. - * Both the functions could be changed to be more readable. - * Returns length of an array. - * Good for debugging because array needs to be closed - * before the length is determined - */ - public DSHandle swiftscript_length(VariableStack stack) - throws ExecutionException { - AbstractDataNode array = (AbstractDataNode) PA_ARRAY.getRawValue(stack); - array.waitFor(); - Map n = array.getArrayValue(); - return new RootDataNode(Types.INT, Integer.valueOf(n.size())); - } + String result = new AbsFile(name).getDir(); + DSHandle handle = new RootDataNode(Types.BOOLEAN, result); + + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceParameter(provid, dn, "file"); + SwiftFunction.logProvenanceResult(provid, handle, "dirname"); + } + + return handle; + } + } + + public static class Length extends AbstractFunction { + private ArgRef array; - public DSHandle swiftscript_existsfile(VariableStack stack) - throws ExecutionException { - DSHandle result = null; - Object[] args = SwiftArg.VARGS.asArray(stack); - String arg = (String) args[0]; - AbsFile file = new AbsFile(arg); - return new RootDataNode(Types.BOOLEAN, file.exists()); - } + @Override + protected Signature getSignature() { + return new Signature(params("array")); + } + + @Override + public Object function(Stack stack) { + AbstractDataNode harray = this.array.getValue(stack); + harray.waitFor(this); + + DSHandle handle = new RootDataNode(Types.INT, Integer.valueOf(harray.getArrayValue().size())); + + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceResult(provid, handle, "length"); + SwiftFunction.logProvenanceParameter(provid, harray, "array"); + } + return handle; + } + } } /* Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -28,12 +28,13 @@ import java.util.HashSet; import java.util.Set; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.lib.Tracer; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; @@ -43,32 +44,24 @@ import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; -public class ReadData extends VDLFunction { +public class ReadData extends SwiftFunction { public static final Logger logger = Logger.getLogger(ReadData.class); - - public static final Arg DEST = new Arg.Positional("dest"); - public static final Arg SRC = new Arg.Positional("src"); - public static boolean warning; - public Tracer tracer; + private ArgRef dest; + private ArgRef src; - static { - setArguments(ReadData.class, new Arg[] { DEST, SRC }); - } - @Override - protected void initializeStatic() { - super.initializeStatic(); - tracer = Tracer.getTracer(this, "SWIFTCALL"); + protected Signature getSignature() { + return new Signature(params("dest", "src")); } - protected Object function(VariableStack stack) throws ExecutionException { - DSHandle dest = (DSHandle) DEST.getValue(stack); - AbstractDataNode src = (AbstractDataNode) SRC.getValue(stack); - if (tracer.isEnabled()) { - tracer.trace(stack, "readData(" + Tracer.unwrapHandle(src) + ")"); - } - src.waitFor(); + public static boolean warning; + + @Override + public Object function(Stack stack) { + AbstractDataNode dest = this.dest.getValue(stack); + AbstractDataNode src = this.src.getValue(stack); + src.waitFor(this); if (src.getType().equals(Types.STRING)) { readData(dest, (String) src.getValue()); } @@ -116,7 +109,7 @@ } } catch (IOException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } @@ -154,7 +147,7 @@ } } catch (InvalidPathException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } @@ -175,7 +168,7 @@ } } catch (InvalidPathException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } @@ -218,7 +211,7 @@ } } catch (InvalidPathException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -25,12 +25,13 @@ import java.io.FileReader; import java.io.IOException; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.lib.Tracer; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; @@ -38,32 +39,25 @@ import org.griphyn.vdl.mapping.PhysicalFormat; import org.griphyn.vdl.type.Types; -public class ReadStructured extends VDLFunction { +public class ReadStructured extends SwiftFunction { public static final Logger logger = Logger.getLogger(ReadStructured.class); - public static final Arg DEST = new Arg.Positional("dest"); - public static final Arg SRC = new Arg.Positional("src"); - public static boolean warning; + private ArgRef dest; + private ArgRef src; - static { - setArguments(ReadStructured.class, new Arg[] { DEST, SRC }); - } - - public Tracer tracer; - - @Override - protected void initializeStatic() { - super.initializeStatic(); - tracer = Tracer.getTracer(this, "SWIFTCALL"); + @Override + protected Signature getSignature() { + return new Signature(params("dest", "src")); } - protected Object function(VariableStack stack) throws ExecutionException { - DSHandle dest = (DSHandle) DEST.getValue(stack); - AbstractDataNode src = (AbstractDataNode) SRC.getValue(stack); - if (tracer.isEnabled()) { - tracer.trace(stack, "readData2(" + Tracer.unwrapHandle(src) + ")"); - } - src.waitFor(); + public static boolean warning; + + + @Override + public Object function(Stack stack) { + AbstractDataNode dest = this.dest.getValue(stack); + AbstractDataNode src = this.src.getValue(stack); + src.waitFor(this); if (src.getType().equals(Types.STRING)) { readData(dest, (String) src.getValue()); } @@ -100,7 +94,7 @@ } } catch (Exception e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,11 +17,15 @@ package org.griphyn.vdl.karajan.lib.swiftscript; +import k.rt.Channel; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.ArrayDataNode; import org.griphyn.vdl.mapping.DSHandle; @@ -47,63 +51,49 @@ %k: Variable sKipped, no output.
%q: Array output */ -public class Sprintf extends VDLFunction { +public class Sprintf extends SwiftFunction { - private static final Logger logger = - Logger.getLogger(Sprintf.class); + private static final Logger logger = Logger.getLogger(Sprintf.class); - static { - setArguments(Sprintf.class, new Arg[] { Arg.VARGS }); + private ArgRef spec; + private ChannelRef c_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("spec", "...")); } + @Override - protected Object function(VariableStack stack) - throws ExecutionException { - AbstractDataNode[] args = waitForAllVargs(stack); + public Object function(Stack stack) { + AbstractDataNode hspec = this.spec.getValue(stack); + hspec.waitFor(this); + Channel args = c_vargs.get(stack); + waitForAll(this, args); + String spec = (String) hspec.getValue(); - String msg = format(args); - logger.debug("generated: " + msg); + String msg = format(spec, args); + if (logger.isDebugEnabled()) { + logger.debug("generated: " + msg); + } DSHandle result = new RootDataNode(Types.STRING); result.setValue(msg); return result; } - public static String format(DSHandle[] args) - throws ExecutionException { - if (! (args[0].getType() == Types.STRING)) - throw new ExecutionException - ("First argument to sprintf() must be a string!"); - - String spec = (String) args[0].getValue(); + public static String format(String spec, Channel args) { logger.debug("spec: " + spec); - DSHandle[] vars = copyArray(args, 1, args.length-1); - StringBuilder output = new StringBuilder(); - format(spec, vars, output); - + format(spec, args, output); return output.toString(); } - - public static DSHandle[] copyArray(DSHandle[] src, - int offset, int length) - { - DSHandle[] result = new DSHandle[length]; - - for (int i = 0; i < length; i++) - result[i] = src[i+offset]; - - return result; - } /** This method can be targeted as a helper function (by @sprintf(), etc.) */ - public static void format(String spec, DSHandle[] vars, - StringBuilder output) - throws ExecutionException - { + public static void format(String spec, Channel vars, StringBuilder output) { int i = 0; int arg = 0; while (i < spec.length()) { @@ -123,53 +113,48 @@ } } - private static int append(char c, int arg, DSHandle[] vars, - StringBuilder output) - throws ExecutionException { + private static int append(char c, int arg, Channel vars, StringBuilder output) { if (c == '%') { output.append('%'); return arg; } - if (arg >= vars.length) { - throw new ExecutionException - ("tracef(): too many specifiers!"); + if (arg >= vars.size()) { + throw new IllegalArgumentException("tracef(): too many specifiers!"); } if (c == 'M') { - append_M(vars[arg], output); + append_M(vars.get(arg), output); } else if (c == 'b') { - append_b(vars[arg], output); + append_b(vars.get(arg), output); } else if (c == 'f') { - append_f(vars[arg], output); + append_f(vars.get(arg), output); } else if (c == 'i') { - append_i(vars[arg], output); + append_i(vars.get(arg), output); } else if (c == 'p') { - output.append(vars[arg].toString()); + output.append(vars.get(arg).toString()); } else if (c == 's') { - append_s(vars[arg], output); + append_s(vars.get(arg), output); } else if (c == 'q') { - append_q(vars[arg], output); + append_q(vars.get(arg), output); } else if (c == 'k') { ; } else { - throw new ExecutionException - ("tracef(): Unknown format: %" + c); + throw new IllegalArgumentException("tracef(): Unknown format: %" + c); } return arg+1; } - private static void append_M(DSHandle arg, StringBuilder output) - throws ExecutionException { + private static void append_M(DSHandle arg, StringBuilder output) { try { synchronized (arg.getRoot()) { - String[] names = VDLFunction.filename(arg); + String[] names = SwiftFunction.filename(arg); if (names.length > 1) output.append(names); else @@ -177,8 +162,7 @@ } } catch (Exception e) { - throw new ExecutionException - ("tracef(%M): Could not lookup: " + arg); + throw new IllegalArgumentException("tracef(%M): Could not lookup: " + arg); } } @@ -188,36 +172,33 @@ output.append(arg.getValue()); } else { - throw new ExecutionException - ("tracef(): %b requires a boolean! " + dshandleDescription(arg)); + throw new IllegalArgumentException("tracef(): %b requires a boolean! " + + dshandleDescription(arg)); } } - private static void append_f(DSHandle arg, StringBuilder output) - throws ExecutionException { + private static void append_f(DSHandle arg, StringBuilder output) { if (arg.getType() == Types.FLOAT) { output.append(arg.getValue()); } else { - throw new ExecutionException - ("tracef(): %f requires a float! " + dshandleDescription(arg)); + throw new IllegalArgumentException("tracef(): %f requires a float! " + + dshandleDescription(arg)); } } - private static void append_i(DSHandle arg, StringBuilder output) - throws ExecutionException { + private static void append_i(DSHandle arg, StringBuilder output) { if (arg.getType() == Types.INT) { Integer d = (Integer) arg.getValue(); output.append(d); } else { - throw new ExecutionException - ("tracef(): %i requires an int! " + dshandleDescription(arg)); + throw new IllegalArgumentException("tracef(): %i requires an int! " + + dshandleDescription(arg)); } } - private static void append_q(DSHandle arg, StringBuilder output) - throws ExecutionException { + private static void append_q(DSHandle arg, StringBuilder output) { if (arg instanceof ArrayDataNode) { ArrayDataNode node = (ArrayDataNode) arg; output.append("["); @@ -234,25 +215,24 @@ } catch (Exception e) { e.printStackTrace(); - throw new ExecutionException - ("trace(%q): Could not get children of: " + arg); + throw new IllegalArgumentException("trace(%q): Could not get children of: " + + arg); } output.append("]"); } else { - throw new ExecutionException - ("tracef(): %q requires an array! " + dshandleDescription(arg)); + throw new IllegalArgumentException("tracef(): %q requires an array! " + + dshandleDescription(arg)); } } - private static void append_s(DSHandle arg, StringBuilder output) - throws ExecutionException { + private static void append_s(DSHandle arg, StringBuilder output) { if (arg.getType() == Types.STRING) { output.append(arg.getValue()); } else { - throw new ExecutionException - ("tracef(): %s requires a string! " + dshandleDescription(arg)); + throw new IllegalArgumentException("tracef(): %s requires a string! " + + dshandleDescription(arg)); } } @@ -273,8 +253,7 @@ output.append('\t'); } else { - throw new ExecutionException - ("tracef(): unknown backslash escape sequence! " + + throw new IllegalArgumentException("tracef(): unknown backslash escape sequence! " + "(\\" + c + ")\n" + "\t in " + spec + " character: " + i); } Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,11 +17,14 @@ package org.griphyn.vdl.karajan.lib.swiftscript; +import k.rt.Channel; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbstractDataNode; /** @@ -34,21 +37,27 @@ 3) allows for consumption of variables without display (%k); 4) does not impose any formatting (commas, etc.).

*/ -public class Tracef extends VDLFunction { +public class Tracef extends SwiftFunction { + private static final Logger logger = Logger.getLogger(Tracef.class); + + private ArgRef spec; + private ChannelRef c_vargs; - private static final Logger logger = - Logger.getLogger(Tracef.class); - - static { - setArguments(Tracef.class, new Arg[] { Arg.VARGS }); + @Override + protected Signature getSignature() { + return new Signature(params("spec", "...")); } + @Override - protected Object function(VariableStack stack) - throws ExecutionException { - AbstractDataNode[] args = waitForAllVargs(stack); - - String msg = Sprintf.format(args); + public Object function(Stack stack) { + AbstractDataNode hspec = this.spec.getValue(stack); + hspec.waitFor(this); + Channel args = c_vargs.get(stack); + waitForAll(this, args); + String spec = (String) hspec.getValue(); + + String msg = Sprintf.format(spec, args); logger.info(msg); System.out.print(msg); return null; Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -25,15 +25,16 @@ import java.util.Map; import java.util.TreeMap; +import k.rt.ExecutionException; +import k.rt.Stack; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.arguments.Arg; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; @@ -41,25 +42,28 @@ import org.griphyn.vdl.type.Types; -public class WriteData extends VDLFunction { +public class WriteData extends SwiftFunction { public static final Logger logger = Logger.getLogger(WriteData.class); - public static final Arg DEST = new Arg.Positional("dest"); - public static final Arg SRC = new Arg.Positional("src"); + private ArgRef dest; + private ArgRef src; + + @Override + protected Signature getSignature() { + return new Signature(params("dest", "src")); + } + public static boolean warning; - static { - setArguments(WriteData.class, new Arg[] { DEST, SRC }); - } - - protected Object function(VariableStack stack) throws ExecutionException { + @Override + public Object function(Stack stack) { // dest needs to be mapped to a file, or a string - DSHandle dest = (DSHandle) DEST.getValue(stack); + AbstractDataNode dest = this.dest.getValue(stack); // src can be any of several forms of value - AbstractDataNode src = (AbstractDataNode) SRC.getValue(stack); + AbstractDataNode src = this.src.getValue(stack); - src.waitFor(); + src.waitFor(this); if (dest.getType().equals(Types.STRING)) { writeData((String)dest.getValue(), src); @@ -109,7 +113,7 @@ } } catch (IOException e) { - throw new ExecutionException(e); + throw new ExecutionException(this, e); } } Modified: branches/faster/src/org/griphyn/vdl/karajan/monitor/SystemState.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/monitor/SystemState.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/monitor/SystemState.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -25,17 +25,20 @@ import java.util.Map; import java.util.Set; -import org.globus.cog.karajan.stack.VariableStack; +import k.rt.Stack; + import org.griphyn.vdl.karajan.monitor.items.StatefulItem; import org.griphyn.vdl.karajan.monitor.items.StatefulItemClass; +import com.sun.org.apache.xpath.internal.VariableStack; + public class SystemState { private Map> classes; private Set listeners; private Map stats; private int total, completed; private long start; - private VariableStack stack; + private Stack stack; private String projectName; public SystemState(String projectName) { @@ -132,11 +135,11 @@ return start; } - public VariableStack getStack() { + public Stack getStack() { return stack; } - public void setStack(VariableStack stack) { + public void setStack(Stack stack) { this.stack = stack; } Modified: branches/faster/src/org/griphyn/vdl/karajan/monitor/items/SummaryItem.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/monitor/items/SummaryItem.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/monitor/items/SummaryItem.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,6 +20,7 @@ */ package org.griphyn.vdl.karajan.monitor.items; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -58,14 +59,8 @@ public int getCount(String key, SystemState state) { if (state.getStack() != null) { - ProgressTicker t = RuntimeStats.getTicker(state.getStack()); - Integer value = t.getSummary().get(key); - if (value != null) { - return value; - } - else { - return 0; - } + // TODO Must get these from log + return -1; } else { return getCount(key); @@ -74,7 +69,8 @@ public synchronized Map getCounts(SystemState state) { if (state.getStack() != null) { - return RuntimeStats.getTicker(state.getStack()).getSummary(); + // TODO Must get these from log + return Collections.emptyMap(); } else { return new HashMap(counts); Modified: branches/faster/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -25,9 +25,9 @@ import org.globus.cog.abstraction.impl.execution.coaster.WorkerShellCommand; import org.globus.cog.abstraction.interfaces.Service; import org.globus.cog.abstraction.interfaces.Task; -import org.globus.cog.karajan.workflow.service.ProtocolException; -import org.globus.cog.karajan.workflow.service.channels.ChannelManager; -import org.globus.cog.karajan.workflow.service.channels.KarajanChannel; +import org.globus.cog.coaster.ProtocolException; +import org.globus.cog.coaster.channels.ChannelManager; +import org.globus.cog.coaster.channels.CoasterChannel; import org.griphyn.vdl.karajan.monitor.monitors.ansi.tui.Dialog; import org.griphyn.vdl.karajan.monitor.monitors.ansi.tui.Terminal; import org.griphyn.vdl.karajan.monitor.monitors.ansi.tui.Terminal.InputHandler; @@ -72,7 +72,7 @@ private String runcmd(String cmd) { try { - KarajanChannel channel = ChannelManager.getManager() + CoasterChannel channel = ChannelManager.getManager() .reserveChannel(contact, cred, LocalRequestManager.INSTANCE); WorkerShellCommand wsc = new WorkerShellCommand(workerId, cmd); wsc.execute(channel); Modified: branches/faster/src/org/griphyn/vdl/karajan/monitor/processors/ExecutionContextProcessor.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/monitor/processors/ExecutionContextProcessor.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/monitor/processors/ExecutionContextProcessor.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -21,8 +21,7 @@ package org.griphyn.vdl.karajan.monitor.processors; import org.apache.log4j.Level; -import org.globus.cog.karajan.stack.VariableStack; -import org.griphyn.vdl.karajan.VDL2ExecutionContext; +import org.griphyn.vdl.karajan.SwiftExecutor; import org.griphyn.vdl.karajan.monitor.SystemState; public class ExecutionContextProcessor extends AbstractMessageProcessor { @@ -32,12 +31,9 @@ } public Class getSupportedSource() { - return VDL2ExecutionContext.class; + return SwiftExecutor.class; } public void processMessage(SystemState state, Object message, Object details) { - if (message instanceof VariableStack) { - state.setStack((VariableStack) message); - } } } Modified: branches/faster/src/org/griphyn/vdl/karajan/monitor/processors/TaskProcessor.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/monitor/processors/TaskProcessor.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/karajan/monitor/processors/TaskProcessor.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -35,7 +35,7 @@ } public Class getSupportedSource() { - return org.globus.cog.karajan.workflow.nodes.grid.AbstractGridNode.class; + return org.globus.cog.karajan.compiled.nodes.grid.AbstractGridNode.class; } public void processMessage(SystemState state, Object message, Object details) { Modified: branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -28,10 +28,12 @@ import java.util.List; import java.util.Map; +import k.rt.Future; +import k.thr.Yield; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.futures.Future; -import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.futures.FutureNotYetAvailable; import org.griphyn.vdl.karajan.DSHandleFutureWrapper; import org.griphyn.vdl.karajan.FutureTracker; import org.griphyn.vdl.karajan.FutureWrapper; @@ -109,7 +111,7 @@ } } - public void init(MappingParamSet params) { + public void init(MappingParamSet params) throws HandleOpenException { throw new UnsupportedOperationException(); } @@ -623,12 +625,15 @@ return DATASET_URI_PREFIX + datasetIDPartialID + ":" + datasetIDCounter; } - public synchronized void waitFor() { + public synchronized void waitFor(Node who) { if (!closed) { if (logger.isDebugEnabled()) { logger.debug("Waiting for " + this); } - throw new FutureNotYetAvailable(getFutureWrapper()); + + Yield y = new FutureNotYetAvailable(getFutureWrapper()); + y.getState().addTraceElement(who); + throw y; } else { if (logger.isDebugEnabled()) { @@ -640,6 +645,24 @@ } } + public synchronized void waitFor() throws OOBYield { + if (!closed) { + if (logger.isDebugEnabled()) { + logger.debug("Waiting for " + this); + } + + throw new OOBYield(new FutureNotYetAvailable(getFutureWrapper()), this); + } + else { + if (logger.isDebugEnabled()) { + logger.debug("Do not need to wait for " + this); + } + if (value instanceof RuntimeException) { + throw (RuntimeException) value; + } + } + } + public void addListener(DSHandleListener listener) { throw new UnsupportedOperationException(); } Modified: branches/faster/src/org/griphyn/vdl/mapping/AbstractMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/AbstractMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/AbstractMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -48,7 +48,7 @@ } } - public void setParams(MappingParamSet params) { + public void setParams(MappingParamSet params) throws HandleOpenException { this.params = params; } Modified: branches/faster/src/org/griphyn/vdl/mapping/ArrayDataNode.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/ArrayDataNode.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/ArrayDataNode.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -23,9 +23,10 @@ import java.util.List; import java.util.Map; -import org.globus.cog.karajan.workflow.futures.Future; -import org.globus.cog.karajan.workflow.futures.FutureList; -import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; +import k.rt.Future; + +import org.globus.cog.karajan.futures.FutureList; +import org.globus.cog.karajan.futures.FutureNotYetAvailable; import org.griphyn.vdl.karajan.ArrayIndexFutureList; import org.griphyn.vdl.karajan.FutureTracker; import org.griphyn.vdl.type.Field; Modified: branches/faster/src/org/griphyn/vdl/mapping/DSHandle.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/DSHandle.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/DSHandle.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -47,7 +47,7 @@ */ public Type getType(); - public void init(MappingParamSet params); + public void init(MappingParamSet params) throws HandleOpenException; public DSHandle getRoot(); Modified: branches/faster/src/org/griphyn/vdl/mapping/DataDependentException.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/DataDependentException.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/DataDependentException.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -35,7 +35,7 @@ } public String getMessage() { - return getVariableName() + " not derived due to errors in data dependencies"; + return getHandle() + " not derived due to errors in data dependencies"; } } Modified: branches/faster/src/org/griphyn/vdl/mapping/DependentException.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/DependentException.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/DependentException.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -41,15 +41,6 @@ public DSHandle getHandle() { return handle; } - - public String getVariableName() { - if (handle instanceof AbstractDataNode) { - return ((AbstractDataNode) handle).getDisplayableName(); - } - else { - return handle.toString(); - } - } public String toString() { return getMessage(); Modified: branches/faster/src/org/griphyn/vdl/mapping/HandleOpenException.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/HandleOpenException.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/HandleOpenException.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -24,11 +24,17 @@ private DSHandle source; public HandleOpenException(DSHandle source) { - super("Handle open: " + source.getType() + " " + source.toString()); this.source = source; } public DSHandle getSource() { return source; } + + @Override + public String getMessage() { + return "Handle open: " + source.getType() + " " + source.toString(); + } + + } Modified: branches/faster/src/org/griphyn/vdl/mapping/Mapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/Mapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/Mapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -54,10 +54,8 @@ */ boolean isStatic(); - void setParams(MappingParamSet params); + void setParams(MappingParamSet params) throws HandleOpenException; - void setParam(MappingParam p, Object value); - Object getParam(MappingParam name); /** Modified: branches/faster/src/org/griphyn/vdl/mapping/MappingParam.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/MappingParam.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/MappingParam.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -61,9 +61,9 @@ /** Returns the value of this parameter. If the value is a dataset, * then the value will be converted to a string. If no value is * specified in the SwiftScript program, then the default value - * will be returned. + * will be returned. */ - public Object getValue(Mapper mapper) { + public Object getValue(Mapper mapper) throws HandleOpenException { Object value = mapper.getParam(this); if (value instanceof AbstractDataNode) { AbstractDataNode handle = (AbstractDataNode) value; @@ -89,7 +89,7 @@ } } - public Object getValue(MappingParamSet params) { + public Object getValue(MappingParamSet params) throws HandleOpenException { Object value = params.get(this); if (value instanceof AbstractDataNode) { AbstractDataNode handle = (AbstractDataNode) value; @@ -120,7 +120,7 @@ /** Returns the mapper parameter as a String. Other data types will be converted to a String as appropriate. */ - public String getStringValue(Mapper mapper) { + public String getStringValue(Mapper mapper) throws HandleOpenException { Object value = getValue(mapper); if (value == null) { return null; @@ -128,7 +128,7 @@ return String.valueOf(value); } - public String getStringValue(MappingParamSet params) { + public String getStringValue(MappingParamSet params) throws HandleOpenException { Object value = getValue(params); if (value == null) { return null; @@ -136,10 +136,6 @@ return String.valueOf(value); } - public void setValue(Mapper mapper, Object value) { - mapper.setParam(this, value); - } - public boolean isPresent(Mapper mapper) { return mapper.getParam(this) != null; } @@ -155,7 +151,7 @@ * match, then 'false' is returned (note an exception is not thrown, * unlike other getValue methods). */ - public boolean getBooleanValue(Mapper mapper) { + public boolean getBooleanValue(Mapper mapper) throws HandleOpenException { Object value = getValue(mapper); if (value instanceof String) { return Boolean.valueOf((String) value).booleanValue(); @@ -172,7 +168,7 @@ * is a string, then the string will be parsed to an int. If the * value is an int, then this will be passed through. Otherwise, * a NumberFormatException will be thrown. */ - public int getIntValue(Mapper mapper) { + public int getIntValue(Mapper mapper) throws HandleOpenException { Object value = getValue(mapper); if (value instanceof String) { return Integer.parseInt((String) value); Added: branches/faster/src/org/griphyn/vdl/mapping/OOBYield.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/OOBYield.java (rev 0) +++ branches/faster/src/org/griphyn/vdl/mapping/OOBYield.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -0,0 +1,30 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jan 28, 2013 + */ +package org.griphyn.vdl.mapping; + +import k.thr.Yield; + +public class OOBYield extends HandleOpenException { + private final Yield y; + + public OOBYield(Yield y, DSHandle h) { + super(h); + this.y = y; + } + + public Yield wrapped() { + return y; + } + + public Yield wrapped(Object traceElement) { + y.getState().addTraceElement(traceElement); + return y; + } +} Modified: branches/faster/src/org/griphyn/vdl/mapping/RootArrayDataNode.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -17,11 +17,11 @@ package org.griphyn.vdl.mapping; +import k.rt.Future; +import k.rt.FutureListener; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.futures.Future; -import org.globus.cog.karajan.workflow.futures.FutureListener; -import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; +import org.globus.cog.karajan.futures.FutureNotYetAvailable; import org.griphyn.vdl.karajan.lib.Tracer; import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Type; @@ -50,7 +50,7 @@ this.dmc = dmc; } - public void init(MappingParamSet params) { + public void init(MappingParamSet params) throws HandleOpenException { this.params = params; if (this.params == null) { initialized(); @@ -60,14 +60,14 @@ } } - private synchronized void innerInit() { + private synchronized void innerInit() throws HandleOpenException { if (logger.isDebugEnabled()) { logger.debug("innerInit: " + this); } waitingMapperParam = params.getFirstOpenParamValue(); if (waitingMapperParam != null) { - waitingMapperParam.getFutureWrapper().addModificationAction(this, null); + waitingMapperParam.getFutureWrapper().addListener(this); if (tracer.isEnabled()) { tracer.trace(getThread(), getDeclarationLine(), getDisplayableName() + " WAIT " + Tracer.getVarName(waitingMapperParam)); @@ -109,8 +109,16 @@ } } - public void futureModified(Future f, VariableStack stack) { - innerInit(); + public void futureUpdated(Future f) { + try { + innerInit(); + } + catch (OOBYield e) { + throw e.wrapped(); + } + catch (HandleOpenException e) { + e.printStackTrace(); + } } public String getParam(MappingParam p) { Modified: branches/faster/src/org/griphyn/vdl/mapping/RootDataNode.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/RootDataNode.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/RootDataNode.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,11 +20,11 @@ */ package org.griphyn.vdl.mapping; +import k.rt.Future; +import k.rt.FutureListener; + import org.apache.log4j.Logger; -import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.futures.Future; -import org.globus.cog.karajan.workflow.futures.FutureListener; -import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; +import org.globus.cog.karajan.futures.FutureNotYetAvailable; import org.griphyn.vdl.karajan.lib.Tracer; import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Type; @@ -56,21 +56,23 @@ setValue(value); } - public void init(MappingParamSet params) { + public void init(MappingParamSet params) throws HandleOpenException { this.params = params; - if(this.params == null) { + if (this.params == null) { initialized(); - } else { + } + else { innerInit(); } } /** must have this.params set to the appropriate parameters before - being called. */ - private synchronized void innerInit() { + being called. + * @throws HandleOpenException */ + private synchronized void innerInit() throws HandleOpenException { waitingMapperParam = params.getFirstOpenParamValue(); if (waitingMapperParam != null) { - waitingMapperParam.getFutureWrapper().addModificationAction(this, null); + waitingMapperParam.getFutureWrapper().addListener(this); if (tracer.isEnabled()) { tracer.trace(getThread(), getDeclarationLine(), getDisplayableName() + " WAIT " + Tracer.getVarName(waitingMapperParam)); @@ -113,8 +115,16 @@ } } - public void futureModified(Future f, VariableStack stack) { - innerInit(); + public void futureUpdated(Future f) { + try { + innerInit(); + } + catch (OOBYield e) { + throw e.wrapped(); + } + catch (HandleOpenException e) { + e.printStackTrace(); + } } @@ -271,9 +281,6 @@ initialized = true; waitingMapperParam = null; if (tracer.isEnabled()) { - if ("sphOut".equals(getDisplayableName())) { - System.out.println(); - } tracer.trace(getThread(), getDeclarationLine(), getDisplayableName() + " INITIALIZED " + params); } } Modified: branches/faster/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -31,6 +31,7 @@ import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.AbstractMapper; +import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidMappingParameterException; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.MappingParamSet; @@ -71,6 +72,8 @@ public static final MappingParam PARAM_PATTERN = new MappingParam("pattern", null); public static final MappingParam PARAM_LOCATION = new MappingParam("location", null); public static final MappingParam PARAM_NOAUTO = new MappingParam("noauto", "false"); + + private String location, prefix, suffix, pattern; public static final Logger logger = Logger.getLogger(AbstractFileMapper.class); @@ -82,6 +85,8 @@ this.elementMapper = elementMapper; } + + /** Creates an AbstractFileMapper without specifying a * FileNameElementMapper. The elementMapper must be specified @@ -99,7 +104,7 @@ this.elementMapper = elementMapper; } - public void setParams(MappingParamSet params) { + public void setParams(MappingParamSet params) throws HandleOpenException { super.setParams(params); if (PARAM_SUFFIX.isPresent(this)) { String suffix = PARAM_SUFFIX.getStringValue(this); @@ -109,22 +114,23 @@ ". Value set was '" + noauto + "'"); } if (!suffix.startsWith(".") && noauto.equals("false")) { - PARAM_SUFFIX.setValue(this, "." + suffix); + params.set(PARAM_SUFFIX, "." + suffix); } } if (PARAM_PATTERN.isPresent(this)) { String pattern = PARAM_PATTERN.getStringValue(this); - PARAM_PATTERN.setValue(this, replaceWildcards(pattern)); + params.set(PARAM_PATTERN, replaceWildcards(pattern)); } + location = PARAM_LOCATION.getStringValue(this); + prefix = PARAM_PREFIX.getStringValue(this); + suffix = PARAM_SUFFIX.getStringValue(this); + pattern = PARAM_PATTERN.getStringValue(this); } public PhysicalFormat map(Path path) { if(logger.isDebugEnabled()) logger.debug("mapper id="+this.hashCode()+" starting to map "+path); StringBuffer sb = new StringBuffer(); - final String location = PARAM_LOCATION.getStringValue(this); - final String prefix = PARAM_PREFIX.getStringValue(this); - final String suffix = PARAM_SUFFIX.getStringValue(this); maybeAppend(sb, location); if (location != null && !location.endsWith("/")) { sb.append('/'); @@ -215,10 +221,6 @@ public Collection existing() { if(logger.isDebugEnabled()) logger.debug("list existing paths for mapper id="+this.hashCode()); - final String location = PARAM_LOCATION.getStringValue(this); - final String prefix = PARAM_PREFIX.getStringValue(this); - final String suffix = PARAM_SUFFIX.getStringValue(this); - final String pattern = PARAM_PATTERN.getStringValue(this); List result = new ArrayList(); final AbsFile f; @@ -287,21 +289,20 @@ public Path rmap(String name) { logger.debug("rmap "+name); - final String prefix = PARAM_PREFIX.getStringValue(this); - - if(prefix!=null) { - if(name.startsWith(prefix)) { + if(prefix != null) { + if (name.startsWith(prefix)) { name = name.substring(prefix.length()); - } else { + } + else { throw new RuntimeException("filename '"+name+"' does not begin with prefix '"+prefix+"'"); } } - final String suffix = PARAM_SUFFIX.getStringValue(this); - if(suffix!=null) { - if(name.endsWith(suffix)) { + if(suffix != null) { + if (name.endsWith(suffix)) { name = name.substring(0,name.length() - suffix.length()); - } else { + } + else { throw new RuntimeException("filename '"+name+"' does not end with suffix '"+suffix+"'"); } } @@ -353,11 +354,11 @@ } public String getLocation() { - return PARAM_LOCATION.getStringValue(this); + return location; } public String getPrefix() { - return PARAM_PREFIX.getStringValue(this); + return prefix; } Modified: branches/faster/src/org/griphyn/vdl/mapping/file/CSVMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/file/CSVMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/file/CSVMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -33,6 +33,7 @@ import org.griphyn.vdl.mapping.AbstractMapper; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.GeneralizedFileFormat; +import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidMappingParameterException; import org.griphyn.vdl.mapping.Mapper; import org.griphyn.vdl.mapping.MappingParam; @@ -67,16 +68,24 @@ private List content = new ArrayList(); /** whether the CSV file has been read already. */ - private boolean read = false; + private boolean read = false; + + private String delim, hdelim; + private boolean header; + private int skip; - public void setParams(MappingParamSet params) { + public void setParams(MappingParamSet params) throws HandleOpenException { super.setParams(params); if (!PARAM_FILE.isPresent(this)) { throw new InvalidMappingParameterException("CSV mapper must have a file parameter."); } - if (!PARAM_HDELIMITER.isPresent(this)) { - PARAM_HDELIMITER.setValue(this, PARAM_DELIMITER.getValue(this)); - } + if (!PARAM_HDELIMITER.isPresent(this)) { + params.set(PARAM_HDELIMITER, PARAM_DELIMITER.getRawValue(this)); + } + delim = PARAM_DELIMITER.getStringValue(this); + hdelim = PARAM_HDELIMITER.getStringValue(this); + header = PARAM_HEADER.getBooleanValue(this); + skip = PARAM_SKIP.getIntValue(this); } private synchronized void readFile() { @@ -86,10 +95,6 @@ String file = getCSVFile(); - String delim = PARAM_DELIMITER.getStringValue(this); - String hdelim = PARAM_HDELIMITER.getStringValue(this); - boolean header = PARAM_HEADER.getBooleanValue(this); - int skip = PARAM_SKIP.getIntValue(this); try { BufferedReader br = new BufferedReader(new FileReader(file)); Modified: branches/faster/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Set; +import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.Mapper; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.MappingParamSet; @@ -41,7 +42,7 @@ super(new ConcurrentElementMapper()); } - public void setParams(MappingParamSet params) { + public void setParams(MappingParamSet params) throws HandleOpenException { String prefix = PARAM_PREFIX.getStringValue(params); prefix = "_concurrent/" + (prefix == null ? "" : prefix + "-") + PARAM_THREAD_PREFIX.getValue(params); Modified: branches/faster/src/org/griphyn/vdl/mapping/file/ExternalMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/file/ExternalMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/file/ExternalMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -25,14 +25,13 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.log4j.Logger; import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractMapper; +import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.Path; @@ -48,7 +47,7 @@ private static final String[] STRING_ARRAY = new String[0]; - public void setParams(MappingParamSet params) { + public void setParams(MappingParamSet params) throws HandleOpenException { super.setParams(params); map = new HashMap(); rmap = new HashMap(); Modified: branches/faster/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -46,7 +46,7 @@ if (!path.isArrayIndex(0)) { return null; } - String location = PARAM_LOCATION.getStringValue(this); + String location = getLocation(); Object index = path.getFirst(); String filename = filenames.get(index); if (filename == null) { Modified: branches/faster/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -18,74 +18,53 @@ package org.griphyn.vdl.mapping.file; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.StringTokenizer; -import java.util.TreeSet; import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractMapper; -import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidMappingParameterException; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; -import org.griphyn.vdl.type.Types; /** Maps a string (separated by space, comma or semicolon) of filenames to an array. */ public class FixedArrayFileMapper extends AbstractMapper { public static final MappingParam PARAM_FILES = new MappingParam("files"); - private List files; + private String[] files; + public FixedArrayFileMapper() { super(); } - public void setParams(MappingParamSet params) { + public void setParams(MappingParamSet params) throws HandleOpenException { super.setParams(params); - DSHandle dn = (DSHandle) PARAM_FILES.getRawValue(this); - if (dn == null) { - throw new InvalidMappingParameterException("Missing required mapper parameter: " - + PARAM_FILES); + String cfiles = PARAM_FILES.getStringValue(this); + if (cfiles == null) { + throw new InvalidMappingParameterException("Missing required mapper parameter: " + + PARAM_FILES); } - if (Types.STRING.equals(dn.getType())) { - String cfiles = (String) dn.getValue(); - - StringTokenizer st = new StringTokenizer(cfiles, " ,;"); - String[] files = new String[st.countTokens()]; - for (int i = 0; st.hasMoreTokens(); i++) { - files[i] = st.nextToken(); - } - this.files = Arrays.asList(files); + StringTokenizer st = new StringTokenizer(cfiles, " ,;"); + files = new String[st.countTokens()]; + for (int i = 0; st.hasMoreTokens(); i++) { + files[i] = st.nextToken(); } - else if (dn.getType().isArray() && Types.STRING.equals(dn.getType().itemType())) { - files = new ArrayList(); - Map m = dn.getArrayValue(); - // must keep order - @SuppressWarnings("unchecked") - Set> s = new TreeSet>((Set>) m.keySet()); - Iterator i = s.iterator(); - while(i.hasNext()) { - Comparable nextKey = (Comparable) i.next(); - files.add((String) m.get(nextKey).getValue()); - } - } - else { - throw new InvalidMappingParameterException("Unrecognized value for " - + PARAM_FILES + " parameter: " + dn.getType() + ". Valid values are a string or an array of strings."); - } + params.set(PARAM_FILES, files); } + protected String[] getFiles() { + return files; + } + public Collection existing() { List l = new ArrayList(); - for (int i = 0; i < files.size(); i++) { + for (int i = 0; i < getFiles().length; i++) { l.add(Path.EMPTY_PATH.addLast(i, true)); } return l; @@ -99,7 +78,7 @@ Object o = path.getFirst(); if (o instanceof Integer) { int index = ((Integer) o).intValue(); - return new AbsFile(files.get(index)); + return new AbsFile(getFiles()[index]); } else { throw new IllegalArgumentException("The fixed array mapper can only be used with an int key array"); Modified: branches/faster/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -28,6 +28,7 @@ import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractMapper; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.Path; @@ -36,16 +37,21 @@ public class RegularExpressionMapper extends AbstractMapper { public static final MappingParam PARAM_SOURCE = new MappingParam("source"); public static final MappingParam PARAM_MATCH = new MappingParam("match"); - public static final MappingParam PARAM_TRANSFORM = new MappingParam("transform"); + public static final MappingParam PARAM_TRANSFORM = new MappingParam("transform"); + + private String match, source, transform; public RegularExpressionMapper() { } - public void setParams(MappingParamSet params) { + public void setParams(MappingParamSet params) throws HandleOpenException { super.setParams(params); if (!PARAM_MATCH.isPresent(this)) { throw new RuntimeException("Missing parameter match!"); - } + } + match = PARAM_MATCH.getStringValue(this); + source = PARAM_SOURCE.getStringValue(this); + transform = PARAM_TRANSFORM.getStringValue(this); } public Collection existing() { @@ -64,9 +70,7 @@ PARAM_MATCH.getName() + "; maybe you meant @filename(" + h.getPathFromRoot() + ")?"); } } - String match = PARAM_MATCH.getStringValue(this); - String source = PARAM_SOURCE.getStringValue(this); - String transform = PARAM_TRANSFORM.getStringValue(this); + Pattern p = Pattern.compile(match); Matcher m = p.matcher(source); if (!m.find()) { @@ -104,7 +108,12 @@ params.put("transform", "\\1_area.\\2"); MappingParamSet mps = new MappingParamSet(); mps.setAll(params); - reMapper.setParams(mps); + try { + reMapper.setParams(mps); + } + catch (HandleOpenException e) { + e.printStackTrace(); + } System.out.println(reMapper.map(Path.EMPTY_PATH)); } } Modified: branches/faster/src/org/griphyn/vdl/mapping/file/SimpleFileMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/file/SimpleFileMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/file/SimpleFileMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -20,6 +20,7 @@ */ package org.griphyn.vdl.mapping.file; +import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.MappingParamSet; @@ -30,7 +31,7 @@ super(); } - public void setParams(MappingParamSet params) { + public void setParams(MappingParamSet params) throws HandleOpenException { super.setParams(params); int precision = PARAM_PADDING.getIntValue(this); setElementMapper(new DefaultFileNameElementMapper(precision)); Modified: branches/faster/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -23,7 +23,9 @@ import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractMapper; +import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.MappingParam; +import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; @@ -32,13 +34,21 @@ public class SingleFileMapper extends AbstractMapper { public static final MappingParam PARAM_FILE = new MappingParam("file"); + + private AbsFile file; public SingleFileMapper() { super(); } - public Collection existing() { - if (new AbsFile(PARAM_FILE.getStringValue(this)).exists()) { + @Override + public void setParams(MappingParamSet params) throws HandleOpenException { + super.setParams(params); + file = new AbsFile(PARAM_FILE.getStringValue(this)); + } + + public Collection existing() { + if (file.exists()) { return Arrays.asList(new Path[] {Path.EMPTY_PATH}); } else { @@ -47,7 +57,7 @@ } public PhysicalFormat map(Path path) { - return new AbsFile(PARAM_FILE.getStringValue(this)); + return file; } public boolean isStatic() { Modified: branches/faster/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -41,16 +41,21 @@ public static final MappingParam PARAM_SOURCE = new MappingParam("source"); public static final MappingParam PARAM_MATCH = new MappingParam("match"); - public static final MappingParam PARAM_TRANSFORM = new MappingParam("transform"); + public static final MappingParam PARAM_TRANSFORM = new MappingParam("transform"); + + private String match, transform; public StructuredRegularExpressionMapper() { } - public void setParams(MappingParamSet params) { + public void setParams(MappingParamSet params) throws HandleOpenException { super.setParams(params); if (!PARAM_MATCH.isPresent(this)) { throw new RuntimeException("Missing parameter match!"); - } + } + + match = PARAM_MATCH.getStringValue(this); + transform = PARAM_TRANSFORM.getStringValue(this); } public Collection existing() { @@ -82,9 +87,6 @@ logger.debug("map(): path: " + path); - String match = PARAM_MATCH.getStringValue(this); - String transform = PARAM_TRANSFORM.getStringValue(this); - DSHandle sourceHandle = (DSHandle) PARAM_SOURCE.getRawValue(this); DSHandle hereHandle; try { Modified: branches/faster/src/org/griphyn/vdl/mapping/file/TestMapper.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/file/TestMapper.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/mapping/file/TestMapper.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -22,8 +22,10 @@ import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractMapper; +import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.Mapper; import org.griphyn.vdl.mapping.MappingParam; +import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; @@ -35,15 +37,27 @@ public static final MappingParam PARAM_STATIC = new MappingParam("static", true); private PhysicalFormat remap, map; + + private boolean remappable, temp, _static; + private String file; @Override + public void setParams(MappingParamSet params) throws HandleOpenException { + super.setParams(params); + remappable = PARAM_REMAPPABLE.getBooleanValue(this); + temp = PARAM_TEMP.getBooleanValue(this); + _static = PARAM_STATIC.getBooleanValue(this); + file = PARAM_FILE.getStringValue(this); + } + + @Override public boolean canBeRemapped(Path path) { - return PARAM_REMAPPABLE.getBooleanValue(this); + return remappable; } @Override public void remap(Path path, Mapper sourceMapper, Path sourcePath) { - if (PARAM_REMAPPABLE.getBooleanValue(this)) { + if (remappable) { remap = sourceMapper.map(sourcePath); System.out.println("Remapping " + path + " -> " + remap); ensureCollectionConsistency(sourceMapper, sourcePath); @@ -56,7 +70,7 @@ @Override public void clean(Path path) { PhysicalFormat pf = map(path); - if (PARAM_TEMP.getBooleanValue(this)) { + if (temp) { System.out.println("Cleaning file " + pf); FileGarbageCollector.getDefault().decreaseUsageCount(pf); } @@ -67,13 +81,13 @@ @Override public boolean isPersistent(Path path) { - return !PARAM_TEMP.getBooleanValue(this); + return !temp; } public PhysicalFormat map(Path path) { if (remap == null) { if (map == null) { - map = new AbsFile(PARAM_FILE.getStringValue(this)); + map = new AbsFile(file); } return map; } @@ -87,6 +101,6 @@ } public boolean isStatic() { - return PARAM_STATIC.getBooleanValue(this); + return _static; } } Modified: branches/faster/src/org/griphyn/vdl/util/VDL2Config.java =================================================================== --- branches/faster/src/org/griphyn/vdl/util/VDL2Config.java 2013-01-29 04:30:59 UTC (rev 6169) +++ branches/faster/src/org/griphyn/vdl/util/VDL2Config.java 2013-01-29 07:31:09 UTC (rev 6170) @@ -115,7 +115,6 @@ put("cdm.broadcast.mode", "file"); put("use.provider.staging", "false"); - put("use.wrapper.staging", "false"); put("ticker.date.format", ""); put("ticker.prefix", "Progress: time:"); From tmoore at ci.uchicago.edu Thu Jan 10 11:28:33 2013 From: tmoore at ci.uchicago.edu (tmoore at ci.uchicago.edu) Date: Thu, 10 Jan 2013 17:28:33 -0000 Subject: [Swift-commit] r6148 - SwiftApps/ParVis/HiRAMTools Message-ID: <20130110172831.2B8579CF80@svn.ci.uchicago.edu> Author: tmoore Date: 2013-01-10 11:28:31 -0600 (Thu, 10 Jan 2013) New Revision: 6148 Modified: SwiftApps/ParVis/HiRAMTools/combine_realization.sh SwiftApps/ParVis/HiRAMTools/makeyearly_2yr.sh SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh SwiftApps/ParVis/HiRAMTools/makeyearly_realization_2yr.sh SwiftApps/ParVis/HiRAMTools/makeyearly_realization_semi.sh SwiftApps/ParVis/HiRAMTools/runpfrepps.sh Log: Updated with new project name Modified: SwiftApps/ParVis/HiRAMTools/combine_realization.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2013-01-09 23:16:09 UTC (rev 6147) +++ SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2013-01-10 17:28:31 UTC (rev 6148) @@ -61,7 +61,7 @@ - SU_Climate + prec_sense default 2.56 10000 Modified: SwiftApps/ParVis/HiRAMTools/makeyearly_2yr.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/makeyearly_2yr.sh 2013-01-09 23:16:09 UTC (rev 6147) +++ SwiftApps/ParVis/HiRAMTools/makeyearly_2yr.sh 2013-01-10 17:28:31 UTC (rev 6148) @@ -39,14 +39,21 @@ echo $fname if echo $fname | egrep -v 'grid_spec|scalar'; then - # Merge 2 quarter-decade files into one half-decade file +#this script has proven unreliable for some reason. I'm going to guess it's a fs problem +#this time attempt to cp the 2 year file to scratch -# ncrcat $qtrdec1 $qtrdec2 5years.nc +cp $the2year ./current2yearfile.nc +SOURCE_FILESIZE=`stat -c %s ./current2yearfile.nc` +echo "Source File Size: ${SOURCE_FILESIZE}" + # Extract each year from the half-decade file echo "processing ${fname}" for (( y=$firstyear; y<$firstyear+2; y++ )); do - ncks -d time,"${y}-01-01 00:00:00","${y}-12-31 23:59:59" $the2year $y.nc + echo "Processing ${fname} for ${y}" + ncks -d time,"${y}-01-01 00:00:00","${y}-12-31 23:59:59" current2yearfile.nc $y.nc + RESULT_FILESIZE==`stat -c %s ${y}.nc` + echo "Result File Size for ${fname}: ${y} - ${RESULT_FILESIZE}" done fi Modified: SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh 2013-01-09 23:16:09 UTC (rev 6147) +++ SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh 2013-01-10 17:28:31 UTC (rev 6148) @@ -66,7 +66,7 @@ - SU_Climate + prec_sense default 2.56 10000 Modified: SwiftApps/ParVis/HiRAMTools/makeyearly_realization_2yr.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/makeyearly_realization_2yr.sh 2013-01-09 23:16:09 UTC (rev 6147) +++ SwiftApps/ParVis/HiRAMTools/makeyearly_realization_2yr.sh 2013-01-10 17:28:31 UTC (rev 6148) @@ -21,7 +21,7 @@ # generate the list of files to process: do every half-decade echo f1 >ncfiles -for y in $(seq 1950 2 1998); do +for y in $(seq 1950 2 1998 ); do startyear=${y}0101; # nextyear=$((y+1))0101 @@ -48,7 +48,7 @@ wrapperlog.always.transfer=true sitedir.keep=true -execution.retries=10 +execution.retries=1 lazy.errors=true #status.mode=provider use.wrapper.staging=false @@ -72,7 +72,7 @@ - SU_Climate + prec_sense default 2.56 10000 @@ -81,7 +81,7 @@ 32 28000 - 00:20:00 + 02:30:00 100 100 1 Modified: SwiftApps/ParVis/HiRAMTools/makeyearly_realization_semi.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/makeyearly_realization_semi.sh 2013-01-09 23:16:09 UTC (rev 6147) +++ SwiftApps/ParVis/HiRAMTools/makeyearly_realization_semi.sh 2013-01-10 17:28:31 UTC (rev 6148) @@ -91,7 +91,7 @@ - SU_Climate + prec_sense default 2.56 10000 Modified: SwiftApps/ParVis/HiRAMTools/runpfrepps.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/runpfrepps.sh 2013-01-09 23:16:09 UTC (rev 6147) +++ SwiftApps/ParVis/HiRAMTools/runpfrepps.sh 2013-01-10 17:28:31 UTC (rev 6148) @@ -53,7 +53,7 @@ - SU_Climate + prec_sense default 2.56