[Swift-commit] r7145 - SwiftApps/Swift-MapRed/mapred_combiner_V3
yadunandb at ci.uchicago.edu
yadunandb at ci.uchicago.edu
Mon Oct 7 17:15:06 CDT 2013
Author: yadunandb
Date: 2013-10-07 17:15:05 -0500 (Mon, 07 Oct 2013)
New Revision: 7145
Added:
SwiftApps/Swift-MapRed/mapred_combiner_V3/dereference.sh
Removed:
SwiftApps/Swift-MapRed/mapred_combiner_V3/dummy
Modified:
SwiftApps/Swift-MapRed/mapred_combiner_V3/LAB_RECORDS
SwiftApps/Swift-MapRed/mapred_combiner_V3/README
SwiftApps/Swift-MapRed/mapred_combiner_V3/beagle.xml
SwiftApps/Swift-MapRed/mapred_combiner_V3/client.c
SwiftApps/Swift-MapRed/mapred_combiner_V3/kway.swift
SwiftApps/Swift-MapRed/mapred_combiner_V3/mapper.sh
SwiftApps/Swift-MapRed/mapred_combiner_V3/reducer.sh
SwiftApps/Swift-MapRed/mapred_combiner_V3/reducer_template.sh
SwiftApps/Swift-MapRed/mapred_combiner_V3/run.sh
SwiftApps/Swift-MapRed/mapred_combiner_V3/teragen.swift
Log:
Updates for combiner and tree
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V3/LAB_RECORDS
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V3/LAB_RECORDS 2013-10-07 21:36:23 UTC (rev 7144)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V3/LAB_RECORDS 2013-10-07 22:15:05 UTC (rev 7145)
@@ -2817,3 +2817,1361 @@
ACTUAL COUNT : 5000000
EXPECTED COUNT: 5000000
---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Could not compile SwiftScript source: line 77:1: expecting EOF, found 'final_result'
+real 0.93
+user 0.85
+sys 0.08
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 42
+Warning: Parameter deref, on line 33, shadows variable of same name on line 45
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1205-qo1iqb0c
+Progress: time: Fri, 04 Oct 2013 12:06:01 -0500
+Progress: time: Fri, 04 Oct 2013 12:06:10 -0500 Stage in:1 Submitted:9
+Progress: time: Fri, 04 Oct 2013 12:06:11 -0500 Submitted:6 Stage out:1 Finished successfully:3
+Progress: time: Fri, 04 Oct 2013 12:06:12 -0500 Submitted:1 Stage out:1 Finished successfully:8
+Progress: time: Fri, 04 Oct 2013 12:06:14 -0500 Stage out:1 Finished successfully:11
+Final status: Fri, 04 Oct 2013 12:06:14 -0500 Finished successfully:14
+real 17.89
+user 24.40
+sys 9.14
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 42
+Warning: Parameter deref, on line 33, shadows variable of same name on line 45
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1208-hayzl915
+Progress: time: Fri, 04 Oct 2013 12:08:03 -0500
+Progress: time: Fri, 04 Oct 2013 12:08:11 -0500 Stage in:1 Submitted:9
+Progress: time: Fri, 04 Oct 2013 12:08:12 -0500 Submitted:4 Stage out:2 Finished successfully:4
+Progress: time: Fri, 04 Oct 2013 12:08:13 -0500 Active:2 Stage out:1 Finished successfully:11
+Progress: time: Fri, 04 Oct 2013 12:08:19 -0500 Stage out:1 Finished successfully:14
+Final status: Fri, 04 Oct 2013 12:08:19 -0500 Finished successfully:16
+real 20.32
+user 21.88
+sys 7.62
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 28, shadows variable of same name on line 42
+Warning: Parameter deref, on line 33, shadows variable of same name on line 45
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1210-z6mpvdt5
+Progress: time: Fri, 04 Oct 2013 12:10:28 -0500
+Progress: time: Fri, 04 Oct 2013 12:10:34 -0500 Stage in:1 Submitted:9
+Progress: time: Fri, 04 Oct 2013 12:10:35 -0500 Submitted:5 Stage out:1 Finished successfully:4
+Progress: time: Fri, 04 Oct 2013 12:10:36 -0500 Active:1 Finished successfully:10
+Progress: time: Fri, 04 Oct 2013 12:10:38 -0500 Stage out:1 Finished successfully:11
+Final status: Fri, 04 Oct 2013 12:10:38 -0500 Finished successfully:14
+real 14.87
+user 21.46
+sys 6.96
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 1000000
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Could not compile SwiftScript source: line 3:1: expecting EOF, found 'import'
+real 0.78
+user 0.80
+sys 0.06
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Could not start execution
+ When processing import kway.swift
+ kway.swift.swift (No such file or directory)
+real 1.90
+user 2.36
+sys 0.08
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter reducer, on line 17, shadows variable of same name on line 45
+Could not start execution
+ Compile error in foreach statement at line 22
+ Compile error in assignment at line 26: You cannot assign value of type fileptr to a variable of type file
+
+real 1.93
+user 2.93
+sys 0.09
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter reducer, on line 17, shadows variable of same name on line 45
+Could not start execution
+ Compile error in foreach statement at line 33
+ Compile error in procedure invocation at line 35
+ Procedure reduce_data is not declared.
+real 2.02
+user 2.92
+sys 0.11
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter reducer, on line 21, shadows variable of same name on line 45
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 43
+Warning: Parameter deref, on line 34, shadows variable of same name on line 46
+Could not start execution
+ Compile error in procedure invocation at line 74
+ Wrong number of procedure output arguments: specified 2 and should be 1
+real 2.00
+user 3.14
+sys 0.09
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 71
+Warning: Parameter deref, on line 34, shadows variable of same name on line 74
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 73
+Could not start execution
+ Compile error in procedure invocation at line 102
+ Wrong number of procedure output arguments: specified 2 and should be 1
+real 1.97
+user 2.80
+sys 0.11
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 71
+Warning: Parameter deref, on line 34, shadows variable of same name on line 74
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 73
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1255-x631g76d
+Progress: time: Fri, 04 Oct 2013 12:55:12 -0500
+Progress: time: Fri, 04 Oct 2013 12:55:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 12:56:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 12:56:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 12:57:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 12:57:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 12:58:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 12:58:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 12:59:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 12:59:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:00:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:00:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:01:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:01:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:02:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:02:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:03:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:03:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:04:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:04:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:05:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:05:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:06:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:06:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:07:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:07:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:08:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:08:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:09:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:09:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:10:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:10:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:11:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:11:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:12:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:12:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:13:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:13:42 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:14:12 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:14:32 -0500 Stage in:1 Submitted:9
+Progress: time: Fri, 04 Oct 2013 13:14:33 -0500 Submitted:5 Active:1 Stage out:1 Finished successfully:3
+Progress: time: Fri, 04 Oct 2013 13:14:35 -0500 Active:2 Stage out:1 Finished successfully:11
+ comb0000.out going to bucket:0
+ comb0002.out going to bucket:1
+ comb0001.out going to bucket:0
+Execution failed:
+ Exception in bash:
+ Arguments: [reducer.sh, comb0002.out]
+ Host: beagle
+ Directory: teragen-20131004-1255-x631g76d/jobs/h/bash-hrbr27gl
+
+Caused by:
+ Error staging in file: /lustre/beagle/yadunandb/Swift-MapRed/mapred_combiner_V3/reducer.sh (No such file or directory)
+org.globus.cog.abstraction.impl.common.execution.JobException: Error staging in file: /lustre/beagle/yadunandb/Swift-MapRed/mapred_combiner_V3/reducer.sh (No such file or directory) (exit code: 520)
+ at org.globus.cog.abstraction.coaster.service.local.JobStatusHandler.requestComplete(JobStatusHandler.java:40)
+ at org.globus.cog.karajan.workflow.service.handlers.RequestHandler.receiveCompleted(RequestHandler.java:88)
+ at org.globus.cog.karajan.workflow.service.channels.AbstractKarajanChannel.handleRequest(AbstractKarajanChannel.java:527)
+ at org.globus.cog.karajan.workflow.service.channels.AbstractStreamKarajanChannel.stepNIO(AbstractStreamKarajanChannel.java:238)
+ at org.globus.cog.karajan.workflow.service.channels.NIOMultiplexer.loop(NIOMultiplexer.java:97)
+ at org.globus.cog.karajan.workflow.service.channels.NIOMultiplexer.run(NIOMultiplexer.java:56)
+ (exit code: 520)
+ reduce_data, teragen.swift, line 57
+ kway, teragen.swift, line 102
+real 1167.82
+user 1459.94
+sys 822.06
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 71
+Warning: Parameter deref, on line 34, shadows variable of same name on line 74
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 73
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1320-2dxclct7
+Progress: time: Fri, 04 Oct 2013 13:20:15 -0500
+Progress: time: Fri, 04 Oct 2013 13:20:29 -0500 Stage in:1 Submitted:19
+Progress: time: Fri, 04 Oct 2013 13:20:30 -0500 Submitted:10 Active:1 Stage out:1 Finished successfully:8
+Progress: time: Fri, 04 Oct 2013 13:20:31 -0500 Stage out:1 Finished successfully:19
+Progress: time: Fri, 04 Oct 2013 13:20:32 -0500 Active:1 Stage out:1 Finished successfully:22
+ comb0000.out going to bucket:0
+ comb0001.out going to bucket:0
+ comb0002.out going to bucket:1
+Progress: time: Fri, 04 Oct 2013 13:20:38 -0500 Active:1 Stage out:1 Finished successfully:24
+Progress: time: Fri, 04 Oct 2013 13:20:43 -0500 Stage out:1 Finished successfully:26
+Final status: Fri, 04 Oct 2013 13:20:43 -0500 Finished successfully:28
+real 32.94
+user 39.98
+sys 16.20
+chunks 20 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 2000000
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:50 X FILESIZE:100
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 71
+Warning: Parameter deref, on line 34, shadows variable of same name on line 74
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 73
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1325-muugqrmd
+Progress: time: Fri, 04 Oct 2013 13:25:14 -0500
+Progress: time: Fri, 04 Oct 2013 13:25:38 -0500 Stage in:1 Submitted:49
+Progress: time: Fri, 04 Oct 2013 13:25:40 -0500 Submitted:45 Active:2 Stage out:1 Finished successfully:2
+Progress: time: Fri, 04 Oct 2013 13:25:41 -0500 Stage in:1 Submitted:38 Active:2 Stage out:2 Finished successfully:7
+Progress: time: Fri, 04 Oct 2013 13:25:42 -0500 Submitted:31 Active:3 Stage out:3 Finished successfully:13
+Progress: time: Fri, 04 Oct 2013 13:25:43 -0500 Submitted:16 Active:7 Stage out:1 Finished successfully:26
+Progress: time: Fri, 04 Oct 2013 13:25:44 -0500 Stage in:2 Submitted:5 Active:2 Stage out:4 Finished successfully:37
+Progress: time: Fri, 04 Oct 2013 13:25:45 -0500 Active:9 Stage out:1 Finished successfully:51
+Progress: time: Fri, 04 Oct 2013 13:25:47 -0500 Active:7 Stage out:1 Finished successfully:53
+Progress: time: Fri, 04 Oct 2013 13:25:49 -0500 Active:5 Stage out:1 Finished successfully:55
+Progress: time: Fri, 04 Oct 2013 13:25:52 -0500 Active:2 Stage out:1 Finished successfully:58
+Progress: time: Fri, 04 Oct 2013 13:25:53 -0500 Active:1 Stage out:1 Finished successfully:59
+ comb0000.out going to bucket:0
+ comb0007.out going to bucket:3
+ comb0001.out going to bucket:0
+ comb0005.out going to bucket:2
+ comb0008.out going to bucket:4
+ comb0004.out going to bucket:2
+ comb0006.out going to bucket:3
+ comb0002.out going to bucket:1
+ comb0009.out going to bucket:4
+ comb0003.out going to bucket:1
+Progress: time: Fri, 04 Oct 2013 13:25:59 -0500 Active:5 Stage out:1 Finished successfully:61
+Progress: time: Fri, 04 Oct 2013 13:26:14 -0500 Active:1 Finished successfully:66
+Progress: time: Fri, 04 Oct 2013 13:26:44 -0500 Active:1 Finished successfully:66
+Progress: time: Fri, 04 Oct 2013 13:27:14 -0500 Active:1 Finished successfully:66
+Progress: time: Fri, 04 Oct 2013 13:27:44 -0500 Active:1 Finished successfully:66
+chunks 50 filesize 100 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 50000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:50 X FILESIZE:1
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 72
+Warning: Parameter deref, on line 34, shadows variable of same name on line 75
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 74
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1330-t51uomd4
+Progress: time: Fri, 04 Oct 2013 13:30:47 -0500
+Progress: time: Fri, 04 Oct 2013 13:31:17 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:31:47 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:32:17 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:32:47 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:33:17 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:33:47 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:34:17 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:34:47 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:35:17 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:35:47 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:36:17 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:36:47 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:37:17 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:37:47 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:38:17 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:38:47 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:39:17 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:39:47 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:40:17 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:40:47 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:41:17 -0500 Submitted:50
+Progress: time: Fri, 04 Oct 2013 13:41:18 -0500 Submitted:49 Stage out:1
+Progress: time: Fri, 04 Oct 2013 13:41:19 -0500 Submitted:37 Active:1 Stage out:1 Finished successfully:11
+Progress: time: Fri, 04 Oct 2013 13:41:20 -0500 Submitted:26 Active:1 Stage out:1 Finished successfully:22
+Progress: time: Fri, 04 Oct 2013 13:41:21 -0500 Submitted:14 Active:1 Stage out:1 Finished successfully:34
+Progress: time: Fri, 04 Oct 2013 13:41:22 -0500 Submitted:2 Stage out:1 Finished successfully:47
+ comb0000.out going to bucket:0
+ comb0001.out going to bucket:0
+bucket[1] contains 0 bucket[0] contains 2 Progress: time: Fri, 04 Oct 2013 13:41:23 -0500 Active:2 Finished successfully:53
+Progress: time: Fri, 04 Oct 2013 13:41:28 -0500 Active:1 Stage out:1 Finished successfully:53
+Progress: time: Fri, 04 Oct 2013 13:41:47 -0500 Active:1 Finished successfully:54
+Progress: time: Fri, 04 Oct 2013 13:42:17 -0500 Active:1 Finished successfully:54
+Progress: time: Fri, 04 Oct 2013 13:42:47 -0500 Active:1 Finished successfully:54
+Progress: time: Fri, 04 Oct 2013 13:43:17 -0500 Active:1 Finished successfully:54
+Progress: time: Fri, 04 Oct 2013 13:43:47 -0500 Active:1 Finished successfully:54
+Progress: time: Fri, 04 Oct 2013 13:44:17 -0500 Active:1 Finished successfully:54
+Progress: time: Fri, 04 Oct 2013 13:44:47 -0500 Active:1 Finished successfully:54
+Progress: time: Fri, 04 Oct 2013 13:45:17 -0500 Active:1 Finished successfully:54
+chunks 50 filesize 1 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 500000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 72
+Warning: Parameter deref, on line 34, shadows variable of same name on line 75
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 74
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1346-89n2onug
+Progress: time: Fri, 04 Oct 2013 13:46:25 -0500
+Progress: time: Fri, 04 Oct 2013 13:46:55 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:47:25 -0500 Submitted:10
+Progress: time: Fri, 04 Oct 2013 13:47:26 -0500 Stage in:1 Submitted:9
+Progress: time: Fri, 04 Oct 2013 13:47:27 -0500 Submitted:4 Active:1 Stage out:1 Finished successfully:4
+Progress: time: Fri, 04 Oct 2013 13:47:28 -0500 Active:1 Stage out:1 Finished successfully:11
+ comb0001.out going to bucket:0
+ comb0000.out going to bucket:0
+bucket[1] contains 0 bucket[0] contains 2 Progress: time: Fri, 04 Oct 2013 13:47:34 -0500 Active:1 Stage out:1 Finished successfully:13
+Progress: time: Fri, 04 Oct 2013 13:47:55 -0500 Active:1 Finished successfully:14
+Progress: time: Fri, 04 Oct 2013 13:48:25 -0500 Active:1 Finished successfully:14
+Progress: time: Fri, 04 Oct 2013 13:48:55 -0500 Active:1 Finished successfully:14
+Progress: time: Fri, 04 Oct 2013 13:49:25 -0500 Active:1 Finished successfully:14
+Progress: time: Fri, 04 Oct 2013 13:49:55 -0500 Active:1 Finished successfully:14
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1350-m7zfpaeg
+Progress: time: Fri, 04 Oct 2013 13:50:13 -0500
+Progress: time: Fri, 04 Oct 2013 13:50:23 -0500 Stage in:1 Submitted:9
+Progress: time: Fri, 04 Oct 2013 13:50:24 -0500 Submitted:5 Stage out:1 Finished successfully:4
+Progress: time: Fri, 04 Oct 2013 13:50:25 -0500 Finished successfully:10
+Progress: time: Fri, 04 Oct 2013 13:50:27 -0500 Stage out:1 Finished successfully:11
+Execution failed:
+ comb0000.out going to bucket:0
+bucket[0] contains 1 tracef(): Unknown format: %d
+ swiftscript:tracef, teragen.swift, line 55
+ kway, teragen.swift, line 104
+real 18.03
+user 25.86
+sys 9.97
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1350-nrke2n45
+Progress: time: Fri, 04 Oct 2013 13:50:45 -0500
+Progress: time: Fri, 04 Oct 2013 13:50:50 -0500 Stage in:1 Submitted:9
+Progress: time: Fri, 04 Oct 2013 13:50:51 -0500 Submitted:5 Stage out:1 Finished successfully:4
+Progress: time: Fri, 04 Oct 2013 13:50:52 -0500 Stage out:1 Finished successfully:10
+Progress: time: Fri, 04 Oct 2013 13:50:53 -0500 Stage out:1 Finished successfully:11
+Execution failed:
+ comb0000.out going to bucket:0
+bucket[0] contains 1 tracef(): Unknown format: %/
+ swiftscript:tracef, teragen.swift, line 55
+ kway, teragen.swift, line 104
+real 13.14
+user 20.16
+sys 6.10
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1351-ojrynh7b
+Progress: time: Fri, 04 Oct 2013 13:51:16 -0500
+Progress: time: Fri, 04 Oct 2013 13:51:23 -0500 Stage in:1 Submitted:9
+Progress: time: Fri, 04 Oct 2013 13:51:24 -0500 Submitted:5 Stage out:1 Finished successfully:4
+Progress: time: Fri, 04 Oct 2013 13:51:25 -0500 Active:1 Finished successfully:10
+Progress: time: Fri, 04 Oct 2013 13:51:26 -0500 Stage out:1 Finished successfully:11
+ Length of array : 1 len mod K_way: 0
+ comb0000.out going to bucket:0
+bucket[0] contains 1 Final status: Fri, 04 Oct 2013 13:51:27 -0500 Finished successfully:15
+real 15.29
+user 22.27
+sys 6.83
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 1000000
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131004-1352-9zoro7ka
+Progress: time: Fri, 04 Oct 2013 13:52:27 -0500
+Progress: time: Fri, 04 Oct 2013 13:52:33 -0500 Stage in:1 Submitted:19
+Progress: time: Fri, 04 Oct 2013 13:52:34 -0500 Submitted:15 Stage out:1 Finished successfully:4
+Progress: time: Fri, 04 Oct 2013 13:52:36 -0500 Submitted:9 Stage out:1 Finished successfully:10
+Progress: time: Fri, 04 Oct 2013 13:52:37 -0500 Submitted:3 Stage out:1 Finished successfully:16
+Progress: time: Fri, 04 Oct 2013 13:52:38 -0500 Active:1 Finished successfully:20
+Progress: time: Fri, 04 Oct 2013 13:52:40 -0500 Stage out:1 Finished successfully:21
+ Length of array : 1 len mod K_way: 0
+ comb0000.out going to bucket:0
+bucket[0] contains 1 items
+Final status: Fri, 04 Oct 2013 13:52:40 -0500 Finished successfully:25
+real 18.15
+user 25.46
+sys 8.64
+chunks 20 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 2000000
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+0.94
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+
+RunID: 20131004-1634-240ueo37
+Progress: time: Fri, 04 Oct 2013 16:34:51 -0500
+Progress: time: Fri, 04 Oct 2013 16:35:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:35:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:36:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:36:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:37:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:37:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:38:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:38:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:39:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:39:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:40:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:40:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:41:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:41:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:42:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:42:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:43:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:43:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:44:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:44:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:45:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:45:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:46:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:46:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:47:21 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:47:51 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 16:48:03 -0500 Stage in:1 Submitted:19
+Progress: time: Fri, 04 Oct 2013 16:48:05 -0500 Stage in:1 Submitted:18 Stage out:1
+Progress: time: Fri, 04 Oct 2013 16:48:06 -0500 Submitted:10 Stage out:3 Finished successfully:7
+Progress: time: Fri, 04 Oct 2013 16:48:08 -0500 Stage in:1 Submitted:1 Stage out:4 Finished successfully:14
+Progress: time: Fri, 04 Oct 2013 16:48:10 -0500 Finished successfully:21
+Progress: time: Fri, 04 Oct 2013 16:48:11 -0500 Stage out:2 Finished successfully:23
+ Length of array : 4 len mod K_way: 2
+ comb0000.out going to bucket:0
+ comb0002.out going to bucket:1
+ comb0001.out going to bucket:0
+ comb0003.out going to bucket:1
+bucket[0] contains 2 items
+bucket[1] contains 2 items
+bucket[2] contains 0 items
+Progress: time: Fri, 04 Oct 2013 16:48:16 -0500 Active:2 Stage out:1 Finished successfully:25
+Progress: time: Fri, 04 Oct 2013 16:48:21 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:48:51 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:49:21 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:49:51 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:50:21 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:50:51 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:51:21 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:51:51 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:52:21 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:52:51 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:53:21 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:53:51 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:54:21 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:54:51 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:55:21 -0500 Active:1 Finished successfully:27
+Progress: time: Fri, 04 Oct 2013 16:55:51 -0500 Active:1 Finished successfully:27
+chunks 20 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+0.94
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+
+RunID: 20131004-1704-ipni41je
+Progress: time: Fri, 04 Oct 2013 17:04:25 -0500
+Progress: time: Fri, 04 Oct 2013 17:04:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:05:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:05:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:06:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:06:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:07:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:07:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:08:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:08:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:09:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:09:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:10:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:10:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:11:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:11:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:12:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:12:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:13:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:13:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:14:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:14:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:15:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:15:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:16:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:16:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:17:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:17:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:18:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:18:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:19:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:19:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:20:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:20:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:21:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:21:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:22:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:22:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:23:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:23:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:24:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:24:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:25:25 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:25:55 -0500 Submitted:20
+Progress: time: Fri, 04 Oct 2013 17:26:25 -0500 Submitted:20
+chunks 20 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Could not start execution
+ Compile error in procedure invocation at line 103
+ Variable p_errs was not declared in this scope.
+real 4.93
+user 2.89
+sys 0.11
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+0.94
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+
+RunID: 20131007-1159-zzt5h0fg
+Progress: time: Mon, 07 Oct 2013 12:00:01 -0500
+Progress: time: Mon, 07 Oct 2013 12:00:09 -0500 Stage in:1 Submitted:9
+Progress: time: Mon, 07 Oct 2013 12:00:10 -0500 Submitted:5 Active:1 Stage out:1 Finished successfully:3
+Progress: time: Mon, 07 Oct 2013 12:00:11 -0500 Active:1 Stage out:1 Finished successfully:8
+Progress: time: Mon, 07 Oct 2013 12:00:13 -0500 Stage out:2 Finished successfully:8
+Progress: time: Mon, 07 Oct 2013 12:00:14 -0500 Active:1 Stage out:1 Finished successfully:12
+Progress: time: Mon, 07 Oct 2013 12:00:20 -0500 Stage out:1 Finished successfully:14
+Execution failed:
+ Exception in bash:
+ Arguments: [dereference.sh, p_final.out]
+ Host: beagle
+ Directory: teragen-20131007-1159-zzt5h0fg/jobs/6/bash-64kaybgl
+
+Caused by:
+ Application /bin/bash failed with an exit code of 1
+ dereference, teragen.swift, line 109
+real 26.66
+user 9.13
+sys 0.44
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:10 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+0.94
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+
+RunID: 20131007-1214-bgf2mn1e
+Progress: time: Mon, 07 Oct 2013 12:14:26 -0500
+Progress: time: Mon, 07 Oct 2013 12:14:34 -0500 Stage in:1 Submitted:9
+Progress: time: Mon, 07 Oct 2013 12:14:35 -0500 Submitted:4 Active:1 Finished successfully:5
+Progress: time: Mon, 07 Oct 2013 12:14:36 -0500 Active:1 Stage out:1 Finished successfully:11
+Progress: time: Mon, 07 Oct 2013 12:14:42 -0500 Stage out:1 Finished successfully:13
+Final status: Mon, 07 Oct 2013 12:14:42 -0500 Finished successfully:15
+real 20.81
+user 9.12
+sys 0.48
+chunks 10 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 1000000
+EXPECTED COUNT: 1000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+0.94
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+
+RunID: 20131007-1401-np3ofaqe
+Progress: time: Mon, 07 Oct 2013 14:01:41 -0500
+Progress: time: Mon, 07 Oct 2013 14:01:44 -0500 Submitting:19 Submitted:1
+Progress: time: Mon, 07 Oct 2013 14:01:52 -0500 Stage in:1 Submitted:19
+Progress: time: Mon, 07 Oct 2013 14:01:53 -0500 Stage in:1 Submitted:14 Stage out:2 Finished successfully:3
+Progress: time: Mon, 07 Oct 2013 14:01:54 -0500 Submitted:7 Active:1 Stage out:1 Finished successfully:11
+Progress: time: Mon, 07 Oct 2013 14:01:55 -0500 Stage out:1 Finished successfully:19
+Progress: time: Mon, 07 Oct 2013 14:01:56 -0500 Active:1 Stage out:1 Finished successfully:21
+Progress: time: Mon, 07 Oct 2013 14:02:02 -0500 Stage out:1 Finished successfully:23
+Final status: Mon, 07 Oct 2013 14:02:02 -0500 Finished successfully:25
+real 35.72
+user 11.84
+sys 1.58
+chunks 20 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 2000000
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+0.94
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+svn: This client is too old to work with working copy '.'. You need
+to get a newer Subversion client, or to downgrade this working copy.
+See http://subversion.tigris.org/faq.html#working-copy-format-change
+for details.
+
+RunID: 20131007-1412-vv6bgvtb
+Progress: time: Mon, 07 Oct 2013 14:12:47 -0500
+Progress: time: Mon, 07 Oct 2013 14:12:49 -0500 Submitting:19 Submitted:1
+Progress: time: Mon, 07 Oct 2013 14:12:57 -0500 Stage in:1 Submitted:19
+Progress: time: Mon, 07 Oct 2013 14:12:58 -0500 Submitted:16 Active:1 Finished successfully:3
+Progress: time: Mon, 07 Oct 2013 14:13:00 -0500 Submitted:13 Stage out:1 Finished successfully:6
+Progress: time: Mon, 07 Oct 2013 14:13:01 -0500 Submitted:8 Active:1 Finished successfully:11
+Progress: time: Mon, 07 Oct 2013 14:13:02 -0500 Submitted:4 Finished successfully:16
+Progress: time: Mon, 07 Oct 2013 14:13:03 -0500 Active:1 Finished successfully:20
+ Length of array : 1 len mod K_way: 0
+ comb0000.out going to bucket:0
+bucket[0] contains 1 items
+Progress: time: Mon, 07 Oct 2013 14:13:04 -0500 Stage out:1 Finished successfully:23
+Execution failed:
+ Exception in bash:
+ Arguments: [dereference.sh, p_final.out]
+ Host: beagle
+ Directory: teragen-20131007-1412-vv6bgvtb/jobs/i/bash-i5cl3cgl
+
+Caused by:
+ Application /bin/bash failed with an exit code of 1
+ dereference, teragen.swift, line 109
+real 31.26
+user 12.73
+sys 1.40
+chunks 20 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1416-6v4l9xb2
+Progress: time: Mon, 07 Oct 2013 14:16:52 -0500
+Progress: time: Mon, 07 Oct 2013 14:16:54 -0500 Submitting:19 Submitted:1
+Progress: time: Mon, 07 Oct 2013 14:17:05 -0500 Stage in:1 Submitted:19
+Progress: time: Mon, 07 Oct 2013 14:17:07 -0500 Submitted:18 Stage out:2
+Progress: time: Mon, 07 Oct 2013 14:17:08 -0500 Submitted:12 Active:1 Stage out:3 Finished successfully:4
+Progress: time: Mon, 07 Oct 2013 14:17:09 -0500 Stage in:1 Submitted:1 Active:2 Stage out:1 Finished successfully:15
+Progress: time: Mon, 07 Oct 2013 14:17:10 -0500 Active:1 Finished successfully:20
+Progress: time: Mon, 07 Oct 2013 14:17:11 -0500 Stage out:1 Finished successfully:24
+ Length of array : 4 len mod K_way: 2
+ comb0000.out going to bucket:0
+ comb0002.out going to bucket:1
+ comb0003.out going to bucket:1
+ comb0001.out going to bucket:0
+bucket[1] contains 2 items
+bucket[2] contains 0 items
+bucket[0] contains 2 items
+Progress: time: Mon, 07 Oct 2013 14:17:17 -0500 Active:1 Stage out:1 Finished successfully:26
+Progress: time: Mon, 07 Oct 2013 14:17:22 -0500 Active:1 Finished successfully:27
+Progress: time: Mon, 07 Oct 2013 14:17:52 -0500 Active:1 Finished successfully:27
+chunks 20 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1420-vhemxuqb
+Progress: time: Mon, 07 Oct 2013 14:20:54 -0500
+Progress: time: Mon, 07 Oct 2013 14:20:56 -0500 Submitting:19 Submitted:1
+Progress: time: Mon, 07 Oct 2013 14:21:03 -0500 Stage in:1 Submitted:19
+Progress: time: Mon, 07 Oct 2013 14:21:04 -0500 Submitted:17 Stage out:1 Finished successfully:2
+Progress: time: Mon, 07 Oct 2013 14:21:05 -0500 Submitted:13 Active:1 Finished successfully:6
+Progress: time: Mon, 07 Oct 2013 14:21:06 -0500 Submitted:10 Stage out:1 Finished successfully:9
+Progress: time: Mon, 07 Oct 2013 14:21:07 -0500 Submitted:6 Finished successfully:14
+Progress: time: Mon, 07 Oct 2013 14:21:08 -0500 Submitted:1 Active:1 Finished successfully:18
+Progress: time: Mon, 07 Oct 2013 14:21:11 -0500 Stage out:1 Finished successfully:21
+ Length of array : 1 len mod K_way: 0
+ comb0000.out going to bucket:0
+bucket[0] contains 1 items
+Final status: Mon, 07 Oct 2013 14:21:12 -0500 Finished successfully:25
+real 28.66
+user 26.51
+sys 2.40
+chunks 20 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 2000000
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1423-rjhx1ljb
+Progress: time: Mon, 07 Oct 2013 14:23:56 -0500
+Progress: time: Mon, 07 Oct 2013 14:23:59 -0500 Submitting:19 Submitted:1
+Progress: time: Mon, 07 Oct 2013 14:24:07 -0500 Stage in:1 Submitted:19
+Progress: time: Mon, 07 Oct 2013 14:24:08 -0500 Submitted:16 Stage out:1 Finished successfully:3
+Progress: time: Mon, 07 Oct 2013 14:24:10 -0500 Submitted:13 Finished successfully:7
+Progress: time: Mon, 07 Oct 2013 14:24:11 -0500 Submitted:9 Stage out:1 Finished successfully:10
+Progress: time: Mon, 07 Oct 2013 14:24:12 -0500 Submitted:5 Stage out:1 Finished successfully:14
+Progress: time: Mon, 07 Oct 2013 14:24:13 -0500 Stage out:1 Finished successfully:19
+Progress: time: Mon, 07 Oct 2013 14:24:15 -0500 Stage out:1 Finished successfully:21
+ Length of array : 1 len mod K_way: 0
+ comb0000.out going to bucket:0
+bucket[0] contains 1 items
+Progress: time: Mon, 07 Oct 2013 14:24:16 -0500 Active:1 Finished successfully:24
+Final status: Mon, 07 Oct 2013 14:24:16 -0500 Finished successfully:25
+real 27.93
+user 27.20
+sys 1.67
+chunks 20 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 2000000
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:100
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1435-3mq4rth3
+Progress: time: Mon, 07 Oct 2013 14:36:01 -0500
+Progress: time: Mon, 07 Oct 2013 14:36:03 -0500 Submitting:19 Submitted:1
+Progress: time: Mon, 07 Oct 2013 14:36:10 -0500 Stage in:1 Submitted:19
+Progress: time: Mon, 07 Oct 2013 14:36:12 -0500 Submitted:18 Stage out:1 Finished successfully:1
+Progress: time: Mon, 07 Oct 2013 14:36:13 -0500 Submitted:16 Stage out:1 Finished successfully:3
+Progress: time: Mon, 07 Oct 2013 14:36:14 -0500 Submitted:14 Stage out:1 Finished successfully:5
+Progress: time: Mon, 07 Oct 2013 14:36:15 -0500 Submitted:12 Stage out:1 Finished successfully:7
+Progress: time: Mon, 07 Oct 2013 14:36:16 -0500 Submitted:10 Stage out:1 Finished successfully:9
+Progress: time: Mon, 07 Oct 2013 14:36:17 -0500 Submitted:8 Stage out:1 Finished successfully:11
+Progress: time: Mon, 07 Oct 2013 14:36:18 -0500 Submitted:5 Active:1 Finished successfully:14
+Progress: time: Mon, 07 Oct 2013 14:36:19 -0500 Active:2 Stage out:1 Finished successfully:17
+Progress: time: Mon, 07 Oct 2013 14:36:21 -0500 Active:2 Stage out:1 Finished successfully:21
+Progress: time: Mon, 07 Oct 2013 14:36:22 -0500 Active:1 Stage out:1 Finished successfully:22
+Progress: time: Mon, 07 Oct 2013 14:36:31 -0500 Active:1 Finished successfully:23
+Progress: time: Mon, 07 Oct 2013 14:36:34 -0500 Stage out:1 Finished successfully:23
+ Length of array : 3 len mod K_way: 1
+ comb0000.out going to bucket:0
+ comb0002.out going to bucket:1
+ comb0001.out going to bucket:0
+bucket[0] contains 2 items
+bucket[1] contains 1 items
+Progress: time: Mon, 07 Oct 2013 14:36:40 -0500 Active:1 Stage out:1 Finished successfully:24
+Progress: time: Mon, 07 Oct 2013 14:36:45 -0500 Stage out:1 Finished successfully:26
+Final status: Mon, 07 Oct 2013 14:36:45 -0500 Finished successfully:28
+real 52.18
+user 60.29
+sys 10.91
+chunks 20 filesize 100 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 20000000
+EXPECTED COUNT: 20000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:50 X FILESIZE:100
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1438-0ajshz7d
+Progress: time: Mon, 07 Oct 2013 14:38:08 -0500
+Progress: time: Mon, 07 Oct 2013 14:38:09 -0500 Selecting site:49 Submitting:1
+Progress: time: Mon, 07 Oct 2013 14:38:10 -0500 Submitting:49 Submitted:1
+Progress: time: Mon, 07 Oct 2013 14:38:37 -0500 Stage in:1 Submitted:49
+Progress: time: Mon, 07 Oct 2013 14:38:38 -0500 Submitted:46 Active:2 Finished successfully:2
+Progress: time: Mon, 07 Oct 2013 14:38:39 -0500 Submitted:41 Active:2 Stage out:1 Finished successfully:6
+Progress: time: Mon, 07 Oct 2013 14:38:40 -0500 Submitted:32 Active:4 Stage out:1 Finished successfully:13
+Progress: time: Mon, 07 Oct 2013 14:38:41 -0500 Submitted:24 Active:5 Stage out:2 Finished successfully:19
+Progress: time: Mon, 07 Oct 2013 14:38:42 -0500 Submitted:17 Active:2 Stage out:5 Finished successfully:26
+Progress: time: Mon, 07 Oct 2013 14:38:44 -0500 Submitted:8 Active:4 Stage out:5 Finished successfully:33
+Progress: time: Mon, 07 Oct 2013 14:38:45 -0500 Active:7 Stage out:1 Finished successfully:42
+Progress: time: Mon, 07 Oct 2013 14:38:46 -0500 Active:8 Stage out:2 Finished successfully:51
+Progress: time: Mon, 07 Oct 2013 14:38:49 -0500 Active:6 Stage out:1 Finished successfully:54
+Progress: time: Mon, 07 Oct 2013 14:38:52 -0500 Active:3 Stage out:1 Finished successfully:57
+Progress: time: Mon, 07 Oct 2013 14:38:54 -0500 Active:1 Stage out:1 Finished successfully:59
+ Length of array : 10 len mod K_way: 5
+ comb0000.out going to bucket:0
+ comb0003.out going to bucket:1
+ comb0004.out going to bucket:2
+ comb0001.out going to bucket:0
+ comb0005.out going to bucket:2
+ comb0009.out going to bucket:4
+ comb0007.out going to bucket:3
+ comb0002.out going to bucket:1
+ comb0008.out going to bucket:4
+ comb0006.out going to bucket:3
+bucket[0] contains 2 items
+bucket[4] contains 2 items
+bucket[2] contains 2 items
+bucket[5] contains 0 items
+bucket[1] contains 2 items
+bucket[3] contains 2 items
+Progress: time: Mon, 07 Oct 2013 14:38:55 -0500 Active:5 Finished successfully:62
+Progress: time: Mon, 07 Oct 2013 14:39:00 -0500 Active:4 Stage out:1 Finished successfully:62
+Progress: time: Mon, 07 Oct 2013 14:39:08 -0500 Active:1 Finished successfully:66
+Progress: time: Mon, 07 Oct 2013 14:39:38 -0500 Active:1 Finished successfully:66
+Progress: time: Mon, 07 Oct 2013 14:40:08 -0500 Active:1 Finished successfully:66
+Progress: time: Mon, 07 Oct 2013 14:40:38 -0500 Active:1 Finished successfully:66
+Progress: time: Mon, 07 Oct 2013 14:41:08 -0500 Active:1 Finished successfully:66
+Progress: time: Mon, 07 Oct 2013 14:41:38 -0500 Active:1 Finished successfully:66
+Progress: time: Mon, 07 Oct 2013 14:42:08 -0500 Active:1 Finished successfully:66
+Progress: time: Mon, 07 Oct 2013 14:42:38 -0500 Active:1 Finished successfully:66
+Progress: time: Mon, 07 Oct 2013 14:43:08 -0500 Active:1 Finished successfully:66
+chunks 50 filesize 100 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 50000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:20, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Could not open: 20.xml
+Execution failed:
+
+ Could not find sites file: 20.xml
+
+real 3.11
+user 5.07
+sys 0.16
+chunks 20 filesize 10 site 20 type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:10
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1454-6ssmruog
+Progress: time: Mon, 07 Oct 2013 14:55:02 -0500
+Progress: time: Mon, 07 Oct 2013 14:55:08 -0500 Stage in:1 Submitted:19
+Progress: time: Mon, 07 Oct 2013 14:55:09 -0500 Submitted:16 Stage out:1 Finished successfully:3
+Progress: time: Mon, 07 Oct 2013 14:55:10 -0500 Submitted:10 Active:1 Finished successfully:9
+Progress: time: Mon, 07 Oct 2013 14:55:11 -0500 Submitted:5 Stage out:1 Finished successfully:14
+Progress: time: Mon, 07 Oct 2013 14:55:12 -0500 Submitted:1 Finished successfully:19
+Progress: time: Mon, 07 Oct 2013 14:55:14 -0500 Stage out:1 Finished successfully:21
+ Length of array : 1 len mod K_way: 0
+ comb0000.out going to bucket:0
+bucket[0] contains 1 items
+Final status: Mon, 07 Oct 2013 14:55:15 -0500 Finished successfully:25
+real 18.28
+user 24.99
+sys 7.78
+chunks 20 filesize 10 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 2000000
+EXPECTED COUNT: 2000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:100
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1455-mr6k67cb
+Progress: time: Mon, 07 Oct 2013 14:55:28 -0500
+Progress: time: Mon, 07 Oct 2013 14:55:33 -0500 Stage in:1 Submitted:19
+Progress: time: Mon, 07 Oct 2013 14:55:34 -0500 Submitted:18 Stage out:1 Finished successfully:1
+Progress: time: Mon, 07 Oct 2013 14:55:35 -0500 Submitted:15 Stage out:1 Finished successfully:4
+Progress: time: Mon, 07 Oct 2013 14:55:36 -0500 Submitted:12 Active:1 Finished successfully:7
+Progress: time: Mon, 07 Oct 2013 14:55:38 -0500 Submitted:10 Stage out:1 Finished successfully:9
+Progress: time: Mon, 07 Oct 2013 14:55:39 -0500 Stage in:1 Submitted:7 Active:1 Finished successfully:11
+Progress: time: Mon, 07 Oct 2013 14:55:40 -0500 Submitted:2 Active:2 Stage out:1 Finished successfully:15
+Progress: time: Mon, 07 Oct 2013 14:55:42 -0500 Active:3 Stage out:1 Finished successfully:21
+Progress: time: Mon, 07 Oct 2013 14:55:44 -0500 Active:1 Stage out:1 Finished successfully:23
+Progress: time: Mon, 07 Oct 2013 14:55:54 -0500 Stage out:1 Finished successfully:24
+ Length of array : 4 len mod K_way: 2
+ comb0000.out going to bucket:0
+ comb0001.out going to bucket:0
+ comb0003.out going to bucket:1
+ comb0002.out going to bucket:1
+bucket[1] contains 2 items
+bucket[2] contains 0 items
+bucket[0] contains 2 items
+Progress: time: Mon, 07 Oct 2013 14:55:58 -0500 Active:3 Finished successfully:25
+Progress: time: Mon, 07 Oct 2013 14:55:59 -0500 Active:2 Stage out:1 Finished successfully:25
+Progress: time: Mon, 07 Oct 2013 14:56:28 -0500 Active:1 Finished successfully:27
+chunks 20 filesize 100 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 20000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:20 X FILESIZE:100
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1459-d99sc8i7
+Progress: time: Mon, 07 Oct 2013 14:59:47 -0500
+Progress: time: Mon, 07 Oct 2013 14:59:54 -0500 Stage in:1 Submitted:19
+Progress: time: Mon, 07 Oct 2013 14:59:55 -0500 Stage in:1 Submitted:16 Active:1 Stage out:1 Finished successfully:1
+Progress: time: Mon, 07 Oct 2013 14:59:56 -0500 Submitted:12 Active:1 Stage out:1 Finished successfully:6
+Progress: time: Mon, 07 Oct 2013 14:59:57 -0500 Stage in:1 Submitted:7 Active:1 Finished successfully:11
+Progress: time: Mon, 07 Oct 2013 14:59:59 -0500 Submitted:3 Active:1 Stage out:1 Finished successfully:15
+Progress: time: Mon, 07 Oct 2013 15:00:00 -0500 Finished successfully:20
+Progress: time: Mon, 07 Oct 2013 15:00:09 -0500 Active:1 Stage out:1 Finished successfully:21
+ Length of array : 2 len-1 mod K_way: 0
+ comb0000.out going to bucket:0
+ comb0001.out going to bucket:0
+bucket[0] contains 2 items
+Progress: time: Mon, 07 Oct 2013 15:00:14 -0500 Stage out:1 Finished successfully:23
+Final status: Mon, 07 Oct 2013 15:00:15 -0500 Finished successfully:26
+real 32.52
+user 40.04
+sys 15.38
+chunks 20 filesize 100 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 20000000
+EXPECTED COUNT: 20000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:50 X FILESIZE:100
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1513-dsiudl0f
+Progress: time: Mon, 07 Oct 2013 15:14:00 -0500
+Progress: time: Mon, 07 Oct 2013 15:14:08 -0500 Stage in:1 Submitted:49
+Progress: time: Mon, 07 Oct 2013 15:14:09 -0500 Submitted:47 Active:1 Stage out:1 Finished successfully:1
+Progress: time: Mon, 07 Oct 2013 15:14:10 -0500 Submitted:43 Active:2 Stage out:1 Finished successfully:4
+Progress: time: Mon, 07 Oct 2013 15:14:11 -0500 Submitted:37 Active:1 Stage out:2 Finished successfully:10
+Progress: time: Mon, 07 Oct 2013 15:14:12 -0500 Stage in:2 Submitted:31 Stage out:2 Finished successfully:15
+Progress: time: Mon, 07 Oct 2013 15:14:13 -0500 Stage in:1 Submitted:25 Active:1 Stage out:2 Finished successfully:21
+Progress: time: Mon, 07 Oct 2013 15:14:14 -0500 Submitted:18 Active:2 Stage out:1 Finished successfully:29
+Progress: time: Mon, 07 Oct 2013 15:14:15 -0500 Stage in:1 Submitted:10 Active:2 Stage out:2 Finished successfully:35
+Progress: time: Mon, 07 Oct 2013 15:14:16 -0500 Submitted:1 Active:4 Stage out:1 Finished successfully:44
+Progress: time: Mon, 07 Oct 2013 15:14:18 -0500 Active:4 Stage out:1 Finished successfully:51
+Progress: time: Mon, 07 Oct 2013 15:14:21 -0500 Active:3 Stage out:1 Finished successfully:52
+Progress: time: Mon, 07 Oct 2013 15:14:29 -0500 Active:2 Stage out:1 Finished successfully:53
+Progress: time: Mon, 07 Oct 2013 15:14:32 -0500 Active:1 Stage out:1 Finished successfully:54
+ Length of array : 5 len-1 mod K_way: 2
+ comb0000.out going to bucket:0
+ comb0003.out going to bucket:1
+ comb0001.out going to bucket:0
+ comb0004.out going to bucket:2
+ comb0002.out going to bucket:1
+bucket[0] contains 2 items
+bucket[2] contains 1 items
+bucket[1] contains 2 items
+Progress: time: Mon, 07 Oct 2013 15:14:37 -0500 Active:2 Stage out:1 Finished successfully:56
+Duplicate mapping found:
+ tmp_err (line 61) and tmp_err (line 61) are both used to write to file://localhost/tmp_err
+ Length of array : 3 len-1 mod K_way: 1
+ bucket0000.out going to bucket:0
+ bucket0001.out going to bucket:0
+ bucket0002.out going to bucket:1
+bucket[1] contains 1 items
+bucket[0] contains 2 items
+Progress: time: Mon, 07 Oct 2013 15:14:43 -0500 Stage out:1 Finished successfully:60
+Progress: time: Mon, 07 Oct 2013 15:14:48 -0500 Stage out:1 Finished successfully:61
+Final status: Mon, 07 Oct 2013 15:14:48 -0500 Finished successfully:63
+real 52.94
+user 55.86
+sys 20.52
+chunks 50 filesize 100 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 13000000
+EXPECTED COUNT: 50000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:30 X FILESIZE:50
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1610-vymvpub4
+Progress: time: Mon, 07 Oct 2013 16:10:41 -0500
+Progress: time: Mon, 07 Oct 2013 16:10:52 -0500 Stage in:1 Submitted:29
+Progress: time: Mon, 07 Oct 2013 16:10:53 -0500 Submitted:27 Stage out:1 Finished successfully:2
+Progress: time: Mon, 07 Oct 2013 16:10:54 -0500 Submitted:21 Active:1 Stage out:1 Finished successfully:7
+Progress: time: Mon, 07 Oct 2013 16:10:56 -0500 Submitted:17 Active:1 Stage out:2 Finished successfully:10
+Progress: time: Mon, 07 Oct 2013 16:10:57 -0500 Submitted:9 Active:1 Stage out:3 Finished successfully:17
+Progress: time: Mon, 07 Oct 2013 16:10:58 -0500 Stage in:1 Submitted:3 Active:1 Stage out:4 Finished successfully:21
+Progress: time: Mon, 07 Oct 2013 16:10:59 -0500 Finished successfully:31
+Progress: time: Mon, 07 Oct 2013 16:11:01 -0500 Active:3 Stage out:1 Finished successfully:32
+Progress: time: Mon, 07 Oct 2013 16:11:03 -0500 Active:1 Stage out:1 Finished successfully:34
+Progress: time: Mon, 07 Oct 2013 16:11:05 -0500 Stage out:1 Finished successfully:35
+ Length of array : 5 len-1 mod K_way: 2
+ comb0003.out going to bucket:1
+ comb0001.out going to bucket:0
+ comb0002.out going to bucket:1
+ comb0004.out going to bucket:2
+ comb0000.out going to bucket:0
+bucket[0] contains 2 items
+bucket[1] contains 2 items
+bucket[2] contains 1 items
+Progress: time: Mon, 07 Oct 2013 16:11:10 -0500 Active:2 Stage out:1 Finished successfully:36
+ Length of array : 3 len-1 mod K_way: 1
+ bucket0000.out going to bucket:0
+ bucket0002.out going to bucket:1
+ bucket0001.out going to bucket:0
+bucket[0] contains 2 items
+bucket[1] contains 1 items
+Progress: time: Mon, 07 Oct 2013 16:11:11 -0500 Stage in:1 Active:1 Finished successfully:39
+Progress: time: Mon, 07 Oct 2013 16:11:16 -0500 Active:1 Stage out:1 Finished successfully:39
+Progress: time: Mon, 07 Oct 2013 16:11:17 -0500 Stage out:1 Finished successfully:41
+Final status: Mon, 07 Oct 2013 16:11:17 -0500 Finished successfully:43
+real 40.96
+user 49.13
+sys 19.60
+chunks 30 filesize 50 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 9000000
+EXPECTED COUNT: 15000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:30 X FILESIZE:100
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 73
+Warning: Parameter deref, on line 34, shadows variable of same name on line 76
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 75
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1615-4azd1d55
+Progress: time: Mon, 07 Oct 2013 16:15:58 -0500
+Progress: time: Mon, 07 Oct 2013 16:16:08 -0500 Stage in:1 Submitted:29
+Progress: time: Mon, 07 Oct 2013 16:16:10 -0500 Submitted:28 Stage out:2
+Progress: time: Mon, 07 Oct 2013 16:16:11 -0500 Submitted:22 Active:2 Stage out:1 Finished successfully:5
+Progress: time: Mon, 07 Oct 2013 16:16:12 -0500 Stage in:1 Submitted:21 Active:1 Stage out:2 Finished successfully:5
+Progress: time: Mon, 07 Oct 2013 16:16:13 -0500 Submitted:16 Active:2 Stage out:1 Finished successfully:11
+Progress: time: Mon, 07 Oct 2013 16:16:15 -0500 Submitted:10 Active:2 Stage out:1 Finished successfully:17
+Progress: time: Mon, 07 Oct 2013 16:16:16 -0500 Submitted:4 Active:2 Stage out:1 Finished successfully:23
+Progress: time: Mon, 07 Oct 2013 16:16:17 -0500 Stage out:1 Finished successfully:29
+Progress: time: Mon, 07 Oct 2013 16:16:26 -0500 Active:2 Stage out:1 Finished successfully:31
+Progress: time: Mon, 07 Oct 2013 16:16:28 -0500 Stage out:1 Finished successfully:33
+ Length of array : 3 len-1 mod K_way: 0
+ comb0000.out going to bucket:0
+ comb0002.out going to bucket:0
+ comb0001.out going to bucket:0
+bucket[0] contains 3 items
+Progress: time: Mon, 07 Oct 2013 16:16:33 -0500 Stage out:1 Finished successfully:34
+Final status: Mon, 07 Oct 2013 16:16:33 -0500 Finished successfully:37
+real 39.56
+user 53.14
+sys 22.53
+chunks 30 filesize 100 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 30000000
+EXPECTED COUNT: 30000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:beagle, CHUNKS:30 X FILESIZE:100
+Warning: Parameter uniq, on line 29, shadows variable of same name on line 75
+Warning: Parameter deref, on line 34, shadows variable of same name on line 78
+Warning: Parameter reducer, on line 39, shadows variable of same name on line 77
+Swift 0.94.1 RC4 swift-r7109 cog-r3802
+
+RunID: 20131007-1712-osyrrn60
+Progress: time: Mon, 07 Oct 2013 17:12:32 -0500
+Progress: time: Mon, 07 Oct 2013 17:12:39 -0500 Stage in:1 Submitted:29
+Progress: time: Mon, 07 Oct 2013 17:12:40 -0500 Submitted:28 Stage out:1 Finished successfully:1
+Progress: time: Mon, 07 Oct 2013 17:12:42 -0500 Submitted:26 Stage out:1 Finished successfully:3
+Progress: time: Mon, 07 Oct 2013 17:12:43 -0500 Submitted:23 Stage out:1 Finished successfully:6
+Progress: time: Mon, 07 Oct 2013 17:12:44 -0500 Submitted:21 Stage out:1 Finished successfully:8
+Progress: time: Mon, 07 Oct 2013 17:12:45 -0500 Submitted:19 Stage out:1 Finished successfully:10
+Progress: time: Mon, 07 Oct 2013 17:12:46 -0500 Submitted:16 Active:1 Stage out:1 Finished successfully:12
+Progress: time: Mon, 07 Oct 2013 17:12:47 -0500 Submitted:11 Active:2 Stage out:1 Finished successfully:16
+Progress: time: Mon, 07 Oct 2013 17:12:49 -0500 Stage in:2 Submitted:8 Stage out:2 Finished successfully:18
+Progress: time: Mon, 07 Oct 2013 17:12:50 -0500 Active:4 Finished successfully:26
+Progress: time: Mon, 07 Oct 2013 17:12:51 -0500 Finished successfully:31
+Progress: time: Mon, 07 Oct 2013 17:12:52 -0500 Active:5 Stage out:1 Finished successfully:31
+Progress: time: Mon, 07 Oct 2013 17:12:53 -0500 Active:3 Stage out:2 Finished successfully:32
+Progress: time: Mon, 07 Oct 2013 17:12:55 -0500 Active:2 Stage out:1 Finished successfully:34
+Progress: time: Mon, 07 Oct 2013 17:13:02 -0500 Active:1 Finished successfully:36
+Progress: time: Mon, 07 Oct 2013 17:13:07 -0500 Stage out:1 Finished successfully:36
+ Length of array : 6 len-1 mod K_way: 1
+ comb0000.out going to bucket:0
+ comb0005.out going to bucket:1
+ comb0001.out going to bucket:0
+ comb0003.out going to bucket:0
+ comb0004.out going to bucket:1
+ comb0002.out going to bucket:0
+bucket[0] contains 4 items
+bucket[1] contains 2 items
+Progress: time: Mon, 07 Oct 2013 17:13:12 -0500 Active:1 Stage out:1 Finished successfully:37
+Progress: time: Mon, 07 Oct 2013 17:13:18 -0500 Stage out:1 Finished successfully:39
+Final status: Mon, 07 Oct 2013 17:13:18 -0500 Finished successfully:41
+real 50.56
+user 62.78
+sys 27.55
+chunks 30 filesize 100 site beagle type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 30000000
+EXPECTED COUNT: 30000000
+---------------------------------------------
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V3/README
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V3/README 2013-10-07 21:36:23 UTC (rev 7144)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V3/README 2013-10-07 22:15:05 UTC (rev 7145)
@@ -1,10 +1,40 @@
-Run teragen
+Swift-Reduce
+============
+Run MapReduce style applications with Swift. Since swift is a file based
+model as opposed to the Key-Value record based approach of MapReduce, there
+are differences in the way the user supplied Map and Reduce functions
+should work.
+map_core.sh
+-----------
+
+The user supplied Map function should be a bash function named mapper().
+It would take any number of arguments from swift and would output it's
+results to stdout. Swift would redirect the output to appropriate files.
+The mapper function should be placed in a file map_core.sh.
+
+reduce_core.sh
+--------------
+
+The Reduce function should be named reduce() and placed in reduce_core.sh.
+This function would be passed intermediate files generated from the map
+stage and output its results to a file of its choosing. The filename of
+the result file is output to stdout.
+
+setup.sh
+--------
+
+Run the setup.sh script to embed the map_core.sh and reduce_core.sh scripts
+in the wrapper scripts used by swift in runtime.
+
+
+Example - Teragen
+=================
+
Record
======
-
-Each record consists of 1 number padded to 100 bytes.
+Each record consists of 1 random number and count, padded to 100 bytes.
So, in the usual run a file of 10^8 records generated would have 10^6 bytes.
@@ -14,5 +44,4 @@
teragen will split the task of generating files containing 10^8 records (~100 MB)
over 10000 tasks, thus generating in total 1 Tera records.
-
./teragen_wrap.sh 10 # Generates 10000000
\ No newline at end of file
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V3/beagle.xml
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V3/beagle.xml 2013-10-07 21:36:23 UTC (rev 7144)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V3/beagle.xml 2013-10-07 22:15:05 UTC (rev 7145)
@@ -12,13 +12,13 @@
<profile namespace="globus" key="maxWalltime">00:20:00</profile>
<profile namespace="globus" key="queue">batch</profile>
<profile namespace="globus" key="userHomeOverride">/lustre/beagle/yadunandb/swiftwork</profile>
- <profile namespace="globus" key="slots">4</profile>
+ <profile namespace="globus" key="slots">10</profile>
<profile namespace="globus" key="maxnodes">1</profile>
<profile namespace="globus" key="nodeGranularity">1</profile>
<profile namespace="karajan" key="jobThrottle">1.00</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
<!-- <filesystem provider="local"/> -->
<!-- <workdirectory>/lustre/beagle/yadunandb/swiftwork</workdirectory> -->
- <workdirectory>/tmp/yadunandb/swiftwork</workdirectory>
+ <workdirectory>/tmp/yadunandb/swiftwork</workdirectory>
</pool>
</config>
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V3/client.c
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V3/client.c 2013-10-07 21:36:23 UTC (rev 7144)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V3/client.c 2013-10-07 22:15:05 UTC (rev 7145)
@@ -70,7 +70,7 @@
if ( (con_res = connect(sockfd, p->ai_addr, p->ai_addrlen)) == -1) {
close(sockfd);
- printf("Connect failed with %d:%s\n", con_res, strerror(errno));
+ fprintf(stderr,"Connect failed with %d:%s\n", con_res, strerror(errno));
perror("client: connect");
continue;
}
@@ -85,30 +85,30 @@
inet_ntop(p->ai_family, get_in_addr((struct sockaddr *)p->ai_addr),
s, sizeof s);
- //printf("client: connecting to %s\n", s);
+ fprintf(stderr,"client: connecting to %s\n", s);
freeaddrinfo(servinfo); // all done with this structure
//Send message to server
- //printf("Client sending contents of file %s to server \n",argv[2]);
+ fprintf(stderr,"Client sending contents of file %s to server \n",argv[2]);
// Read file contents into a buffer;
if ( (f_in = fopen(argv[2],"r")) == NULL ){
- printf("Opening[%s] failed: %s\n", argv[2], strerror(errno));
+ fprintf(stderr, "Opening[%s] failed: %s\n", argv[2], strerror(errno));
exit(errno);
};
fseek(f_in, 0, SEEK_END);
file_size = ftell(f_in); // Find length
- //printf("File size: %d\n", file_size);
+ fprintf(stderr, "File size: %d\n", file_size);
fseek(f_in, 0, SEEK_SET);
- buffer = (char *) malloc (file_size);
+ buffer = (char *) malloc ((sizeof (char)) * (file_size+2));
fread(buffer, file_size, 1, f_in);
buffer[file_size] = '\0';
fclose(f_in);
- //printf("Content of file : \n%s\n", buffer);
+ fprintf(stderr, "Content of file : \n%s\n", buffer);
if (send(sockfd, buffer, file_size, 0) == -1)
perror("send");
@@ -116,7 +116,7 @@
//Receive message back from server
reply_buf = (char *)malloc(MAXDATASIZE);
- //printf("Client receiving message from server \n");
+ fprintf(stderr, "Client receiving message from server \n");
if ((numbytes = recv(sockfd, reply_buf, MAXDATASIZE-1, 0)) == -1) {
perror("recv");
exit(1);
Added: SwiftApps/Swift-MapRed/mapred_combiner_V3/dereference.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V3/dereference.sh (rev 0)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V3/dereference.sh 2013-10-07 22:15:05 UTC (rev 7145)
@@ -0,0 +1,20 @@
+#!/bin/bash
+# The derefernce script takes one fileptr as input
+# and dumps the content of the file to stdout.
+TOKEN=($(cat $1))
+REMOTE=${TOKEN[0]}
+FILE=${TOKEN[1]}
+TARGETFILE=$(basename $FILE)
+REPODIR=$(dirname $FILE)
+echo "$REMOTE $HOSTNAME" 1>&2
+if [ "$REMOTE" == "$(hostname -f)" ]
+then # Move to an identifiable file if local
+ cat $FILE
+else # Fetch to an identifiable file if remote
+ echo "chirp_get $REMOTE $TARGETFILE $FILE" 1>&2
+ chirp_get $REMOTE $TARGETFILE $FILE
+ cat $FILE
+ rm $FILE
+fi
+
+
Property changes on: SwiftApps/Swift-MapRed/mapred_combiner_V3/dereference.sh
___________________________________________________________________
Added: svn:executable
+ *
Deleted: SwiftApps/Swift-MapRed/mapred_combiner_V3/dummy
===================================================================
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V3/kway.swift
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V3/kway.swift 2013-10-07 21:36:23 UTC (rev 7144)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V3/kway.swift 2013-10-07 22:15:05 UTC (rev 7145)
@@ -1,46 +1,89 @@
type file;
type script;
+type fileptr;
+/*
app (file out, file err) comb_data (script comb, file array[])
{
bash @comb @array stdout=@out stderr=@err;
}
+*/
+app (fileptr out, file err) kway_reduce (script comb, fileptr array[])
+{
+ bash @comb @array stdout=@out stderr=@err;
+}
-// This stage is done without any file pointers as the
-// data movement is expected to be minimal.
-(fileptr final) kway (script comb, int K_way, fileptr array[])
+// This stage is done with file pointers so as to keep
+// data movement to a minimum. In the test-case being
+// used this is going to give a performance hit as the
+// actual data file is as large as the fileptr and a chirp_get
+// is used to fetch the file.
+(fileptr final) kway (script reducer, int K_way, fileptr array[])
{
- file next_level[];
- int len = @length(array);
- file bucket[][];
- foreach index, i in [0:len-1]
- {
- int bucketid = index %/ K_way;
- bucket[bucketid][index %% K_way] = array[index];
+ file next_level[];
+ int len = @length(array);
+ fileptr bucket[][];
+
+ foreach index, i in [0:len-1]
+ {
+ int bucketid = index %/ K_way;
+ bucket[bucketid][index %% K_way] = array[index];
tracef(" %s going to bucket:%i \n", @array[index], bucketid);
- }
+ }
+
+ fileptr bucket_out[] <simple_mapper; prefix="bucket", suffix=".out">;
+ file bucket_err[] <simple_mapper; prefix="bucket", suffix=".err">;
+
+ foreach id in [0:(len%/K_way)]
+ {
+ (bucket_out[id], bucket_err[id])= kway_reduce(reducer, bucket[id]);
+ }
+ file tmp_err <"tmp_err">;
+ if ( @length(bucket_out) > K_way ){
+ final = kway(reducer, K_way, bucket_out);
+ }else{
+ (final, tmp_err) = kway_reduce(reducer, bucket_out);
+ }
+}
- file bucket_out[] <simple_mapper; prefix="bucket", suffix=".out">;
- file bucket_err[] <simple_mapper; prefix="bucket", suffix=".err">;
- foreach id in [0:(len%/K_way)]
- {
- (bucket_out[id], bucket_err[id]) = comb_data (comb, bucket[id]);
- }
- file tmp_err <"tmp_err">;
- if ( @length(bucket_out) > K_way ){
- final = kway(comb, bucket_out);
- }else{
- (final, tmp_err) = comb_data(comb, bucket_out);
- }
+(fileptr final) kway_2 (script reducer, int K_way, fileptr array[])
+{
+
+ int len = @length(array);
+ fileptr bucket[][];
+ file next_level[];
+ file tmp_err <"tmp_err">;
+
+ foreach index, i in [0:len-1]
+ {
+ int bucketid = index %/ K_way;
+ bucket[bucketid][index %% K_way] = array[index];
+ tracef(" %s going to bucket:%i \n", @array[index], bucketid);
+ }
+
+ fileptr bucket_out[] <simple_mapper; prefix="bucket", suffix=".out">;
+ file bucket_err[] <simple_mapper; prefix="bucket", suffix=".err">;
+
+ foreach id in [0:(len%/K_way)]
+ {
+ (bucket_out[id], bucket_err[id])= kway_reduce(reducer, bucket[id]);
+ }
+
+ if ( @length(bucket_out) > K_way ){
+ final = kway(reducer, K_way, bucket_out);
+ }else{
+ (final, tmp_err) = kway_reduce(reducer, bucket_out);
+ }
}
-file inputs[] <filesys_mapper; prefix="tgen", suffix=".out">;
-file final <"final">;
+fileptr inputs[] <filesys_mapper; prefix="tgen", suffix=".out">;
+fileptr final <"final">;
foreach i,ind in inputs{
tracef("File[%i] : %s\n", ind, at filename(i));
}
script dummy <"dummy.sh">;
-final = kway(dummy, 10, inputs);
+final = kway(dummy, 5, inputs);
+
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V3/mapper.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V3/mapper.sh 2013-10-07 21:36:23 UTC (rev 7144)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V3/mapper.sh 2013-10-07 22:15:05 UTC (rev 7145)
@@ -2,9 +2,6 @@
# By default with ARG1:100 and SLICESIZE=10000, this script will generate
# 10^6 records.
-
-#TODO: Design, what args would the user supply, which gets passed on to the
-#actual user function
ARG1=1
[ ! -z $1 ] && ARG1=$1
@@ -34,7 +31,6 @@
FILE=$PWD/$FILE
fi
-# The user supplied map function
#USER_MAP
mapper (){
ARG1=$1;
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V3/reducer.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V3/reducer.sh 2013-10-07 21:36:23 UTC (rev 7144)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V3/reducer.sh 2013-10-07 22:15:05 UTC (rev 7145)
@@ -1,5 +1,5 @@
#!/bin/bash
-# The combiner is expecting to receive filenames as args
+# The reducer is expecting to receive filenames as args
# Each file would contains strings in the format HOSTNAME FILEPATH
# The files passed here are filepointers which need to be resolved
@@ -39,6 +39,8 @@
echo "chirp_get $REMOTE $TARGETFILE $REPODIR/$ID.$RANDOM.imd2" 1>&2
chirp_get $REMOTE $TARGETFILE $REPODIR/$ID.$RANDOM.imd2
fi
+
+
}
for file_token in ${FILES[*]};
@@ -51,6 +53,9 @@
FILE=${TOKEN[1]}
CHIRPDIR=$(dirname $FILE)
echo "$HOSTNAME $CHIRPDIR $CHIRPDIR/$ID*imd2 " 1>&2
-cat $(reduce $CHIRPDIR/$ID*imd2)
-rm -rf $CHIRPDIR/$ID*{imd,imd2} &> /dev/null &
+#cat $(reduce $CHIRPDIR/$ID*imd2)
+RESULT_FILE=$(reduce $CHIRPDIR/$ID*imd2)
+echo "$HOSTNAME $RESULT_FILE"
+echo "Content of $RESULT_FILE : $(cat $RESULT_FILE)" 1>&2
+rm -rf $CHIRPDIR/$ID*imd2 &> /dev/null &
exit 0
\ No newline at end of file
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V3/reducer_template.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V3/reducer_template.sh 2013-10-07 21:36:23 UTC (rev 7144)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V3/reducer_template.sh 2013-10-07 22:15:05 UTC (rev 7145)
@@ -1,5 +1,5 @@
#!/bin/bash
-# The combiner is expecting to receive filenames as args
+# The reducer is expecting to receive filenames as args
# Each file would contains strings in the format HOSTNAME FILEPATH
# The files passed here are filepointers which need to be resolved
@@ -31,6 +31,8 @@
echo "chirp_get $REMOTE $TARGETFILE $REPODIR/$ID.$RANDOM.imd2" 1>&2
chirp_get $REMOTE $TARGETFILE $REPODIR/$ID.$RANDOM.imd2
fi
+
+
}
for file_token in ${FILES[*]};
@@ -43,6 +45,7 @@
FILE=${TOKEN[1]}
CHIRPDIR=$(dirname $FILE)
echo "$HOSTNAME $CHIRPDIR $CHIRPDIR/$ID*imd2 " 1>&2
-cat $(reduce $CHIRPDIR/$ID*imd2)
-rm -rf $CHIRPDIR/$ID*{imd,imd2} &> /dev/null &
+#cat $(reduce $CHIRPDIR/$ID*imd2)
+echo "$HOSTNAME $(reduce $CHIRPDIR/$ID*imd2)"
+rm -rf $CHIRPDIR/$ID*imd2 &> /dev/null &
exit 0
\ No newline at end of file
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V3/run.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V3/run.sh 2013-10-07 21:36:23 UTC (rev 7144)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V3/run.sh 2013-10-07 22:15:05 UTC (rev 7145)
@@ -18,8 +18,9 @@
if [ "$SITE" == "beagle" ]
then
# ./compile
- cp exec_server /lustre/beagle/yadunandb/bin/
- cp exec_client /lustre/beagle/yadunandb/bin/
+# cp exec_server /lustre/beagle/yadunandb/bin/
+# cp exec_client /lustre/beagle/yadunandb/bin/
+ echo "Skipping compile and copy"
fi;
#Call as check_error $? <ERR_CODE> <Error message>
check_error()
Modified: SwiftApps/Swift-MapRed/mapred_combiner_V3/teragen.swift
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V3/teragen.swift 2013-10-07 21:36:23 UTC (rev 7144)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V3/teragen.swift 2013-10-07 22:15:05 UTC (rev 7145)
@@ -1,3 +1,4 @@
+//import "kway";
type file;
type script;
@@ -20,7 +21,7 @@
bash @comb node_string stdout=@out stderr=@err;
}
-app (file out, file err) reduce_data (script comb, fileptr array[])
+app (fileptr out, file err) reduce_data (script comb, fileptr array[])
{
bash @comb @array stdout=@out stderr=@err;
}
@@ -30,7 +31,43 @@
bash @uniq @filepointers stdout=@out stderr=@err;
}
+app (file out, file err) dereference (script deref, fileptr ptr)
+{
+ bash @deref @ptr stdout=@out stderr=@err;
+}
+(fileptr final) kway (script reducer, int K_way, fileptr array[])
+{
+ file next_level[];
+ int len = @length(array);
+ fileptr bucket[][];
+
+ foreach index, i in [0:len-1]
+ {
+ int bucketid = index %/ K_way;
+ bucket[bucketid][index %% K_way] = array[index];
+ tracef(" %s going to bucket:%i \n", @array[index], bucketid);
+ }
+
+ // fileptr bucket_out[] <simple_mapper; prefix="bucket", suffix=".out">;
+ // file bucket_err[] <simple_mapper; prefix="bucket", suffix=".err">;
+ fileptr bucket_out[];
+ file bucket_err[];
+
+ tracef(" Length of array : %i len-1 mod K_way: %i \n", len, (len-1)%/K_way);
+ foreach id in [0:((len-1)%/K_way)]
+ {
+ (bucket_out[id], bucket_err[id])= reduce_data (reducer, bucket[id]);
+ tracef("bucket[%i] contains %i items \n", id, @length(bucket[id]));
+ }
+ file tmp_err;
+ if ( @length(bucket_out) > K_way ){
+ final = kway(reducer, K_way, bucket_out);
+ }else{
+ (final, tmp_err) = reduce_data (reducer, bucket_out);
+ }
+}
+
int files = @toInt(@arg("files","0"));
int fsize = @toInt(@arg("filesize","10")); // This would make 10^4 records per file
@@ -38,6 +75,7 @@
script uniq <"node_uniq.sh">;
script combiner <"combiner.sh">;
script reducer <"reducer.sh">;
+script deref <"dereference.sh">;
fileptr tgen_out[] <simple_mapper; prefix="tgen", suffix=".out">;
file tgen_err[] <simple_mapper; prefix="tgen", suffix=".err">;
@@ -62,9 +100,14 @@
// This is the final reduce stage, here results from all nodes are
// brought to a single node for reduction
// Tree based implementation could be used.
-file final <"final_result">;
-file errs <"final_errors">;
-(final, errs) = reduce_data(reducer, comb_out);
+fileptr p_final <"p_final.out">;
+file p_errs <"p_final.err">;
+//(p_final, p_errs) = reduce_data(reducer, comb_out);
+p_final = kway(reducer, 4, comb_out);
+
+file f_out <"final_result">;
+file f_err <"final_errors">;
+(f_out, f_err) = dereference (deref, p_final);
// This version of reduce_data uses the results from map stage directly
//(final, errs) = reduce_data(reducer, tgen_out);
More information about the Swift-commit
mailing list